diff --git a/.travis.yml b/.travis.yml index e2211a7..2399b91 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: go go: - - 1.9 - 1.11 env: - "PATH=/home/travis/gopath/bin:$PATH" diff --git a/Dockerfile b/Dockerfile index 7d66955..f1929ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9.2-alpine AS builder +FROM golang:1.11.4-alpine3.8 AS builder LABEL maintainer="joona@kuori.org" RUN apk add --update gcc musl-dev git diff --git a/Gopkg.lock b/Gopkg.lock index 9584248..713c7af 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -2,12 +2,12 @@ [[projects]] - digest = "1:289dd4d7abfb3ad2b5f728fbe9b1d5c1bf7d265a3eb9ef92869af1f7baba4c7a" + digest = "1:e4b30804a381d7603b8a344009987c1ba351c26043501b23b8c7ce21f0b67474" name = "github.com/BurntSushi/toml" packages = ["."] pruneopts = "" - revision = "b26d9c308763d68093482582cea63d69be07a0f0" - version = "v0.3.0" + revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005" + version = "v0.3.1" [[projects]] digest = "1:9ceecb4271682fd824475d451b6abf02b99ad04e72f8de3408a9d8b7fd15b933" @@ -18,12 +18,12 @@ version = "v1.5" [[projects]] - digest = "1:56c130d885a4aacae1dd9c7b71cfe39912c7ebc1ff7d2b46083c8812996dc43b" + digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77" name = "github.com/davecgh/go-spew" packages = ["spew"] pruneopts = "" - revision = "346938d642f2ec3594ed81d874461961cd0faa76" - version = "v1.1.0" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" [[projects]] branch = "master" @@ -34,12 +34,12 @@ revision = "8d10e4a1bae52cd8b81ffdec3445890d6dccab3d" [[projects]] - digest = "1:55848e643a99a9dfceb19e090ce67111328fbb1780f34c62a0430994ff85fb90" + digest = "1:2defd14040b1ca18840524e9d0f9cfab0ea8d2f2d396b343d221f8bc8e21ab9d" name = "github.com/fatih/structs" packages = ["."] pruneopts = "" - revision = "a720dfa8df582c51dee1b36feabb906bde1588bd" - version = "v1.0" + revision = "4966fc68f5b7593aafa6cbbba2d65ec6e1416047" + version = "v1.1.0" [[projects]] branch = "master" @@ -58,20 +58,20 @@ revision = "6f8212e8d10df7383609d3c377ca08884d8f3ec0" [[projects]] - branch = "master" - digest = "1:9abc49f39e3e23e262594bb4fb70abf74c0c99e94f99153f43b143805e850719" + digest = "1:cea4aa2038169ee558bf507d5ea02c94ca85bcca28a4c7bb99fd59b31e43a686" name = "github.com/google/go-querystring" packages = ["query"] pruneopts = "" - revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a" + revision = "44c6ddd0a2342c386950e880b658017258da92fc" + version = "v1.0.0" [[projects]] - digest = "1:c1d7e883c50a26ea34019320d8ae40fad86c9e5d56e63a1ba2cb618cef43e986" + digest = "1:a25a2c5ae694b01713fb6cd03c3b1ac1ccc1902b9f0a922680a88ec254f968e1" name = "github.com/google/uuid" packages = ["."] pruneopts = "" - revision = "064e2069ce9c359c118179501254f67d7d37ba24" - version = "0.2" + revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8" + version = "v1.1.0" [[projects]] digest = "1:788735d9cba7f51b7cc86e6c7ba7b40b0b7c7b374bf3a0b5fa7c929fa2af2da8" @@ -82,15 +82,15 @@ version = "v1.1.0" [[projects]] - digest = "1:3c818dada3e41bdb0f509f78e6775610f1bb179449ec8c4c86a45fae35460f3f" + digest = "1:deeb6536ebbcb66ab11213ecfaac9daadfcd004dd057446a4980ffd6c15c36e2" name = "github.com/julienschmidt/httprouter" packages = ["."] pruneopts = "" - revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669" - version = "v1.1" + revision = "348b672cd90d8190f8240323e372ecd1e66b59dc" + version = "v1.2.0" [[projects]] - digest = "1:4d0614a5d2e5e394368521b087428b2996ae95ebc4699afabc61adac9b7cec38" + digest = "1:064eb4d2277fb6a23944380644268b99acc0ae0576c72d0b6d7288d3c2fe4a1e" name = "github.com/klauspost/compress" packages = [ "flate", @@ -98,51 +98,59 @@ "zlib", ] pruneopts = "" - revision = "b939724e787a27c0005cabe3f78e7ed7987ac74f" - version = "v1.4.0" + revision = "30be6041bed523c18e269a700ebd9c2ea9328574" + version = "v1.4.1" [[projects]] - digest = "1:f0117357f14b0a625ddbbe25e23637291ac0276402fcfd25fc447422456364ce" + digest = "1:198af4ad42013a3779477b597458f860a6a99e57e1674fa4f61eb04b8b5b768f" name = "github.com/klauspost/cpuid" packages = ["."] pruneopts = "" - revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da" - version = "v1.1" + revision = "e7e905edc00ea8827e58662220139109efea09db" + version = "v1.2.0" + +[[projects]] + digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3" + name = "github.com/konsorten/go-windows-terminal-sequences" + packages = ["."] + pruneopts = "" + revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242" + version = "v1.0.1" [[projects]] branch = "master" - digest = "1:09792d732b079867772cdbabdf7dc54ef9f9d04c998a9ce6226657151fccbb94" + digest = "1:bc36cd980d0069137800b505af4937c6109f4dc7cefe39d7c2efc7c8d51528d6" name = "github.com/lib/pq" packages = [ ".", "oid", ] pruneopts = "" - revision = "90697d60dd844d5ef6ff15135d0203f65d2f53b8" + revision = "9eb73efc1fcc404148b56765b0d3f61d9a5ef8ee" [[projects]] - digest = "1:bc03901fc8f0965ccba8bc453eae21a9b04f95999eab664c7de6dc7290f4e8f4" + digest = "1:8bbdb2b3dce59271877770d6fe7dcbb8362438fa7d2e1e1f688e4bf2aac72706" name = "github.com/mattn/go-sqlite3" packages = ["."] pruneopts = "" - revision = "25ecb14adfc7543176f7d85291ec7dba82c6f7e4" - version = "v1.9.0" + revision = "c7c4067b79cc51e6dfdcef5c702e74b1e0fa7c75" + version = "v1.10.0" [[projects]] - digest = "1:4c8d8358c45ba11ab7bb15df749d4df8664ff1582daead28bae58cf8cbe49890" + digest = "1:103c1001b79009329824bc60f93f8f7150bf1ad31f18d29a476deb8a2de0f002" name = "github.com/miekg/dns" packages = ["."] pruneopts = "" - revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" - version = "v1.0.8" + revision = "8fc2e5773bbd308ca2fcc962fd8d25c1bd0f6743" + version = "v1.1.4" [[projects]] - branch = "master" digest = "1:fe67641b990bdc1802f8a1e462a4924210a8762a8a17b72e09656049c906b871" name = "github.com/moul/http2curl" packages = ["."] pruneopts = "" revision = "9ac6cf4d929b2fa8fd2d2e6dec5bb0feb4f4911d" + version = "v1.0.0" [[projects]] digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411" @@ -153,12 +161,12 @@ version = "v1.0.0" [[projects]] - digest = "1:78c9cf43ddeacd0e472f412082227a0fac2ae107ee60e9112156f9371f9912cf" + digest = "1:5f47c69f85311c4dc292be6cc995a0a3fe8337a6ce38ef4f71e5b7efd5ad42e0" name = "github.com/rs/cors" packages = ["."] pruneopts = "" - revision = "3fb1b69b103a84de38a19c3c6ec073dd6caa4d3f" - version = "v1.5.0" + revision = "9a47f48565a795472d43519dd49aac781f3034fb" + version = "v1.6.0" [[projects]] digest = "1:3962f553b77bf6c03fc07cd687a22dd3b00fe11aa14d31194f5505f5bb65cdc8" @@ -169,38 +177,38 @@ version = "v1.0.0" [[projects]] - digest = "1:3fcbf733a8d810a21265a7f2fe08a3353db2407da052b233f8b204b5afc03d9b" + digest = "1:9a3c631555e0351fdc4e696577bb63afd90c399d782a8462dba9d100d7021db3" name = "github.com/sirupsen/logrus" packages = [ ".", "hooks/test", ] pruneopts = "" - revision = "3e01752db0189b9157070a0e1668a620f9a85da2" - version = "v1.0.6" + revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5" + version = "v1.3.0" [[projects]] - digest = "1:c587772fb8ad29ad4db67575dad25ba17a51f072ff18a22b4f0257a4d9c24f75" + digest = "1:381bcbeb112a51493d9d998bbba207a529c73dbb49b3fd789e48c63fac1f192c" name = "github.com/stretchr/testify" packages = [ "assert", "require", ] pruneopts = "" - revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" - version = "v1.2.2" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" [[projects]] - branch = "master" digest = "1:857a9ecd5cb13379ecc8f798f6e6b6b574c98b9355657d91e068275f1120aaf7" name = "github.com/valyala/bytebufferpool" packages = ["."] pruneopts = "" revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7" + version = "v1.0.0" [[projects]] branch = "master" - digest = "1:ed95b0b73de30dc5507de18b0cd6a2b79dc0ef685eefeb27fd1386f7a4e04f2b" + digest = "1:5838606075eb2e11d5b99d4526e36863a9315f006d322abfbf5a7fff73250271" name = "github.com/valyala/fasthttp" packages = [ ".", @@ -208,7 +216,7 @@ "stackless", ] pruneopts = "" - revision = "e5f51c11919d4f66400334047b897ef0a94c6f3c" + revision = "aaec9b0fe2d01bd3ff761345ba47bb94a9b0dc59" [[projects]] branch = "master" @@ -227,12 +235,12 @@ revision = "bd5ef7bd5415a7ac448318e64f11a24cd21e594b" [[projects]] - branch = "master" - digest = "1:0dd2250939dcf18c0b1a7b0e364e13b083331f723a6853ff2950a0c4273ff6e2" + digest = "1:c2c74271189fe48e23c25216570008b7ecc5782688d6efd81325886ac6f62ad0" name = "github.com/xeipuuv/gojsonschema" packages = ["."] pruneopts = "" - revision = "6cd6dcbc9e7514bea255a9241665a6c2d0b37fb4" + revision = "f971f3cd73b2899de6923801c147f075263e0c50" + version = "v1.1.0" [[projects]] branch = "master" @@ -263,7 +271,7 @@ [[projects]] branch = "master" - digest = "1:4cae11053a5fc8e7b08228fcc14d161d3e60b64ba508a8b216937da472690991" + digest = "1:cbc1ebc01ec2ceb2c3cc7b9e33357dbc3eff173335f02d9f01603f14102602a7" name = "golang.org/x/crypto" packages = [ "acme", @@ -275,11 +283,11 @@ "ssh/terminal", ] pruneopts = "" - revision = "de0752318171da717af4ce24d0a2e8626afaeb11" + revision = "b8fe1690c61389d7d2a8074a507d1d40c5d30448" [[projects]] branch = "master" - digest = "1:67c2d940f2d5c017ef88e9847709dca9b38d5fe82f1e33fb42ace515219f22f1" + digest = "1:1e9704e5379e68ac473c28aeb3f7e7cd4036ae8a246bf0285b5ebdbb8e0cfacf" name = "golang.org/x/net" packages = [ "bpf", @@ -291,18 +299,18 @@ "publicsuffix", ] pruneopts = "" - revision = "f9ce57c11b242f0f1599cf25c89d8cb02c45295a" + revision = "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf" [[projects]] branch = "master" - digest = "1:8812fbc18f45708b5580ed61267fefe5eeb29b36773bdfaad48b0843e3810c02" + digest = "1:4d6f20f5bc52af99569b1be5ea55d4e22646d5e022b92f2d3d75347628614734" name = "golang.org/x/sys" packages = [ "unix", "windows", ] pruneopts = "" - revision = "f0d5e33068cb57c22a181f5df0ffda885309eb5a" + revision = "7ae0202eb74c2b534255c71b5a15fa4115aabbcc" [[projects]] digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4" diff --git a/README.md b/README.md index 817786d..8306c66 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ See the INSTALL section for information on how to do this. ## Installation -1) Install [Go 1.9 or newer](https://golang.org/doc/install). +1) Install [Go 1.11 or newer](https://golang.org/doc/install). 2) Install acme-dns: `go get github.com/joohoi/acme-dns/...`. This will install acme-dns to `~/go/bin/acme-dns`. diff --git a/dns.go b/dns.go index b703911..ba3746f 100644 --- a/dns.go +++ b/dns.go @@ -84,11 +84,7 @@ func (d *DNSServer) handleRequest(w dns.ResponseWriter, r *dns.Msg) { if r.Opcode == dns.OpcodeQuery { d.readQuery(m) - } else if r.Opcode == dns.OpcodeUpdate { - log.Debug("Refusing DNS Dynamic update request") - m.MsgHdr.Rcode = dns.RcodeRefused } - w.WriteMsg(m) } diff --git a/dns_test.go b/dns_test.go index 9c9138a..7826b27 100644 --- a/dns_test.go +++ b/dns_test.go @@ -116,22 +116,6 @@ func TestEDNS(t *testing.T) { } } -func TestOpcodeUpdate(t *testing.T) { - msg := new(dns.Msg) - msg.Id = dns.Id() - msg.Question = make([]dns.Question, 1) - msg.Question[0] = dns.Question{Name: dns.Fqdn("auth.example.org"), Qtype: dns.TypeANY, Qclass: dns.ClassINET} - msg.MsgHdr.Opcode = dns.OpcodeUpdate - in, err := dns.Exchange(msg, "127.0.0.1:15353") - if err != nil || in == nil { - t.Errorf("Encountered an error with UPDATE request") - } else if err == nil { - if in.Rcode != dns.RcodeRefused { - t.Errorf("Expected RCODE Refused from UPDATE request, but got [%s] instead", dns.RcodeToString[in.Rcode]) - } - } -} - func TestResolveCNAME(t *testing.T) { resolv := resolver{server: "127.0.0.1:15353"} expected := "cn.example.org. 3600 IN CNAME something.example.org." diff --git a/vendor/github.com/BurntSushi/toml/COPYING b/vendor/github.com/BurntSushi/toml/COPYING index 5a8e332..01b5743 100644 --- a/vendor/github.com/BurntSushi/toml/COPYING +++ b/vendor/github.com/BurntSushi/toml/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING index 5a8e332..01b5743 100644 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-decoder/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING index 5a8e332..01b5743 100644 --- a/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING +++ b/vendor/github.com/BurntSushi/toml/cmd/toml-test-encoder/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING b/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING index 5a8e332..01b5743 100644 --- a/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING +++ b/vendor/github.com/BurntSushi/toml/cmd/tomlv/COPYING @@ -1,14 +1,21 @@ - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - Version 2, December 2004 +The MIT License (MIT) - Copyright (C) 2004 Sam Hocevar +Copyright (c) 2013 TOML authors - Everyone is permitted to copy and distribute verbatim or modified - copies of this license document, and changing it is allowed as long - as the name is changed. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. You just DO WHAT THE FUCK YOU WANT TO. +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/BurntSushi/toml/decode_test.go b/vendor/github.com/BurntSushi/toml/decode_test.go index 0c36b33..95bc987 100644 --- a/vendor/github.com/BurntSushi/toml/decode_test.go +++ b/vendor/github.com/BurntSushi/toml/decode_test.go @@ -16,6 +16,8 @@ age = 250 andrew = "gallant" kait = "brady" now = 1987-07-05T05:45:00Z +nowEast = 2017-06-22T16:15:21+08:00 +nowWest = 2017-06-22T02:14:36-06:00 yesOrNo = true pi = 3.14 colors = [ @@ -38,6 +40,8 @@ cauchy = "cat 2" Pi float64 YesOrNo bool Now time.Time + NowEast time.Time + NowWest time.Time Andrew string Kait string My map[string]cats @@ -53,11 +57,21 @@ cauchy = "cat 2" if err != nil { panic(err) } + nowEast, err := time.Parse("2006-01-02T15:04:05-07:00", "2017-06-22T16:15:21+08:00") + if err != nil { + panic(err) + } + nowWest, err := time.Parse("2006-01-02T15:04:05-07:00", "2017-06-22T02:14:36-06:00") + if err != nil { + panic(err) + } var answer = simple{ Age: 250, Andrew: "gallant", Kait: "brady", Now: now, + NowEast: nowEast, + NowWest: nowWest, YesOrNo: true, Pi: 3.14, Colors: [][]string{ diff --git a/vendor/github.com/BurntSushi/toml/lex.go b/vendor/github.com/BurntSushi/toml/lex.go index 6dee7fc..e0a742a 100644 --- a/vendor/github.com/BurntSushi/toml/lex.go +++ b/vendor/github.com/BurntSushi/toml/lex.go @@ -775,7 +775,7 @@ func lexDatetime(lx *lexer) stateFn { return lexDatetime } switch r { - case '-', 'T', ':', '.', 'Z': + case '-', 'T', ':', '.', 'Z', '+': return lexDatetime } diff --git a/vendor/github.com/davecgh/go-spew/.travis.yml b/vendor/github.com/davecgh/go-spew/.travis.yml index 984e073..1f4cbf5 100644 --- a/vendor/github.com/davecgh/go-spew/.travis.yml +++ b/vendor/github.com/davecgh/go-spew/.travis.yml @@ -1,14 +1,28 @@ language: go +go_import_path: github.com/davecgh/go-spew go: - - 1.5.4 - - 1.6.3 - - 1.7 + - 1.6.x + - 1.7.x + - 1.8.x + - 1.9.x + - 1.10.x + - tip +sudo: false install: - - go get -v golang.org/x/tools/cmd/cover + - go get -v github.com/alecthomas/gometalinter + - gometalinter --install script: - - go test -v -tags=safe ./spew - - go test -v -tags=testcgo ./spew -covermode=count -coverprofile=profile.cov + - export PATH=$PATH:$HOME/gopath/bin + - export GORACE="halt_on_error=1" + - test -z "$(gometalinter --disable-all + --enable=gofmt + --enable=golint + --enable=vet + --enable=gosimple + --enable=unconvert + --deadline=4m ./spew | tee /dev/stderr)" + - go test -v -race -tags safe ./spew + - go test -v -race -tags testcgo ./spew -covermode=atomic -coverprofile=profile.cov after_success: - go get -v github.com/mattn/goveralls - - export PATH=$PATH:$HOME/gopath/bin - goveralls -coverprofile=profile.cov -service=travis-ci diff --git a/vendor/github.com/davecgh/go-spew/LICENSE b/vendor/github.com/davecgh/go-spew/LICENSE index c836416..bc52e96 100644 --- a/vendor/github.com/davecgh/go-spew/LICENSE +++ b/vendor/github.com/davecgh/go-spew/LICENSE @@ -2,7 +2,7 @@ ISC License Copyright (c) 2012-2016 Dave Collins -Permission to use, copy, modify, and distribute this software for any +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. diff --git a/vendor/github.com/davecgh/go-spew/README.md b/vendor/github.com/davecgh/go-spew/README.md index 2624304..f6ed02c 100644 --- a/vendor/github.com/davecgh/go-spew/README.md +++ b/vendor/github.com/davecgh/go-spew/README.md @@ -1,12 +1,9 @@ go-spew ======= -[![Build Status](https://img.shields.io/travis/davecgh/go-spew.svg)] -(https://travis-ci.org/davecgh/go-spew) [![ISC License] -(http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) [![Coverage Status] -(https://img.shields.io/coveralls/davecgh/go-spew.svg)] -(https://coveralls.io/r/davecgh/go-spew?branch=master) - +[![Build Status](https://img.shields.io/travis/davecgh/go-spew.svg)](https://travis-ci.org/davecgh/go-spew) +[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) +[![Coverage Status](https://img.shields.io/coveralls/davecgh/go-spew.svg)](https://coveralls.io/r/davecgh/go-spew?branch=master) Go-spew implements a deep pretty printer for Go data structures to aid in debugging. A comprehensive suite of tests with 100% test coverage is provided @@ -21,8 +18,7 @@ post about it ## Documentation -[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)] -(http://godoc.org/github.com/davecgh/go-spew/spew) +[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/davecgh/go-spew/spew) Full `go doc` style documentation for the project can be viewed online without installing this package by using the excellent GoDoc site here: diff --git a/vendor/github.com/davecgh/go-spew/spew/bypass.go b/vendor/github.com/davecgh/go-spew/spew/bypass.go index 8a4a658..7929947 100644 --- a/vendor/github.com/davecgh/go-spew/spew/bypass.go +++ b/vendor/github.com/davecgh/go-spew/spew/bypass.go @@ -16,7 +16,9 @@ // when the code is not running on Google App Engine, compiled by GopherJS, and // "-tags safe" is not added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. -// +build !js,!appengine,!safe,!disableunsafe +// Go versions prior to 1.4 are disabled because they use a different layout +// for interfaces which make the implementation of unsafeReflectValue more complex. +// +build !js,!appengine,!safe,!disableunsafe,go1.4 package spew @@ -34,80 +36,49 @@ const ( ptrSize = unsafe.Sizeof((*byte)(nil)) ) -var ( - // offsetPtr, offsetScalar, and offsetFlag are the offsets for the - // internal reflect.Value fields. These values are valid before golang - // commit ecccf07e7f9d which changed the format. The are also valid - // after commit 82f48826c6c7 which changed the format again to mirror - // the original format. Code in the init function updates these offsets - // as necessary. - offsetPtr = uintptr(ptrSize) - offsetScalar = uintptr(0) - offsetFlag = uintptr(ptrSize * 2) +type flag uintptr - // flagKindWidth and flagKindShift indicate various bits that the - // reflect package uses internally to track kind information. - // - // flagRO indicates whether or not the value field of a reflect.Value is - // read-only. - // - // flagIndir indicates whether the value field of a reflect.Value is - // the actual data or a pointer to the data. - // - // These values are valid before golang commit 90a7c3c86944 which - // changed their positions. Code in the init function updates these - // flags as necessary. - flagKindWidth = uintptr(5) - flagKindShift = uintptr(flagKindWidth - 1) - flagRO = uintptr(1 << 0) - flagIndir = uintptr(1 << 1) +var ( + // flagRO indicates whether the value field of a reflect.Value + // is read-only. + flagRO flag + + // flagAddr indicates whether the address of the reflect.Value's + // value may be taken. + flagAddr flag ) -func init() { - // Older versions of reflect.Value stored small integers directly in the - // ptr field (which is named val in the older versions). Versions - // between commits ecccf07e7f9d and 82f48826c6c7 added a new field named - // scalar for this purpose which unfortunately came before the flag - // field, so the offset of the flag field is different for those - // versions. - // - // This code constructs a new reflect.Value from a known small integer - // and checks if the size of the reflect.Value struct indicates it has - // the scalar field. When it does, the offsets are updated accordingly. - vv := reflect.ValueOf(0xf00) - if unsafe.Sizeof(vv) == (ptrSize * 4) { - offsetScalar = ptrSize * 2 - offsetFlag = ptrSize * 3 - } +// flagKindMask holds the bits that make up the kind +// part of the flags field. In all the supported versions, +// it is in the lower 5 bits. +const flagKindMask = flag(0x1f) - // Commit 90a7c3c86944 changed the flag positions such that the low - // order bits are the kind. This code extracts the kind from the flags - // field and ensures it's the correct type. When it's not, the flag - // order has been changed to the newer format, so the flags are updated - // accordingly. - upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag) - upfv := *(*uintptr)(upf) - flagKindMask := uintptr((1<>flagKindShift != uintptr(reflect.Int) { - flagKindShift = 0 - flagRO = 1 << 5 - flagIndir = 1 << 6 +// Different versions of Go have used different +// bit layouts for the flags type. This table +// records the known combinations. +var okFlags = []struct { + ro, addr flag +}{{ + // From Go 1.4 to 1.5 + ro: 1 << 5, + addr: 1 << 7, +}, { + // Up to Go tip. + ro: 1<<5 | 1<<6, + addr: 1 << 8, +}} - // Commit adf9b30e5594 modified the flags to separate the - // flagRO flag into two bits which specifies whether or not the - // field is embedded. This causes flagIndir to move over a bit - // and means that flagRO is the combination of either of the - // original flagRO bit and the new bit. - // - // This code detects the change by extracting what used to be - // the indirect bit to ensure it's set. When it's not, the flag - // order has been changed to the newer format, so the flags are - // updated accordingly. - if upfv&flagIndir == 0 { - flagRO = 3 << 5 - flagIndir = 1 << 7 - } +var flagValOffset = func() uintptr { + field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") + if !ok { + panic("reflect.Value has no flag field") } + return field.Offset +}() + +// flagField returns a pointer to the flag field of a reflect.Value. +func flagField(v *reflect.Value) *flag { + return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset)) } // unsafeReflectValue converts the passed reflect.Value into a one that bypasses @@ -119,34 +90,56 @@ func init() { // This allows us to check for implementations of the Stringer and error // interfaces to be used for pretty printing ordinarily unaddressable and // inaccessible values such as unexported struct fields. -func unsafeReflectValue(v reflect.Value) (rv reflect.Value) { - indirects := 1 - vt := v.Type() - upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr) - rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag)) - if rvf&flagIndir != 0 { - vt = reflect.PtrTo(v.Type()) - indirects++ - } else if offsetScalar != 0 { - // The value is in the scalar field when it's not one of the - // reference types. - switch vt.Kind() { - case reflect.Uintptr: - case reflect.Chan: - case reflect.Func: - case reflect.Map: - case reflect.Ptr: - case reflect.UnsafePointer: - default: - upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + - offsetScalar) +func unsafeReflectValue(v reflect.Value) reflect.Value { + if !v.IsValid() || (v.CanInterface() && v.CanAddr()) { + return v + } + flagFieldPtr := flagField(&v) + *flagFieldPtr &^= flagRO + *flagFieldPtr |= flagAddr + return v +} + +// Sanity checks against future reflect package changes +// to the type or semantics of the Value.flag field. +func init() { + field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") + if !ok { + panic("reflect.Value has no flag field") + } + if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() { + panic("reflect.Value flag field has changed kind") + } + type t0 int + var t struct { + A t0 + // t0 will have flagEmbedRO set. + t0 + // a will have flagStickyRO set + a t0 + } + vA := reflect.ValueOf(t).FieldByName("A") + va := reflect.ValueOf(t).FieldByName("a") + vt0 := reflect.ValueOf(t).FieldByName("t0") + + // Infer flagRO from the difference between the flags + // for the (otherwise identical) fields in t. + flagPublic := *flagField(&vA) + flagWithRO := *flagField(&va) | *flagField(&vt0) + flagRO = flagPublic ^ flagWithRO + + // Infer flagAddr from the difference between a value + // taken from a pointer and not. + vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A") + flagNoPtr := *flagField(&vA) + flagPtr := *flagField(&vPtrA) + flagAddr = flagNoPtr ^ flagPtr + + // Check that the inferred flags tally with one of the known versions. + for _, f := range okFlags { + if flagRO == f.ro && flagAddr == f.addr { + return } } - - pv := reflect.NewAt(vt, upv) - rv = pv - for i := 0; i < indirects; i++ { - rv = rv.Elem() - } - return rv + panic("reflect.Value read-only flag has changed semantics") } diff --git a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go index 1fe3cf3..205c28d 100644 --- a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go +++ b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go @@ -16,7 +16,7 @@ // when the code is running on Google App Engine, compiled by GopherJS, or // "-tags safe" is added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. -// +build js appengine safe disableunsafe +// +build js appengine safe disableunsafe !go1.4 package spew diff --git a/vendor/github.com/davecgh/go-spew/spew/common.go b/vendor/github.com/davecgh/go-spew/spew/common.go index 7c519ff..1be8ce9 100644 --- a/vendor/github.com/davecgh/go-spew/spew/common.go +++ b/vendor/github.com/davecgh/go-spew/spew/common.go @@ -180,7 +180,7 @@ func printComplex(w io.Writer, c complex128, floatPrecision int) { w.Write(closeParenBytes) } -// printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x' +// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x' // prefix to Writer w. func printHexPtr(w io.Writer, p uintptr) { // Null pointer. diff --git a/vendor/github.com/davecgh/go-spew/spew/dump.go b/vendor/github.com/davecgh/go-spew/spew/dump.go index df1d582..f78d89f 100644 --- a/vendor/github.com/davecgh/go-spew/spew/dump.go +++ b/vendor/github.com/davecgh/go-spew/spew/dump.go @@ -35,16 +35,16 @@ var ( // cCharRE is a regular expression that matches a cgo char. // It is used to detect character arrays to hexdump them. - cCharRE = regexp.MustCompile("^.*\\._Ctype_char$") + cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`) // cUnsignedCharRE is a regular expression that matches a cgo unsigned // char. It is used to detect unsigned character arrays to hexdump // them. - cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$") + cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`) // cUint8tCharRE is a regular expression that matches a cgo uint8_t. // It is used to detect uint8_t arrays to hexdump them. - cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$") + cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`) ) // dumpState contains information about the state of a dump operation. @@ -143,10 +143,10 @@ func (d *dumpState) dumpPtr(v reflect.Value) { // Display dereferenced value. d.w.Write(openParenBytes) switch { - case nilFound == true: + case nilFound: d.w.Write(nilAngleBytes) - case cycleFound == true: + case cycleFound: d.w.Write(circularBytes) default: diff --git a/vendor/github.com/davecgh/go-spew/spew/dump_test.go b/vendor/github.com/davecgh/go-spew/spew/dump_test.go index 5aad9c7..4a31a2e 100644 --- a/vendor/github.com/davecgh/go-spew/spew/dump_test.go +++ b/vendor/github.com/davecgh/go-spew/spew/dump_test.go @@ -768,7 +768,7 @@ func addUintptrDumpTests() { func addUnsafePointerDumpTests() { // Null pointer. - v := unsafe.Pointer(uintptr(0)) + v := unsafe.Pointer(nil) nv := (*unsafe.Pointer)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) diff --git a/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go b/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go index 6ab1808..108baa5 100644 --- a/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go +++ b/vendor/github.com/davecgh/go-spew/spew/dumpcgo_test.go @@ -82,18 +82,20 @@ func addCgoDumpTests() { v5Len := fmt.Sprintf("%d", v5l) v5Cap := fmt.Sprintf("%d", v5c) v5t := "[6]testdata._Ctype_uint8_t" + v5t2 := "[6]testdata._Ctype_uchar" v5s := "(len=" + v5Len + " cap=" + v5Cap + ") " + "{\n 00000000 74 65 73 74 35 00 " + " |test5.|\n}" - addDumpTest(v5, "("+v5t+") "+v5s+"\n") + addDumpTest(v5, "("+v5t+") "+v5s+"\n", "("+v5t2+") "+v5s+"\n") // C typedefed unsigned char array. v6, v6l, v6c := testdata.GetCgoTypdefedUnsignedCharArray() v6Len := fmt.Sprintf("%d", v6l) v6Cap := fmt.Sprintf("%d", v6c) v6t := "[6]testdata._Ctype_custom_uchar_t" + v6t2 := "[6]testdata._Ctype_uchar" v6s := "(len=" + v6Len + " cap=" + v6Cap + ") " + "{\n 00000000 74 65 73 74 36 00 " + " |test6.|\n}" - addDumpTest(v6, "("+v6t+") "+v6s+"\n") + addDumpTest(v6, "("+v6t+") "+v6s+"\n", "("+v6t2+") "+v6s+"\n") } diff --git a/vendor/github.com/davecgh/go-spew/spew/format.go b/vendor/github.com/davecgh/go-spew/spew/format.go index c49875b..b04edb7 100644 --- a/vendor/github.com/davecgh/go-spew/spew/format.go +++ b/vendor/github.com/davecgh/go-spew/spew/format.go @@ -182,10 +182,10 @@ func (f *formatState) formatPtr(v reflect.Value) { // Display dereferenced value. switch { - case nilFound == true: + case nilFound: f.fs.Write(nilAngleBytes) - case cycleFound == true: + case cycleFound: f.fs.Write(circularShortBytes) default: diff --git a/vendor/github.com/davecgh/go-spew/spew/format_test.go b/vendor/github.com/davecgh/go-spew/spew/format_test.go index f9b93ab..87ee965 100644 --- a/vendor/github.com/davecgh/go-spew/spew/format_test.go +++ b/vendor/github.com/davecgh/go-spew/spew/format_test.go @@ -1083,7 +1083,7 @@ func addUintptrFormatterTests() { func addUnsafePointerFormatterTests() { // Null pointer. - v := unsafe.Pointer(uintptr(0)) + v := unsafe.Pointer(nil) nv := (*unsafe.Pointer)(nil) pv := &v vAddr := fmt.Sprintf("%p", pv) @@ -1536,14 +1536,14 @@ func TestPrintSortedKeys(t *testing.T) { t.Errorf("Sorted keys mismatch 3:\n %v %v", s, expected) } - s = cfg.Sprint(map[testStruct]int{testStruct{1}: 1, testStruct{3}: 3, testStruct{2}: 2}) + s = cfg.Sprint(map[testStruct]int{{1}: 1, {3}: 3, {2}: 2}) expected = "map[ts.1:1 ts.2:2 ts.3:3]" if s != expected { t.Errorf("Sorted keys mismatch 4:\n %v %v", s, expected) } if !spew.UnsafeDisabled { - s = cfg.Sprint(map[testStructP]int{testStructP{1}: 1, testStructP{3}: 3, testStructP{2}: 2}) + s = cfg.Sprint(map[testStructP]int{{1}: 1, {3}: 3, {2}: 2}) expected = "map[ts.1:1 ts.2:2 ts.3:3]" if s != expected { t.Errorf("Sorted keys mismatch 5:\n %v %v", s, expected) diff --git a/vendor/github.com/davecgh/go-spew/spew/internal_test.go b/vendor/github.com/davecgh/go-spew/spew/internal_test.go index 20a9cfe..e312b4f 100644 --- a/vendor/github.com/davecgh/go-spew/spew/internal_test.go +++ b/vendor/github.com/davecgh/go-spew/spew/internal_test.go @@ -36,10 +36,7 @@ type dummyFmtState struct { } func (dfs *dummyFmtState) Flag(f int) bool { - if f == int('+') { - return true - } - return false + return f == int('+') } func (dfs *dummyFmtState) Precision() (int, bool) { diff --git a/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go b/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go index a0c612e..80dc221 100644 --- a/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go +++ b/vendor/github.com/davecgh/go-spew/spew/internalunsafe_test.go @@ -16,7 +16,7 @@ // when the code is not running on Google App Engine, compiled by GopherJS, and // "-tags safe" is not added to the go build command line. The "disableunsafe" // tag is deprecated and thus should not be used. -// +build !js,!appengine,!safe,!disableunsafe +// +build !js,!appengine,!safe,!disableunsafe,go1.4 /* This test file is part of the spew package rather than than the spew_test @@ -30,7 +30,6 @@ import ( "bytes" "reflect" "testing" - "unsafe" ) // changeKind uses unsafe to intentionally change the kind of a reflect.Value to @@ -38,13 +37,13 @@ import ( // fallback code which punts to the standard fmt library for new types that // might get added to the language. func changeKind(v *reflect.Value, readOnly bool) { - rvf := (*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + offsetFlag)) - *rvf = *rvf | ((1<= 6 && (name == "" || name == ifs.Name) { - copy(nodeID[:], ifs.HardwareAddr) - ifname = ifs.Name - return true - } + iname, addr := getHardwareInterface(name) // null implementation for js + if iname != "" && addr != nil { + ifname = iname + copy(nodeID[:], addr) + return true } // We found no interfaces with a valid hardware address. If name @@ -94,9 +83,6 @@ func SetNodeID(id []byte) bool { // NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is // not valid. The NodeID is only well defined for version 1 and 2 UUIDs. func (uuid UUID) NodeID() []byte { - if len(uuid) != 16 { - return nil - } var node [6]byte copy(node[:], uuid[10:]) return node[:] diff --git a/vendor/github.com/google/uuid/node_js.go b/vendor/github.com/google/uuid/node_js.go new file mode 100644 index 0000000..24b78ed --- /dev/null +++ b/vendor/github.com/google/uuid/node_js.go @@ -0,0 +1,12 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build js + +package uuid + +// getHardwareInterface returns nil values for the JS version of the code. +// This remvoves the "net" dependency, because it is not used in the browser. +// Using the "net" library inflates the size of the transpiled JS code by 673k bytes. +func getHardwareInterface(name string) (string, []byte) { return "", nil } diff --git a/vendor/github.com/google/uuid/node_net.go b/vendor/github.com/google/uuid/node_net.go new file mode 100644 index 0000000..0cbbcdd --- /dev/null +++ b/vendor/github.com/google/uuid/node_net.go @@ -0,0 +1,33 @@ +// Copyright 2017 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !js + +package uuid + +import "net" + +var interfaces []net.Interface // cached list of interfaces + +// getHardwareInterface returns the name and hardware address of interface name. +// If name is "" then the name and hardware address of one of the system's +// interfaces is returned. If no interfaces are found (name does not exist or +// there are no interfaces) then "", nil is returned. +// +// Only addresses of at least 6 bytes are returned. +func getHardwareInterface(name string) (string, []byte) { + if interfaces == nil { + var err error + interfaces, err = net.Interfaces() + if err != nil { + return "", nil + } + } + for _, ifs := range interfaces { + if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) { + return ifs.Name, ifs.HardwareAddr + } + } + return "", nil +} diff --git a/vendor/github.com/google/uuid/seq_test.go b/vendor/github.com/google/uuid/seq_test.go index 853a4aa..4f6c549 100644 --- a/vendor/github.com/google/uuid/seq_test.go +++ b/vendor/github.com/google/uuid/seq_test.go @@ -16,7 +16,7 @@ var regressions = flag.Bool("regressions", false, "run uuid regression tests") // TestClockSeqRace tests for a particular race condition of returning two // identical Version1 UUIDs. The duration of 1 minute was chosen as the race -// condition, before being fixed, nearly always occured in under 30 seconds. +// condition, before being fixed, nearly always occurred in under 30 seconds. func TestClockSeqRace(t *testing.T) { if !*regressions { t.Skip("skipping regression tests") diff --git a/vendor/github.com/google/uuid/sql.go b/vendor/github.com/google/uuid/sql.go index 528ad0d..f326b54 100644 --- a/vendor/github.com/google/uuid/sql.go +++ b/vendor/github.com/google/uuid/sql.go @@ -13,35 +13,36 @@ import ( // Currently, database types that map to string and []byte are supported. Please // consult database-specific driver documentation for matching types. func (uuid *UUID) Scan(src interface{}) error { - switch src.(type) { + switch src := src.(type) { + case nil: + return nil + case string: // if an empty UUID comes from a table, we return a null UUID - if src.(string) == "" { + if src == "" { return nil } // see Parse for required string format - u, err := Parse(src.(string)) - + u, err := Parse(src) if err != nil { return fmt.Errorf("Scan: %v", err) } *uuid = u - case []byte: - b := src.([]byte) + case []byte: // if an empty UUID comes from a table, we return a null UUID - if len(b) == 0 { + if len(src) == 0 { return nil } // assumes a simple slice of bytes if 16 bytes // otherwise attempts to parse - if len(b) != 16 { - return uuid.Scan(string(b)) + if len(src) != 16 { + return uuid.Scan(string(src)) } - copy((*uuid)[:], b) + copy((*uuid)[:], src) default: return fmt.Errorf("Scan: unable to scan type %T into UUID", src) diff --git a/vendor/github.com/google/uuid/sql_test.go b/vendor/github.com/google/uuid/sql_test.go index c193196..1803dfd 100644 --- a/vendor/github.com/google/uuid/sql_test.go +++ b/vendor/github.com/google/uuid/sql_test.go @@ -10,9 +10,9 @@ import ( ) func TestScan(t *testing.T) { - var stringTest string = "f47ac10b-58cc-0372-8567-0e02b2c3d479" - var badTypeTest int = 6 - var invalidTest string = "f47ac10b-58cc-0372-8567-0e02b2c3d4" + stringTest := "f47ac10b-58cc-0372-8567-0e02b2c3d479" + badTypeTest := 6 + invalidTest := "f47ac10b-58cc-0372-8567-0e02b2c3d4" byteTest := make([]byte, 16) byteTestUUID := Must(Parse(stringTest)) @@ -90,6 +90,17 @@ func TestScan(t *testing.T) { t.Error("UUID was not nil after scanning empty byte slice") } } + + uuid = UUID{} + err = (&uuid).Scan(nil) + if err != nil { + t.Fatal(err) + } + for _, v := range uuid { + if v != 0 { + t.Error("UUID was not nil after scanning nil") + } + } } func TestValue(t *testing.T) { diff --git a/vendor/github.com/google/uuid/time.go b/vendor/github.com/google/uuid/time.go index fd7fe0a..e6ef06c 100644 --- a/vendor/github.com/google/uuid/time.go +++ b/vendor/github.com/google/uuid/time.go @@ -86,7 +86,7 @@ func clockSequence() int { return int(clockSeq & 0x3fff) } -// SetClockSeq sets the clock sequence to the lower 14 bits of seq. Setting to +// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to // -1 causes a new sequence to be generated. func SetClockSequence(seq int) { defer timeMu.Unlock() @@ -100,9 +100,9 @@ func setClockSequence(seq int) { randomBits(b[:]) // clock sequence seq = int(b[0])<<8 | int(b[1]) } - old_seq := clockSeq + oldSeq := clockSeq clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant - if old_seq != clockSeq { + if oldSeq != clockSeq { lasttime = 0 } } diff --git a/vendor/github.com/google/uuid/uuid.go b/vendor/github.com/google/uuid/uuid.go index b7b9ced..524404c 100644 --- a/vendor/github.com/google/uuid/uuid.go +++ b/vendor/github.com/google/uuid/uuid.go @@ -1,4 +1,4 @@ -// Copyright 2016 Google Inc. All rights reserved. +// Copyright 2018 Google Inc. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. @@ -35,20 +35,43 @@ const ( var rander = rand.Reader // random function -// Parse decodes s into a UUID or returns an error. Both the UUID form of -// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and -// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded. +// Parse decodes s into a UUID or returns an error. Both the standard UUID +// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the +// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex +// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx. func Parse(s string) (UUID, error) { var uuid UUID - if len(s) != 36 { - if len(s) != 36+9 { - return uuid, fmt.Errorf("invalid UUID length: %d", len(s)) - } + switch len(s) { + // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36: + + // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: if strings.ToLower(s[:9]) != "urn:uuid:" { return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9]) } s = s[9:] + + // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + case 36 + 2: + s = s[1:] + + // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + case 32: + var ok bool + for i := range uuid { + uuid[i], ok = xtob(s[i*2], s[i*2+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(s)) } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { return uuid, errors.New("invalid UUID format") } @@ -58,11 +81,11 @@ func Parse(s string) (UUID, error) { 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} { - if v, ok := xtob(s[x], s[x+1]); !ok { + v, ok := xtob(s[x], s[x+1]) + if !ok { return uuid, errors.New("invalid UUID format") - } else { - uuid[i] = v } + uuid[i] = v } return uuid, nil } @@ -70,15 +93,29 @@ func Parse(s string) (UUID, error) { // ParseBytes is like Parse, except it parses a byte slice instead of a string. func ParseBytes(b []byte) (UUID, error) { var uuid UUID - if len(b) != 36 { - if len(b) != 36+9 { - return uuid, fmt.Errorf("invalid UUID length: %d", len(b)) - } + switch len(b) { + case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx + case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) { return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9]) } b = b[9:] + case 36 + 2: // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} + b = b[1:] + case 32: // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + var ok bool + for i := 0; i < 32; i += 2 { + uuid[i/2], ok = xtob(b[i], b[i+1]) + if !ok { + return uuid, errors.New("invalid UUID format") + } + } + return uuid, nil + default: + return uuid, fmt.Errorf("invalid UUID length: %d", len(b)) } + // s is now at least 36 bytes long + // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx if b[8] != '-' || b[13] != '-' || b[18] != '-' || b[23] != '-' { return uuid, errors.New("invalid UUID format") } @@ -88,15 +125,32 @@ func ParseBytes(b []byte) (UUID, error) { 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} { - if v, ok := xtob(b[x], b[x+1]); !ok { + v, ok := xtob(b[x], b[x+1]) + if !ok { return uuid, errors.New("invalid UUID format") - } else { - uuid[i] = v } + uuid[i] = v } return uuid, nil } +// MustParse is like Parse but panics if the string cannot be parsed. +// It simplifies safe initialization of global variables holding compiled UUIDs. +func MustParse(s string) UUID { + uuid, err := Parse(s) + if err != nil { + panic(`uuid: Parse(` + s + `): ` + err.Error()) + } + return uuid +} + +// FromBytes creates a new UUID from a byte slice. Returns an error if the slice +// does not have a length of 16. The bytes are copied from the slice. +func FromBytes(b []byte) (uuid UUID, err error) { + err = uuid.UnmarshalBinary(b) + return uuid, err +} + // Must returns uuid if err is nil and panics otherwise. func Must(uuid UUID, err error) UUID { if err != nil { @@ -123,7 +177,7 @@ func (uuid UUID) URN() string { } func encodeHex(dst []byte, uuid UUID) { - hex.Encode(dst[:], uuid[:4]) + hex.Encode(dst, uuid[:4]) dst[8] = '-' hex.Encode(dst[9:13], uuid[4:6]) dst[13] = '-' @@ -176,7 +230,7 @@ func (v Variant) String() string { return fmt.Sprintf("BadVariant%d", int(v)) } -// SetRand sets the random number generator to r, which implents io.Reader. +// SetRand sets the random number generator to r, which implements io.Reader. // If r.Read returns an error when the package requests random data then // a panic will be issued. // diff --git a/vendor/github.com/google/uuid/uuid_test.go b/vendor/github.com/google/uuid/uuid_test.go index 70986ff..e7876f1 100644 --- a/vendor/github.com/google/uuid/uuid_test.go +++ b/vendor/github.com/google/uuid/uuid_test.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "os" + "runtime" "strings" "testing" "time" @@ -58,12 +59,22 @@ var tests = []test{ {"f47ac10b-58cc-4372-e567-0e02b2c3d479", 4, Future, true}, {"f47ac10b-58cc-4372-f567-0e02b2c3d479", 4, Future, true}, + {"f47ac10b158cc-5372-a567-0e02b2c3d479", 0, Invalid, false}, {"f47ac10b-58cc25372-a567-0e02b2c3d479", 0, Invalid, false}, {"f47ac10b-58cc-53723a567-0e02b2c3d479", 0, Invalid, false}, {"f47ac10b-58cc-5372-a56740e02b2c3d479", 0, Invalid, false}, {"f47ac10b-58cc-5372-a567-0e02-2c3d479", 0, Invalid, false}, {"g47ac10b-58cc-4372-a567-0e02b2c3d479", 0, Invalid, false}, + + + {"{f47ac10b-58cc-0372-8567-0e02b2c3d479}", 0, RFC4122, true}, + {"{f47ac10b-58cc-0372-8567-0e02b2c3d479", 0, Invalid, false}, + {"f47ac10b-58cc-0372-8567-0e02b2c3d479}", 0, Invalid, false}, + + {"f47ac10b58cc037285670e02b2c3d479", 0, RFC4122, true}, + {"f47ac10b58cc037285670e02b2c3d4790", 0, Invalid, false}, + {"f47ac10b58cc037285670e02b2c3d47", 0, Invalid, false}, } var constants = []struct { @@ -121,6 +132,25 @@ func TestUUID(t *testing.T) { } } +func TestFromBytes(t *testing.T) { + b := []byte{ + 0x7d, 0x44, 0x48, 0x40, + 0x9d, 0xc0, + 0x11, 0xd1, + 0xb2, 0x45, + 0x5f, 0xfd, 0xce, 0x74, 0xfa, 0xd2, + } + uuid, err := FromBytes(b) + if err != nil { + t.Fatalf("%s", err) + } + for i := 0; i < len(uuid); i++ { + if b[i] != uuid[i] { + t.Fatalf("FromBytes() got %v expected %v\b", uuid[:], b) + } + } +} + func TestConstants(t *testing.T) { for x, tt := range constants { v, ok := tt.c.(fmt.Stringer) @@ -246,7 +276,7 @@ func TestCoding(t *testing.T) { if err != nil { t.Errorf("Parse returned unexpected error %v", err) } - if data != data { + if data != uuid { t.Errorf("%s: decoded to %s, expected %s", text, uuid, data) } } @@ -286,7 +316,7 @@ func TestVersion1(t *testing.T) { case t1 > t2 && q1 == q2: t.Error("time reversed") case t1 < t2 && q1 != q2: - t.Error("clock sequence chaned unexpectedly") + t.Error("clock sequence changed unexpectedly") } } @@ -302,8 +332,10 @@ func TestNode(t *testing.T) { if !SetNodeInterface("") { t.Error("SetNodeInterface failed") } - if ni := NodeInterface(); ni == "" { - t.Error("NodeInterface returned an empty string") + if runtime.GOARCH != "js" { + if ni := NodeInterface(); ni == "" { + t.Error("NodeInterface returned an empty string") + } } ni := NodeID() @@ -328,7 +360,7 @@ func TestNode(t *testing.T) { } if ni := NodeInterface(); ni != "user" { - t.Errorf("got inteface %q, want %q", ni, "user") + t.Errorf("got interface %q, want %q", ni, "user") } } @@ -372,8 +404,10 @@ func TestNodeID(t *testing.T) { nid := []byte{1, 2, 3, 4, 5, 6} SetNodeInterface("") s := NodeInterface() - if s == "" || s == "user" { - t.Errorf("NodeInterface %q after SetInteface", s) + if runtime.GOARCH != "js" { + if s == "" || s == "user" { + t.Errorf("NodeInterface %q after SetInterface", s) + } } node1 := NodeID() if node1 == nil { @@ -424,7 +458,7 @@ func TestDCE(t *testing.T) { type badRand struct{} func (r badRand) Read(buf []byte) (int, error) { - for i, _ := range buf { + for i := range buf { buf[i] = byte(i) } return len(buf), nil @@ -435,13 +469,13 @@ func TestBadRand(t *testing.T) { uuid1 := New() uuid2 := New() if uuid1 != uuid2 { - t.Errorf("execpted duplicates, got %q and %q", uuid1, uuid2) + t.Errorf("expected duplicates, got %q and %q", uuid1, uuid2) } SetRand(nil) uuid1 = New() uuid2 = New() if uuid1 == uuid2 { - t.Errorf("unexecpted duplicates, got %q", uuid1) + t.Errorf("unexpected duplicates, got %q", uuid1) } } @@ -471,7 +505,6 @@ func parseBytesUnsafe(b []byte) (UUID, error) { return Parse(*(*string)(unsafe.Pointer(&b))) } - func BenchmarkParseBytesUnsafe(b *testing.B) { for i := 0; i < b.N; i++ { _, err := parseBytesUnsafe(asBytes) diff --git a/vendor/github.com/google/uuid/version1.go b/vendor/github.com/google/uuid/version1.go index 22dc07c..199a1ac 100644 --- a/vendor/github.com/google/uuid/version1.go +++ b/vendor/github.com/google/uuid/version1.go @@ -13,7 +13,7 @@ import ( // or SetNodeInterface then it will be set automatically. If the NodeID cannot // be set NewUUID returns nil. If clock sequence has not been set by // SetClockSequence then it will be set automatically. If GetTime fails to -// return the current NewUUID returns Nil and an error. +// return the current NewUUID returns nil and an error. // // In most cases, New should be used. func NewUUID() (UUID, error) { diff --git a/vendor/github.com/google/uuid/version4.go b/vendor/github.com/google/uuid/version4.go index 390dd2c..84af91c 100644 --- a/vendor/github.com/google/uuid/version4.go +++ b/vendor/github.com/google/uuid/version4.go @@ -6,7 +6,7 @@ package uuid import "io" -// New is creates a new random UUID or panics. New is equivalent to +// New creates a new random UUID or panics. New is equivalent to // the expression // // uuid.Must(uuid.NewRandom()) @@ -14,12 +14,12 @@ func New() UUID { return Must(NewRandom()) } -// NewRandom returns a Random (Version 4) UUID or panics. +// NewRandom returns a Random (Version 4) UUID. // // The strength of the UUIDs is based on the strength of the crypto/rand // package. // -// A note about uniqueness derived from from the UUID Wikipedia entry: +// A note about uniqueness derived from the UUID Wikipedia entry: // // Randomly generated UUIDs have 122 random bits. One's annual risk of being // hit by a meteorite is estimated to be one chance in 17 billion, that diff --git a/vendor/github.com/julienschmidt/httprouter/.travis.yml b/vendor/github.com/julienschmidt/httprouter/.travis.yml index a4d6cc5..477807d 100644 --- a/vendor/github.com/julienschmidt/httprouter/.travis.yml +++ b/vendor/github.com/julienschmidt/httprouter/.travis.yml @@ -1,8 +1,18 @@ sudo: false language: go go: - - 1.1 - - 1.2 - - 1.3 - - 1.4 + - 1.7 + - 1.8 + - 1.9 + - "1.10" - tip +before_install: + - go get golang.org/x/tools/cmd/cover + - go get github.com/mattn/goveralls + - go get github.com/golang/lint/golint +script: + - go test -v -covermode=count -coverprofile=coverage.out + - go vet ./... + - test -z "$(gofmt -d -s . | tee /dev/stderr)" + - test -z "$(golint ./... | tee /dev/stderr)" + - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci diff --git a/vendor/github.com/julienschmidt/httprouter/README.md b/vendor/github.com/julienschmidt/httprouter/README.md index 9875c70..eabf4aa 100644 --- a/vendor/github.com/julienschmidt/httprouter/README.md +++ b/vendor/github.com/julienschmidt/httprouter/README.md @@ -1,58 +1,37 @@ -# HttpRouter [![Build Status](https://travis-ci.org/julienschmidt/httprouter.png?branch=master)](https://travis-ci.org/julienschmidt/httprouter) [![Coverage](http://gocover.io/_badge/github.com/julienschmidt/httprouter?0)](http://gocover.io/github.com/julienschmidt/httprouter) [![GoDoc](http://godoc.org/github.com/julienschmidt/httprouter?status.png)](http://godoc.org/github.com/julienschmidt/httprouter) +# HttpRouter [![Build Status](https://travis-ci.org/julienschmidt/httprouter.svg?branch=master)](https://travis-ci.org/julienschmidt/httprouter) [![Coverage Status](https://coveralls.io/repos/github/julienschmidt/httprouter/badge.svg?branch=master)](https://coveralls.io/github/julienschmidt/httprouter?branch=master) [![GoDoc](https://godoc.org/github.com/julienschmidt/httprouter?status.svg)](http://godoc.org/github.com/julienschmidt/httprouter) -HttpRouter is a lightweight high performance HTTP request router -(also called *multiplexer* or just *mux* for short) for [Go](http://golang.org/). +HttpRouter is a lightweight high performance HTTP request router (also called *multiplexer* or just *mux* for short) for [Go](https://golang.org/). -In contrast to the [default mux](http://golang.org/pkg/net/http/#ServeMux) of Go's net/http package, this router supports -variables in the routing pattern and matches against the request method. -It also scales better. +In contrast to the [default mux](https://golang.org/pkg/net/http/#ServeMux) of Go's `net/http` package, this router supports variables in the routing pattern and matches against the request method. It also scales better. -The router is optimized for high performance and a small memory footprint. -It scales well even with very long paths and a large number of routes. -A compressing dynamic trie (radix tree) structure is used for efficient matching. +The router is optimized for high performance and a small memory footprint. It scales well even with very long paths and a large number of routes. A compressing dynamic trie (radix tree) structure is used for efficient matching. ## Features -**Only explicit matches:** With other routers, like [http.ServeMux](http://golang.org/pkg/net/http/#ServeMux), -a requested URL path could match multiple patterns. Therefore they have some -awkward pattern priority rules, like *longest match* or *first registered, -first matched*. By design of this router, a request can only match exactly one -or no route. As a result, there are also no unintended matches, which makes it -great for SEO and improves the user experience. -**Stop caring about trailing slashes:** Choose the URL style you like, the -router automatically redirects the client if a trailing slash is missing or if -there is one extra. Of course it only does so, if the new path has a handler. -If you don't like it, you can [turn off this behavior](http://godoc.org/github.com/julienschmidt/httprouter#Router.RedirectTrailingSlash). +**Only explicit matches:** With other routers, like [`http.ServeMux`](https://golang.org/pkg/net/http/#ServeMux), a requested URL path could match multiple patterns. Therefore they have some awkward pattern priority rules, like *longest match* or *first registered, first matched*. By design of this router, a request can only match exactly one or no route. As a result, there are also no unintended matches, which makes it great for SEO and improves the user experience. -**Path auto-correction:** Besides detecting the missing or additional trailing -slash at no extra cost, the router can also fix wrong cases and remove -superfluous path elements (like `../` or `//`). -Is [CAPTAIN CAPS LOCK](http://www.urbandictionary.com/define.php?term=Captain+Caps+Lock) one of your users? -HttpRouter can help him by making a case-insensitive look-up and redirecting him -to the correct URL. +**Stop caring about trailing slashes:** Choose the URL style you like, the router automatically redirects the client if a trailing slash is missing or if there is one extra. Of course it only does so, if the new path has a handler. If you don't like it, you can [turn off this behavior](https://godoc.org/github.com/julienschmidt/httprouter#Router.RedirectTrailingSlash). -**Parameters in your routing pattern:** Stop parsing the requested URL path, -just give the path segment a name and the router delivers the dynamic value to -you. Because of the design of the router, path parameters are very cheap. +**Path auto-correction:** Besides detecting the missing or additional trailing slash at no extra cost, the router can also fix wrong cases and remove superfluous path elements (like `../` or `//`). Is [CAPTAIN CAPS LOCK](http://www.urbandictionary.com/define.php?term=Captain+Caps+Lock) one of your users? HttpRouter can help him by making a case-insensitive look-up and redirecting him to the correct URL. -**Zero Garbage:** The matching and dispatching process generates zero bytes of -garbage. In fact, the only heap allocations that are made, is by building the -slice of the key-value pairs for path parameters. If the request path contains -no parameters, not a single heap allocation is necessary. +**Parameters in your routing pattern:** Stop parsing the requested URL path, just give the path segment a name and the router delivers the dynamic value to you. Because of the design of the router, path parameters are very cheap. -**Best Performance:** [Benchmarks speak for themselves](https://github.com/julienschmidt/go-http-routing-benchmark). -See below for technical details of the implementation. +**Zero Garbage:** The matching and dispatching process generates zero bytes of garbage. The only heap allocations that are made are building the slice of the key-value pairs for path parameters, and building new context and request objects (the latter only in the standard `Handler`/`HandlerFunc` api). In the 3-argument API, if the request path contains no parameters not a single heap allocation is necessary. -**No more server crashes:** You can set a [Panic handler](http://godoc.org/github.com/julienschmidt/httprouter#Router.PanicHandler) to deal with panics -occurring during handling a HTTP request. The router then recovers and lets the -PanicHandler log what happened and deliver a nice error page. +**Best Performance:** [Benchmarks speak for themselves](https://github.com/julienschmidt/go-http-routing-benchmark). See below for technical details of the implementation. -Of course you can also set **custom [NotFound](http://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) and [MethodNotAllowed](http://godoc.org/github.com/julienschmidt/httprouter#Router.MethodNotAllowed) handlers** and [**serve static files**](http://godoc.org/github.com/julienschmidt/httprouter#Router.ServeFiles). +**No more server crashes:** You can set a [Panic handler](https://godoc.org/github.com/julienschmidt/httprouter#Router.PanicHandler) to deal with panics occurring during handling a HTTP request. The router then recovers and lets the `PanicHandler` log what happened and deliver a nice error page. + +**Perfect for APIs:** The router design encourages to build sensible, hierarchical RESTful APIs. Moreover it has builtin native support for [OPTIONS requests](http://zacstewart.com/2012/04/14/http-options-method.html) and `405 Method Not Allowed` replies. + +Of course you can also set **custom [`NotFound`](https://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) and [`MethodNotAllowed`](https://godoc.org/github.com/julienschmidt/httprouter#Router.MethodNotAllowed) handlers** and [**serve static files**](https://godoc.org/github.com/julienschmidt/httprouter#Router.ServeFiles). ## Usage + This is just a quick introduction, view the [GoDoc](http://godoc.org/github.com/julienschmidt/httprouter) for details. Let's start with a trivial example: + ```go package main @@ -81,12 +60,11 @@ func main() { ``` ### Named parameters -As you can see, `:name` is a *named parameter*. -The values are accessible via `httprouter.Params`, which is just a slice of `httprouter.Param`s. -You can get the value of a parameter either by its index in the slice, or by using the `ByName(name)` method: -`:name` can be retrived by `ByName("name")`. + +As you can see, `:name` is a *named parameter*. The values are accessible via `httprouter.Params`, which is just a slice of `httprouter.Param`s. You can get the value of a parameter either by its index in the slice, or by using the `ByName(name)` method: `:name` can be retrived by `ByName("name")`. Named parameters only match a single path segment: + ``` Pattern: /user/:user @@ -99,9 +77,9 @@ Pattern: /user/:user **Note:** Since this router has only explicit matches, you can not register static routes and parameters for the same path segment. For example you can not register the patterns `/user/new` and `/user/:user` for the same request method at the same time. The routing of different request methods is independent from each other. ### Catch-All parameters -The second type are *catch-all* parameters and have the form `*name`. -Like the name suggests, they match everything. -Therefore they must always be at the **end** of the pattern: + +The second type are *catch-all* parameters and have the form `*name`. Like the name suggests, they match everything. Therefore they must always be at the **end** of the pattern: + ``` Pattern: /src/*filepath @@ -111,11 +89,8 @@ Pattern: /src/*filepath ``` ## How does it work? -The router relies on a tree structure which makes heavy use of *common prefixes*, -it is basically a *compact* [*prefix tree*](http://en.wikipedia.org/wiki/Trie) -(or just [*Radix tree*](http://en.wikipedia.org/wiki/Radix_tree)). -Nodes with a common prefix also share a common parent. Here is a short example -what the routing tree for the `GET` request method could look like: + +The router relies on a tree structure which makes heavy use of *common prefixes*, it is basically a *compact* [*prefix tree*](https://en.wikipedia.org/wiki/Trie) (or just [*Radix tree*](https://en.wikipedia.org/wiki/Radix_tree)). Nodes with a common prefix also share a common parent. Here is a short example what the routing tree for the `GET` request method could look like: ``` Priority Path Handle @@ -130,33 +105,15 @@ Priority Path Handle 1 | └team\ *<7> 1 └contact\ *<8> ``` -Every `*` represents the memory address of a handler function (a pointer). -If you follow a path trough the tree from the root to the leaf, you get the -complete route path, e.g `\blog\:post\`, where `:post` is just a placeholder -([*parameter*](#named-parameters)) for an actual post name. Unlike hash-maps, a -tree structure also allows us to use dynamic parts like the `:post` parameter, -since we actually match against the routing patterns instead of just comparing -hashes. [As benchmarks show](https://github.com/julienschmidt/go-http-routing-benchmark), -this works very well and efficient. -Since URL paths have a hierarchical structure and make use only of a limited set -of characters (byte values), it is very likely that there are a lot of common -prefixes. This allows us to easily reduce the routing into ever smaller problems. -Moreover the router manages a separate tree for every request method. -For one thing it is more space efficient than holding a method->handle map in -every single node, for another thing is also allows us to greatly reduce the -routing problem before even starting the look-up in the prefix-tree. +Every `*` represents the memory address of a handler function (a pointer). If you follow a path trough the tree from the root to the leaf, you get the complete route path, e.g `\blog\:post\`, where `:post` is just a placeholder ([*parameter*](#named-parameters)) for an actual post name. Unlike hash-maps, a tree structure also allows us to use dynamic parts like the `:post` parameter, since we actually match against the routing patterns instead of just comparing hashes. [As benchmarks show](https://github.com/julienschmidt/go-http-routing-benchmark), this works very well and efficient. -For even better scalability, the child nodes on each tree level are ordered by -priority, where the priority is just the number of handles registered in sub -nodes (children, grandchildren, and so on..). -This helps in two ways: +Since URL paths have a hierarchical structure and make use only of a limited set of characters (byte values), it is very likely that there are a lot of common prefixes. This allows us to easily reduce the routing into ever smaller problems. Moreover the router manages a separate tree for every request method. For one thing it is more space efficient than holding a method->handle map in every single node, it also allows us to greatly reduce the routing problem before even starting the look-up in the prefix-tree. -1. Nodes which are part of the most routing paths are evaluated first. This -helps to make as much routes as possible to be reachable as fast as possible. -2. It is some sort of cost compensation. The longest reachable path (highest -cost) can always be evaluated first. The following scheme visualizes the tree -structure. Nodes are evaluated from top to bottom and from left to right. +For even better scalability, the child nodes on each tree level are ordered by priority, where the priority is just the number of handles registered in sub nodes (children, grandchildren, and so on..). This helps in two ways: + +1. Nodes which are part of the most routing paths are evaluated first. This helps to make as much routes as possible to be reachable as fast as possible. +2. It is some sort of cost compensation. The longest reachable path (highest cost) can always be evaluated first. The following scheme visualizes the tree structure. Nodes are evaluated from top to bottom and from left to right. ``` ├------------ @@ -168,47 +125,38 @@ structure. Nodes are evaluated from top to bottom and from left to right. └- ``` +## Why doesn't this work with `http.Handler`? -## Why doesn't this work with http.Handler? -**It does!** The router itself implements the http.Handler interface. -Moreover the router provides convenient [adapters for http.Handler](http://godoc.org/github.com/julienschmidt/httprouter#Router.Handler)s and [http.HandlerFunc](http://godoc.org/github.com/julienschmidt/httprouter#Router.HandlerFunc)s -which allows them to be used as a [httprouter.Handle](http://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) when registering a route. -The only disadvantage is, that no parameter values can be retrieved when a -http.Handler or http.HandlerFunc is used, since there is no efficient way to -pass the values with the existing function parameters. -Therefore [httprouter.Handle](http://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) has a third function parameter. +**It does!** The router itself implements the `http.Handler` interface. Moreover the router provides convenient [adapters for `http.Handler`](https://godoc.org/github.com/julienschmidt/httprouter#Router.Handler)s and [`http.HandlerFunc`](https://godoc.org/github.com/julienschmidt/httprouter#Router.HandlerFunc)s which allows them to be used as a [`httprouter.Handle`](https://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) when registering a route. The only disadvantage is, that no parameter values can be retrieved when a `http.Handler` or `http.HandlerFunc` is used, since there is no efficient way to pass the values with the existing function parameters. Therefore [`httprouter.Handle`](https://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) has a third function parameter. Just try it out for yourself, the usage of HttpRouter is very straightforward. The package is compact and minimalistic, but also probably one of the easiest routers to set up. - ## Where can I find Middleware *X*? -This package just provides a very efficient request router with a few extra -features. The router is just a [http.Handler](http://golang.org/pkg/net/http/#Handler), -you can chain any http.Handler compatible middleware before the router, -for example the [Gorilla handlers](http://www.gorillatoolkit.org/pkg/handlers). -Or you could [just write your own](http://justinas.org/writing-http-middleware-in-go/), -it's very easy! + +This package just provides a very efficient request router with a few extra features. The router is just a [`http.Handler`](https://golang.org/pkg/net/http/#Handler), you can chain any http.Handler compatible middleware before the router, for example the [Gorilla handlers](http://www.gorillatoolkit.org/pkg/handlers). Or you could [just write your own](https://justinas.org/writing-http-middleware-in-go/), it's very easy! Alternatively, you could try [a web framework based on HttpRouter](#web-frameworks-based-on-httprouter). ### Multi-domain / Sub-domains + Here is a quick example: Does your server serve multiple domains / hosts? You want to use sub-domains? Define a router per host! + ```go // We need an object that implements the http.Handler interface. // Therefore we need a type for which we implement the ServeHTTP method. // We just use a map here, in which we map host names (with port) to http.Handlers type HostSwitch map[string]http.Handler -// Implement the ServerHTTP method on our new type +// Implement the ServeHTTP method on our new type func (hs HostSwitch) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Check if a http.Handler is registered for the given host. // If yes, use it to handle the request. if handler := hs[r.Host]; handler != nil { handler.ServeHTTP(w, r) } else { - // Handle host names for wich no handler is registered + // Handle host names for which no handler is registered http.Error(w, "Forbidden", 403) // Or Redirect? } } @@ -230,94 +178,89 @@ func main() { ``` ### Basic Authentication -Another quick example: Basic Authentification (RFC 2617) for handles: + +Another quick example: Basic Authentication (RFC 2617) for handles: ```go package main import ( - "bytes" - "encoding/base64" - "fmt" - "github.com/julienschmidt/httprouter" - "net/http" - "log" - "strings" + "fmt" + "log" + "net/http" + + "github.com/julienschmidt/httprouter" ) -func BasicAuth(h httprouter.Handle, user, pass []byte) httprouter.Handle { +func BasicAuth(h httprouter.Handle, requiredUser, requiredPassword string) httprouter.Handle { return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { - const basicAuthPrefix string = "Basic " - // Get the Basic Authentication credentials - auth := r.Header.Get("Authorization") - if strings.HasPrefix(auth, basicAuthPrefix) { - // Check credentials - payload, err := base64.StdEncoding.DecodeString(auth[len(basicAuthPrefix):]) - if err == nil { - pair := bytes.SplitN(payload, []byte(":"), 2) - if len(pair) == 2 && - bytes.Equal(pair[0], user) && - bytes.Equal(pair[1], pass) { + user, password, hasAuth := r.BasicAuth() - // Delegate request to the given handle - h(w, r, ps) - return - } - } + if hasAuth && user == requiredUser && password == requiredPassword { + // Delegate request to the given handle + h(w, r, ps) + } else { + // Request Basic Authentication otherwise + w.Header().Set("WWW-Authenticate", "Basic realm=Restricted") + http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) } - - // Request Basic Authentication otherwise - w.Header().Set("WWW-Authenticate", "Basic realm=Restricted") - http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized) } } func Index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - fmt.Fprint(w, "Not protected!\n") + fmt.Fprint(w, "Not protected!\n") } func Protected(w http.ResponseWriter, r *http.Request, _ httprouter.Params) { - fmt.Fprint(w, "Protected!\n") + fmt.Fprint(w, "Protected!\n") } func main() { - user := []byte("gordon") - pass := []byte("secret!") - - router := httprouter.New() - router.GET("/", Index) - router.GET("/protected/", BasicAuth(Protected, user, pass)) + user := "gordon" + pass := "secret!" - log.Fatal(http.ListenAndServe(":8080", router)) + router := httprouter.New() + router.GET("/", Index) + router.GET("/protected/", BasicAuth(Protected, user, pass)) + + log.Fatal(http.ListenAndServe(":8080", router)) } ``` ## Chaining with the NotFound handler -**NOTE: It might be required to set [Router.HandleMethodNotAllowed](http://godoc.org/github.com/julienschmidt/httprouter#Router.HandleMethodNotAllowed) to `false` to avoid problems.** +**NOTE: It might be required to set [`Router.HandleMethodNotAllowed`](https://godoc.org/github.com/julienschmidt/httprouter#Router.HandleMethodNotAllowed) to `false` to avoid problems.** -You can use another [http.HandlerFunc](http://golang.org/pkg/net/http/#HandlerFunc), for example another router, to handle requests which could not be matched by this router by using the [Router.NotFound](http://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) handler. This allows chaining. +You can use another [`http.Handler`](https://golang.org/pkg/net/http/#Handler), for example another router, to handle requests which could not be matched by this router by using the [`Router.NotFound`](https://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) handler. This allows chaining. ### Static files -The `NotFound` handler can for example be used to serve static files from the root path `/` (like an index.html file along with other assets): + +The `NotFound` handler can for example be used to serve static files from the root path `/` (like an `index.html` file along with other assets): + ```go // Serve static files from the ./public directory -router.NotFound = http.FileServer(http.Dir("public")).ServeHTTP +router.NotFound = http.FileServer(http.Dir("public")) ``` But this approach sidesteps the strict core rules of this router to avoid routing problems. A cleaner approach is to use a distinct sub-path for serving files, like `/static/*filepath` or `/files/*filepath`. ## Web Frameworks based on HttpRouter + If the HttpRouter is a bit too minimalistic for you, you might try one of the following more high-level 3rd-party web frameworks building upon the HttpRouter package: + * [Ace](https://github.com/plimble/ace): Blazing fast Go Web Framework -* [api2go](https://github.com/univedo/api2go): A JSON API Implementation for Go +* [api2go](https://github.com/manyminds/api2go): A JSON API Implementation for Go * [Gin](https://github.com/gin-gonic/gin): Features a martini-like API with much better performance * [Goat](https://github.com/bahlo/goat): A minimalistic REST API server in Go +* [goMiddlewareChain](https://github.com/TobiEiss/goMiddlewareChain): An express.js-like-middleware-chain * [Hikaru](https://github.com/najeira/hikaru): Supports standalone and Google AppEngine * [Hitch](https://github.com/nbio/hitch): Hitch ties httprouter, [httpcontext](https://github.com/nbio/httpcontext), and middleware up in a bow +* [httpway](https://github.com/corneldamian/httpway): Simple middleware extension with context for httprouter and a server with gracefully shutdown support * [kami](https://github.com/guregu/kami): A tiny web framework using x/net/context * [Medeina](https://github.com/imdario/medeina): Inspired by Ruby's Roda and Cuba * [Neko](https://github.com/rocwong/neko): A lightweight web application framework for Golang +* [River](https://github.com/abiosoft/river): River is a simple and lightweight REST server * [Roxanna](https://github.com/iamthemuffinman/Roxanna): An amalgamation of httprouter, better logging, and hot reload * [siesta](https://github.com/VividCortex/siesta): Composable HTTP handlers with contexts +* [xmux](https://github.com/rs/xmux): xmux is a httprouter fork on top of xhandler (net/context aware) diff --git a/vendor/github.com/julienschmidt/httprouter/params_go17.go b/vendor/github.com/julienschmidt/httprouter/params_go17.go new file mode 100644 index 0000000..abb4209 --- /dev/null +++ b/vendor/github.com/julienschmidt/httprouter/params_go17.go @@ -0,0 +1,38 @@ +// +build go1.7 + +package httprouter + +import ( + "context" + "net/http" +) + +type paramsKey struct{} + +// ParamsKey is the request context key under which URL params are stored. +// +// This is only present from go 1.7. +var ParamsKey = paramsKey{} + +// Handler is an adapter which allows the usage of an http.Handler as a +// request handle. With go 1.7+, the Params will be available in the +// request context under ParamsKey. +func (r *Router) Handler(method, path string, handler http.Handler) { + r.Handle(method, path, + func(w http.ResponseWriter, req *http.Request, p Params) { + ctx := req.Context() + ctx = context.WithValue(ctx, ParamsKey, p) + req = req.WithContext(ctx) + handler.ServeHTTP(w, req) + }, + ) +} + +// ParamsFromContext pulls the URL parameters from a request context, +// or returns nil if none are present. +// +// This is only present from go 1.7. +func ParamsFromContext(ctx context.Context) Params { + p, _ := ctx.Value(ParamsKey).(Params) + return p +} diff --git a/vendor/github.com/julienschmidt/httprouter/params_legacy.go b/vendor/github.com/julienschmidt/httprouter/params_legacy.go new file mode 100644 index 0000000..0278a44 --- /dev/null +++ b/vendor/github.com/julienschmidt/httprouter/params_legacy.go @@ -0,0 +1,16 @@ +// +build !go1.7 + +package httprouter + +import "net/http" + +// Handler is an adapter which allows the usage of an http.Handler as a +// request handle. With go 1.7+, the Params will be available in the +// request context under ParamsKey. +func (r *Router) Handler(method, path string, handler http.Handler) { + r.Handle(method, path, + func(w http.ResponseWriter, req *http.Request, _ Params) { + handler.ServeHTTP(w, req) + }, + ) +} diff --git a/vendor/github.com/julienschmidt/httprouter/path.go b/vendor/github.com/julienschmidt/httprouter/path.go index 486134d..0331c7e 100644 --- a/vendor/github.com/julienschmidt/httprouter/path.go +++ b/vendor/github.com/julienschmidt/httprouter/path.go @@ -41,7 +41,7 @@ func CleanPath(p string) string { buf[0] = '/' } - trailing := n > 2 && p[n-1] == '/' + trailing := n > 1 && p[n-1] == '/' // A bit more clunky without a 'lazybuf' like the path package, but the loop // gets completely inlined (bufApp). So in contrast to the path package this @@ -59,11 +59,11 @@ func CleanPath(p string) string { case p[r] == '.' && p[r+1] == '/': // . element - r++ + r += 2 case p[r] == '.' && p[r+1] == '.' && (r+2 == n || p[r+2] == '/'): // .. element: remove to last / - r += 2 + r += 3 if w > 1 { // can backtrack diff --git a/vendor/github.com/julienschmidt/httprouter/path_test.go b/vendor/github.com/julienschmidt/httprouter/path_test.go index c4ceda5..75fde32 100644 --- a/vendor/github.com/julienschmidt/httprouter/path_test.go +++ b/vendor/github.com/julienschmidt/httprouter/path_test.go @@ -22,6 +22,7 @@ var cleanTests = []struct { // missing root {"", "/"}, + {"a/", "/a/"}, {"abc", "/abc"}, {"abc/def", "/abc/def"}, {"a/b/c", "/a/b/c"}, diff --git a/vendor/github.com/julienschmidt/httprouter/router.go b/vendor/github.com/julienschmidt/httprouter/router.go index 155b871..558e139 100644 --- a/vendor/github.com/julienschmidt/httprouter/router.go +++ b/vendor/github.com/julienschmidt/httprouter/router.go @@ -55,7 +55,7 @@ // // Catch-all parameters match anything until the path end, including the // directory index (the '/' before the catch-all). Since they match anything -// until the end, catch-all paramerters must always be the final path element. +// until the end, catch-all parameters must always be the final path element. // Path: /files/*filepath // // Requests: @@ -138,14 +138,20 @@ type Router struct { // handler. HandleMethodNotAllowed bool - // Configurable http.HandlerFunc which is called when no matching route is - // found. If it is not set, http.NotFound is used. - NotFound http.HandlerFunc + // If enabled, the router automatically replies to OPTIONS requests. + // Custom OPTIONS handlers take priority over automatic replies. + HandleOPTIONS bool - // Configurable http.HandlerFunc which is called when a request + // Configurable http.Handler which is called when no matching route is + // found. If it is not set, http.NotFound is used. + NotFound http.Handler + + // Configurable http.Handler which is called when a request // cannot be routed and HandleMethodNotAllowed is true. // If it is not set, http.Error with http.StatusMethodNotAllowed is used. - MethodNotAllowed http.HandlerFunc + // The "Allow" header with allowed request methods is set before the handler + // is called. + MethodNotAllowed http.Handler // Function to handle panics recovered from http handlers. // It should be used to generate a error page and return the http error code @@ -165,6 +171,7 @@ func New() *Router { RedirectTrailingSlash: true, RedirectFixedPath: true, HandleMethodNotAllowed: true, + HandleOPTIONS: true, } } @@ -229,16 +236,6 @@ func (r *Router) Handle(method, path string, handle Handle) { root.addRoute(path, handle) } -// Handler is an adapter which allows the usage of an http.Handler as a -// request handle. -func (r *Router) Handler(method, path string, handler http.Handler) { - r.Handle(method, path, - func(w http.ResponseWriter, req *http.Request, _ Params) { - handler.ServeHTTP(w, req) - }, - ) -} - // HandlerFunc is an adapter which allows the usage of an http.HandlerFunc as a // request handle. func (r *Router) HandlerFunc(method, path string, handler http.HandlerFunc) { @@ -286,15 +283,53 @@ func (r *Router) Lookup(method, path string) (Handle, Params, bool) { return nil, nil, false } +func (r *Router) allowed(path, reqMethod string) (allow string) { + if path == "*" { // server-wide + for method := range r.trees { + if method == "OPTIONS" { + continue + } + + // add request method to list of allowed methods + if len(allow) == 0 { + allow = method + } else { + allow += ", " + method + } + } + } else { // specific path + for method := range r.trees { + // Skip the requested method - we already tried this one + if method == reqMethod || method == "OPTIONS" { + continue + } + + handle, _, _ := r.trees[method].getValue(path) + if handle != nil { + // add request method to list of allowed methods + if len(allow) == 0 { + allow = method + } else { + allow += ", " + method + } + } + } + } + if len(allow) > 0 { + allow += ", OPTIONS" + } + return +} + // ServeHTTP makes the router implement the http.Handler interface. func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { if r.PanicHandler != nil { defer r.recv(w, req) } - if root := r.trees[req.Method]; root != nil { - path := req.URL.Path + path := req.URL.Path + if root := r.trees[req.Method]; root != nil { if handle, ps, tsr := root.getValue(path); handle != nil { handle(w, req, ps) return @@ -331,18 +366,19 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { } } - // Handle 405 - if r.HandleMethodNotAllowed { - for method := range r.trees { - // Skip the requested method - we already tried this one - if method == req.Method { - continue - } - - handle, _, _ := r.trees[method].getValue(req.URL.Path) - if handle != nil { + if req.Method == "OPTIONS" && r.HandleOPTIONS { + // Handle OPTIONS requests + if allow := r.allowed(path, req.Method); len(allow) > 0 { + w.Header().Set("Allow", allow) + return + } + } else { + // Handle 405 + if r.HandleMethodNotAllowed { + if allow := r.allowed(path, req.Method); len(allow) > 0 { + w.Header().Set("Allow", allow) if r.MethodNotAllowed != nil { - r.MethodNotAllowed(w, req) + r.MethodNotAllowed.ServeHTTP(w, req) } else { http.Error(w, http.StatusText(http.StatusMethodNotAllowed), @@ -356,7 +392,7 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { // Handle 404 if r.NotFound != nil { - r.NotFound(w, req) + r.NotFound.ServeHTTP(w, req) } else { http.NotFound(w, req) } diff --git a/vendor/github.com/julienschmidt/httprouter/router_test.go b/vendor/github.com/julienschmidt/httprouter/router_test.go index 9dc6296..b774a4e 100644 --- a/vendor/github.com/julienschmidt/httprouter/router_test.go +++ b/vendor/github.com/julienschmidt/httprouter/router_test.go @@ -68,11 +68,11 @@ func TestRouter(t *testing.T) { } type handlerStruct struct { - handeled *bool + handled *bool } func (h handlerStruct) ServeHTTP(w http.ResponseWriter, r *http.Request) { - *h.handeled = true + *h.handled = true } func TestRouterAPI(t *testing.T) { @@ -174,26 +174,175 @@ func TestRouterRoot(t *testing.T) { } } +func TestRouterChaining(t *testing.T) { + router1 := New() + router2 := New() + router1.NotFound = router2 + + fooHit := false + router1.POST("/foo", func(w http.ResponseWriter, req *http.Request, _ Params) { + fooHit = true + w.WriteHeader(http.StatusOK) + }) + + barHit := false + router2.POST("/bar", func(w http.ResponseWriter, req *http.Request, _ Params) { + barHit = true + w.WriteHeader(http.StatusOK) + }) + + r, _ := http.NewRequest("POST", "/foo", nil) + w := httptest.NewRecorder() + router1.ServeHTTP(w, r) + if !(w.Code == http.StatusOK && fooHit) { + t.Errorf("Regular routing failed with router chaining.") + t.FailNow() + } + + r, _ = http.NewRequest("POST", "/bar", nil) + w = httptest.NewRecorder() + router1.ServeHTTP(w, r) + if !(w.Code == http.StatusOK && barHit) { + t.Errorf("Chained routing failed with router chaining.") + t.FailNow() + } + + r, _ = http.NewRequest("POST", "/qax", nil) + w = httptest.NewRecorder() + router1.ServeHTTP(w, r) + if !(w.Code == http.StatusNotFound) { + t.Errorf("NotFound behavior failed with router chaining.") + t.FailNow() + } +} + +func TestRouterOPTIONS(t *testing.T) { + handlerFunc := func(_ http.ResponseWriter, _ *http.Request, _ Params) {} + + router := New() + router.POST("/path", handlerFunc) + + // test not allowed + // * (server) + r, _ := http.NewRequest("OPTIONS", "*", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusOK) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } + + // path + r, _ = http.NewRequest("OPTIONS", "/path", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusOK) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } + + r, _ = http.NewRequest("OPTIONS", "/doesnotexist", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusNotFound) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } + + // add another method + router.GET("/path", handlerFunc) + + // test again + // * (server) + r, _ = http.NewRequest("OPTIONS", "*", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusOK) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, GET, OPTIONS" && allow != "GET, POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } + + // path + r, _ = http.NewRequest("OPTIONS", "/path", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusOK) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, GET, OPTIONS" && allow != "GET, POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } + + // custom handler + var custom bool + router.OPTIONS("/path", func(w http.ResponseWriter, r *http.Request, _ Params) { + custom = true + }) + + // test again + // * (server) + r, _ = http.NewRequest("OPTIONS", "*", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusOK) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, GET, OPTIONS" && allow != "GET, POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } + if custom { + t.Error("custom handler called on *") + } + + // path + r, _ = http.NewRequest("OPTIONS", "/path", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusOK) { + t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } + if !custom { + t.Error("custom handler not called") + } +} + func TestRouterNotAllowed(t *testing.T) { handlerFunc := func(_ http.ResponseWriter, _ *http.Request, _ Params) {} router := New() router.POST("/path", handlerFunc) - // Test not allowed + // test not allowed r, _ := http.NewRequest("GET", "/path", nil) w := httptest.NewRecorder() router.ServeHTTP(w, r) if !(w.Code == http.StatusMethodNotAllowed) { t.Errorf("NotAllowed handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) } + // add another method + router.DELETE("/path", handlerFunc) + router.OPTIONS("/path", handlerFunc) // must be ignored + + // test again + r, _ = http.NewRequest("GET", "/path", nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, r) + if !(w.Code == http.StatusMethodNotAllowed) { + t.Errorf("NotAllowed handling failed: Code=%d, Header=%v", w.Code, w.Header()) + } else if allow := w.Header().Get("Allow"); allow != "POST, DELETE, OPTIONS" && allow != "DELETE, POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } + + // test custom handler w = httptest.NewRecorder() responseText := "custom method" - router.MethodNotAllowed = func(w http.ResponseWriter, req *http.Request) { + router.MethodNotAllowed = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusTeapot) w.Write([]byte(responseText)) - } + }) router.ServeHTTP(w, r) if got := w.Body.String(); !(got == responseText) { t.Errorf("unexpected response got %q want %q", got, responseText) @@ -201,6 +350,9 @@ func TestRouterNotAllowed(t *testing.T) { if w.Code != http.StatusTeapot { t.Errorf("unexpected response code %d want %d", w.Code, http.StatusTeapot) } + if allow := w.Header().Get("Allow"); allow != "POST, DELETE, OPTIONS" && allow != "DELETE, POST, OPTIONS" { + t.Error("unexpected Allow header value: " + allow) + } } func TestRouterNotFound(t *testing.T) { @@ -212,35 +364,35 @@ func TestRouterNotFound(t *testing.T) { router.GET("/", handlerFunc) testRoutes := []struct { - route string - code int - header string + route string + code int + location string }{ - {"/path/", 301, "map[Location:[/path]]"}, // TSR -/ - {"/dir", 301, "map[Location:[/dir/]]"}, // TSR +/ - {"", 301, "map[Location:[/]]"}, // TSR +/ - {"/PATH", 301, "map[Location:[/path]]"}, // Fixed Case - {"/DIR/", 301, "map[Location:[/dir/]]"}, // Fixed Case - {"/PATH/", 301, "map[Location:[/path]]"}, // Fixed Case -/ - {"/DIR", 301, "map[Location:[/dir/]]"}, // Fixed Case +/ - {"/../path", 301, "map[Location:[/path]]"}, // CleanPath - {"/nope", 404, ""}, // NotFound + {"/path/", 301, "/path"}, // TSR -/ + {"/dir", 301, "/dir/"}, // TSR +/ + {"", 301, "/"}, // TSR +/ + {"/PATH", 301, "/path"}, // Fixed Case + {"/DIR/", 301, "/dir/"}, // Fixed Case + {"/PATH/", 301, "/path"}, // Fixed Case -/ + {"/DIR", 301, "/dir/"}, // Fixed Case +/ + {"/../path", 301, "/path"}, // CleanPath + {"/nope", 404, ""}, // NotFound } for _, tr := range testRoutes { r, _ := http.NewRequest("GET", tr.route, nil) w := httptest.NewRecorder() router.ServeHTTP(w, r) - if !(w.Code == tr.code && (w.Code == 404 || fmt.Sprint(w.Header()) == tr.header)) { - t.Errorf("NotFound handling route %s failed: Code=%d, Header=%v", tr.route, w.Code, w.Header()) + if !(w.Code == tr.code && (w.Code == 404 || fmt.Sprint(w.Header().Get("Location")) == tr.location)) { + t.Errorf("NotFound handling route %s failed: Code=%d, Header=%v", tr.route, w.Code, w.Header().Get("Location")) } } // Test custom not found handler var notFound bool - router.NotFound = func(rw http.ResponseWriter, r *http.Request) { + router.NotFound = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) { rw.WriteHeader(404) notFound = true - } + }) r, _ := http.NewRequest("GET", "/nope", nil) w := httptest.NewRecorder() router.ServeHTTP(w, r) diff --git a/vendor/github.com/julienschmidt/httprouter/tree.go b/vendor/github.com/julienschmidt/httprouter/tree.go index a15bc2c..a8fa98b 100644 --- a/vendor/github.com/julienschmidt/httprouter/tree.go +++ b/vendor/github.com/julienschmidt/httprouter/tree.go @@ -7,6 +7,7 @@ package httprouter import ( "strings" "unicode" + "unicode/utf8" ) func min(a, b int) int { @@ -33,9 +34,10 @@ func countParams(path string) uint8 { type nodeType uint8 const ( - static nodeType = 0 - param nodeType = 1 - catchAll nodeType = 2 + static nodeType = iota // default + root + param + catchAll ) type node struct { @@ -58,9 +60,7 @@ func (n *node) incrementChildPrio(pos int) int { newPos := pos for newPos > 0 && n.children[newPos-1].priority < prio { // swap node positions - tmpN := n.children[newPos-1] - n.children[newPos-1] = n.children[newPos] - n.children[newPos] = tmpN + n.children[newPos-1], n.children[newPos] = n.children[newPos], n.children[newPos-1] newPos-- } @@ -105,6 +105,7 @@ func (n *node) addRoute(path string, handle Handle) { child := node{ path: n.path[i:], wildChild: n.wildChild, + nType: static, indices: n.indices, children: n.children, handle: n.handle, @@ -141,16 +142,25 @@ func (n *node) addRoute(path string, handle Handle) { numParams-- // Check if the wildcard matches - if len(path) >= len(n.path) && n.path == path[:len(n.path)] { - // check for longer wildcard, e.g. :name and :names - if len(n.path) >= len(path) || path[len(n.path)] == '/' { - continue walk + if len(path) >= len(n.path) && n.path == path[:len(n.path)] && + // Check for longer wildcard, e.g. :name and :names + (len(n.path) >= len(path) || path[len(n.path)] == '/') { + continue walk + } else { + // Wildcard conflict + var pathSeg string + if n.nType == catchAll { + pathSeg = path + } else { + pathSeg = strings.SplitN(path, "/", 2)[0] } + prefix := fullPath[:strings.Index(fullPath, pathSeg)] + n.path + panic("'" + pathSeg + + "' in new path '" + fullPath + + "' conflicts with existing wildcard '" + n.path + + "' in existing prefix '" + prefix + + "'") } - - panic("path segment '" + path + - "' conflicts with existing wildcard '" + n.path + - "' in path '" + fullPath + "'") } c := path[0] @@ -187,7 +197,7 @@ func (n *node) addRoute(path string, handle Handle) { } else if i == len(path) { // Make node a (in-path) leaf if n.handle != nil { - panic("a handle is already registered for path ''" + fullPath + "'") + panic("a handle is already registered for path '" + fullPath + "'") } n.handle = handle } @@ -195,6 +205,7 @@ func (n *node) addRoute(path string, handle Handle) { } } else { // Empty tree n.insertChild(numParams, path, fullPath, handle) + n.nType = root } } @@ -317,7 +328,7 @@ func (n *node) insertChild(numParams uint8, path, fullPath string, handle Handle // made if a handle exists with an extra (without the) trailing slash for the // given path. func (n *node) getValue(path string) (handle Handle, p Params, tsr bool) { -walk: // Outer loop for walking the tree +walk: // outer loop for walking the tree for { if len(path) > len(n.path) { if path[:len(n.path)] == n.path { @@ -411,6 +422,11 @@ walk: // Outer loop for walking the tree return } + if path == "/" && n.wildChild && n.nType != root { + tsr = true + return + } + // No handle found. Check if a handle for this path + a // trailing slash exists for trailing slash recommendation for i := 0; i < len(n.indices); i++ { @@ -439,34 +455,117 @@ walk: // Outer loop for walking the tree // It returns the case-corrected path and a bool indicating whether the lookup // was successful. func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPath []byte, found bool) { - ciPath = make([]byte, 0, len(path)+1) // preallocate enough memory + return n.findCaseInsensitivePathRec( + path, + strings.ToLower(path), + make([]byte, 0, len(path)+1), // preallocate enough memory for new path + [4]byte{}, // empty rune buffer + fixTrailingSlash, + ) +} - // Outer loop for walking the tree - for len(path) >= len(n.path) && strings.ToLower(path[:len(n.path)]) == strings.ToLower(n.path) { - path = path[len(n.path):] +// shift bytes in array by n bytes left +func shiftNRuneBytes(rb [4]byte, n int) [4]byte { + switch n { + case 0: + return rb + case 1: + return [4]byte{rb[1], rb[2], rb[3], 0} + case 2: + return [4]byte{rb[2], rb[3]} + case 3: + return [4]byte{rb[3]} + default: + return [4]byte{} + } +} + +// recursive case-insensitive lookup function used by n.findCaseInsensitivePath +func (n *node) findCaseInsensitivePathRec(path, loPath string, ciPath []byte, rb [4]byte, fixTrailingSlash bool) ([]byte, bool) { + loNPath := strings.ToLower(n.path) + +walk: // outer loop for walking the tree + for len(loPath) >= len(loNPath) && (len(loNPath) == 0 || loPath[1:len(loNPath)] == loNPath[1:]) { + // add common path to result ciPath = append(ciPath, n.path...) - if len(path) > 0 { + if path = path[len(n.path):]; len(path) > 0 { + loOld := loPath + loPath = loPath[len(loNPath):] + // If this node does not have a wildcard (param or catchAll) child, // we can just look up the next child node and continue to walk down // the tree if !n.wildChild { - r := unicode.ToLower(rune(path[0])) - for i, index := range n.indices { - // must use recursive approach since both index and - // ToLower(index) could exist. We must check both. - if r == unicode.ToLower(index) { - out, found := n.children[i].findCaseInsensitivePath(path, fixTrailingSlash) - if found { - return append(ciPath, out...), true + // skip rune bytes already processed + rb = shiftNRuneBytes(rb, len(loNPath)) + + if rb[0] != 0 { + // old rune not finished + for i := 0; i < len(n.indices); i++ { + if n.indices[i] == rb[0] { + // continue with child node + n = n.children[i] + loNPath = strings.ToLower(n.path) + continue walk + } + } + } else { + // process a new rune + var rv rune + + // find rune start + // runes are up to 4 byte long, + // -4 would definitely be another rune + var off int + for max := min(len(loNPath), 3); off < max; off++ { + if i := len(loNPath) - off; utf8.RuneStart(loOld[i]) { + // read rune from cached lowercase path + rv, _ = utf8.DecodeRuneInString(loOld[i:]) + break + } + } + + // calculate lowercase bytes of current rune + utf8.EncodeRune(rb[:], rv) + // skipp already processed bytes + rb = shiftNRuneBytes(rb, off) + + for i := 0; i < len(n.indices); i++ { + // lowercase matches + if n.indices[i] == rb[0] { + // must use a recursive approach since both the + // uppercase byte and the lowercase byte might exist + // as an index + if out, found := n.children[i].findCaseInsensitivePathRec( + path, loPath, ciPath, rb, fixTrailingSlash, + ); found { + return out, true + } + break + } + } + + // same for uppercase rune, if it differs + if up := unicode.ToUpper(rv); up != rv { + utf8.EncodeRune(rb[:], up) + rb = shiftNRuneBytes(rb, off) + + for i := 0; i < len(n.indices); i++ { + // uppercase matches + if n.indices[i] == rb[0] { + // continue with child node + n = n.children[i] + loNPath = strings.ToLower(n.path) + continue walk + } } } } // Nothing found. We can recommend to redirect to the same URL // without a trailing slash if a leaf exists for that path - found = (fixTrailingSlash && path == "/" && n.handle != nil) - return + return ciPath, (fixTrailingSlash && path == "/" && n.handle != nil) } n = n.children[0] @@ -484,8 +583,11 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa // we need to go deeper! if k < len(path) { if len(n.children) > 0 { - path = path[k:] + // continue with child node n = n.children[0] + loNPath = strings.ToLower(n.path) + loPath = loPath[k:] + path = path[k:] continue } @@ -493,7 +595,7 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa if fixTrailingSlash && len(path) == k+1 { return ciPath, true } - return + return ciPath, false } if n.handle != nil { @@ -506,7 +608,7 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa return append(ciPath, '/'), true } } - return + return ciPath, false case catchAll: return append(ciPath, path...), true @@ -531,11 +633,11 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa (n.nType == catchAll && n.children[0].handle != nil) { return append(ciPath, '/'), true } - return + return ciPath, false } } } - return + return ciPath, false } } @@ -545,11 +647,10 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa if path == "/" { return ciPath, true } - if len(path)+1 == len(n.path) && n.path[len(path)] == '/' && - strings.ToLower(path) == strings.ToLower(n.path[:len(path)]) && - n.handle != nil { + if len(loPath)+1 == len(loNPath) && loNPath[len(loPath)] == '/' && + loPath[1:] == loNPath[1:len(loPath)] && n.handle != nil { return append(ciPath, n.path...), true } } - return + return ciPath, false } diff --git a/vendor/github.com/julienschmidt/httprouter/tree_test.go b/vendor/github.com/julienschmidt/httprouter/tree_test.go index 64f26d1..e89d945 100644 --- a/vendor/github.com/julienschmidt/httprouter/tree_test.go +++ b/vendor/github.com/julienschmidt/httprouter/tree_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "reflect" + "regexp" "strings" "testing" ) @@ -22,7 +23,7 @@ func printChildren(n *node, prefix string) { } } -// Used as a workaround since we can't compare functions or their adresses +// Used as a workaround since we can't compare functions or their addresses var fakeHandlerValue string func fakeHandler(val string) Handle { @@ -89,7 +90,7 @@ func checkMaxParams(t *testing.T, n *node) uint8 { maxParams = params } } - if n.nType != static && !n.wildChild { + if n.nType > root && !n.wildChild { maxParams++ } @@ -394,6 +395,9 @@ func TestTreeTrailingSlashRedirect(t *testing.T) { "/1/:id/2", "/aa", "/a/", + "/admin", + "/admin/:category", + "/admin/:category/:page", "/doc", "/doc/go_faq.html", "/doc/go1.html", @@ -423,6 +427,9 @@ func TestTreeTrailingSlashRedirect(t *testing.T) { "/0/go/", "/1/go", "/a", + "/admin/", + "/admin/config/", + "/admin/config/permissions/", "/doc/", } for _, route := range tsrRoutes { @@ -452,6 +459,24 @@ func TestTreeTrailingSlashRedirect(t *testing.T) { } } +func TestTreeRootTrailingSlashRedirect(t *testing.T) { + tree := &node{} + + recv := catchPanic(func() { + tree.addRoute("/:test", fakeHandler("/:test")) + }) + if recv != nil { + t.Fatalf("panic inserting test route: %v", recv) + } + + handler, _, tsr := tree.getValue("/") + if handler != nil { + t.Fatalf("non-nil handler") + } else if tsr { + t.Errorf("expected no TSR recommendation") + } +} + func TestTreeFindCaseInsensitivePath(t *testing.T) { tree := &node{} @@ -478,6 +503,16 @@ func TestTreeFindCaseInsensitivePath(t *testing.T) { "/doc/go/away", "/no/a", "/no/b", + "/Π", + "/u/apfêl/", + "/u/äpfêl/", + "/u/öpfêl", + "/v/Äpfêl/", + "/v/Öpfêl", + "/w/♬", // 3 byte + "/w/♭/", // 3 byte, last byte differs + "/w/𠜎", // 4 byte + "/w/𠜏/", // 4 byte } for _, route := range routes { @@ -556,6 +591,20 @@ func TestTreeFindCaseInsensitivePath(t *testing.T) { {"/DOC/", "/doc", true, true}, {"/NO", "", false, true}, {"/DOC/GO", "", false, true}, + {"/π", "/Π", true, false}, + {"/π/", "/Π", true, true}, + {"/u/ÄPFÊL/", "/u/äpfêl/", true, false}, + {"/u/ÄPFÊL", "/u/äpfêl/", true, true}, + {"/u/ÖPFÊL/", "/u/öpfêl", true, true}, + {"/u/ÖPFÊL", "/u/öpfêl", true, false}, + {"/v/äpfêL/", "/v/Äpfêl/", true, false}, + {"/v/äpfêL", "/v/Äpfêl/", true, true}, + {"/v/öpfêL/", "/v/Öpfêl", true, true}, + {"/v/öpfêL", "/v/Öpfêl", true, false}, + {"/w/♬/", "/w/♬", true, true}, + {"/w/♭", "/w/♭/", true, true}, + {"/w/𠜎/", "/w/𠜎", true, true}, + {"/w/𠜏", "/w/𠜏/", true, true}, } // With fixTrailingSlash = true for _, test := range tests { @@ -609,3 +658,42 @@ func TestTreeInvalidNodeType(t *testing.T) { t.Fatalf("Expected panic '"+panicMsg+"', got '%v'", recv) } } + +func TestTreeWildcardConflictEx(t *testing.T) { + conflicts := [...]struct { + route string + segPath string + existPath string + existSegPath string + }{ + {"/who/are/foo", "/foo", `/who/are/\*you`, `/\*you`}, + {"/who/are/foo/", "/foo/", `/who/are/\*you`, `/\*you`}, + {"/who/are/foo/bar", "/foo/bar", `/who/are/\*you`, `/\*you`}, + {"/conxxx", "xxx", `/con:tact`, `:tact`}, + {"/conooo/xxx", "ooo", `/con:tact`, `:tact`}, + } + + for _, conflict := range conflicts { + // I have to re-create a 'tree', because the 'tree' will be + // in an inconsistent state when the loop recovers from the + // panic which threw by 'addRoute' function. + tree := &node{} + routes := [...]string{ + "/con:tact", + "/who/are/*you", + "/who/foo/hello", + } + + for _, route := range routes { + tree.addRoute(route, fakeHandler(route)) + } + + recv := catchPanic(func() { + tree.addRoute(conflict.route, fakeHandler(conflict.route)) + }) + + if !regexp.MustCompile(fmt.Sprintf("'%s' in new path .* conflicts with existing wildcard '%s' in existing prefix '%s'", conflict.segPath, conflict.existSegPath, conflict.existPath)).MatchString(fmt.Sprint(recv)) { + t.Fatalf("invalid wildcard conflict error (%v)", recv) + } + } +} diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 90f2407..280d548 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -14,6 +14,7 @@ It offers slightly better compression at lower compression settings, and up to 3 # changelog +* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression). * Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below. * Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0). * Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change. diff --git a/vendor/github.com/klauspost/compress/flate/crc32_amd64.go b/vendor/github.com/klauspost/compress/flate/crc32_amd64.go index 70a6095..8298d30 100644 --- a/vendor/github.com/klauspost/compress/flate/crc32_amd64.go +++ b/vendor/github.com/klauspost/compress/flate/crc32_amd64.go @@ -1,5 +1,6 @@ //+build !noasm //+build !appengine +//+build !gccgo // Copyright 2015, Klaus Post, see LICENSE for details. diff --git a/vendor/github.com/klauspost/compress/flate/crc32_amd64.s b/vendor/github.com/klauspost/compress/flate/crc32_amd64.s index 2fb2079..a799437 100644 --- a/vendor/github.com/klauspost/compress/flate/crc32_amd64.s +++ b/vendor/github.com/klauspost/compress/flate/crc32_amd64.s @@ -1,5 +1,6 @@ //+build !noasm //+build !appengine +//+build !gccgo // Copyright 2015, Klaus Post, see LICENSE for details. diff --git a/vendor/github.com/klauspost/compress/flate/crc32_noasm.go b/vendor/github.com/klauspost/compress/flate/crc32_noasm.go index bd98bd5..dcf43bd 100644 --- a/vendor/github.com/klauspost/compress/flate/crc32_noasm.go +++ b/vendor/github.com/klauspost/compress/flate/crc32_noasm.go @@ -1,4 +1,4 @@ -//+build !amd64 noasm appengine +//+build !amd64 noasm appengine gccgo // Copyright 2015, Klaus Post, see LICENSE for details. diff --git a/vendor/github.com/klauspost/compress/fse/fse.go b/vendor/github.com/klauspost/compress/fse/fse.go index 57cfda3..490e680 100644 --- a/vendor/github.com/klauspost/compress/fse/fse.go +++ b/vendor/github.com/klauspost/compress/fse/fse.go @@ -14,6 +14,7 @@ package fse import ( "errors" "fmt" + "math/bits" ) const ( @@ -136,3 +137,7 @@ func (s *Scratch) prepare(in []byte) (*Scratch, error) { func tableStep(tableSize uint32) uint32 { return (tableSize >> 1) + (tableSize >> 3) + 3 } + +func highBits(val uint32) (n uint32) { + return uint32(bits.Len32(val) - 1) +} diff --git a/vendor/github.com/klauspost/compress/fse/highbit_go19.go b/vendor/github.com/klauspost/compress/fse/highbit_go19.go deleted file mode 100644 index e16ecd5..0000000 --- a/vendor/github.com/klauspost/compress/fse/highbit_go19.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build go1.9 - -package fse - -import "math/bits" - -func highBits(val uint32) (n uint32) { - return uint32(bits.Len32(val) - 1) -} diff --git a/vendor/github.com/klauspost/compress/fse/highbit_old.go b/vendor/github.com/klauspost/compress/fse/highbit_old.go deleted file mode 100644 index 7bb313a..0000000 --- a/vendor/github.com/klauspost/compress/fse/highbit_old.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !go1.9 - -package fse - -var deBruijnClz = [...]uint8{0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31} - -// highBits returns the highest set bit -func highBits(v uint32) (n uint32) { - v |= v >> 1 - v |= v >> 2 - v |= v >> 4 - v |= v >> 8 - v |= v >> 16 - return uint32(deBruijnClz[(v*0x07C4ACDD)>>27]) -} diff --git a/vendor/github.com/klauspost/compress/huff0/README.md b/vendor/github.com/klauspost/compress/huff0/README.md new file mode 100644 index 0000000..0a8448c --- /dev/null +++ b/vendor/github.com/klauspost/compress/huff0/README.md @@ -0,0 +1,87 @@ +# Huff0 entropy compression + +This package provides Huff0 encoding and decoding as used in zstd. + +[Huff0](https://github.com/Cyan4973/FiniteStateEntropy#new-generation-entropy-coders), +a Huffman codec designed for modern CPU, featuring OoO (Out of Order) operations on multiple ALU +(Arithmetic Logic Unit), achieving extremely fast compression and decompression speeds. + +This can be used for compressing input with a lot of similar input values to the smallest number of bytes. +This does not perform any multi-byte [dictionary coding](https://en.wikipedia.org/wiki/Dictionary_coder) as LZ coders, +but it can be used as a secondary step to compressors (like Snappy) that does not do entropy encoding. + +* [Godoc documentation](https://godoc.org/github.com/klauspost/compress/huff0) + +THIS PACKAGE IS NOT CONSIDERED STABLE AND API OR ENCODING MAY CHANGE IN THE FUTURE. + +## News + + * Mar 2018: First implementation released. Consider this beta software for now. + +# Usage + +This package provides a low level interface that allows to compress single independent blocks. + +Each block is separate, and there is no built in integrity checks. +This means that the caller should keep track of block sizes and also do checksums if needed. + +Compressing a block is done via the [`Compress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress1X) and +[`Compress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Compress4X) functions. +You must provide input and will receive the output and maybe an error. + +These error values can be returned: + +| Error | Description | +|---------------------|-----------------------------------------------------------------------------| +| `` | Everything ok, output is returned | +| `ErrIncompressible` | Returned when input is judged to be too hard to compress | +| `ErrUseRLE` | Returned from the compressor when the input is a single byte value repeated | +| `ErrTooBig` | Returned if the input block exceeds the maximum allowed size (128 Kib) | +| `(error)` | An internal error occurred. | + + +As can be seen above some of there are errors that will be returned even under normal operation so it is important to handle these. + +To reduce allocations you can provide a [`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object +that can be re-used for successive calls. Both compression and decompression accepts a `Scratch` object, and the same +object can be used for both. + +Be aware, that when re-using a `Scratch` object that the *output* buffer is also re-used, so if you are still using this +you must set the `Out` field in the scratch to nil. The same buffer is used for compression and decompression output. + +The `Scratch` object will retain state that allows to re-use previous tables for encoding and decoding. + +## Tables and re-use + +Huff0 allows for reusing tables from the previous block to save space if that is expected to give better/faster results. + +The Scratch object allows you to set a [`ReusePolicy`](https://godoc.org/github.com/klauspost/compress/huff0#ReusePolicy) +that controls this behaviour. See the documentation for details. This can be altered between each block. + +Do however note that this information is *not* stored in the output block and it is up to the users of the package to +record whether [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable) should be called, +based on the boolean reported back from the CompressXX call. + +If you want to store the table separate from the data, you can access them as `OutData` and `OutTable` on the +[`Scratch`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch) object. + +## Decompressing + +The first part of decoding is to initialize the decoding table through [`ReadTable`](https://godoc.org/github.com/klauspost/compress/huff0#ReadTable). +This will initialize the decoding tables. +You can supply the complete block to `ReadTable` and it will return the data part of the block +which can be given to the decompressor. + +Decompressing is done by calling the [`Decompress1X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress1X) +or [`Decompress4X`](https://godoc.org/github.com/klauspost/compress/huff0#Scratch.Decompress4X) function. + +You must provide the output from the compression stage, at exactly the size you got back. If you receive an error back +your input was likely corrupted. + +It is important to note that a successful decoding does *not* mean your output matches your original input. +There are no integrity checks, so relying on errors from the decompressor does not assure your data is valid. + +# Contributing + +Contributions are always welcome. Be aware that adding public functions will require good justification and breaking +changes will likely not be accepted. If in doubt open an issue before writing the PR. \ No newline at end of file diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index c9d8e0e..ec0c3fc 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -67,21 +67,25 @@ func (b *bitWriter) flush() { v := b.nBits >> 3 switch v { case 0: + return case 1: b.out = append(b.out, byte(b.bitContainer), ) + b.bitContainer >>= 1 << 3 case 2: b.out = append(b.out, byte(b.bitContainer), byte(b.bitContainer>>8), ) + b.bitContainer >>= 2 << 3 case 3: b.out = append(b.out, byte(b.bitContainer), byte(b.bitContainer>>8), byte(b.bitContainer>>16), ) + b.bitContainer >>= 3 << 3 case 4: b.out = append(b.out, byte(b.bitContainer), @@ -89,6 +93,7 @@ func (b *bitWriter) flush() { byte(b.bitContainer>>16), byte(b.bitContainer>>24), ) + b.bitContainer >>= 4 << 3 case 5: b.out = append(b.out, byte(b.bitContainer), @@ -97,6 +102,7 @@ func (b *bitWriter) flush() { byte(b.bitContainer>>24), byte(b.bitContainer>>32), ) + b.bitContainer >>= 5 << 3 case 6: b.out = append(b.out, byte(b.bitContainer), @@ -106,6 +112,7 @@ func (b *bitWriter) flush() { byte(b.bitContainer>>32), byte(b.bitContainer>>40), ) + b.bitContainer >>= 6 << 3 case 7: b.out = append(b.out, byte(b.bitContainer), @@ -116,6 +123,7 @@ func (b *bitWriter) flush() { byte(b.bitContainer>>40), byte(b.bitContainer>>48), ) + b.bitContainer >>= 7 << 3 case 8: b.out = append(b.out, byte(b.bitContainer), @@ -127,10 +135,12 @@ func (b *bitWriter) flush() { byte(b.bitContainer>>48), byte(b.bitContainer>>56), ) + b.bitContainer = 0 + b.nBits = 0 + return default: panic(fmt.Errorf("bits (%d) > 64", b.nBits)) } - b.bitContainer >>= v << 3 b.nBits &= 7 } diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 22bee74..9531d86 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -6,6 +6,10 @@ import ( "sync" ) +// Compress1X will compress the input. +// The output can be decoded using Decompress1X. +// Supply a Scratch object. The scratch object contains state about re-use, +// So when sharing across independent encodes, be sure to set the re-use policy. func Compress1X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { s, err = s.prepare(in) if err != nil { @@ -14,6 +18,11 @@ func Compress1X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { return compress(in, s, s.compress1X) } +// Compress4X will compress the input. The input is split into 4 independent blocks +// and compressed similar to Compress1X. +// The output can be decoded using Decompress4X. +// Supply a Scratch object. The scratch object contains state about re-use, +// So when sharing across independent encodes, be sure to set the re-use policy. func Compress4X(in []byte, s *Scratch) (out []byte, reUsed bool, err error) { s, err = s.prepare(in) if err != nil { @@ -80,8 +89,7 @@ func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error) return nil, false, err } - if !s.canUseTable(s.cTable) { - // TODO: Not needed. + if false && !s.canUseTable(s.cTable) { panic("invalid table generated") } @@ -512,7 +520,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { // renorm totalCost totalCost >>= largestBits - maxNbBits /* note : totalCost is necessarily a multiple of baseCost */ - /* repay normalized cost */ + // repay normalized cost { const noSymbol = 0xF0F0F0F0 var rankLast [tableLogMax + 2]uint32 @@ -528,7 +536,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { if huffNode[pos].nbBits >= currentNbBits { continue } - currentNbBits = huffNode[pos].nbBits /* < maxNbBits */ + currentNbBits = huffNode[pos].nbBits // < maxNbBits rankLast[maxNbBits-currentNbBits] = uint32(pos) } } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 436cc41..b6ede8b 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -29,7 +29,7 @@ type dEntryDouble struct { // ReadTable will read a table from the input. // The size of the input may be larger than the table definition. // Any content remaining after the table definition will be returned. -// if no Scratch is provided a new one is allocated. +// If no Scratch is provided a new one is allocated. // The returned Scratch can be used for decoding input using this table. func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { s, err = s.prepare(in) @@ -150,8 +150,9 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { } // Decompress1X will decompress a 1X encoded stream. -// The supplied input must match the end of a block exactly. -// Before this is called, the table must be initialized with ReadTable. +// The length of the supplied input must match the end of a block exactly. +// Before this is called, the table must be initialized with ReadTable unless +// the encoder re-used the table. func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { if len(s.dt.single) == 0 { return nil, errors.New("no table loaded") @@ -206,8 +207,9 @@ func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { } // Decompress4X will decompress a 4X encoded stream. -// Before this is called, the table must be initialized with ReadTable. -// The supplied input must match the end of a block exactly. +// Before this is called, the table must be initialized with ReadTable unless +// the encoder re-used the table. +// The length of the supplied input must match the end of a block exactly. // The destination size of the uncompressed data must be known and provided. func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { if len(s.dt.single) == 0 { diff --git a/vendor/github.com/klauspost/compress/huff0/highbit_go19.go b/vendor/github.com/klauspost/compress/huff0/highbit_go19.go deleted file mode 100644 index 9d54ba9..0000000 --- a/vendor/github.com/klauspost/compress/huff0/highbit_go19.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build go1.9 - -package huff0 - -import "math/bits" - -func highBit32(val uint32) (n uint32) { - return uint32(bits.Len32(val) - 1) -} diff --git a/vendor/github.com/klauspost/compress/huff0/highbit_old.go b/vendor/github.com/klauspost/compress/huff0/highbit_old.go deleted file mode 100644 index 5da31b1..0000000 --- a/vendor/github.com/klauspost/compress/huff0/highbit_old.go +++ /dev/null @@ -1,18 +0,0 @@ -// +build !go1.9 - -package huff0 - -var deBruijnClz = [...]uint8{0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31} - -// highBit32 returns the highest set bit -func highBit32(v uint32) (n uint32) { - v |= v >> 1 - v |= v >> 2 - v |= v >> 4 - v |= v >> 8 - v |= v >> 16 - return uint32(deBruijnClz[(v*0x07C4ACDD)>>27]) -} diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index a9a93aa..1535ca2 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -1,9 +1,13 @@ +// Package huff0 provides fast huffman encoding as used in zstd. +// +// See README.md at https://github.com/klauspost/compress/tree/master/huff0 for details. package huff0 import ( "errors" "fmt" "math" + "math/bits" "github.com/klauspost/compress/fse" ) @@ -230,3 +234,7 @@ func (s *Scratch) minSize(total int) int { } return int(nbBits) >> 3 } + +func highBit32(val uint32) (n uint32) { + return uint32(bits.Len32(val) - 1) +} diff --git a/vendor/github.com/klauspost/cpuid/.travis.yml b/vendor/github.com/klauspost/cpuid/.travis.yml index 2332ea6..630192d 100644 --- a/vendor/github.com/klauspost/cpuid/.travis.yml +++ b/vendor/github.com/klauspost/cpuid/.travis.yml @@ -6,10 +6,9 @@ os: - linux - osx go: - - 1.5.x - - 1.6.x - - 1.7.x - 1.8.x + - 1.9.x + - 1.10.x - master script: diff --git a/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt b/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt new file mode 100644 index 0000000..2ef4714 --- /dev/null +++ b/vendor/github.com/klauspost/cpuid/CONTRIBUTING.txt @@ -0,0 +1,35 @@ +Developer Certificate of Origin +Version 1.1 + +Copyright (C) 2015- Klaus Post & Contributors. +Email: klauspost@gmail.com + +Everyone is permitted to copy and distribute verbatim copies of this +license document, but changing it is not allowed. + + +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the best + of my knowledge, is covered under an appropriate open source + license and I have the right under that license to submit that + work with modifications, whether created in whole or in part + by me, under the same open source license (unless I am + permitted to submit under a different license), as indicated + in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including all + personal information I submit with it, including my sign-off) is + maintained indefinitely and may be redistributed consistent with + this project or the open source license(s) involved. diff --git a/vendor/github.com/klauspost/cpuid/cpuid.go b/vendor/github.com/klauspost/cpuid/cpuid.go index 44e5094..60c681b 100644 --- a/vendor/github.com/klauspost/cpuid/cpuid.go +++ b/vendor/github.com/klauspost/cpuid/cpuid.go @@ -76,6 +76,8 @@ const ( RDTSCP // RDTSCP Instruction CX16 // CMPXCHG16B Instruction SGX // Software Guard Extensions + IBPB // Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB) + STIBP // Single Thread Indirect Branch Predictors // Performance indicators SSE2SLOW // SSE2 is supported, but usually not faster @@ -131,6 +133,8 @@ var flagNames = map[Flags]string{ RDTSCP: "RDTSCP", // RDTSCP Instruction CX16: "CX16", // CMPXCHG16B Instruction SGX: "SGX", // Software Guard Extensions + IBPB: "IBPB", // Indirect Branch Restricted Speculation and Indirect Branch Predictor Barrier + STIBP: "STIBP", // Single Thread Indirect Branch Predictors // Performance indicators SSE2SLOW: "SSE2SLOW", // SSE2 supported, but usually not faster @@ -450,7 +454,7 @@ func (c CPUInfo) CX16() bool { // TSX is split into HLE (Hardware Lock Elision) and RTM (Restricted Transactional Memory) detection. // So TSX simply checks that. func (c CPUInfo) TSX() bool { - return c.Features&(MPX|RTM) == MPX|RTM + return c.Features&(HLE|RTM) == HLE|RTM } // Atom indicates an Atom processor @@ -854,7 +858,7 @@ func support() Flags { // Check AVX2, AVX2 requires OS support, but BMI1/2 don't. if mfi >= 7 { - _, ebx, ecx, _ := cpuidex(7, 0) + _, ebx, ecx, edx := cpuidex(7, 0) if (rval&AVX) != 0 && (ebx&0x00000020) != 0 { rval |= AVX2 } @@ -888,6 +892,12 @@ func support() Flags { if ebx&(1<<29) != 0 { rval |= SHA } + if edx&(1<<26) != 0 { + rval |= IBPB + } + if edx&(1<<27) != 0 { + rval |= STIBP + } // Only detect AVX-512 features if XGETBV is supported if c&((1<<26)|(1<<27)) == (1<<26)|(1<<27) { diff --git a/vendor/github.com/klauspost/cpuid/private/cpuid.go b/vendor/github.com/klauspost/cpuid/private/cpuid.go index 2171214..b799515 100644 --- a/vendor/github.com/klauspost/cpuid/private/cpuid.go +++ b/vendor/github.com/klauspost/cpuid/private/cpuid.go @@ -70,6 +70,8 @@ const ( rdtscp // RDTSCP Instruction cx16 // CMPXCHG16B Instruction sgx // Software Guard Extensions + ibpb // Indirect Branch Restricted Speculation (IBRS) and Indirect Branch Predictor Barrier (IBPB) + stibp // Single Thread Indirect Branch Predictors // Performance indicators sse2slow // SSE2 is supported, but usually not faster @@ -125,6 +127,8 @@ var flagNames = map[flags]string{ rdtscp: "RDTSCP", // RDTSCP Instruction cx16: "CX16", // CMPXCHG16B Instruction sgx: "SGX", // Software Guard Extensions + ibpb: "IBPB", // Indirect Branch Restricted Speculation and Indirect Branch Predictor Barrier + stibp: "STIBP", // Single Thread Indirect Branch Predictors // Performance indicators sse2slow: "SSE2SLOW", // SSE2 supported, but usually not faster @@ -848,7 +852,7 @@ func support() flags { // Check AVX2, AVX2 requires OS support, but BMI1/2 don't. if mfi >= 7 { - _, ebx, ecx, _ := cpuidex(7, 0) + _, ebx, ecx, edx := cpuidex(7, 0) if (rval&avx) != 0 && (ebx&0x00000020) != 0 { rval |= avx2 } @@ -882,6 +886,12 @@ func support() flags { if ebx&(1<<29) != 0 { rval |= sha } + if edx&(1<<26) != 0 { + rval |= ibpb + } + if edx&(1<<27) != 0 { + rval |= stibp + } // Only detect AVX-512 features if XGETBV is supported if c&((1<<26)|(1<<27)) == (1<<26)|(1<<27) { diff --git a/vendor/github.com/klauspost/cpuid/testdata/cpuid_data.zip b/vendor/github.com/klauspost/cpuid/testdata/cpuid_data.zip index 885925f..6943243 100644 Binary files a/vendor/github.com/klauspost/cpuid/testdata/cpuid_data.zip and b/vendor/github.com/klauspost/cpuid/testdata/cpuid_data.zip differ diff --git a/vendor/github.com/konsorten/go-windows-terminal-sequences/LICENSE b/vendor/github.com/konsorten/go-windows-terminal-sequences/LICENSE new file mode 100644 index 0000000..14127cd --- /dev/null +++ b/vendor/github.com/konsorten/go-windows-terminal-sequences/LICENSE @@ -0,0 +1,9 @@ +(The MIT License) + +Copyright (c) 2017 marvin + konsorten GmbH (open-source@konsorten.de) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/konsorten/go-windows-terminal-sequences/README.md b/vendor/github.com/konsorten/go-windows-terminal-sequences/README.md new file mode 100644 index 0000000..949b77e --- /dev/null +++ b/vendor/github.com/konsorten/go-windows-terminal-sequences/README.md @@ -0,0 +1,40 @@ +# Windows Terminal Sequences + +This library allow for enabling Windows terminal color support for Go. + +See [Console Virtual Terminal Sequences](https://docs.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences) for details. + +## Usage + +```go +import ( + "syscall" + + sequences "github.com/konsorten/go-windows-terminal-sequences" +) + +func main() { + sequences.EnableVirtualTerminalProcessing(syscall.Stdout, true) +} + +``` + +## Authors + +The tool is sponsored by the [marvin + konsorten GmbH](http://www.konsorten.de). + +We thank all the authors who provided code to this library: + +* Felix Kollmann + +## License + +(The MIT License) + +Copyright (c) 2018 marvin + konsorten GmbH (open-source@konsorten.de) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/konsorten/go-windows-terminal-sequences/go.mod b/vendor/github.com/konsorten/go-windows-terminal-sequences/go.mod new file mode 100644 index 0000000..716c613 --- /dev/null +++ b/vendor/github.com/konsorten/go-windows-terminal-sequences/go.mod @@ -0,0 +1 @@ +module github.com/konsorten/go-windows-terminal-sequences diff --git a/vendor/github.com/konsorten/go-windows-terminal-sequences/sequences.go b/vendor/github.com/konsorten/go-windows-terminal-sequences/sequences.go new file mode 100644 index 0000000..ef18d8f --- /dev/null +++ b/vendor/github.com/konsorten/go-windows-terminal-sequences/sequences.go @@ -0,0 +1,36 @@ +// +build windows + +package sequences + +import ( + "syscall" + "unsafe" +) + +var ( + kernel32Dll *syscall.LazyDLL = syscall.NewLazyDLL("Kernel32.dll") + setConsoleMode *syscall.LazyProc = kernel32Dll.NewProc("SetConsoleMode") +) + +func EnableVirtualTerminalProcessing(stream syscall.Handle, enable bool) error { + const ENABLE_VIRTUAL_TERMINAL_PROCESSING uint32 = 0x4 + + var mode uint32 + err := syscall.GetConsoleMode(syscall.Stdout, &mode) + if err != nil { + return err + } + + if enable { + mode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING + } else { + mode &^= ENABLE_VIRTUAL_TERMINAL_PROCESSING + } + + ret, _, err := setConsoleMode.Call(uintptr(unsafe.Pointer(stream)), uintptr(mode)) + if ret == 0 { + return err + } + + return nil +} diff --git a/vendor/github.com/konsorten/go-windows-terminal-sequences/sequences_test.go b/vendor/github.com/konsorten/go-windows-terminal-sequences/sequences_test.go new file mode 100644 index 0000000..aad41c5 --- /dev/null +++ b/vendor/github.com/konsorten/go-windows-terminal-sequences/sequences_test.go @@ -0,0 +1,48 @@ +// +build windows + +package sequences + +import ( + "fmt" + "os" + "syscall" + "testing" +) + +func TestStdoutSequencesOn(t *testing.T) { + err := EnableVirtualTerminalProcessing(syscall.Stdout, true) + if err != nil { + t.Fatalf("Failed to enable VTP: %v", err) + } + defer EnableVirtualTerminalProcessing(syscall.Stdout, false) + + fmt.Fprintf(os.Stdout, "\x1b[34mHello \x1b[35mWorld\x1b[0m!\n") +} + +func TestStdoutSequencesOff(t *testing.T) { + err := EnableVirtualTerminalProcessing(syscall.Stdout, false) + if err != nil { + t.Fatalf("Failed to enable VTP: %v", err) + } + + fmt.Fprintf(os.Stdout, "\x1b[34mHello \x1b[35mWorld\x1b[0m!\n") +} + +func TestStderrSequencesOn(t *testing.T) { + err := EnableVirtualTerminalProcessing(syscall.Stderr, true) + if err != nil { + t.Fatalf("Failed to enable VTP: %v", err) + } + defer EnableVirtualTerminalProcessing(syscall.Stderr, false) + + fmt.Fprintf(os.Stderr, "\x1b[34mHello \x1b[35mWorld\x1b[0m!\n") +} + +func TestStderrSequencesOff(t *testing.T) { + err := EnableVirtualTerminalProcessing(syscall.Stderr, false) + if err != nil { + t.Fatalf("Failed to enable VTP: %v", err) + } + + fmt.Fprintf(os.Stderr, "\x1b[34mHello \x1b[35mWorld\x1b[0m!\n") +} diff --git a/vendor/github.com/lib/pq/.travis.sh b/vendor/github.com/lib/pq/.travis.sh index a297dc4..21a5264 100755 --- a/vendor/github.com/lib/pq/.travis.sh +++ b/vendor/github.com/lib/pq/.travis.sh @@ -80,7 +80,7 @@ megacheck_install() { } golint_install() { - go get github.com/golang/lint/golint + go get golang.org/x/lint/golint } $1 diff --git a/vendor/github.com/lib/pq/.travis.yml b/vendor/github.com/lib/pq/.travis.yml index 18556e0..f030580 100644 --- a/vendor/github.com/lib/pq/.travis.yml +++ b/vendor/github.com/lib/pq/.travis.yml @@ -1,9 +1,9 @@ language: go go: - - 1.8.x - 1.9.x - 1.10.x + - 1.11.x - master sudo: true @@ -44,7 +44,7 @@ script: - > goimports -d -e $(find -name '*.go') | awk '{ print } END { exit NR == 0 ? 0 : 1 }' - go vet ./... - - megacheck -go 1.8 ./... + - megacheck -go 1.9 ./... - golint ./... - PQTEST_BINARY_PARAMETERS=no go test -race -v ./... - PQTEST_BINARY_PARAMETERS=yes go test -race -v ./... diff --git a/vendor/github.com/lib/pq/README.md b/vendor/github.com/lib/pq/README.md index d71f3c2..385fe73 100644 --- a/vendor/github.com/lib/pq/README.md +++ b/vendor/github.com/lib/pq/README.md @@ -10,7 +10,7 @@ ## Docs For detailed documentation and basic usage examples, please see the package -documentation at . +documentation at . ## Tests diff --git a/vendor/github.com/lib/pq/bench_test.go b/vendor/github.com/lib/pq/bench_test.go index 33d7a02..b375498 100644 --- a/vendor/github.com/lib/pq/bench_test.go +++ b/vendor/github.com/lib/pq/bench_test.go @@ -1,5 +1,3 @@ -// +build go1.1 - package pq import ( diff --git a/vendor/github.com/lib/pq/conn_go18.go b/vendor/github.com/lib/pq/conn_go18.go index a5254f2..81c9ee4 100644 --- a/vendor/github.com/lib/pq/conn_go18.go +++ b/vendor/github.com/lib/pq/conn_go18.go @@ -1,5 +1,3 @@ -// +build go1.8 - package pq import ( diff --git a/vendor/github.com/lib/pq/doc.go b/vendor/github.com/lib/pq/doc.go index a1b0297..2a60054 100644 --- a/vendor/github.com/lib/pq/doc.go +++ b/vendor/github.com/lib/pq/doc.go @@ -239,7 +239,7 @@ for more information). Note that the channel name will be truncated to 63 bytes by the PostgreSQL server. You can find a complete, working example of Listener usage at -http://godoc.org/github.com/lib/pq/example/listen. +https://godoc.org/github.com/lib/pq/example/listen. */ package pq diff --git a/vendor/github.com/lib/pq/go.mod b/vendor/github.com/lib/pq/go.mod new file mode 100644 index 0000000..edf0b34 --- /dev/null +++ b/vendor/github.com/lib/pq/go.mod @@ -0,0 +1 @@ +module github.com/lib/pq diff --git a/vendor/github.com/lib/pq/go18_test.go b/vendor/github.com/lib/pq/go18_test.go index 1a88a5b..72cd71f 100644 --- a/vendor/github.com/lib/pq/go18_test.go +++ b/vendor/github.com/lib/pq/go18_test.go @@ -1,5 +1,3 @@ -// +build go1.8 - package pq import ( diff --git a/vendor/github.com/lib/pq/notify.go b/vendor/github.com/lib/pq/notify.go index 947d189..850bb90 100644 --- a/vendor/github.com/lib/pq/notify.go +++ b/vendor/github.com/lib/pq/notify.go @@ -725,6 +725,9 @@ func (l *Listener) Close() error { } l.isClosed = true + // Unblock calls to Listen() + l.reconnectCond.Broadcast() + return nil } diff --git a/vendor/github.com/lib/pq/rows_test.go b/vendor/github.com/lib/pq/rows_test.go index 3033bc0..b3420a2 100644 --- a/vendor/github.com/lib/pq/rows_test.go +++ b/vendor/github.com/lib/pq/rows_test.go @@ -1,5 +1,3 @@ -// +build go1.8 - package pq import ( diff --git a/vendor/github.com/lib/pq/ssl.go b/vendor/github.com/lib/pq/ssl.go index e1a326a..d902084 100644 --- a/vendor/github.com/lib/pq/ssl.go +++ b/vendor/github.com/lib/pq/ssl.go @@ -58,7 +58,13 @@ func ssl(o values) (func(net.Conn) (net.Conn, error), error) { if err != nil { return nil, err } - sslRenegotiation(&tlsConf) + + // Accept renegotiation requests initiated by the backend. + // + // Renegotiation was deprecated then removed from PostgreSQL 9.5, but + // the default configuration of older versions has it enabled. Redshift + // also initiates renegotiations and cannot be reconfigured. + tlsConf.Renegotiation = tls.RenegotiateFreelyAsClient return func(conn net.Conn) (net.Conn, error) { client := tls.Client(conn, &tlsConf) diff --git a/vendor/github.com/lib/pq/ssl_go1.7.go b/vendor/github.com/lib/pq/ssl_go1.7.go deleted file mode 100644 index d7ba43b..0000000 --- a/vendor/github.com/lib/pq/ssl_go1.7.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build go1.7 - -package pq - -import "crypto/tls" - -// Accept renegotiation requests initiated by the backend. -// -// Renegotiation was deprecated then removed from PostgreSQL 9.5, but -// the default configuration of older versions has it enabled. Redshift -// also initiates renegotiations and cannot be reconfigured. -func sslRenegotiation(conf *tls.Config) { - conf.Renegotiation = tls.RenegotiateFreelyAsClient -} diff --git a/vendor/github.com/lib/pq/ssl_renegotiation.go b/vendor/github.com/lib/pq/ssl_renegotiation.go deleted file mode 100644 index 85ed5e4..0000000 --- a/vendor/github.com/lib/pq/ssl_renegotiation.go +++ /dev/null @@ -1,8 +0,0 @@ -// +build !go1.7 - -package pq - -import "crypto/tls" - -// Renegotiation is not supported by crypto/tls until Go 1.7. -func sslRenegotiation(*tls.Config) {} diff --git a/vendor/github.com/mattn/go-sqlite3/.travis.yml b/vendor/github.com/mattn/go-sqlite3/.travis.yml index a21c817..2ae08be 100644 --- a/vendor/github.com/mattn/go-sqlite3/.travis.yml +++ b/vendor/github.com/mattn/go-sqlite3/.travis.yml @@ -12,18 +12,18 @@ env: matrix: - GOTAGS= - GOTAGS=libsqlite3 - - GOTAGS="sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable" + - GOTAGS="sqlite_allow_uri_authority sqlite_app_armor sqlite_foreign_keys sqlite_fts5 sqlite_icu sqlite_introspect sqlite_json sqlite_secure_delete sqlite_see sqlite_stat4 sqlite_trace sqlite_userauth sqlite_vacuum_incr sqlite_vtable sqlite_unlock_notify" - GOTAGS=sqlite_vacuum_full go: - 1.9.x - 1.10.x + - 1.11.x before_install: - | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew update - brew upgrade icu4c fi - | go get github.com/smartystreets/goconvey diff --git a/vendor/github.com/mattn/go-sqlite3/README.md b/vendor/github.com/mattn/go-sqlite3/README.md index 37d017a..207f1cd 100644 --- a/vendor/github.com/mattn/go-sqlite3/README.md +++ b/vendor/github.com/mattn/go-sqlite3/README.md @@ -67,6 +67,7 @@ This is also known as a DSN string. (Data Source Name). Options are append after the filename of the SQLite database. The database filename and options are seperated by an `?` (Question Mark). +Options should be URL-encoded (see [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)). This also applies when using an in-memory database instead of a file. @@ -198,7 +199,7 @@ Additional information: # Google Cloud Platform -Building on GCP is not possible because `Google Cloud Platform does not allow `gcc` to be executed. +Building on GCP is not possible because Google Cloud Platform does not allow `gcc` to be executed. Please work only with compiled final binaries. @@ -290,7 +291,7 @@ For example the TDM-GCC Toolchain can be found [here](ttps://sourceforge.net/pro When receiving a compile time error referencing recompile with `-FPIC` then you are probably using a hardend system. - You can copile the library on a hardend system with the following command. + You can compile the library on a hardend system with the following command. ```bash go build -ldflags '-extldflags=-fno-PIC' @@ -473,7 +474,7 @@ For an example see [shaxbee/go-spatialite](https://github.com/shaxbee/go-spatial For more information see [#289](https://github.com/mattn/go-sqlite3/issues/289) -- Trying to execure a `.` (dot) command throws an error. +- Trying to execute a `.` (dot) command throws an error. Error: `Error: near ".": syntax error` Dot command are part of SQLite3 CLI not of this library. diff --git a/vendor/github.com/mattn/go-sqlite3/callback.go b/vendor/github.com/mattn/go-sqlite3/callback.go index 5a735c0..2c68973 100644 --- a/vendor/github.com/mattn/go-sqlite3/callback.go +++ b/vendor/github.com/mattn/go-sqlite3/callback.go @@ -77,6 +77,12 @@ func updateHookTrampoline(handle uintptr, op int, db *C.char, table *C.char, row callback(op, C.GoString(db), C.GoString(table), rowid) } +//export authorizerTrampoline +func authorizerTrampoline(handle uintptr, op int, arg1 *C.char, arg2 *C.char, arg3 *C.char) int { + callback := lookupHandle(handle).(func(int, string, string, string) int) + return callback(op, C.GoString(arg1), C.GoString(arg2), C.GoString(arg3)) +} + // Use handles to avoid passing Go pointers to C. type handleVal struct { diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c index f077152..7763197 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.c @@ -1,7 +1,7 @@ #ifndef USE_LIBSQLITE3 /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.24.0. By combining all the individual C code files into this +** version 3.25.2. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -56,6 +56,12 @@ #define CTIMEOPT_VAL_(opt) #opt #define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) +/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This +** option requires a separate macro because legal values contain a single +** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ +#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 +#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) + /* ** An array of names of all compile-time options. This array should ** be sorted A-Z. @@ -139,7 +145,7 @@ static const char * const sqlite3azCompileOpt[] = { "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), #endif #ifdef SQLITE_DEFAULT_LOOKASIDE - "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOOKASIDE), + "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), #endif #if SQLITE_DEFAULT_MEMSTATUS "DEFAULT_MEMSTATUS", @@ -1151,9 +1157,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.24.0" -#define SQLITE_VERSION_NUMBER 3024000 -#define SQLITE_SOURCE_ID "2018-06-04 19:24:41 c7ee0833225bfd8c5ec2f9bf62b97c4e04d03bd9566366d5221ac8fb199a87ca" +#define SQLITE_VERSION "3.25.2" +#define SQLITE_VERSION_NUMBER 3025002 +#define SQLITE_SOURCE_ID "2018-09-25 19:08:10 fb90e7189ae6d62e77ba3a308ca5d683f90bbe633cf681865365b8e92792d1c7" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -1500,6 +1506,7 @@ SQLITE_API int sqlite3_exec( */ #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -1539,6 +1546,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) @@ -1914,7 +1922,8 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the ** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary -** write ahead log and shared memory files used for transaction control +** write ahead log ([WAL file]) and shared memory +** files used for transaction control ** are automatically deleted when the latest connection to the database ** closes. Setting persistent WAL mode causes those files to persist after ** close. Persisting the files is useful when other processes that do not @@ -2100,6 +2109,26 @@ struct sqlite3_io_methods { ** a file lock using the xLock or xShmLock methods of the VFS to wait ** for up to M milliseconds before failing, where M is the single ** unsigned integer parameter. +** +**
  • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provide a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -2135,6 +2164,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 #define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 #define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -3149,6 +3179,12 @@ struct sqlite3_mem_methods { ** with no schema and no content. The following process works even for ** a badly corrupted database file: **
      +**
    1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. **
    2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); **
    3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); **
    4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); @@ -3297,12 +3333,17 @@ SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); ** program, the value returned reflects the number of rows modified by the ** previous INSERT, UPDATE or DELETE statement within the same trigger. ** -** See also the [sqlite3_total_changes()] interface, the -** [count_changes pragma], and the [changes() SQL function]. -** ** If a separate thread makes changes on the same database connection ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. +** +** See also: +**
        +**
      • the [sqlite3_total_changes()] interface +**
      • the [count_changes pragma] +**
      • the [changes() SQL function] +**
      • the [data_version pragma] +**
      */ SQLITE_API int sqlite3_changes(sqlite3*); @@ -3320,13 +3361,26 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** count, but those made as part of REPLACE constraint resolution are ** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. -** -** See also the [sqlite3_changes()] interface, the -** [count_changes pragma], and the [total_changes() SQL function]. ** +** This the [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. +** ** If a separate thread makes changes on the same database connection ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. +** +** See also: +**
        +**
      • the [sqlite3_changes()] interface +**
      • the [count_changes pragma] +**
      • the [changes() SQL function] +**
      • the [data_version pragma] +**
      • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
      */ SQLITE_API int sqlite3_total_changes(sqlite3*); @@ -4382,13 +4436,24 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** [database connection] D failed, then the sqlite3_errcode(D) interface ** returns the numeric [result code] or [extended result code] for that ** API call. -** If the most recent API call was successful, -** then the return value from sqlite3_errcode() is undefined. ** ^The sqlite3_extended_errcode() ** interface is the same except that it always returns the ** [extended result code] even when extended result codes are ** disabled. ** +** The values returned by sqlite3_errcode() and/or +** sqlite3_extended_errcode() might change with each API call. +** Except, there are some interfaces that are guaranteed to never +** change the value of the error code. The error-code preserving +** interfaces are: +** +**
        +**
      • sqlite3_errcode() +**
      • sqlite3_extended_errcode() +**
      • sqlite3_errmsg() +**
      • sqlite3_errmsg16() +**
      +** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language ** text that describes the error, as either UTF-8 or UTF-16 respectively. ** ^(Memory to hold the error message string is managed internally. @@ -5542,11 +5607,25 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** -** ^(If a memory allocation error occurs during the evaluation of any -** of these routines, a default value is returned. The default value -** is either the integer 0, the floating point number 0.0, or a NULL -** pointer. Subsequent calls to [sqlite3_errcode()] will return -** [SQLITE_NOMEM].)^ +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
        +**
      • sqlite3_column_blob() +**
      • sqlite3_column_text() +**
      • sqlite3_column_text16() +**
      • sqlite3_column_bytes() +**
      • sqlite3_column_bytes16() +**
      +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); @@ -5623,11 +5702,13 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only differences between -** these routines are the text encoding expected for -** the second parameter (the name of the function being created) -** and the presence or absence of a destructor callback for -** the application data pointer. +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database @@ -5673,7 +5754,8 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc ** callback only; NULL pointers must be passed as the xStep and xFinal @@ -5682,15 +5764,24 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** SQL function or aggregate, pass NULL pointers for all three function ** callbacks. ** -** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, -** then it is destructor for the application data pointer. -** The destructor is invoked when the function is deleted, either by being -** overloaded or when the database connection closes.)^ -** ^The destructor is also invoked if the call to -** sqlite3_create_function_v2() fails. -** ^When the destructor callback of the tenth parameter is invoked, it -** is passed a single argument which is a copy of the application data -** pointer which was the fifth parameter to sqlite3_create_function_v2(). +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -5743,6 +5834,18 @@ SQLITE_API int sqlite3_create_function_v2( void (*xFinal)(sqlite3_context*), void(*xDestroy)(void*) ); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -5885,6 +5988,28 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
        +**
      • sqlite3_value_blob() +**
      • sqlite3_value_text() +**
      • sqlite3_value_text16() +**
      • sqlite3_value_text16le() +**
      • sqlite3_value_text16be() +**
      • sqlite3_value_bytes() +**
      • sqlite3_value_bytes16() +**
      +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -7351,6 +7476,7 @@ struct sqlite3_index_info { #define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 #define SQLITE_INDEX_CONSTRAINT_ISNULL 71 #define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 /* ** CAPI3REF: Register A Virtual Table Implementation @@ -8027,6 +8153,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files ** METHOD: sqlite3 +** KEYWORDS: {file control} ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -8041,11 +8168,18 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. ** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes ** a pointer to the underlying [sqlite3_file] object to be written into -** the space pointed to by the 4th parameter. ^The [SQLITE_FCNTL_FILE_POINTER] -** case is a short-circuit path which does not actually invoke the -** underlying sqlite3_io_methods.xFileControl method. +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. ** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error @@ -9864,7 +9998,6 @@ SQLITE_API int sqlite3_system_errno(sqlite3*); /* ** CAPI3REF: Database Snapshot ** KEYWORDS: {snapshot} {sqlite3_snapshot} -** EXPERIMENTAL ** ** An instance of the snapshot object records the state of a [WAL mode] ** database for some specific point in history. @@ -9881,11 +10014,6 @@ SQLITE_API int sqlite3_system_errno(sqlite3*); ** version of the database file so that it is possible to later open a new read ** transaction that sees that historical version of the database rather than ** the most recent version. -** -** The constructor for this object is [sqlite3_snapshot_get()]. The -** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer -** to an historical snapshot (if possible). The destructor for -** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. */ typedef struct sqlite3_snapshot { unsigned char hidden[48]; @@ -9893,7 +10021,7 @@ typedef struct sqlite3_snapshot { /* ** CAPI3REF: Record A Database Snapshot -** EXPERIMENTAL +** CONSTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a ** new [sqlite3_snapshot] object that records the current state of @@ -9909,7 +10037,7 @@ typedef struct sqlite3_snapshot { ** in this case. ** **
        -**
      • The database handle must be in [autocommit mode]. +**
      • The database handle must not be in [autocommit mode]. ** **
      • Schema S of [database connection] D must be a [WAL mode] database. ** @@ -9932,7 +10060,7 @@ typedef struct sqlite3_snapshot { ** to avoid a memory leak. ** ** The [sqlite3_snapshot_get()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( sqlite3 *db, @@ -9942,24 +10070,35 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( /* ** CAPI3REF: Start a read transaction on an historical snapshot -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** -** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a -** read transaction for schema S of -** [database connection] D such that the read transaction -** refers to historical [snapshot] P, rather than the most -** recent change to the database. -** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success -** or an appropriate [error code] if it fails. +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. ** -** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be -** the first operation following the [BEGIN] that takes the schema S -** out of [autocommit mode]. -** ^In other words, schema S must not currently be in -** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the -** database connection D must be out of [autocommit mode]. -** ^A [snapshot] will fail to open if it has been overwritten by a -** [checkpoint]. ** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the ** database connection D does not know that the database file for ** schema S is in [WAL mode]. A database connection might not know @@ -9970,7 +10109,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** database connection in order to make it ready to use snapshots.) ** ** The [sqlite3_snapshot_open()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( sqlite3 *db, @@ -9980,20 +10119,20 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( /* ** CAPI3REF: Destroy a snapshot -** EXPERIMENTAL +** DESTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. ** The application must eventually free every [sqlite3_snapshot] object ** using this routine to avoid a memory leak. ** ** The [sqlite3_snapshot_free()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** ** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages ** of two valid snapshot handles. @@ -10012,6 +10151,9 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** Otherwise, this API returns a negative value if P1 refers to an older ** snapshot than P2, zero if the two handles refer to the same database ** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, @@ -10020,23 +10162,26 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( /* ** CAPI3REF: Recover snapshots from a wal file -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** -** If all connections disconnect from a database file but do not perform -** a checkpoint, the existing wal file is opened along with the database -** file the next time the database is opened. At this point it is only -** possible to successfully call sqlite3_snapshot_open() to open the most -** recent snapshot of the database (the one at the head of the wal file), -** even though the wal file may contain other valid snapshots for which -** clients have sqlite3_snapshot handles. +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. ** -** This function attempts to scan the wal file associated with database zDb +** This function attempts to scan the WAL file associated with database zDb ** of database handle db and make all valid snapshots available to ** sqlite3_snapshot_open(). It is an error if there is already a read -** transaction open on the database, or if the database is not a wal mode +** transaction open on the database, or if the database is not a WAL mode ** database. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); @@ -10147,7 +10292,7 @@ SQLITE_API int sqlite3_deserialize( ** in the P argument is held in memory obtained from [sqlite3_malloc64()] ** and that SQLite should take ownership of this memory and automatically ** free it when it has finished using it. Without this flag, the caller -** is resposible for freeing any dynamically allocated memory. +** is responsible for freeing any dynamically allocated memory. ** ** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to ** grow the size of the database using calls to [sqlite3_realloc64()]. This @@ -12325,7 +12470,7 @@ struct Fts5ExtensionApi { ** This way, even if the tokenizer does not provide synonyms ** when tokenizing query text (it should not - to do would be ** inefficient), it doesn't matter if the user queries for -** 'first + place' or '1st + place', as there are entires in the +** 'first + place' or '1st + place', as there are entries in the ** FTS index corresponding to both forms of the first token. **
    ** @@ -12353,7 +12498,7 @@ struct Fts5ExtensionApi { ** extra data to the FTS index or require FTS5 to query for multiple terms, ** so it is efficient in terms of disk space and query speed. However, it ** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: +** token "first" is substituted for "1st" by the tokenizer, then the query: ** ** ** ... MATCH '1s*' @@ -13217,94 +13362,104 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_REPLACE 73 #define TK_RESTRICT 74 #define TK_ROW 75 -#define TK_TRIGGER 76 -#define TK_VACUUM 77 -#define TK_VIEW 78 -#define TK_VIRTUAL 79 -#define TK_WITH 80 -#define TK_REINDEX 81 -#define TK_RENAME 82 -#define TK_CTIME_KW 83 -#define TK_ANY 84 -#define TK_BITAND 85 -#define TK_BITOR 86 -#define TK_LSHIFT 87 -#define TK_RSHIFT 88 -#define TK_PLUS 89 -#define TK_MINUS 90 -#define TK_STAR 91 -#define TK_SLASH 92 -#define TK_REM 93 -#define TK_CONCAT 94 -#define TK_COLLATE 95 -#define TK_BITNOT 96 -#define TK_ON 97 -#define TK_INDEXED 98 -#define TK_STRING 99 -#define TK_JOIN_KW 100 -#define TK_CONSTRAINT 101 -#define TK_DEFAULT 102 -#define TK_NULL 103 -#define TK_PRIMARY 104 -#define TK_UNIQUE 105 -#define TK_CHECK 106 -#define TK_REFERENCES 107 -#define TK_AUTOINCR 108 -#define TK_INSERT 109 -#define TK_DELETE 110 -#define TK_UPDATE 111 -#define TK_SET 112 -#define TK_DEFERRABLE 113 -#define TK_FOREIGN 114 -#define TK_DROP 115 -#define TK_UNION 116 -#define TK_ALL 117 -#define TK_EXCEPT 118 -#define TK_INTERSECT 119 -#define TK_SELECT 120 -#define TK_VALUES 121 -#define TK_DISTINCT 122 -#define TK_DOT 123 -#define TK_FROM 124 -#define TK_JOIN 125 -#define TK_USING 126 -#define TK_ORDER 127 -#define TK_GROUP 128 -#define TK_HAVING 129 -#define TK_LIMIT 130 -#define TK_WHERE 131 -#define TK_INTO 132 -#define TK_NOTHING 133 -#define TK_FLOAT 134 -#define TK_BLOB 135 -#define TK_INTEGER 136 -#define TK_VARIABLE 137 -#define TK_CASE 138 -#define TK_WHEN 139 -#define TK_THEN 140 -#define TK_ELSE 141 -#define TK_INDEX 142 -#define TK_ALTER 143 -#define TK_ADD 144 -#define TK_TRUEFALSE 145 -#define TK_ISNOT 146 -#define TK_FUNCTION 147 -#define TK_COLUMN 148 -#define TK_AGG_FUNCTION 149 -#define TK_AGG_COLUMN 150 -#define TK_UMINUS 151 -#define TK_UPLUS 152 -#define TK_TRUTH 153 -#define TK_REGISTER 154 -#define TK_VECTOR 155 -#define TK_SELECT_COLUMN 156 -#define TK_IF_NULL_ROW 157 -#define TK_ASTERISK 158 -#define TK_SPAN 159 -#define TK_END_OF_FILE 160 -#define TK_UNCLOSED_STRING 161 -#define TK_SPACE 162 -#define TK_ILLEGAL 163 +#define TK_ROWS 76 +#define TK_TRIGGER 77 +#define TK_VACUUM 78 +#define TK_VIEW 79 +#define TK_VIRTUAL 80 +#define TK_WITH 81 +#define TK_CURRENT 82 +#define TK_FOLLOWING 83 +#define TK_PARTITION 84 +#define TK_PRECEDING 85 +#define TK_RANGE 86 +#define TK_UNBOUNDED 87 +#define TK_REINDEX 88 +#define TK_RENAME 89 +#define TK_CTIME_KW 90 +#define TK_ANY 91 +#define TK_BITAND 92 +#define TK_BITOR 93 +#define TK_LSHIFT 94 +#define TK_RSHIFT 95 +#define TK_PLUS 96 +#define TK_MINUS 97 +#define TK_STAR 98 +#define TK_SLASH 99 +#define TK_REM 100 +#define TK_CONCAT 101 +#define TK_COLLATE 102 +#define TK_BITNOT 103 +#define TK_ON 104 +#define TK_INDEXED 105 +#define TK_STRING 106 +#define TK_JOIN_KW 107 +#define TK_CONSTRAINT 108 +#define TK_DEFAULT 109 +#define TK_NULL 110 +#define TK_PRIMARY 111 +#define TK_UNIQUE 112 +#define TK_CHECK 113 +#define TK_REFERENCES 114 +#define TK_AUTOINCR 115 +#define TK_INSERT 116 +#define TK_DELETE 117 +#define TK_UPDATE 118 +#define TK_SET 119 +#define TK_DEFERRABLE 120 +#define TK_FOREIGN 121 +#define TK_DROP 122 +#define TK_UNION 123 +#define TK_ALL 124 +#define TK_EXCEPT 125 +#define TK_INTERSECT 126 +#define TK_SELECT 127 +#define TK_VALUES 128 +#define TK_DISTINCT 129 +#define TK_DOT 130 +#define TK_FROM 131 +#define TK_JOIN 132 +#define TK_USING 133 +#define TK_ORDER 134 +#define TK_GROUP 135 +#define TK_HAVING 136 +#define TK_LIMIT 137 +#define TK_WHERE 138 +#define TK_INTO 139 +#define TK_NOTHING 140 +#define TK_FLOAT 141 +#define TK_BLOB 142 +#define TK_INTEGER 143 +#define TK_VARIABLE 144 +#define TK_CASE 145 +#define TK_WHEN 146 +#define TK_THEN 147 +#define TK_ELSE 148 +#define TK_INDEX 149 +#define TK_ALTER 150 +#define TK_ADD 151 +#define TK_WINDOW 152 +#define TK_OVER 153 +#define TK_FILTER 154 +#define TK_TRUEFALSE 155 +#define TK_ISNOT 156 +#define TK_FUNCTION 157 +#define TK_COLUMN 158 +#define TK_AGG_FUNCTION 159 +#define TK_AGG_COLUMN 160 +#define TK_UMINUS 161 +#define TK_UPLUS 162 +#define TK_TRUTH 163 +#define TK_REGISTER 164 +#define TK_VECTOR 165 +#define TK_SELECT_COLUMN 166 +#define TK_IF_NULL_ROW 167 +#define TK_ASTERISK 168 +#define TK_SPAN 169 +#define TK_END_OF_FILE 170 +#define TK_UNCLOSED_STRING 171 +#define TK_SPACE 172 +#define TK_ILLEGAL 173 /* The token codes above must all fit in 8 bits */ #define TKFLG_MASK 0xff @@ -13578,7 +13733,8 @@ typedef INT16_TYPE LogEst; # if defined(__SIZEOF_POINTER__) # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(_M_ARM) || defined(__arm__) || defined(__x86) + defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ + (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else # define SQLITE_PTRSIZE 8 @@ -13619,7 +13775,7 @@ typedef INT16_TYPE LogEst; # if defined(i386) || defined(__i386__) || defined(_M_IX86) || \ defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__) + defined(__arm__) || defined(_M_ARM64) # define SQLITE_BYTEORDER 1234 # elif defined(sparc) || defined(__ppc__) # define SQLITE_BYTEORDER 4321 @@ -13874,6 +14030,7 @@ typedef struct NameContext NameContext; typedef struct Parse Parse; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RenameToken RenameToken; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; @@ -13894,8 +14051,35 @@ typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; +typedef struct Window Window; typedef struct With With; + +/* +** The bitmask datatype defined below is used for various optimizations. +** +** Changing this from a 64-bit to a 32-bit type limits the number of +** tables in a join to 32 instead of 64. But it also reduces the size +** of the library by 738 bytes on ix86. +*/ +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif + +/* +** The number of bits in a Bitmask. "BMS" means "BitMask Size". +*/ +#define BMS ((int)(sizeof(Bitmask)*8)) + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define ALLBITS ((Bitmask)-1) + /* A VList object records a mapping between parameters/variables/wildcards ** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer ** variable number associated with that parameter. See the format description @@ -13991,7 +14175,7 @@ SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*); SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int,int*); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); @@ -14214,6 +14398,9 @@ struct BtreePayload { SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, int flags, int seekResult); SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3BtreeSkipNext(BtCursor*); +#endif SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); @@ -14381,7 +14568,8 @@ struct VdbeOp { u64 cycles; /* Total time spent executing this instruction */ #endif #ifdef SQLITE_VDBE_COVERAGE - int iSrcLine; /* Source-code line that generated this opcode */ + u32 iSrcLine; /* Source-code line that generated this opcode + ** with flags in the upper 8 bits */ #endif }; typedef struct VdbeOp VdbeOp; @@ -14482,52 +14670,52 @@ typedef struct VdbeOpList VdbeOpList; #define OP_AutoCommit 1 #define OP_Transaction 2 #define OP_SorterNext 3 /* jump */ -#define OP_PrevIfOpen 4 /* jump */ -#define OP_NextIfOpen 5 /* jump */ -#define OP_Prev 6 /* jump */ -#define OP_Next 7 /* jump */ -#define OP_Checkpoint 8 -#define OP_JournalMode 9 -#define OP_Vacuum 10 -#define OP_VFilter 11 /* jump, synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 12 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 13 /* jump */ -#define OP_Gosub 14 /* jump */ -#define OP_InitCoroutine 15 /* jump */ -#define OP_Yield 16 /* jump */ -#define OP_MustBeInt 17 /* jump */ -#define OP_Jump 18 /* jump */ +#define OP_Prev 4 /* jump */ +#define OP_Next 5 /* jump */ +#define OP_Checkpoint 6 +#define OP_JournalMode 7 +#define OP_Vacuum 8 +#define OP_VFilter 9 /* jump, synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 10 /* synopsis: data=r[P3@P2] */ +#define OP_Goto 11 /* jump */ +#define OP_Gosub 12 /* jump */ +#define OP_InitCoroutine 13 /* jump */ +#define OP_Yield 14 /* jump */ +#define OP_MustBeInt 15 /* jump */ +#define OP_Jump 16 /* jump */ +#define OP_Once 17 /* jump */ +#define OP_If 18 /* jump */ #define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_Once 20 /* jump */ -#define OP_If 21 /* jump */ -#define OP_IfNot 22 /* jump */ -#define OP_IfNullRow 23 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ -#define OP_SeekLT 24 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekLE 25 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGE 26 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekGT 27 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NoConflict 28 /* jump, synopsis: key=r[P3@P4] */ -#define OP_NotFound 29 /* jump, synopsis: key=r[P3@P4] */ -#define OP_Found 30 /* jump, synopsis: key=r[P3@P4] */ -#define OP_SeekRowid 31 /* jump, synopsis: intkey=r[P3] */ -#define OP_NotExists 32 /* jump, synopsis: intkey=r[P3] */ -#define OP_Last 33 /* jump */ -#define OP_IfSmaller 34 /* jump */ -#define OP_SorterSort 35 /* jump */ -#define OP_Sort 36 /* jump */ -#define OP_Rewind 37 /* jump */ -#define OP_IdxLE 38 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGT 39 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxLT 40 /* jump, synopsis: key=r[P3@P4] */ -#define OP_IdxGE 41 /* jump, synopsis: key=r[P3@P4] */ -#define OP_RowSetRead 42 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_IfNot 20 /* jump */ +#define OP_IfNullRow 21 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 22 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 23 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 24 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 25 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 30 /* jump, synopsis: intkey=r[P3] */ +#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 32 /* jump */ +#define OP_IfSmaller 33 /* jump */ +#define OP_SorterSort 34 /* jump */ +#define OP_Sort 35 /* jump */ +#define OP_Rewind 36 /* jump */ +#define OP_IdxLE 37 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 38 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 39 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGE 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 41 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 42 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ #define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ #define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_RowSetTest 45 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 46 /* jump */ -#define OP_FkIfZero 47 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 48 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ -#define OP_IfNotZero 49 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_Program 45 /* jump */ +#define OP_FkIfZero 46 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 47 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IfNotZero 48 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 49 /* jump, synopsis: if (--r[P1])==0 goto P2 */ #define OP_IsNull 50 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ #define OP_NotNull 51 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ #define OP_Ne 52 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ @@ -14537,119 +14725,121 @@ typedef struct VdbeOpList VdbeOpList; #define OP_Lt 56 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ #define OP_ElseNotEq 58 /* jump, same as TK_ESCAPE */ -#define OP_DecrJumpZero 59 /* jump, synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 60 /* jump */ -#define OP_VNext 61 /* jump */ -#define OP_Init 62 /* jump, synopsis: Start at P2 */ -#define OP_Return 63 -#define OP_EndCoroutine 64 -#define OP_HaltIfNull 65 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 66 -#define OP_Integer 67 /* synopsis: r[P2]=P1 */ -#define OP_Int64 68 /* synopsis: r[P2]=P4 */ -#define OP_String 69 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 70 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 71 /* synopsis: r[P1]=NULL */ -#define OP_Blob 72 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 73 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 74 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 75 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 76 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 77 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 78 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 79 -#define OP_AddImm 80 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 81 -#define OP_Cast 82 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 83 -#define OP_Compare 84 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_BitAnd 85 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 86 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 87 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 89 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 90 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 91 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 92 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 93 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 94 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_IsTrue 95 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ -#define OP_BitNot 96 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ -#define OP_Offset 97 /* synopsis: r[P3] = sqlite_offset(P1) */ -#define OP_Column 98 /* synopsis: r[P3]=PX */ -#define OP_String8 99 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_Affinity 100 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 101 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_Count 102 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 103 -#define OP_SetCookie 104 -#define OP_ReopenIdx 105 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 106 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 107 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenDup 108 -#define OP_OpenAutoindex 109 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 110 /* synopsis: nColumn=P2 */ -#define OP_SorterOpen 111 -#define OP_SequenceTest 112 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ -#define OP_OpenPseudo 113 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 114 -#define OP_ColumnsUsed 115 -#define OP_Sequence 116 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 117 /* synopsis: r[P2]=rowid */ -#define OP_Insert 118 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 119 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 120 -#define OP_ResetCount 121 -#define OP_SorterCompare 122 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 123 /* synopsis: r[P2]=data */ -#define OP_RowData 124 /* synopsis: r[P2]=data */ -#define OP_Rowid 125 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 126 -#define OP_SeekEnd 127 -#define OP_SorterInsert 128 /* synopsis: key=r[P2] */ -#define OP_IdxInsert 129 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 130 /* synopsis: key=r[P2@P3] */ -#define OP_DeferredSeek 131 /* synopsis: Move P3 to P1.rowid if needed */ -#define OP_IdxRowid 132 /* synopsis: r[P2]=rowid */ -#define OP_Destroy 133 -#define OP_Real 134 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_Clear 135 -#define OP_ResetSorter 136 -#define OP_CreateBtree 137 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ -#define OP_SqlExec 138 -#define OP_ParseSchema 139 -#define OP_LoadAnalysis 140 -#define OP_DropTable 141 -#define OP_DropIndex 142 -#define OP_DropTrigger 143 -#define OP_IntegrityCk 144 -#define OP_RowSetAdd 145 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 146 -#define OP_FkCounter 147 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 148 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 149 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggStep0 150 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep 151 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggFinal 152 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 153 -#define OP_TableLock 154 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 155 -#define OP_VCreate 156 -#define OP_VDestroy 157 -#define OP_VOpen 158 -#define OP_VColumn 159 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 160 -#define OP_Pagecount 161 -#define OP_MaxPgcnt 162 -#define OP_PureFunc0 163 -#define OP_Function0 164 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_PureFunc 165 -#define OP_Function 166 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Trace 167 -#define OP_CursorHint 168 -#define OP_Noop 169 -#define OP_Explain 170 -#define OP_Abortable 171 +#define OP_IncrVacuum 59 /* jump */ +#define OP_VNext 60 /* jump */ +#define OP_Init 61 /* jump, synopsis: Start at P2 */ +#define OP_PureFunc0 62 +#define OP_Function0 63 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_PureFunc 64 +#define OP_Function 65 /* synopsis: r[P3]=func(r[P2@P5]) */ +#define OP_Return 66 +#define OP_EndCoroutine 67 +#define OP_HaltIfNull 68 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 69 +#define OP_Integer 70 /* synopsis: r[P2]=P1 */ +#define OP_Int64 71 /* synopsis: r[P2]=P4 */ +#define OP_String 72 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_Null 73 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 74 /* synopsis: r[P1]=NULL */ +#define OP_Blob 75 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 76 /* synopsis: r[P2]=parameter(P1,P4) */ +#define OP_Move 77 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 78 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 79 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 80 /* synopsis: r[P2]=r[P1] */ +#define OP_ResultRow 81 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 82 +#define OP_AddImm 83 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 84 +#define OP_Cast 85 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 86 +#define OP_Compare 87 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 88 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_Offset 89 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 90 /* synopsis: r[P3]=PX */ +#define OP_Affinity 91 /* synopsis: affinity(r[P1@P2]) */ +#define OP_BitAnd 92 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 93 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 94 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 96 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 97 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 98 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 99 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 100 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 101 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_MakeRecord 102 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_BitNot 103 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_Count 104 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 105 +#define OP_String8 106 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_SetCookie 107 +#define OP_ReopenIdx 108 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 109 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenWrite 110 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenDup 111 +#define OP_OpenAutoindex 112 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 113 /* synopsis: nColumn=P2 */ +#define OP_SorterOpen 114 +#define OP_SequenceTest 115 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 116 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 117 +#define OP_ColumnsUsed 118 +#define OP_SeekHit 119 /* synopsis: seekHit=P2 */ +#define OP_Sequence 120 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 121 /* synopsis: r[P2]=rowid */ +#define OP_Insert 122 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_InsertInt 123 /* synopsis: intkey=P3 data=r[P2] */ +#define OP_Delete 124 +#define OP_ResetCount 125 +#define OP_SorterCompare 126 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 127 /* synopsis: r[P2]=data */ +#define OP_RowData 128 /* synopsis: r[P2]=data */ +#define OP_Rowid 129 /* synopsis: r[P2]=rowid */ +#define OP_NullRow 130 +#define OP_SeekEnd 131 +#define OP_SorterInsert 132 /* synopsis: key=r[P2] */ +#define OP_IdxInsert 133 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 134 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 135 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 136 /* synopsis: r[P2]=rowid */ +#define OP_Destroy 137 +#define OP_Clear 138 +#define OP_ResetSorter 139 +#define OP_CreateBtree 140 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_Real 141 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_SqlExec 142 +#define OP_ParseSchema 143 +#define OP_LoadAnalysis 144 +#define OP_DropTable 145 +#define OP_DropIndex 146 +#define OP_DropTrigger 147 +#define OP_IntegrityCk 148 +#define OP_RowSetAdd 149 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 150 +#define OP_FkCounter 151 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 152 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 153 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 154 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 155 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 156 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 157 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 158 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 159 +#define OP_TableLock 160 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 161 +#define OP_VCreate 162 +#define OP_VDestroy 163 +#define OP_VOpen 164 +#define OP_VColumn 165 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 166 +#define OP_Pagecount 167 +#define OP_MaxPgcnt 168 +#define OP_Trace 169 +#define OP_CursorHint 170 +#define OP_Noop 171 +#define OP_Explain 172 +#define OP_Abortable 173 /* Properties such as "out2" or "jump" that are specified in ** comments following the "case" for each opcode in the vdbe.c @@ -14662,28 +14852,28 @@ typedef struct VdbeOpList VdbeOpList; #define OPFLG_OUT2 0x10 /* out2: P2 is an output */ #define OPFLG_OUT3 0x20 /* out3: P3 is an output */ #define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 8 */ 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x03, 0x03, 0x01,\ +/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x00, 0x10,\ +/* 8 */ 0x00, 0x01, 0x00, 0x01, 0x01, 0x01, 0x03, 0x03,\ +/* 16 */ 0x01, 0x01, 0x03, 0x12, 0x03, 0x01, 0x09, 0x09,\ /* 24 */ 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09,\ -/* 32 */ 0x09, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 40 */ 0x01, 0x01, 0x23, 0x26, 0x26, 0x0b, 0x01, 0x01,\ +/* 32 */ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,\ +/* 40 */ 0x01, 0x23, 0x0b, 0x26, 0x26, 0x01, 0x01, 0x03,\ /* 48 */ 0x03, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 56 */ 0x0b, 0x0b, 0x01, 0x03, 0x01, 0x01, 0x01, 0x02,\ -/* 64 */ 0x02, 0x08, 0x00, 0x10, 0x10, 0x10, 0x10, 0x00,\ -/* 72 */ 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 80 */ 0x02, 0x02, 0x02, 0x00, 0x00, 0x26, 0x26, 0x26,\ -/* 88 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x12,\ -/* 96 */ 0x12, 0x20, 0x00, 0x10, 0x00, 0x00, 0x10, 0x10,\ -/* 104 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 112 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 120 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,\ -/* 128 */ 0x04, 0x04, 0x00, 0x00, 0x10, 0x10, 0x10, 0x00,\ -/* 136 */ 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 144 */ 0x00, 0x06, 0x10, 0x00, 0x04, 0x1a, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 160 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 168 */ 0x00, 0x00, 0x00, 0x00,} +/* 56 */ 0x0b, 0x0b, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00,\ +/* 64 */ 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10, 0x10,\ +/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x10, 0x00, 0x00, 0x10,\ +/* 80 */ 0x10, 0x00, 0x00, 0x02, 0x02, 0x02, 0x00, 0x00,\ +/* 88 */ 0x12, 0x20, 0x00, 0x00, 0x26, 0x26, 0x26, 0x26,\ +/* 96 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x00, 0x12,\ +/* 104 */ 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 112 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 120 */ 0x10, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 128 */ 0x00, 0x10, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00,\ +/* 136 */ 0x10, 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ +/* 144 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x10, 0x00,\ +/* 152 */ 0x04, 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 160 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ +/* 168 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00,} /* The sqlite3P2Values() routine is able to run faster if it knows ** the value of the largest JUMP opcode. The smaller the maximum @@ -14691,7 +14881,7 @@ typedef struct VdbeOpList VdbeOpList; ** generated this include file strives to group all JUMP opcodes ** together near the beginning of the list. */ -#define SQLITE_MX_JUMP_OPCODE 62 /* Maximum JUMP opcode */ +#define SQLITE_MX_JUMP_OPCODE 61 /* Maximum JUMP opcode */ /************** End of opcodes.h *********************************************/ /************** Continuing where we left off in vdbe.h ***********************/ @@ -14765,9 +14955,6 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); -#ifdef SQLITE_COVERAGE_TEST -SQLITE_PRIVATE int sqlite3VdbeLabelHasBeenResolved(Vdbe*,int); -#endif SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); @@ -14789,6 +14976,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +SQLITE_PRIVATE int sqlite3BlobCompare(const Mem*, const Mem*); SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); @@ -14844,23 +15032,52 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); ** ** VdbeCoverageNeverTaken(v) // Previous branch is never taken ** +** VdbeCoverageNeverNull(v) // Previous three-way branch is only +** // taken on the first two ways. The +** // NULL option is not possible +** +** VdbeCoverageEqNe(v) // Previous OP_Jump is only interested +** // in distingishing equal and not-equal. +** ** Every VDBE branch operation must be tagged with one of the macros above. ** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and ** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() ** routine in vdbe.c, alerting the developer to the missed tag. +** +** During testing, the test application will invoke +** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback +** routine that is invoked as each bytecode branch is taken. The callback +** contains the sqlite3.c source line number ov the VdbeCoverage macro and +** flags to indicate whether or not the branch was taken. The test application +** is responsible for keeping track of this and reporting byte-code branches +** that are never taken. +** +** See the VdbeBranchTaken() macro and vdbeTakeBranch() function in the +** vdbe.c source file for additional information. */ #ifdef SQLITE_VDBE_COVERAGE SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); # define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) # define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) -# define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); -# define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); +# define VdbeCoverageAlwaysTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x5000000); +# define VdbeCoverageNeverTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x6000000); +# define VdbeCoverageNeverNull(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageNeverNullIf(v,x) \ + if(x)sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageEqNe(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x8000000); # define VDBE_OFFSET_LINENO(x) (__LINE__+x) #else # define VdbeCoverage(v) # define VdbeCoverageIf(v,x) # define VdbeCoverageAlwaysTaken(v) # define VdbeCoverageNeverTaken(v) +# define VdbeCoverageNeverNull(v) +# define VdbeCoverageNeverNullIf(v,x) +# define VdbeCoverageEqNe(v) # define VDBE_OFFSET_LINENO(x) 0 #endif @@ -14870,6 +15087,10 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const ch # define sqlite3VdbeScanStatus(a,b,c,d,e) #endif +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); +#endif + #endif /* SQLITE_VDBE_H */ /************** End of vdbe.h ************************************************/ @@ -15064,6 +15285,8 @@ SQLITE_PRIVATE int sqlite3PagerUseWal(Pager *pPager, Pgno); SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager); # endif #else # define sqlite3PagerUseWal(x,y) 0 @@ -16047,6 +16270,7 @@ struct sqlite3 { #define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee*/ #define SQLITE_TriggerEQP 0x01000000 /* Show trigger EXPLAIN QUERY PLAN */ #define SQLITE_ResetDatabase 0x02000000 /* Reset the database */ +#define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ /* Flags used only if debugging */ #ifdef SQLITE_DEBUG @@ -16071,7 +16295,7 @@ struct sqlite3 { ** selectively disable various optimizations. */ #define SQLITE_QueryFlattener 0x0001 /* Query flattening */ -#define SQLITE_ColumnCache 0x0002 /* Column cache */ + /* 0x0002 available for reuse */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ #define SQLITE_DistinctOpt 0x0010 /* DISTINCT using indexes */ @@ -16085,6 +16309,8 @@ struct sqlite3 { /* TH3 expects the Stat34 ^^^^^^ value to be 0x0800. Don't change it */ #define SQLITE_PushDown 0x1000 /* The push-down optimization */ #define SQLITE_SimplifyJoin 0x2000 /* Convert LEFT JOIN to JOIN */ +#define SQLITE_SkipScan 0x4000 /* Skip-scans */ +#define SQLITE_PropagateConst 0x8000 /* The constant propagation opt */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* @@ -16123,11 +16349,13 @@ struct sqlite3 { */ struct FuncDef { i8 nArg; /* Number of arguments. -1 means unlimited */ - u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ + u32 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ + void (*xValue)(sqlite3_context*); /* Current agg value */ + void (*xInverse)(sqlite3_context*,int,sqlite3_value**); /* inverse agg-step */ const char *zName; /* SQL name of the function. */ union { FuncDef *pHash; /* Next with a different name but the same hash */ @@ -16184,6 +16412,8 @@ struct FuncDestructor { ** single query - might change over time */ #define SQLITE_FUNC_AFFINITY 0x4000 /* Built-in affinity() function */ #define SQLITE_FUNC_OFFSET 0x8000 /* Built-in sqlite_offset() function */ +#define SQLITE_FUNC_WINDOW 0x10000 /* Built-in window-only function */ +#define SQLITE_FUNC_WINDOW_SIZE 0x20000 /* Requires partition size as arg. */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -16218,6 +16448,12 @@ struct FuncDestructor { ** are interpreted in the same way as the first 4 parameters to ** FUNCTION(). ** +** WFUNCTION(zName, nArg, iArg, xStep, xFinal, xValue, xInverse) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** ** LIKEFUNC(zName, nArg, pArg, flags) ** Used to create a scalar function definition of a function zName ** that accepts nArg arguments and is implemented by a call to C @@ -16228,31 +16464,35 @@ struct FuncDestructor { */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \ - 0, 0, xFunc, 0, #zName, {0} } + 0, 0, xFunc, 0, 0, 0, #zName, {0} } #define PURE_DATE(zName, nArg, iArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ - (void*)&sqlite3Config, 0, xFunc, 0, #zName, {0} } + (void*)&sqlite3Config, 0, xFunc, 0, 0, 0, #zName, {0} } #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, #zName, } + pArg, 0, xFunc, 0, 0, 0, #zName, } #define LIKEFUNC(zName, nArg, arg, flags) \ {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, #zName, {0} } -#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ + (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } +#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,0,#zName, {0}} #define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xFinal,0,#zName, {0}} + +#define WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|f, \ + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,xInverse,#zName, {0}} /* ** All current savepoints are stored in a linked list starting at @@ -16738,6 +16978,7 @@ struct Index { tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif + Bitmask colNotIdxed; /* 0 for unindexed columns in pTab */ }; /* @@ -16776,9 +17017,11 @@ struct IndexSample { ** Each token coming out of the lexer is an instance of ** this structure. Tokens are also used as part of an expression. ** -** Note if Token.z==0 then Token.dyn and Token.n are undefined and -** may contain random values. Do not make any assumptions about Token.dyn -** and Token.n when Token.z==0. +** The memory that "z" points to is owned by other objects. Take care +** that the owner of the "z" string does not deallocate the string before +** the Token goes out of scope! Very often, the "z" points to some place +** in the middle of the Parse.zSql text. But it might also point to a +** static string. */ struct Token { const char *z; /* Text of the token. Not NULL-terminated! */ @@ -16953,6 +17196,9 @@ struct Expr { AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ Table *pTab; /* Table for TK_COLUMN expressions. Can be NULL ** for a column of an index on an expression */ +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin; /* Window definition for window functions */ +#endif }; /* @@ -16961,7 +17207,7 @@ struct Expr { #define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_HasFunc 0x000004 /* Contains one or more functions of any kind */ - /* 0x000008 // available for use */ +#define EP_FixedCol 0x000008 /* TK_Column with a known fixed value */ #define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ #define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ @@ -17083,31 +17329,6 @@ struct IdList { int nId; /* Number of identifiers on the list */ }; -/* -** The bitmask datatype defined below is used for various optimizations. -** -** Changing this from a 64-bit to a 32-bit type limits the number of -** tables in a join to 32 instead of 64. But it also reduces the size -** of the library by 738 bytes on ix86. -*/ -#ifdef SQLITE_BITMASK_TYPE - typedef SQLITE_BITMASK_TYPE Bitmask; -#else - typedef u64 Bitmask; -#endif - -/* -** The number of bits in a Bitmask. "BMS" means "BitMask Size". -*/ -#define BMS ((int)(sizeof(Bitmask)*8)) - -/* -** A bit in a Bitmask -*/ -#define MASKBIT(n) (((Bitmask)1)<<(n)) -#define MASKBIT32(n) (((unsigned int)1)<<(n)) -#define ALLBITS ((Bitmask)-1) - /* ** The following structure describes the FROM clause of a SELECT statement. ** Each table or subquery in the FROM clause is a separate element of @@ -17239,6 +17460,7 @@ struct NameContext { int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ u16 ncFlags; /* Zero or more NC_* flags defined below */ + Select *pWinSelect; /* SELECT statement for any window functions */ }; /* @@ -17261,6 +17483,7 @@ struct NameContext { #define NC_UUpsert 0x0200 /* True if uNC.pUpsert is used */ #define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ #define NC_Complex 0x2000 /* True if a function or subquery seen */ +#define NC_AllowWin 0x4000 /* Window functions are allowed here */ /* ** An instance of the following object describes a single ON CONFLICT @@ -17315,9 +17538,7 @@ struct Select { LogEst nSelectRow; /* Estimated number of result rows */ u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ -#if SELECTTRACE_ENABLED - char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ -#endif + u32 selId; /* Unique identifier number for this SELECT */ int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -17328,6 +17549,10 @@ struct Select { Select *pNext; /* Next select to the left in a compound */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ With *pWith; /* WITH clause attached to this select. Or NULL. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin; /* List of window functions */ + Window *pWinDefn; /* List of named window definitions */ +#endif }; /* @@ -17471,13 +17696,6 @@ struct AutoincInfo { int regCtr; /* Memory register holding the rowid counter */ }; -/* -** Size of the column cache -*/ -#ifndef SQLITE_N_COLCACHE -# define SQLITE_N_COLCACHE 10 -#endif - /* ** At least one instance of the following structure is created for each ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE @@ -17553,7 +17771,6 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 nColCache; /* Number of entries in aColCache[] */ int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ @@ -17563,8 +17780,6 @@ struct Parse { int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ int iSelfTab; /* Table associated with an index on expr, or negative ** of the base register during check-constraint eval */ - int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ - int iCacheCnt; /* Counter used to generate aColCache[].lru values */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ ExprList *pConstExpr;/* Constant expressions */ @@ -17574,9 +17789,7 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ -#if SELECTTRACE_ENABLED - int nSelect; /* Number of SELECT statements seen */ -#endif + int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ @@ -17596,17 +17809,9 @@ struct Parse { ** Fields above must be initialized to zero. The fields that follow, ** down to the beginning of the recursive section, do not need to be ** initialized as they will be set before being used. The boundary is - ** determined by offsetof(Parse,aColCache). + ** determined by offsetof(Parse,aTempReg). **************************************************************************/ - struct yColCache { - int iTable; /* Table cursor number */ - i16 iColumn; /* Table column number */ - u8 tempReg; /* iReg is a temp register that needs to be freed */ - int iLevel; /* Nesting level */ - int iReg; /* Reg with value of this column. 0 means none. */ - int lru; /* Least recently used entry has the smallest value */ - } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ int aTempReg[8]; /* Holding area for temporary registers */ Token sNameToken; /* Token with unqualified schema object name */ @@ -17621,8 +17826,10 @@ struct Parse { ynVar nVar; /* Number of '?' variables seen in the SQL so far */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ u8 explain; /* True if the EXPLAIN flag is found on the query */ +#if !(defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE)) + u8 eParseMode; /* PARSE_MODE_XXX constant */ +#endif #ifndef SQLITE_OMIT_VIRTUALTABLE - u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ int nVtabLock; /* Number of virtual tables to lock */ #endif int nHeight; /* Expression tree height of current sub-select */ @@ -17633,6 +17840,7 @@ struct Parse { Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ const char *zTail; /* All SQL text past the last semicolon parsed */ Table *pNewTable; /* A table being constructed by CREATE TABLE */ + Index *pNewIndex; /* An index being constructed by CREATE INDEX */ Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -17643,12 +17851,20 @@ struct Parse { TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ With *pWithToFree; /* Free this WITH object at the end of the parse */ +#ifndef SQLITE_OMIT_ALTERTABLE + RenameToken *pRename; /* Tokens subject to renaming by ALTER TABLE */ +#endif }; +#define PARSE_MODE_NORMAL 0 +#define PARSE_MODE_DECLARE_VTAB 1 +#define PARSE_MODE_RENAME_COLUMN 2 +#define PARSE_MODE_RENAME_TABLE 3 + /* ** Sizes and pointers of various parts of the Parse object. */ -#define PARSE_HDR_SZ offsetof(Parse,aColCache) /* Recursive part w/o aColCache*/ +#define PARSE_HDR_SZ offsetof(Parse,aTempReg) /* Recursive part w/o aColCache*/ #define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ #define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ #define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ @@ -17659,7 +17875,19 @@ struct Parse { #ifdef SQLITE_OMIT_VIRTUALTABLE #define IN_DECLARE_VTAB 0 #else - #define IN_DECLARE_VTAB (pParse->declareVtab) + #define IN_DECLARE_VTAB (pParse->eParseMode==PARSE_MODE_DECLARE_VTAB) +#endif + +#if defined(SQLITE_OMIT_ALTERTABLE) + #define IN_RENAME_OBJECT 0 +#else + #define IN_RENAME_OBJECT (pParse->eParseMode>=PARSE_MODE_RENAME_COLUMN) +#endif + +#if defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE) + #define IN_SPECIAL_PARSE 0 +#else + #define IN_SPECIAL_PARSE (pParse->eParseMode!=PARSE_MODE_NORMAL) #endif /* @@ -17838,8 +18066,14 @@ typedef struct { char **pzErrMsg; /* Error message stored here */ int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ int rc; /* Result code stored here */ + u32 mInitFlags; /* Flags controlling error messages */ } InitData; +/* +** Allowed values for mInitFlags +*/ +#define INITFLAG_AlterTable 0x0001 /* This is a reparse after ALTER TABLE */ + /* ** Structure containing global configuration data for the SQLite library. ** @@ -17890,7 +18124,7 @@ struct Sqlite3Config { /* The following callback (if not NULL) is invoked on every VDBE branch ** operation. Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE. */ - void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ + void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx); /* Callback */ void *pVdbeBranchArg; /* 1st argument */ #endif #ifndef SQLITE_UNTESTABLE @@ -17941,6 +18175,9 @@ struct Walker { struct IdxExprTrans *pIdxTrans; /* Convert idxed expr to column */ ExprList *pGroupBy; /* GROUP BY clause */ Select *pSelect; /* HAVING to WHERE clause ctx */ + struct WindowRewrite *pRewrite; /* Window rewrite context */ + struct WhereConst *pConst; /* WHERE clause constants */ + struct RenameCtx *pRename; /* RENAME COLUMN context */ } u; }; @@ -17991,6 +18228,68 @@ struct TreeView { }; #endif /* SQLITE_DEBUG */ +/* +** This object is used in varioius ways, all related to window functions +** +** (1) A single instance of this structure is attached to the +** the Expr.pWin field for each window function in an expression tree. +** This object holds the information contained in the OVER clause, +** plus additional fields used during code generation. +** +** (2) All window functions in a single SELECT form a linked-list +** attached to Select.pWin. The Window.pFunc and Window.pExpr +** fields point back to the expression that is the window function. +** +** (3) The terms of the WINDOW clause of a SELECT are instances of this +** object on a linked list attached to Select.pWinDefn. +** +** The uses (1) and (2) are really the same Window object that just happens +** to be accessible in two different ways. Use (3) is are separate objects. +*/ +struct Window { + char *zName; /* Name of window (may be NULL) */ + ExprList *pPartition; /* PARTITION BY clause */ + ExprList *pOrderBy; /* ORDER BY clause */ + u8 eType; /* TK_RANGE or TK_ROWS */ + u8 eStart; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 eEnd; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + Expr *pStart; /* Expression for " PRECEDING" */ + Expr *pEnd; /* Expression for " FOLLOWING" */ + Window *pNextWin; /* Next window function belonging to this SELECT */ + Expr *pFilter; /* The FILTER expression */ + FuncDef *pFunc; /* The function */ + int iEphCsr; /* Partition buffer or Peer buffer */ + int regAccum; + int regResult; + int csrApp; /* Function cursor (used by min/max) */ + int regApp; /* Function register (also used by min/max) */ + int regPart; /* First in a set of registers holding PARTITION BY + ** and ORDER BY values for the window */ + Expr *pOwner; /* Expression object this window is attached to */ + int nBufferCol; /* Number of columns in buffer table */ + int iArgCol; /* Offset of first argument for this function */ +}; + +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*); +SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); +SQLITE_PRIVATE int sqlite3WindowCompare(Parse*, Window*, Window*); +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Window*); +SQLITE_PRIVATE void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int); +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse*, Select*); +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse*, struct SrcList_item*); +SQLITE_PRIVATE void sqlite3WindowUpdate(Parse*, Window*, Window*, FuncDef*); +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p); +SQLITE_PRIVATE void sqlite3WindowFunctions(void); +#else +# define sqlite3WindowDelete(a,b) +# define sqlite3WindowFunctions() +# define sqlite3WindowAttach(a,b,c) +#endif + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -18078,9 +18377,7 @@ SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') #endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8); -#endif /* ** Internal function prototypes @@ -18205,6 +18502,10 @@ SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, co SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView*, const Window*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView*, const Window*, u8); +#endif #endif @@ -18229,7 +18530,7 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*, int); SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); @@ -18241,6 +18542,7 @@ SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); +SQLITE_PRIVATE int sqlite3InitOne(sqlite3*, int, char**, u32); SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); #ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3*,const char *zName); @@ -18290,8 +18592,9 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); #endif -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*); +SQLITE_PRIVATE void sqlite3RowSetDelete(void*); +SQLITE_PRIVATE void sqlite3RowSetClear(void*); SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); @@ -18310,6 +18613,7 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3*, Index*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); @@ -18319,7 +18623,7 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); #endif SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*); SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); @@ -18354,7 +18658,7 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); -SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); @@ -18364,15 +18668,8 @@ SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); #define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse*); -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*); -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); @@ -18460,11 +18757,6 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,IdList*); SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,Select*,int); -#if SELECTTRACE_ENABLED -SQLITE_PRIVATE void sqlite3SelectSetName(Select*,const char*); -#else -# define sqlite3SelectSetName(A,B) -#endif SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); @@ -18493,12 +18785,12 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, i SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, Select*,u8,Upsert*, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,ExprList*, Expr*, u8, const char*,const char*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*, +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, const char*,const char*); SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); @@ -18613,6 +18905,7 @@ SQLITE_PRIVATE int sqlite3MemdbInit(void); SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); +SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq*); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, Expr *pExpr); @@ -18665,9 +18958,10 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, int, int); SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); SQLITE_PRIVATE void sqlite3AlterFunctions(void); SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); +SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr *, int, int); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); @@ -18680,6 +18974,10 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *, Table *, int, int); SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *); SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *, SrcList *); +SQLITE_PRIVATE void *sqlite3RenameTokenMap(Parse*, void*, Token*); +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse*, void *pTo, void *pFrom); +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse*, ExprList*); SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); SQLITE_PRIVATE char sqlite3AffinityType(const char*, Column*); SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); @@ -18698,12 +18996,17 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3*,int,int); SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList(Parse*, ExprList*, int, int); + #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); #endif SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), - void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), FuncDestructor *pDestructor ); SQLITE_PRIVATE void sqlite3NoopDestructor(void*); @@ -18744,6 +19047,7 @@ SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(u64), Parse*); SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); #endif SQLITE_PRIVATE void sqlite3Parser(void*, int, Token); +SQLITE_PRIVATE int sqlite3ParserFallback(int); #ifdef YYTRACKMAXSTACKDEPTH SQLITE_PRIVATE int sqlite3ParserStackPeak(void*); #endif @@ -19444,6 +19748,7 @@ struct VdbeCursor { Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ + Bool seekHit:1; /* See the OP_SeekHit and OP_IfNoHope opcodes */ Btree *pBtx; /* Separate file holding temporary table */ i64 seqCount; /* Sequence counter */ int *aAltMap; /* Mapping from table to index column numbers */ @@ -19527,6 +19832,9 @@ struct VdbeFrame { void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ AuxData *pAuxData; /* Linked list of auxdata allocations */ +#if SQLITE_DEBUG + u32 iFrameMagic; /* magic number for sanity checking */ +#endif int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ @@ -19537,6 +19845,13 @@ struct VdbeFrame { int nDbChange; /* Value of db->nChange */ }; +/* Magic number for sanity checking on VdbeFrame objects */ +#define SQLITE_FRAME_MAGIC 0x879fb71e + +/* +** Return a pointer to the array of registers allocated for use +** by a VdbeFrame. +*/ #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) /* @@ -19551,8 +19866,6 @@ struct sqlite3_value { int nZero; /* Extra zero bytes when MEM_Zero and MEM_Blob set */ const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ - RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ - VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ @@ -19567,7 +19880,7 @@ struct sqlite3_value { void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ - void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ + u16 mScopyFlags; /* flags value immediately after the shallow copy */ #endif }; @@ -19596,8 +19909,8 @@ struct sqlite3_value { #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ #define MEM_AffMask 0x001f /* Mask of affinity bits */ -#define MEM_RowSet 0x0020 /* Value is a RowSet object */ -#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ +/* Available 0x0020 */ +/* Available 0x0040 */ #define MEM_Undefined 0x0080 /* Value is undefined */ #define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ #define MEM_TypeMask 0xc1ff /* Mask of type bits */ @@ -19624,7 +19937,7 @@ struct sqlite3_value { ** that needs to be deallocated to avoid a leak. */ #define VdbeMemDynamic(X) \ - (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) + (((X)->flags&(MEM_Agg|MEM_Dyn))!=0) /* ** Clear any existing type flags from a Mem and replace them with f @@ -19744,9 +20057,9 @@ struct Vdbe { u8 errorAction; /* Recovery action to do in case of an error */ u8 minWriteFileFormat; /* Minimum file format for writable database files */ u8 prepFlags; /* SQLITE_PREPARE_* flags */ - bft expired:1; /* True if the VM needs to be recompiled */ - bft doingRerun:1; /* True if rerunning after an auto-reprepare */ + bft expired:2; /* 1: recompile VM immediately 2: when convenient */ bft explain:2; /* True if EXPLAIN present on SQL command */ + bft doingRerun:1; /* True if rerunning after an auto-reprepare */ bft changeCntOn:1; /* True to update the change-counter */ bft runOnlyOnce:1; /* Automatically expire on reset */ bft usesStmtJournal:1; /* True if uses a statement journal */ @@ -19807,9 +20120,6 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); -#endif SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); @@ -19840,7 +20150,10 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetPointer(Mem*, void*, const char*, void(*)(v SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); @@ -19854,11 +20167,18 @@ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); +#endif SQLITE_PRIVATE const char *sqlite3OpcodeName(int); SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); -SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeFrameIsValid(VdbeFrame*); +#endif +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void*); /* Destructor on Mem */ +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); /* Actually deletes the Frame */ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int); @@ -21956,9 +22276,12 @@ SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ ** Unregister a VFS so that it is no longer accessible. */ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ -#if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; #endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); sqlite3_mutex_leave(mutex); @@ -27289,7 +27612,12 @@ SQLITE_API void sqlite3_str_vappendf( if( bufpt==0 ){ bufpt = ""; }else if( xtype==etDYNSTRING ){ - if( pAccum->nChar==0 && pAccum->mxAlloc && width==0 && precision<0 ){ + if( pAccum->nChar==0 + && pAccum->mxAlloc + && width==0 + && precision<0 + && pAccum->accError==0 + ){ /* Special optimization for sqlite3_mprintf("%z..."): ** Extend an existing memory allocation rather than creating ** a new one. */ @@ -27987,21 +28315,13 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewPush(pView, 1); } do{ -#if SELECTTRACE_ENABLED sqlite3TreeViewLine(pView, - "SELECT%s%s (%s/%p) selFlags=0x%x nSelectRow=%d", + "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), - p->zSelName, p, p->selFlags, + p->selId, p, p->selFlags, (int)p->nSelectRow ); -#else - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, - (int)p->nSelectRow - ); -#endif if( cnt++ ) sqlite3TreeViewPop(pView); if( p->pPrior ){ n = 1000; @@ -28013,8 +28333,23 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m if( p->pHaving ) n++; if( p->pOrderBy ) n++; if( p->pLimit ) n++; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) n++; + if( p->pWinDefn ) n++; +#endif } sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + Window *pX; + pView = sqlite3TreeViewPush(pView, (n--)>0); + sqlite3TreeViewLine(pView, "window-functions"); + for(pX=p->pWin; pX; pX=pX->pNextWin){ + sqlite3TreeViewWinFunc(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(pView); + } +#endif if( p->pSrc && p->pSrc->nSrc ){ int i; pView = sqlite3TreeViewPush(pView, (n--)>0); @@ -28064,6 +28399,16 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewExpr(pView, p->pHaving, 0); sqlite3TreeViewPop(pView); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWinDefn ){ + Window *pX; + sqlite3TreeViewItem(pView, "WINDOW", (n--)>0); + for(pX=p->pWinDefn; pX; pX=pX->pNextWin){ + sqlite3TreeViewWindow(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(pView); + } +#endif if( p->pOrderBy ){ sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); } @@ -28091,6 +28436,83 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m sqlite3TreeViewPop(pView); } +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a description of starting or stopping bounds +*/ +SQLITE_PRIVATE void sqlite3TreeViewBound( + TreeView *pView, /* View context */ + u8 eBound, /* UNBOUNDED, CURRENT, PRECEDING, FOLLOWING */ + Expr *pExpr, /* Value for PRECEDING or FOLLOWING */ + u8 moreToFollow /* True if more to follow */ +){ + switch( eBound ){ + case TK_UNBOUNDED: { + sqlite3TreeViewItem(pView, "UNBOUNDED", moreToFollow); + sqlite3TreeViewPop(pView); + break; + } + case TK_CURRENT: { + sqlite3TreeViewItem(pView, "CURRENT", moreToFollow); + sqlite3TreeViewPop(pView); + break; + } + case TK_PRECEDING: { + sqlite3TreeViewItem(pView, "PRECEDING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(pView); + break; + } + case TK_FOLLOWING: { + sqlite3TreeViewItem(pView, "FOLLOWING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(pView); + break; + } + } +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ + pView = sqlite3TreeViewPush(pView, more); + if( pWin->zName ){ + sqlite3TreeViewLine(pView, "OVER %s", pWin->zName); + }else{ + sqlite3TreeViewLine(pView, "OVER"); + } + if( pWin->pPartition ){ + sqlite3TreeViewExprList(pView, pWin->pPartition, 1, "PARTITION-BY"); + } + if( pWin->pOrderBy ){ + sqlite3TreeViewExprList(pView, pWin->pOrderBy, 1, "ORDER-BY"); + } + if( pWin->eType ){ + sqlite3TreeViewItem(pView, pWin->eType==TK_RANGE ? "RANGE" : "ROWS", 0); + sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); + sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window Function object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView *pView, const Window *pWin, u8 more){ + pView = sqlite3TreeViewPush(pView, more); + sqlite3TreeViewLine(pView, "WINFUNC %s(%d)", + pWin->pFunc->zName, pWin->pFunc->nArg); + sqlite3TreeViewWindow(pView, pWin, 0); + sqlite3TreeViewPop(pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** Generate a human-readable explanation of an expression tree. */ @@ -28128,6 +28550,9 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView, "{%d:%d}%s", pExpr->iTable, pExpr->iColumn, zFlgs); } + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + } break; } case TK_INTEGER: { @@ -28241,10 +28666,17 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_AGG_FUNCTION: case TK_FUNCTION: { ExprList *pFarg; /* List of function arguments */ + Window *pWin; if( ExprHasProperty(pExpr, EP_TokenOnly) ){ pFarg = 0; + pWin = 0; }else{ pFarg = pExpr->x.pList; +#ifndef SQLITE_OMIT_WINDOWFUNC + pWin = pExpr->pWin; +#else + pWin = 0; +#endif } if( pExpr->op==TK_AGG_FUNCTION ){ sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", @@ -28253,8 +28685,13 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); } if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, 0, 0); + sqlite3TreeViewExprList(pView, pFarg, pWin!=0, 0); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + sqlite3TreeViewWindow(pView, pWin, 0); + } +#endif break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -31286,52 +31723,52 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 1 */ "AutoCommit" OpHelp(""), /* 2 */ "Transaction" OpHelp(""), /* 3 */ "SorterNext" OpHelp(""), - /* 4 */ "PrevIfOpen" OpHelp(""), - /* 5 */ "NextIfOpen" OpHelp(""), - /* 6 */ "Prev" OpHelp(""), - /* 7 */ "Next" OpHelp(""), - /* 8 */ "Checkpoint" OpHelp(""), - /* 9 */ "JournalMode" OpHelp(""), - /* 10 */ "Vacuum" OpHelp(""), - /* 11 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 12 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 13 */ "Goto" OpHelp(""), - /* 14 */ "Gosub" OpHelp(""), - /* 15 */ "InitCoroutine" OpHelp(""), - /* 16 */ "Yield" OpHelp(""), - /* 17 */ "MustBeInt" OpHelp(""), - /* 18 */ "Jump" OpHelp(""), + /* 4 */ "Prev" OpHelp(""), + /* 5 */ "Next" OpHelp(""), + /* 6 */ "Checkpoint" OpHelp(""), + /* 7 */ "JournalMode" OpHelp(""), + /* 8 */ "Vacuum" OpHelp(""), + /* 9 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 10 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 11 */ "Goto" OpHelp(""), + /* 12 */ "Gosub" OpHelp(""), + /* 13 */ "InitCoroutine" OpHelp(""), + /* 14 */ "Yield" OpHelp(""), + /* 15 */ "MustBeInt" OpHelp(""), + /* 16 */ "Jump" OpHelp(""), + /* 17 */ "Once" OpHelp(""), + /* 18 */ "If" OpHelp(""), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "Once" OpHelp(""), - /* 21 */ "If" OpHelp(""), - /* 22 */ "IfNot" OpHelp(""), - /* 23 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), - /* 24 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 25 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 26 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 27 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 28 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 29 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 30 */ "Found" OpHelp("key=r[P3@P4]"), - /* 31 */ "SeekRowid" OpHelp("intkey=r[P3]"), - /* 32 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 33 */ "Last" OpHelp(""), - /* 34 */ "IfSmaller" OpHelp(""), - /* 35 */ "SorterSort" OpHelp(""), - /* 36 */ "Sort" OpHelp(""), - /* 37 */ "Rewind" OpHelp(""), - /* 38 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 39 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 40 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 41 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 42 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 20 */ "IfNot" OpHelp(""), + /* 21 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 22 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 25 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 29 */ "Found" OpHelp("key=r[P3@P4]"), + /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 32 */ "Last" OpHelp(""), + /* 33 */ "IfSmaller" OpHelp(""), + /* 34 */ "SorterSort" OpHelp(""), + /* 35 */ "Sort" OpHelp(""), + /* 36 */ "Rewind" OpHelp(""), + /* 37 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 38 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 39 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 40 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 41 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 42 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 45 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 46 */ "Program" OpHelp(""), - /* 47 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 48 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), - /* 49 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 45 */ "Program" OpHelp(""), + /* 46 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 47 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 48 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 49 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), /* 50 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), /* 51 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), /* 52 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), @@ -31341,119 +31778,121 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 56 */ "Lt" OpHelp("IF r[P3]=r[P1]"), /* 58 */ "ElseNotEq" OpHelp(""), - /* 59 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 60 */ "IncrVacuum" OpHelp(""), - /* 61 */ "VNext" OpHelp(""), - /* 62 */ "Init" OpHelp("Start at P2"), - /* 63 */ "Return" OpHelp(""), - /* 64 */ "EndCoroutine" OpHelp(""), - /* 65 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 66 */ "Halt" OpHelp(""), - /* 67 */ "Integer" OpHelp("r[P2]=P1"), - /* 68 */ "Int64" OpHelp("r[P2]=P4"), - /* 69 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 70 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 71 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 72 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 73 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 74 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 75 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 76 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 77 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 78 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 79 */ "CollSeq" OpHelp(""), - /* 80 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 81 */ "RealAffinity" OpHelp(""), - /* 82 */ "Cast" OpHelp("affinity(r[P1])"), - /* 83 */ "Permutation" OpHelp(""), - /* 84 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 85 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 86 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 87 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 89 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 90 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 91 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 92 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 93 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 94 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 95 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), - /* 96 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), - /* 97 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), - /* 98 */ "Column" OpHelp("r[P3]=PX"), - /* 99 */ "String8" OpHelp("r[P2]='P4'"), - /* 100 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 101 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 102 */ "Count" OpHelp("r[P2]=count()"), - /* 103 */ "ReadCookie" OpHelp(""), - /* 104 */ "SetCookie" OpHelp(""), - /* 105 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), - /* 106 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 107 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 108 */ "OpenDup" OpHelp(""), - /* 109 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 110 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 111 */ "SorterOpen" OpHelp(""), - /* 112 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 113 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 114 */ "Close" OpHelp(""), - /* 115 */ "ColumnsUsed" OpHelp(""), - /* 116 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 117 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 118 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 119 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 120 */ "Delete" OpHelp(""), - /* 121 */ "ResetCount" OpHelp(""), - /* 122 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 123 */ "SorterData" OpHelp("r[P2]=data"), - /* 124 */ "RowData" OpHelp("r[P2]=data"), - /* 125 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 126 */ "NullRow" OpHelp(""), - /* 127 */ "SeekEnd" OpHelp(""), - /* 128 */ "SorterInsert" OpHelp("key=r[P2]"), - /* 129 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 130 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 131 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), - /* 132 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 133 */ "Destroy" OpHelp(""), - /* 134 */ "Real" OpHelp("r[P2]=P4"), - /* 135 */ "Clear" OpHelp(""), - /* 136 */ "ResetSorter" OpHelp(""), - /* 137 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), - /* 138 */ "SqlExec" OpHelp(""), - /* 139 */ "ParseSchema" OpHelp(""), - /* 140 */ "LoadAnalysis" OpHelp(""), - /* 141 */ "DropTable" OpHelp(""), - /* 142 */ "DropIndex" OpHelp(""), - /* 143 */ "DropTrigger" OpHelp(""), - /* 144 */ "IntegrityCk" OpHelp(""), - /* 145 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 146 */ "Param" OpHelp(""), - /* 147 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 148 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 149 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 150 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 151 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 152 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 153 */ "Expire" OpHelp(""), - /* 154 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 155 */ "VBegin" OpHelp(""), - /* 156 */ "VCreate" OpHelp(""), - /* 157 */ "VDestroy" OpHelp(""), - /* 158 */ "VOpen" OpHelp(""), - /* 159 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 160 */ "VRename" OpHelp(""), - /* 161 */ "Pagecount" OpHelp(""), - /* 162 */ "MaxPgcnt" OpHelp(""), - /* 163 */ "PureFunc0" OpHelp(""), - /* 164 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), - /* 165 */ "PureFunc" OpHelp(""), - /* 166 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 167 */ "Trace" OpHelp(""), - /* 168 */ "CursorHint" OpHelp(""), - /* 169 */ "Noop" OpHelp(""), - /* 170 */ "Explain" OpHelp(""), - /* 171 */ "Abortable" OpHelp(""), + /* 59 */ "IncrVacuum" OpHelp(""), + /* 60 */ "VNext" OpHelp(""), + /* 61 */ "Init" OpHelp("Start at P2"), + /* 62 */ "PureFunc0" OpHelp(""), + /* 63 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), + /* 64 */ "PureFunc" OpHelp(""), + /* 65 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), + /* 66 */ "Return" OpHelp(""), + /* 67 */ "EndCoroutine" OpHelp(""), + /* 68 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 69 */ "Halt" OpHelp(""), + /* 70 */ "Integer" OpHelp("r[P2]=P1"), + /* 71 */ "Int64" OpHelp("r[P2]=P4"), + /* 72 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 73 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 74 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 75 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 76 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), + /* 77 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 78 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 79 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 80 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 81 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 82 */ "CollSeq" OpHelp(""), + /* 83 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 84 */ "RealAffinity" OpHelp(""), + /* 85 */ "Cast" OpHelp("affinity(r[P1])"), + /* 86 */ "Permutation" OpHelp(""), + /* 87 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 88 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 89 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 90 */ "Column" OpHelp("r[P3]=PX"), + /* 91 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 92 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 93 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 94 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 96 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 97 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 98 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 99 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 100 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 101 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 102 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), + /* 103 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 104 */ "Count" OpHelp("r[P2]=count()"), + /* 105 */ "ReadCookie" OpHelp(""), + /* 106 */ "String8" OpHelp("r[P2]='P4'"), + /* 107 */ "SetCookie" OpHelp(""), + /* 108 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), + /* 109 */ "OpenRead" OpHelp("root=P2 iDb=P3"), + /* 110 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 111 */ "OpenDup" OpHelp(""), + /* 112 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 113 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 114 */ "SorterOpen" OpHelp(""), + /* 115 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 116 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 117 */ "Close" OpHelp(""), + /* 118 */ "ColumnsUsed" OpHelp(""), + /* 119 */ "SeekHit" OpHelp("seekHit=P2"), + /* 120 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 121 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 122 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 123 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), + /* 124 */ "Delete" OpHelp(""), + /* 125 */ "ResetCount" OpHelp(""), + /* 126 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 127 */ "SorterData" OpHelp("r[P2]=data"), + /* 128 */ "RowData" OpHelp("r[P2]=data"), + /* 129 */ "Rowid" OpHelp("r[P2]=rowid"), + /* 130 */ "NullRow" OpHelp(""), + /* 131 */ "SeekEnd" OpHelp(""), + /* 132 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 133 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 134 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 135 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 136 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 137 */ "Destroy" OpHelp(""), + /* 138 */ "Clear" OpHelp(""), + /* 139 */ "ResetSorter" OpHelp(""), + /* 140 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 141 */ "Real" OpHelp("r[P2]=P4"), + /* 142 */ "SqlExec" OpHelp(""), + /* 143 */ "ParseSchema" OpHelp(""), + /* 144 */ "LoadAnalysis" OpHelp(""), + /* 145 */ "DropTable" OpHelp(""), + /* 146 */ "DropIndex" OpHelp(""), + /* 147 */ "DropTrigger" OpHelp(""), + /* 148 */ "IntegrityCk" OpHelp(""), + /* 149 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 150 */ "Param" OpHelp(""), + /* 151 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 152 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 153 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 154 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 155 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 156 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 157 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 158 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 159 */ "Expire" OpHelp(""), + /* 160 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 161 */ "VBegin" OpHelp(""), + /* 162 */ "VCreate" OpHelp(""), + /* 163 */ "VDestroy" OpHelp(""), + /* 164 */ "VOpen" OpHelp(""), + /* 165 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 166 */ "VRename" OpHelp(""), + /* 167 */ "Pagecount" OpHelp(""), + /* 168 */ "MaxPgcnt" OpHelp(""), + /* 169 */ "Trace" OpHelp(""), + /* 170 */ "CursorHint" OpHelp(""), + /* 171 */ "Noop" OpHelp(""), + /* 172 */ "Explain" OpHelp(""), + /* 173 */ "Abortable" OpHelp(""), }; return azName[i]; } @@ -32182,7 +32621,11 @@ static struct unix_syscall { #define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) #if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +# ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +# else { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +# endif #else { "ioctl", (sqlite3_syscall_ptr)0, 0 }, #endif @@ -32363,12 +32806,25 @@ static int robust_open(const char *z, int f, mode_t m){ ** unixEnterMutex() ** assert( unixMutexHeld() ); ** unixEnterLeave() +** +** To prevent deadlock, the global unixBigLock must must be acquired +** before the unixInodeInfo.pLockMutex mutex, if both are held. It is +** OK to get the pLockMutex without holding unixBigLock first, but if +** that happens, the unixBigLock mutex must not be acquired until after +** pLockMutex is released. +** +** OK: enter(unixBigLock), enter(pLockInfo) +** OK: enter(unixBigLock) +** OK: enter(pLockInfo) +** ERROR: enter(pLockInfo), enter(unixBigLock) */ static sqlite3_mutex *unixBigLock = 0; static void unixEnterMutex(void){ + assert( sqlite3_mutex_notheld(unixBigLock) ); /* Not a recursive mutex */ sqlite3_mutex_enter(unixBigLock); } static void unixLeaveMutex(void){ + assert( sqlite3_mutex_held(unixBigLock) ); sqlite3_mutex_leave(unixBigLock); } #ifdef SQLITE_DEBUG @@ -32769,16 +33225,34 @@ struct unixFileId { ** A single inode can have multiple file descriptors, so each unixFile ** structure contains a pointer to an instance of this object and this ** object keeps a count of the number of unixFile pointing to it. +** +** Mutex rules: +** +** (1) Only the pLockMutex mutex must be held in order to read or write +** any of the locking fields: +** nShared, nLock, eFileLock, bProcessLock, pUnused +** +** (2) When nRef>0, then the following fields are unchanging and can +** be read (but not written) without holding any mutex: +** fileId, pLockMutex +** +** (3) With the exceptions above, all the fields may only be read +** or written while holding the global unixBigLock mutex. +** +** Deadlock prevention: The global unixBigLock mutex may not +** be acquired while holding the pLockMutex mutex. If both unixBigLock +** and pLockMutex are needed, then unixBigLock must be acquired first. */ struct unixInodeInfo { struct unixFileId fileId; /* The lookup key */ - int nShared; /* Number of SHARED locks held */ - unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ - unsigned char bProcessLock; /* An exclusive process lock is held */ + sqlite3_mutex *pLockMutex; /* Hold this mutex for... */ + int nShared; /* Number of SHARED locks held */ + int nLock; /* Number of outstanding file locks */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ int nRef; /* Number of pointers to this structure */ unixShmNode *pShmNode; /* Shared memory associated with this inode */ - int nLock; /* Number of outstanding file locks */ - UnixUnusedFd *pUnused; /* Unused file descriptors to close */ unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ unixInodeInfo *pPrev; /* .... doubly linked */ #if SQLITE_ENABLE_LOCKING_STYLE @@ -32794,7 +33268,21 @@ struct unixInodeInfo { ** A lists of all unixInodeInfo objects. */ static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ -static unsigned int nUnusedFd = 0; /* Total unused file descriptors */ + +#ifdef SQLITE_DEBUG +/* +** True if the inode mutex is held, or not. Used only within assert() +** to help verify correct mutex usage. +*/ +int unixFileMutexHeld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_held(pFile->pInode->pLockMutex); +} +int unixFileMutexNotheld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_notheld(pFile->pInode->pLockMutex); +} +#endif /* ** @@ -32900,11 +33388,11 @@ static void closePendingFds(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; UnixUnusedFd *p; UnixUnusedFd *pNext; + assert( unixFileMutexHeld(pFile) ); for(p=pInode->pUnused; p; p=pNext){ pNext = p->pNext; robust_close(pFile, p->fd, __LINE__); sqlite3_free(p); - nUnusedFd--; } pInode->pUnused = 0; } @@ -32918,11 +33406,14 @@ static void closePendingFds(unixFile *pFile){ static void releaseInodeInfo(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); + assert( unixFileMutexNotheld(pFile) ); if( ALWAYS(pInode) ){ pInode->nRef--; if( pInode->nRef==0 ){ assert( pInode->pShmNode==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); closePendingFds(pFile); + sqlite3_mutex_leave(pInode->pLockMutex); if( pInode->pPrev ){ assert( pInode->pPrev->pNext==pInode ); pInode->pPrev->pNext = pInode->pNext; @@ -32934,10 +33425,10 @@ static void releaseInodeInfo(unixFile *pFile){ assert( pInode->pNext->pPrev==pInode ); pInode->pNext->pPrev = pInode->pPrev; } + sqlite3_mutex_free(pInode->pLockMutex); sqlite3_free(pInode); } } - assert( inodeList!=0 || nUnusedFd==0 ); } /* @@ -33007,7 +33498,6 @@ static int findInodeInfo( #else fileId.ino = (u64)statbuf.st_ino; #endif - assert( inodeList!=0 || nUnusedFd==0 ); pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ pInode = pInode->pNext; @@ -33019,6 +33509,13 @@ static int findInodeInfo( } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pInode->pLockMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pInode->pLockMutex==0 ){ + sqlite3_free(pInode); + return SQLITE_NOMEM_BKPT; + } + } pInode->nRef = 1; pInode->pNext = inodeList; pInode->pPrev = 0; @@ -33097,7 +33594,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ assert( pFile ); assert( pFile->eFileLock<=SHARED_LOCK ); - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + sqlite3_mutex_enter(pFile->pInode->pLockMutex); /* Check if a thread in this process holds such a lock */ if( pFile->pInode->eFileLock>SHARED_LOCK ){ @@ -33122,7 +33619,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ } #endif - unixLeaveMutex(); + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); *pResOut = reserved; @@ -33188,8 +33685,8 @@ static int osSetPosixAdvisoryLock( static int unixFileLock(unixFile *pFile, struct flock *pLock){ int rc; unixInodeInfo *pInode = pFile->pInode; - assert( unixMutexHeld() ); assert( pInode!=0 ); + assert( sqlite3_mutex_held(pInode->pLockMutex) ); if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; @@ -33308,8 +33805,8 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* This mutex is needed because pFile->pInode is shared across threads */ - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. @@ -33452,7 +33949,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } end_lock: - unixLeaveMutex(); + sqlite3_mutex_leave(pInode->pLockMutex); OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; @@ -33465,11 +33962,11 @@ end_lock: static void setPendingFd(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; UnixUnusedFd *p = pFile->pPreallocatedUnused; + assert( unixFileMutexHeld(pFile) ); p->pNext = pInode->pUnused; pInode->pUnused = p; pFile->h = -1; pFile->pPreallocatedUnused = 0; - nUnusedFd++; } /* @@ -33500,8 +33997,8 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); @@ -33627,14 +34124,14 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ */ pInode->nLock--; assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - closePendingFds(pFile); - } + if( pInode->nLock==0 ) closePendingFds(pFile); } end_unlock: - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } return rc; } @@ -33705,15 +34202,20 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + unixInodeInfo *pInode = pFile->pInode; + + assert( pInode!=0 ); verifyDbFile(pFile); unixUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); /* unixFile.pInode is always valid here. Otherwise, a different close ** routine (e.g. nolockClose()) would be called instead. */ assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); - if( ALWAYS(pFile->pInode) && pFile->pInode->nLock ){ + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ /* If there are outstanding locks, do not actually close the file just ** yet because that would clear those locks. Instead, add the file ** descriptor to pInode->pUnused list. It will be automatically closed @@ -33721,6 +34223,7 @@ static int unixClose(sqlite3_file *id){ */ setPendingFd(pFile); } + sqlite3_mutex_leave(pInode->pLockMutex); releaseInodeInfo(pFile); rc = closeUnixFile(id); unixLeaveMutex(); @@ -34318,6 +34821,7 @@ static int semXClose(sqlite3_file *id) { unixFile *pFile = (unixFile*)id; semXUnlock(id, NO_LOCK); assert( pFile ); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); releaseInodeInfo(pFile); unixLeaveMutex(); @@ -34432,8 +34936,7 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ *pResOut = 1; return SQLITE_OK; } - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ - + sqlite3_mutex_enter(pFile->pInode->pLockMutex); /* Check if a thread in this process holds such a lock */ if( pFile->pInode->eFileLock>SHARED_LOCK ){ reserved = 1; @@ -34457,7 +34960,7 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ } } - unixLeaveMutex(); + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); *pResOut = reserved; @@ -34520,8 +35023,8 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* This mutex is needed because pFile->pInode is shared across threads */ - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. @@ -34657,7 +35160,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ } afp_end_lock: - unixLeaveMutex(); + sqlite3_mutex_leave(pInode->pLockMutex); OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; @@ -34689,8 +35192,8 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); @@ -34759,14 +35262,14 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( rc==SQLITE_OK ){ pInode->nLock--; assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - closePendingFds(pFile); - } + if( pInode->nLock==0 ) closePendingFds(pFile); } } - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } return rc; } @@ -34778,14 +35281,20 @@ static int afpClose(sqlite3_file *id) { unixFile *pFile = (unixFile*)id; assert( id!=0 ); afpUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); + if( pFile->pInode ){ + unixInodeInfo *pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); } releaseInodeInfo(pFile); sqlite3_free(pFile->lockingContext); @@ -36091,6 +36600,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Check to see if a unixShmNode object already exists. Reuse an existing ** one if present. Create a new one if necessary. */ + assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); pInode = pDbFd->pInode; pShmNode = pInode->pShmNode; @@ -36473,6 +36983,9 @@ static void unixShmBarrier( ){ UNUSED_PARAMETER(fd); sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + assert( fd->pMethods->xLock==nolockLock + || unixFileMutexNotheld((unixFile*)fd) + ); unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } @@ -36514,6 +37027,7 @@ static int unixShmUnmap( /* If pShmNode->nRef has reached 0, then close the underlying ** shared-memory file, too */ + assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); assert( pShmNode->nRef>0 ); pShmNode->nRef--; @@ -36840,7 +37354,7 @@ IOMETHODS( IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 3, /* shared memory is disabled */ + 3, /* shared memory and mmap are enabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ @@ -37336,7 +37850,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** ** Even if a subsequent open() call does succeed, the consequences of ** not searching for a reusable file descriptor are not dire. */ - if( nUnusedFd>0 && 0==osStat(zPath, &sStat) ){ + if( inodeList!=0 && 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; pInode = inodeList; @@ -37346,12 +37860,14 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ } if( pInode ){ UnixUnusedFd **pp; + assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); + sqlite3_mutex_enter(pInode->pLockMutex); for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); pUnused = *pp; if( pUnused ){ - nUnusedFd--; *pp = pUnused->pNext; } + sqlite3_mutex_leave(pInode->pLockMutex); } } unixLeaveMutex(); @@ -42554,6 +43070,9 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ DWORD lastErrno; +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_int64 oldMmapSize; +#endif assert( pFile ); SimulateIOError(return SQLITE_IOERR_TRUNCATE); @@ -42569,6 +43088,15 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + oldMmapSize = pFile->mmapSize; + }else{ + oldMmapSize = 0; + } + winUnmapfile(pFile); +#endif + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( winSeekFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, @@ -42581,12 +43109,12 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } #if SQLITE_MAX_MMAP_SIZE>0 - /* If the file was truncated to a size smaller than the currently - ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. - */ - if( pFile->pMapRegion && nBytemmapSize ){ - pFile->mmapSize = nByte; + if( rc==SQLITE_OK && oldMmapSize>0 ){ + if( oldMmapSize>nByte ){ + winMapfile(pFile, -1); + }else{ + winMapfile(pFile, oldMmapSize); + } } #endif @@ -45774,8 +46302,8 @@ SQLITE_API int sqlite3_os_end(void){ ** This file also implements interface sqlite3_serialize() and ** sqlite3_deserialize(). */ -#ifdef SQLITE_ENABLE_DESERIALIZE /* #include "sqliteInt.h" */ +#ifdef SQLITE_ENABLE_DESERIALIZE /* ** Forward declaration of objects used by this utility @@ -49023,30 +49551,23 @@ struct RowSet { #define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ /* -** Turn bulk memory into a RowSet object. N bytes of memory -** are available at pSpace. The db pointer is used as a memory context -** for any subsequent allocations that need to occur. -** Return a pointer to the new RowSet object. -** -** It must be the case that N is sufficient to make a Rowset. If not -** an assertion fault occurs. -** -** If N is larger than the minimum, use the surplus as an initial -** allocation of entries available to be filled. +** Allocate a RowSet object. Return NULL if a memory allocation +** error occurs. */ -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ - RowSet *p; - assert( N >= ROUND8(sizeof(*p)) ); - p = pSpace; - p->pChunk = 0; - p->db = db; - p->pEntry = 0; - p->pLast = 0; - p->pForest = 0; - p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); - p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); - p->rsFlags = ROWSET_SORTED; - p->iBatch = 0; +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db){ + RowSet *p = sqlite3DbMallocRawNN(db, sizeof(*p)); + if( p ){ + int N = sqlite3DbMallocSize(db, p); + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->rsFlags = ROWSET_SORTED; + p->iBatch = 0; + } return p; } @@ -49055,7 +49576,8 @@ SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int ** the RowSet has allocated over its lifetime. This routine is ** the destructor for the RowSet. */ -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ +SQLITE_PRIVATE void sqlite3RowSetClear(void *pArg){ + RowSet *p = (RowSet*)pArg; struct RowSetChunk *pChunk, *pNextChunk; for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ pNextChunk = pChunk->pNextChunk; @@ -49069,6 +49591,16 @@ SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ p->rsFlags = ROWSET_SORTED; } +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +SQLITE_PRIVATE void sqlite3RowSetDelete(void *pArg){ + sqlite3RowSetClear(pArg); + sqlite3DbFree(((RowSet*)pArg)->db, pArg); +} + /* ** Allocate a new RowSetEntry object that is associated with the ** given RowSet. Return a pointer to the new and completely uninitialized @@ -49556,6 +50088,8 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal); #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -50549,8 +51083,12 @@ static int assert_pager_state(Pager *p){ ** to "print *pPager" in gdb: ** ** (gdb) printf "%s", print_pager_state(pPager) +** +** This routine has external linkage in order to suppress compiler warnings +** about an unused function. It is enclosed within SQLITE_DEBUG and so does +** not appear in normal builds. */ -static char *print_pager_state(Pager *p){ +char *print_pager_state(Pager *p){ static char zRet[1024]; sqlite3_snprintf(1024, zRet, @@ -51316,7 +51854,6 @@ static void pager_reset(Pager *pPager){ ** Return the pPager->iDataVersion value */ SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager *pPager){ - assert( pPager->eState>PAGER_OPEN ); return pPager->iDataVersion; } @@ -55934,9 +56471,10 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** backup in progress needs to be restarted. */ sqlite3BackupRestart(pPager->pBackup); }else{ + PgHdr *pList; if( pagerUseWal(pPager) ){ - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); PgHdr *pPageOne = 0; + pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList==0 ){ /* Must have at least one page for the WAL commit flag. ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ @@ -55957,14 +56495,14 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** should be used. No rollback journal is created if batch-atomic-write ** is enabled. */ - sqlite3_file *fd = pPager->fd; #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE - const int bBatch = zMaster==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ + sqlite3_file *fd = pPager->fd; + int bBatch = zMaster==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC) && !pPager->noSync && sqlite3JournalIsInMemory(pPager->jfd); #else -# define bBatch 0 +# define bBatch 0 #endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE @@ -56016,15 +56554,16 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( } } } -#else +#else /* SQLITE_ENABLE_ATOMIC_WRITE */ #ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE if( zMaster ){ rc = sqlite3JournalCreate(pPager->jfd); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + assert( bBatch==0 ); } #endif rc = pager_incr_changecounter(pPager, 0); -#endif +#endif /* !SQLITE_ENABLE_ATOMIC_WRITE */ if( rc!=SQLITE_OK ) goto commit_phase_one_exit; /* Write the master journal name into the journal file. If a master @@ -56048,24 +56587,36 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + pList = sqlite3PcacheDirtyList(pPager->pPCache); +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE if( bBatch ){ - /* The pager is now in DBMOD state. But regardless of what happens - ** next, attempting to play the journal back into the database would - ** be unsafe. Close it now to make sure that does not happen. */ - sqlite3OsClose(pPager->jfd); rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); - if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - } - rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); - if( bBatch ){ if( rc==SQLITE_OK ){ - rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + } } - if( rc!=SQLITE_OK ){ - sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + + if( (rc&0xFF)==SQLITE_IOERR && rc!=SQLITE_IOERR_NOMEM ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + goto commit_phase_one_exit; + } + bBatch = 0; + }else{ + sqlite3OsClose(pPager->jfd); } } +#endif /* SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + if( bBatch==0 ){ + rc = pager_write_pagelist(pPager, pList); + } if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); goto commit_phase_one_exit; @@ -56817,13 +57368,6 @@ SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ -#ifdef SQLITE_DEBUG - /* The print_pager_state() routine is intended to be used by the debugger - ** only. We invoke it once here to suppress a compiler warning. */ - print_pager_state(pPager); -#endif - - /* The eMode parameter is always valid */ assert( eMode==PAGER_JOURNALMODE_DELETE || eMode==PAGER_JOURNALMODE_TRUNCATE @@ -57192,6 +57736,38 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager){ } return rc; } + +/* +** The caller currently has a read transaction open on the database. +** If this is not a WAL database, SQLITE_ERROR is returned. Otherwise, +** this function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotCheck(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3PagerSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){ + assert( pPager->pWal ); + return sqlite3WalSnapshotUnlock(pPager->pWal); +} + #endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ @@ -57473,6 +58049,18 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; # define WALTRACE(X) #endif +/* +** WAL mode depends on atomic aligned 32-bit loads and stores in a few +** places. The following macros try to make this explicit. +*/ +#if GCC_VESRION>=5004000 +# define AtomicLoad(PTR) __atomic_load_n((PTR),__ATOMIC_RELAXED) +# define AtomicStore(PTR,VAL) __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED) +#else +# define AtomicLoad(PTR) (*(PTR)) +# define AtomicStore(PTR,VAL) (*(PTR) = (VAL)) +#endif + /* ** The maximum (and only) versions of the wal and wal-index formats ** that may be interpreted by this version of SQLite. @@ -58095,48 +58683,51 @@ static int walNextHash(int iPriorHash){ return (iPriorHash+1)&(HASHTABLE_NSLOT-1); } +/* +** An instance of the WalHashLoc object is used to describe the location +** of a page hash table in the wal-index. This becomes the return value +** from walHashGet(). +*/ +typedef struct WalHashLoc WalHashLoc; +struct WalHashLoc { + volatile ht_slot *aHash; /* Start of the wal-index hash table */ + volatile u32 *aPgno; /* aPgno[1] is the page of first frame indexed */ + u32 iZero; /* One less than the frame number of first indexed*/ +}; + /* ** Return pointers to the hash table and page number array stored on ** page iHash of the wal-index. The wal-index is broken into 32KB pages ** numbered starting from 0. ** -** Set output variable *paHash to point to the start of the hash table -** in the wal-index file. Set *piZero to one less than the frame +** Set output variable pLoc->aHash to point to the start of the hash table +** in the wal-index file. Set pLoc->iZero to one less than the frame ** number of the first frame indexed by this hash table. If a ** slot in the hash table is set to N, it refers to frame number -** (*piZero+N) in the log. +** (pLoc->iZero+N) in the log. ** -** Finally, set *paPgno so that *paPgno[1] is the page number of the -** first frame indexed by the hash table, frame (*piZero+1). +** Finally, set pLoc->aPgno so that pLoc->aPgno[1] is the page number of the +** first frame indexed by the hash table, frame (pLoc->iZero+1). */ static int walHashGet( Wal *pWal, /* WAL handle */ int iHash, /* Find the iHash'th table */ - volatile ht_slot **paHash, /* OUT: Pointer to hash index */ - volatile u32 **paPgno, /* OUT: Pointer to page number array */ - u32 *piZero /* OUT: Frame associated with *paPgno[0] */ + WalHashLoc *pLoc /* OUT: Hash table location */ ){ int rc; /* Return code */ - volatile u32 *aPgno; - rc = walIndexPage(pWal, iHash, &aPgno); + rc = walIndexPage(pWal, iHash, &pLoc->aPgno); assert( rc==SQLITE_OK || iHash>0 ); if( rc==SQLITE_OK ){ - u32 iZero; - volatile ht_slot *aHash; - - aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; + pLoc->aHash = (volatile ht_slot *)&pLoc->aPgno[HASHTABLE_NPAGE]; if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; - iZero = 0; + pLoc->aPgno = &pLoc->aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + pLoc->iZero = 0; }else{ - iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + pLoc->iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; } - - *paPgno = &aPgno[-1]; - *paHash = aHash; - *piZero = iZero; + pLoc->aPgno = &pLoc->aPgno[-1]; } return rc; } @@ -58182,9 +58773,7 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ ** actually needed. */ static void walCleanupHash(Wal *pWal){ - volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ - volatile u32 *aPgno = 0; /* Page number array for hash table */ - u32 iZero = 0; /* frame == (aHash[x]+iZero) */ + WalHashLoc sLoc; /* Hash table location */ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ @@ -58202,24 +58791,24 @@ static void walCleanupHash(Wal *pWal){ */ assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame. */ - iLimit = pWal->hdr.mxFrame - iZero; + iLimit = pWal->hdr.mxFrame - sLoc.iZero; assert( iLimit>0 ); for(i=0; iiLimit ){ - aHash[i] = 0; + if( sLoc.aHash[i]>iLimit ){ + sLoc.aHash[i] = 0; } } /* Zero the entries in the aPgno array that correspond to frames with ** frame numbers greater than pWal->hdr.mxFrame. */ - nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); - memset((void *)&aPgno[iLimit+1], 0, nByte); + nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit+1]); + memset((void *)&sLoc.aPgno[iLimit+1], 0, nByte); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the every entry in the mapping region is still reachable @@ -58229,10 +58818,10 @@ static void walCleanupHash(Wal *pWal){ int j; /* Loop counter */ int iKey; /* Hash key */ for(j=1; j<=iLimit; j++){ - for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==j ) break; + for(iKey=walHash(sLoc.aPgno[j]);sLoc.aHash[iKey];iKey=walNextHash(iKey)){ + if( sLoc.aHash[iKey]==j ) break; } - assert( aHash[iKey]==j ); + assert( sLoc.aHash[iKey]==j ); } } #endif /* SQLITE_ENABLE_EXPENSIVE_ASSERT */ @@ -58245,11 +58834,9 @@ static void walCleanupHash(Wal *pWal){ */ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int rc; /* Return code */ - u32 iZero = 0; /* One less than frame number of aPgno[1] */ - volatile u32 *aPgno = 0; /* Page number array */ - volatile ht_slot *aHash = 0; /* Hash table */ + WalHashLoc sLoc; /* Wal-index hash table location */ - rc = walHashGet(pWal, walFramePage(iFrame), &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(iFrame), &sLoc); /* Assuming the wal-index file was successfully mapped, populate the ** page number array and hash table entry. @@ -58259,15 +58846,16 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ int idx; /* Value to write to hash-table slot */ int nCollide; /* Number of hash collisions */ - idx = iFrame - iZero; + idx = iFrame - sLoc.iZero; assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the ** entire hash table and aPgno[] array before proceeding. */ if( idx==1 ){ - int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); - memset((void*)&aPgno[1], 0, nByte); + int nByte = (int)((u8 *)&sLoc.aHash[HASHTABLE_NSLOT] + - (u8 *)&sLoc.aPgno[1]); + memset((void*)&sLoc.aPgno[1], 0, nByte); } /* If the entry in aPgno[] is already set, then the previous writer @@ -58276,18 +58864,18 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ ** Remove the remnants of that writers uncommitted transaction from ** the hash-table before writing any new entries. */ - if( aPgno[idx] ){ + if( sLoc.aPgno[idx] ){ walCleanupHash(pWal); - assert( !aPgno[idx] ); + assert( !sLoc.aPgno[idx] ); } /* Write the aPgno[] array entry and the hash-table slot. */ nCollide = idx; - for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - aPgno[idx] = iPage; - aHash[iKey] = (ht_slot)idx; + sLoc.aPgno[idx] = iPage; + sLoc.aHash[iKey] = (ht_slot)idx; #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the number of entries in the hash table exactly equals @@ -58296,7 +58884,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ { int i; /* Loop counter */ int nEntry = 0; /* Number of entries in the hash table */ - for(i=0; iaSegment[p->nSegment])[iZero]; - iZero++; + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero]; + sLoc.iZero++; for(j=0; jaSegment[i].iZero = iZero; + walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry); + p->aSegment[i].iZero = sLoc.iZero; p->aSegment[i].nEntry = nEntry; p->aSegment[i].aIndex = aIndex; - p->aSegment[i].aPgno = (u32 *)aPgno; + p->aSegment[i].aPgno = (u32 *)sLoc.aPgno; } } sqlite3_free(aTmp); @@ -59050,7 +59638,6 @@ static int walCheckpoint( if( pIter && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0),1))==SQLITE_OK ){ - i64 nSize; /* Current size of database file */ u32 nBackfill = pInfo->nBackfill; pInfo->nBackfillAttempted = mxSafeFrame; @@ -59063,6 +59650,7 @@ static int walCheckpoint( */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); @@ -59770,7 +60358,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #endif for(i=1; iaReadMark[i]; + u32 thisMark = AtomicLoad(pInfo->aReadMark+i); if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; @@ -59783,7 +60371,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ for(i=1; iaReadMark[i] = mxFrame; + mxReadMark = AtomicStore(pInfo->aReadMark+i,mxFrame); mxI = i; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); break; @@ -59835,9 +60423,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** we can guarantee that the checkpointer that set nBackfill could not ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ - pWal->minFrame = pInfo->nBackfill+1; + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark + if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ walUnlockShared(pWal, WAL_READ_LOCK(mxI)); @@ -59888,16 +60476,14 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ }else{ u32 i = pInfo->nBackfillAttempted; for(i=pInfo->nBackfillAttempted; i>pInfo->nBackfill; i--){ - volatile ht_slot *dummy; - volatile u32 *aPgno; /* Array of page numbers */ - u32 iZero; /* Frame corresponding to aPgno[0] */ + WalHashLoc sLoc; /* Hash table location */ u32 pgno; /* Page number in db file */ i64 iDbOff; /* Offset of db file entry */ i64 iWalOff; /* Offset of wal file entry */ - rc = walHashGet(pWal, walFramePage(i), &dummy, &aPgno, &iZero); + rc = walHashGet(pWal, walFramePage(i), &sLoc); if( rc!=SQLITE_OK ) break; - pgno = aPgno[i-iZero]; + pgno = sLoc.aPgno[i-sLoc.iZero]; iDbOff = (i64)(pgno-1) * szPage; if( iDbOff+szPage<=szDb ){ @@ -59938,7 +60524,7 @@ SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ ** ** If the database contents have changes since the previous read ** transaction, then *pChanged is set to 1 before returning. The -** Pager layer will use this to know that is cache is stale and +** Pager layer will use this to know that its cache is stale and ** needs to be flushed. */ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ @@ -60000,7 +60586,7 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ /* Check that the wal file has not been wrapped. Assuming that it has ** not, also check that no checkpointer has attempted to checkpoint any ** frames beyond pSnapshot->mxFrame. If either of these conditions are - ** true, return SQLITE_BUSY_SNAPSHOT. Otherwise, overwrite pWal->hdr + ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr ** with *pSnapshot and set *pChanged as appropriate for opening the ** snapshot. */ if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) @@ -60010,11 +60596,12 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); *pChanged = bChanged; }else{ - rc = SQLITE_BUSY_SNAPSHOT; + rc = SQLITE_ERROR_SNAPSHOT; } /* Release the shared CKPT lock obtained above. */ walUnlockShared(pWal, WAL_CKPT_LOCK); + pWal->minFrame = 1; } @@ -60098,21 +60685,20 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( */ iMinHash = walFramePage(pWal->minFrame); for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ - volatile ht_slot *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ - u32 iZero; /* Frame number corresponding to aPgno[0] */ + WalHashLoc sLoc; /* Hash table location */ int iKey; /* Hash slot index */ int nCollide; /* Number of hash collisions remaining */ int rc; /* Error code */ - rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, iHash, &sLoc); if( rc!=SQLITE_OK ){ return rc; } nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + for(iKey=walHash(pgno); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ + u32 iFrame = sLoc.aHash[iKey] + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame + && sLoc.aPgno[sLoc.aHash[iKey]]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } @@ -60987,6 +61573,43 @@ SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1; return 0; } + +/* +** The caller currently has a read transaction open on the database. +** This function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ + int rc; + rc = walLockShared(pWal, WAL_CKPT_LOCK); + if( rc==SQLITE_OK ){ + WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; + if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + || pNew->mxFramenBackfillAttempted + ){ + rc = SQLITE_ERROR_SNAPSHOT; + walUnlockShared(pWal, WAL_CKPT_LOCK); + } + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3WalSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal){ + assert( pWal ); + walUnlockShared(pWal, WAL_CKPT_LOCK); +} + + #endif /* SQLITE_ENABLE_SNAPSHOT */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -65332,7 +65955,7 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** when A already has a read lock, we encourage A to give up and let B ** proceed. */ -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ BtShared *pBt = p->pBt; int rc = SQLITE_OK; @@ -65348,6 +65971,12 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + if( (p->db->flags & SQLITE_ResetDatabase) + && sqlite3PagerIsreadonly(pBt->pPager)==0 + ){ + pBt->btsFlags &= ~BTS_READ_ONLY; + } + /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ rc = SQLITE_READONLY; @@ -65407,6 +66036,11 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); + }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ + /* if there was no transaction opened when this function was + ** called and SQLITE_BUSY_SNAPSHOT is returned, change the error + ** code to SQLITE_BUSY. */ + rc = SQLITE_BUSY; } } } @@ -65458,14 +66092,18 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } } - trans_begun: - if( rc==SQLITE_OK && wrflag ){ - /* This call makes sure that the pager has the correct number of - ** open savepoints. If the second parameter is greater than 0 and - ** the sub-journal is not already open, then it will be opened here. - */ - rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + if( rc==SQLITE_OK ){ + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + } } btreeIntegrity(p); @@ -67212,6 +67850,23 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ return rc; } +/* +** This function is a no-op if cursor pCur does not point to a valid row. +** Otherwise, if pCur is valid, configure it so that the next call to +** sqlite3BtreeNext() is a no-op. +*/ +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3BtreeSkipNext(BtCursor *pCur){ + /* We believe that the cursor must always be in the valid state when + ** this routine is called, but the proof is difficult, so we add an + ** ALWaYS() test just in case we are wrong. */ + if( ALWAYS(pCur->eState==CURSOR_VALID) ){ + pCur->eState = CURSOR_SKIPNEXT; + pCur->skipNext = 1; + } +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. @@ -67616,7 +68271,16 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ pPage = pCur->pPage; idx = ++pCur->ix; - assert( pPage->isInit ); + if( !pPage->isInit ){ + /* The only known way for this to happen is for there to be a + ** recursive SQL function that does a DELETE operation as part of a + ** SELECT which deletes content out from under an active cursor + ** in a corrupt database file where the table being DELETE-ed from + ** has pages in common with the table being queried. See TH3 + ** module cov1/btree78.test testcase 220 (2018-06-08) for an + ** example. */ + return SQLITE_CORRUPT_BKPT; + } /* If the database file is corrupt, it is possible for the value of idx ** to be invalid here. This can only occur if a second cursor modifies @@ -71328,8 +71992,7 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Also check that the page number is in bounds. */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ - if( iPage==0 ) return 1; - if( iPage>pCheck->nPage ){ + if( iPage>pCheck->nPage || iPage==0 ){ checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } @@ -71384,17 +72047,12 @@ static void checkList( ){ int i; int expected = N; - int iFirst = iPage; - while( N-- > 0 && pCheck->mxErr ){ + int nErrAtStart = pCheck->nErr; + while( iPage!=0 && pCheck->mxErr ){ DbPage *pOvflPage; unsigned char *pOvflData; - if( iPage<1 ){ - checkAppendMsg(pCheck, - "%d of %d pages missing from overflow list starting at %d", - N+1, expected, iFirst); - break; - } if( checkRef(pCheck, iPage) ) break; + N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ checkAppendMsg(pCheck, "failed to get page %d", iPage); break; @@ -71438,10 +72096,12 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); - - if( isFreeList && N<(iPage!=0) ){ - checkAppendMsg(pCheck, "free-page count in header is too small"); - } + } + if( N && nErrAtStart==pCheck->nErr ){ + checkAppendMsg(pCheck, + "%s is %d but should be %d", + isFreeList ? "size" : "overflow list length", + expected-N, expected); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -71835,6 +72495,24 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( /* Check all the tables. */ +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pBt->autoVacuum ){ + int mx = 0; + int mxInHdr; + for(i=0; (int)ipPage1->aData[52]); + if( mx!=mxInHdr ){ + checkAppendMsg(&sCheck, + "max rootpage (%d) disagrees with header (%d)", + mx, mxInHdr + ); + } + }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){ + checkAppendMsg(&sCheck, + "incremental_vacuum enabled with a max rootpage of zero" + ); + } +#endif testcase( pBt->db->flags & SQLITE_CellSizeCk ); pBt->db->flags &= ~SQLITE_CellSizeCk; for(i=0; (int)ibtsFlags &= ~BTS_NO_WAL; if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; - rc = sqlite3BtreeBeginTrans(pBtree, 0); + rc = sqlite3BtreeBeginTrans(pBtree, 0, 0); if( rc==SQLITE_OK ){ u8 *aData = pBt->pPage1->aData; if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ - rc = sqlite3BtreeBeginTrans(pBtree, 2); + rc = sqlite3BtreeBeginTrans(pBtree, 2, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc==SQLITE_OK ){ @@ -72560,7 +73238,7 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ ** before this function exits. */ if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ - rc = sqlite3BtreeBeginTrans(p->pSrc, 0); + rc = sqlite3BtreeBeginTrans(p->pSrc, 0, 0); bCloseTrans = 1; } @@ -72576,10 +73254,10 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ /* Lock the destination database, if it is not locked already. */ if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2, + (int*)&p->iDestSchema)) ){ p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); } /* Do not allow backup if the destination database is in WAL mode @@ -73023,8 +73701,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ if( p->flags & MEM_Null ){ /* Cannot be both MEM_Null and some other type */ - assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob - |MEM_RowSet|MEM_Frame|MEM_Agg))==0 ); + assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob|MEM_Agg))==0 ); /* If MEM_Null is set, then either the value is a pure NULL (the usual ** case) or it is a pointer set using sqlite3_bind_pointer() or @@ -73137,7 +73814,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ #ifndef SQLITE_OMIT_UTF16 int rc; #endif - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ @@ -73170,7 +73847,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ */ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already @@ -73258,7 +73935,7 @@ static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ */ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){ if( ExpandBlob(pMem) ) return SQLITE_NOMEM; if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){ @@ -73283,7 +73960,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ int nByte; assert( pMem->flags & MEM_Zero ); assert( pMem->flags&MEM_Blob ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); /* Set nByte to the number of bytes required to store the expanded blob. */ @@ -73338,7 +74015,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ assert( !(fg&MEM_Zero) ); assert( !(fg&(MEM_Str|MEM_Blob)) ); assert( fg&(MEM_Int|MEM_Real) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -73396,6 +74073,35 @@ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ return ctx.isError; } +/* +** Memory cell pAccum contains the context of an aggregate function. +** This routine calls the xValue method for that function and stores +** the results in memory cell pMem. +** +** SQLITE_ERROR is returned if xValue() reports an error. SQLITE_OK +** otherwise. +*/ +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem *pAccum, Mem *pOut, FuncDef *pFunc){ + sqlite3_context ctx; + Mem t; + assert( pFunc!=0 ); + assert( pFunc->xValue!=0 ); + assert( (pAccum->flags & MEM_Null)!=0 || pFunc==pAccum->u.pDef ); + assert( pAccum->db==0 || sqlite3_mutex_held(pAccum->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pAccum->db; + sqlite3VdbeMemSetNull(pOut); + ctx.pOut = pOut; + ctx.pMem = pAccum; + ctx.pFunc = pFunc; + pFunc->xValue(&ctx); + return ctx.isError; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** If the memory cell contains a value that must be freed by ** invoking the external callback in Mem.xDel, then this routine @@ -73414,15 +74120,8 @@ static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ testcase( p->flags & MEM_Dyn ); } if( p->flags&MEM_Dyn ){ - assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); - }else if( p->flags&MEM_RowSet ){ - sqlite3RowSetClear(p->u.pRowSet); - }else if( p->flags&MEM_Frame ){ - VdbeFrame *pFrame = p->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; } p->flags = MEM_Null; } @@ -73570,7 +74269,7 @@ SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ i64 ix; assert( pMem->flags & MEM_Real ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -73597,7 +74296,7 @@ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ */ SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); pMem->u.i = sqlite3VdbeIntValue(pMem); @@ -73815,26 +74514,36 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ } #endif +#ifdef SQLITE_DEBUG +/* +** Return true if the Mem holds a RowSet object. This routine is intended +** for use inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem *pMem){ + return (pMem->flags&(MEM_Blob|MEM_Dyn))==(MEM_Blob|MEM_Dyn) + && pMem->xDel==sqlite3RowSetDelete; +} +#endif + /* ** Delete any previous value and set the value of pMem to be an ** empty boolean index. +** +** Return SQLITE_OK on success and SQLITE_NOMEM if a memory allocation +** error occurs. */ -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem *pMem){ sqlite3 *db = pMem->db; + RowSet *p; assert( db!=0 ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); - if( db->mallocFailed ){ - pMem->flags = MEM_Null; - pMem->szMalloc = 0; - }else{ - assert( pMem->zMalloc ); - pMem->szMalloc = sqlite3DbMallocSize(db, pMem->zMalloc); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, pMem->szMalloc); - assert( pMem->u.pRowSet!=0 ); - pMem->flags = MEM_RowSet; - } + p = sqlite3RowSetInit(db); + if( p==0 ) return SQLITE_NOMEM; + pMem->z = (char*)p; + pMem->flags = MEM_Blob|MEM_Dyn; + pMem->xDel = sqlite3RowSetDelete; + return SQLITE_OK; } /* @@ -73867,7 +74576,21 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ Mem *pX; for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ - pX->flags |= MEM_Undefined; + /* If pX is marked as a shallow copy of pMem, then verify that + ** no significant changes have been made to pX since the OP_SCopy. + ** A significant change would indicated a missed call to this + ** function for pX. Minor changes, such as adding or removing a + ** dual type, are allowed, as long as the underlying value is the + ** same. */ + u16 mFlags = pMem->flags & pX->flags & pX->mScopyFlags; + assert( (mFlags&MEM_Int)==0 || pMem->u.i==pX->u.i ); + assert( (mFlags&MEM_Real)==0 || pMem->u.r==pX->u.r ); + assert( (mFlags&MEM_Str)==0 || (pMem->n==pX->n && pMem->z==pX->z) ); + assert( (mFlags&MEM_Blob)==0 || sqlite3BlobCompare(pMem,pX)==0 ); + + /* pMem is the register that is changing. But also mark pX as + ** undefined so that we can quickly detect the shallow-copy error */ + pX->flags = MEM_Undefined; pX->pScopyFrom = 0; } } @@ -73888,7 +74611,7 @@ static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); } SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ - assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); assert( pTo->db==pFrom->db ); if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } memcpy(pTo, pFrom, MEMCELLSIZE); @@ -73906,7 +74629,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int sr SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; - assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; @@ -73964,7 +74687,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( u16 flags = 0; /* New value for pMem->flags */ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ if( !z ){ @@ -74086,7 +74809,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); zData = (char *)sqlite3BtreePayloadFetch(pCur, &available); assert( zData!=0 ); @@ -74110,7 +74833,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert( pVal!=0 ); assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); assert( (pVal->flags & (MEM_Null))==0 ); if( pVal->flags & (MEM_Blob|MEM_Str) ){ if( ExpandBlob(pVal) ) return 0; @@ -74153,7 +74876,7 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ if( !pVal ) return 0; assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ assert( sqlite3VdbeMemConsistentDualRep(pVal) ); return pVal->z; @@ -74720,11 +75443,11 @@ SQLITE_PRIVATE int sqlite3Stat4Column( int iCol, /* Column to extract */ sqlite3_value **ppVal /* OUT: Extracted value */ ){ - u32 t; /* a column type code */ + u32 t = 0; /* a column type code */ int nHdr; /* Size of the header in the record */ int iHdr; /* Next unread header byte */ int iField; /* Next unread data byte */ - int szField; /* Size of the current data field */ + int szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ @@ -75017,14 +75740,6 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ #endif #ifdef SQLITE_DEBUG if( p->db->flags & SQLITE_VdbeAddopTrace ){ - int jj, kk; - Parse *pParse = p->pParse; - for(jj=kk=0; jjnColCache; jj++){ - struct yColCache *x = pParse->aColCache + jj; - printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn); - kk++; - } - if( kk ) printf("\n"); sqlite3VdbePrintOp(0, i, &p->aOp[i]); test_addop_breakpoint(); } @@ -75148,7 +75863,7 @@ SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse *pParse){ SQLITE_PRIVATE void sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ if( pParse->explain==2 ){ char *zMsg; - Vdbe *v = pParse->pVdbe; + Vdbe *v; va_list ap; int iThis; va_start(ap, zFmt); @@ -75269,19 +75984,6 @@ SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ } } -#ifdef SQLITE_COVERAGE_TEST -/* -** Return TRUE if and only if the label x has already been resolved. -** Return FALSE (zero) if label x is still unresolved. -** -** This routine is only used inside of testcase() macros, and so it -** only exists when measuring test coverage. -*/ -SQLITE_PRIVATE int sqlite3VdbeLabelHasBeenResolved(Vdbe *v, int x){ - return v->pParse->aLabel && v->pParse->aLabel[ADDR(x)]>=0; -} -#endif /* SQLITE_COVERAGE_TEST */ - /* ** Mark the VDBE as one that can only be run one time. */ @@ -75513,7 +76215,6 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ break; } case OP_Next: - case OP_NextIfOpen: case OP_SorterNext: { pOp->p4.xAdvance = sqlite3BtreeNext; pOp->p4type = P4_ADVANCE; @@ -75523,8 +76224,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( pOp->p2>=0 ); break; } - case OP_Prev: - case OP_PrevIfOpen: { + case OP_Prev: { pOp->p4.xAdvance = sqlite3BtreePrevious; pOp->p4type = P4_ADVANCE; /* The code generator never codes any of these opcodes as a jump @@ -76439,7 +77139,7 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ /* ** Print a single opcode. This routine is used for debugging only. */ -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ char *zP4; char zPtr[50]; char zCom[100]; @@ -76508,9 +77208,8 @@ static void releaseMemArray(Mem *p, int N){ */ testcase( p->flags & MEM_Agg ); testcase( p->flags & MEM_Dyn ); - testcase( p->flags & MEM_Frame ); - testcase( p->flags & MEM_RowSet ); - if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ + testcase( p->xDel==sqlite3VdbeFrameMemDel ); + if( p->flags&(MEM_Agg|MEM_Dyn) ){ sqlite3VdbeMemRelease(p); }else if( p->szMalloc ){ sqlite3DbFreeNN(db, p->zMalloc); @@ -76522,6 +77221,35 @@ static void releaseMemArray(Mem *p, int N){ } } +#ifdef SQLITE_DEBUG +/* +** Verify that pFrame is a valid VdbeFrame pointer. Return true if it is +** and false if something is wrong. +** +** This routine is intended for use inside of assert() statements only. +*/ +SQLITE_PRIVATE int sqlite3VdbeFrameIsValid(VdbeFrame *pFrame){ + if( pFrame->iFrameMagic!=SQLITE_FRAME_MAGIC ) return 0; + return 1; +} +#endif + + +/* +** This is a destructor on a Mem object (which is really an sqlite3_value) +** that deletes the Frame object that is attached to it as a blob. +** +** This routine does not delete the Frame right away. It merely adds the +** frame to a list of frames to be deleted when the Vdbe halts. +*/ +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void *pArg){ + VdbeFrame *pFrame = (VdbeFrame*)pArg; + assert( sqlite3VdbeFrameIsValid(pFrame) ); + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; +} + + /* ** Delete a VdbeFrame object and its contents. VdbeFrame objects are ** allocated by the OP_Program opcode in sqlite3VdbeExec(). @@ -76530,6 +77258,7 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ int i; Mem *aMem = VdbeFrameMem(p); VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; + assert( sqlite3VdbeFrameIsValid(p) ); for(i=0; inChildCsr; i++){ sqlite3VdbeFreeCursor(p->v, apCsr[i]); } @@ -77826,7 +78555,7 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ */ sqlite3VdbeHalt(p); - /* If the VDBE has be run even partially, then transfer the error code + /* If the VDBE has been run even partially, then transfer the error code ** and error message from the VDBE into the main database structure. But ** if the VDBE has just been set to run but has not actually executed any ** instructions yet, leave the main database error information unchanged. @@ -78738,7 +79467,7 @@ static int isAllZero(const char *z, int n){ ** is less than, equal to, or greater than the second, respectively. ** If one blob is a prefix of the other, then the shorter is the lessor. */ -static SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ int c; int n1 = pB1->n; int n2 = pB2->n; @@ -78808,7 +79537,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C f1 = pMem1->flags; f2 = pMem2->flags; combined_flags = f1|f2; - assert( (combined_flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem1) && !sqlite3VdbeMemIsRowSet(pMem2) ); /* If one value is NULL, it is less than the other. If both values ** are NULL, return 0. @@ -78953,7 +79682,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( u32 idx1; /* Offset of first type in header */ int rc = 0; /* Return value */ Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ - KeyInfo *pKeyInfo = pPKey2->pKeyInfo; + KeyInfo *pKeyInfo; const unsigned char *aKey1 = (const unsigned char *)pKey1; Mem mem1; @@ -79048,7 +79777,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( (d1+mem1.n) > (unsigned)nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ - }else if( pKeyInfo->aColl[i] ){ + }else if( (pKeyInfo = pPKey2->pKeyInfo)->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; @@ -79099,7 +79828,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( } if( rc!=0 ){ - if( pKeyInfo->aSortOrder[i] ){ + if( pPKey2->pKeyInfo->aSortOrder[i] ){ rc = -rc; } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); @@ -79108,10 +79837,11 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( } i++; + if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && inField && d1<=(unsigned)nKey1 ); + }while( idx1<(unsigned)szHdr1 && d1<=(unsigned)nKey1 ); /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -79123,7 +79853,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( ** value. */ assert( CORRUPT_DB || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) - || pKeyInfo->db->mallocFailed + || pPKey2->pKeyInfo->db->mallocFailed ); pPKey2->eqSeen = 1; return pPKey2->default_rc; @@ -79449,7 +80179,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); + *res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, pUnpacked, 0); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } @@ -79481,11 +80211,19 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ ** programs obsolete. Removing user-defined functions or collating ** sequences, or changing an authorization function are the types of ** things that make prepared statements obsolete. +** +** If iCode is 1, then expiration is advisory. The statement should +** be reprepared before being restarted, but if it is already running +** it is allowed to run to completion. +** +** Internally, this function just sets the Vdbe.expired flag on all +** prepared statements. The flag is set to 1 for an immediate expiration +** and set to 2 for an advisory expiration. */ -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ Vdbe *p; for(p = db->pVdbe; p; p=p->pNext){ - p->expired = 1; + p->expired = iCode+1; } } @@ -80645,7 +81383,7 @@ static const Mem *columnNullValue(void){ /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG /* .pScopyFrom = */ (Mem*)0, - /* .pFiller = */ (void*)0, + /* .mScopyFlags= */ 0, #endif }; return &nullMem; @@ -81955,32 +82693,56 @@ SQLITE_API int sqlite3_found_count = 0; ** feature is used for test suite validation only and does not appear an ** production builds. ** -** M is an integer, 2 or 3, that indices how many different ways the -** branch can go. It is usually 2. "I" is the direction the branch -** goes. 0 means falls through. 1 means branch is taken. 2 means the -** second alternative branch is taken. +** M is an integer between 2 and 4. 2 indicates a ordinary two-way +** branch (I=0 means fall through and I=1 means taken). 3 indicates +** a 3-way branch where the third way is when one of the operands is +** NULL. 4 indicates the OP_Jump instruction which has three destinations +** depending on whether the first operand is less than, equal to, or greater +** than the second. ** ** iSrcLine is the source code line (from the __LINE__ macro) that -** generated the VDBE instruction. This instrumentation assumes that all -** source code is in a single file (the amalgamation). Special values 1 -** and 2 for the iSrcLine parameter mean that this particular branch is -** always taken or never taken, respectively. +** generated the VDBE instruction combined with flag bits. The source +** code line number is in the lower 24 bits of iSrcLine and the upper +** 8 bytes are flags. The lower three bits of the flags indicate +** values for I that should never occur. For example, if the branch is +** always taken, the flags should be 0x05 since the fall-through and +** alternate branch are never taken. If a branch is never taken then +** flags should be 0x06 since only the fall-through approach is allowed. +** +** Bit 0x04 of the flags indicates an OP_Jump opcode that is only +** interested in equal or not-equal. In other words, I==0 and I==2 +** should be treated the same. +** +** Since only a line number is retained, not the filename, this macro +** only works for amalgamation builds. But that is ok, since these macros +** should be no-ops except for special builds used to measure test coverage. */ #if !defined(SQLITE_VDBE_COVERAGE) # define VdbeBranchTaken(I,M) #else # define VdbeBranchTaken(I,M) vdbeTakeBranch(pOp->iSrcLine,I,M) - static void vdbeTakeBranch(int iSrcLine, u8 I, u8 M){ - if( iSrcLine<=2 && ALWAYS(iSrcLine>0) ){ - M = iSrcLine; - /* Assert the truth of VdbeCoverageAlwaysTaken() and - ** VdbeCoverageNeverTaken() */ - assert( (M & I)==I ); - }else{ - if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ - sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, - iSrcLine,I,M); + static void vdbeTakeBranch(u32 iSrcLine, u8 I, u8 M){ + u8 mNever; + assert( I<=2 ); /* 0: fall through, 1: taken, 2: alternate taken */ + assert( M<=4 ); /* 2: two-way branch, 3: three-way branch, 4: OP_Jump */ + assert( I> 24; + assert( (I & mNever)==0 ); + if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + I |= mNever; + if( M==2 ) I |= 0x04; + if( M==4 ){ + I |= 0x08; + if( (mNever&0x08)!=0 && (I&0x05)!=0) I |= 0x05; /*NO_TEST*/ } + sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, + iSrcLine&0xffffff, I, M); } #endif @@ -82311,7 +83073,7 @@ static void memTracePrint(Mem *p){ }else if( p->flags & MEM_Real ){ printf(" r:%g", p->u.r); #endif - }else if( p->flags & MEM_RowSet ){ + }else if( sqlite3VdbeMemIsRowSet(p) ){ printf(" (rowset)"); }else{ char zBuf[200]; @@ -83065,6 +83827,9 @@ case OP_Null: { /* out2 */ assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; pOut->n = 0; +#ifdef SQLITE_DEBUG + pOut->uTemp = 0; +#endif while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); @@ -83186,6 +83951,7 @@ case OP_Copy: { pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); while( 1 ){ + memAboutToChange(p, pOut); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); Deephemeralize(pOut); #ifdef SQLITE_DEBUG @@ -83218,7 +83984,8 @@ case OP_SCopy: { /* out2 */ assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; + pOut->pScopyFrom = pIn1; + pOut->mScopyFlags = pIn1->flags; #endif break; } @@ -83852,7 +84619,12 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ if( (flags1 | flags3)&MEM_Str ){ if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); - testcase( flags3!=pIn3->flags ); /* Possible if pIn1==pIn3 */ + assert( flags3==pIn3->flags ); + /* testcase( flags3!=pIn3->flags ); + ** this used to be possible with pIn1==pIn3, but not since + ** the column cache was removed. The following assignment + ** is essentially a no-op. But, it provides defense-in-depth + ** in case our analysis is incorrect, so it is left in. */ flags3 = pIn3->flags; } if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ @@ -84066,11 +84838,11 @@ case OP_Compare: { */ case OP_Jump: { /* jump */ if( iCompare<0 ){ - VdbeBranchTaken(0,3); pOp = &aOp[pOp->p1 - 1]; + VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ - VdbeBranchTaken(1,3); pOp = &aOp[pOp->p2 - 1]; + VdbeBranchTaken(1,4); pOp = &aOp[pOp->p2 - 1]; }else{ - VdbeBranchTaken(2,3); pOp = &aOp[pOp->p3 - 1]; + VdbeBranchTaken(2,4); pOp = &aOp[pOp->p3 - 1]; } break; } @@ -84167,7 +84939,7 @@ case OP_Not: { /* same as TK_NOT, in1, out2 */ } /* Opcode: BitNot P1 P2 * * * -** Synopsis: r[P1]= ~r[P1] +** Synopsis: r[P2]= ~r[P1] ** ** Interpret the content of register P1 as an integer. Store the ** ones-complement of the P1 value into register P2. If P1 holds @@ -84982,7 +85754,7 @@ case OP_Savepoint: { } } if( isSchemaChange ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); sqlite3ResetAllSchemasOfConnection(db); db->mDbFlags |= DBFLAG_SchemaChange; } @@ -85124,8 +85896,7 @@ case OP_AutoCommit: { */ case OP_Transaction: { Btree *pBt; - int iMeta; - int iGen; + int iMeta = 0; assert( p->bIsReader ); assert( p->readOnly==0 || pOp->p2==0 ); @@ -85138,7 +85909,7 @@ case OP_Transaction: { pBt = db->aDb[pOp->p1].pBt; if( pBt ){ - rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); + rc = sqlite3BtreeBeginTrans(pBt, pOp->p2, &iMeta); testcase( rc==SQLITE_BUSY_SNAPSHOT ); testcase( rc==SQLITE_BUSY_RECOVERY ); if( rc!=SQLITE_OK ){ @@ -85171,19 +85942,17 @@ case OP_Transaction: { p->nStmtDefCons = db->nDeferredCons; p->nStmtDefImmCons = db->nDeferredImmCons; } - - /* Gather the schema version number for checking: + } + assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); + if( pOp->p5 + && (iMeta!=pOp->p3 + || db->aDb[pOp->p1].pSchema->iGeneration!=pOp->p4.i) + ){ + /* ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema ** version is checked to ensure that the schema has not changed since the ** SQL statement was prepared. */ - sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); - iGen = db->aDb[pOp->p1].pSchema->iGeneration; - }else{ - iGen = iMeta = 0; - } - assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); - if( pOp->p5 && (iMeta!=pOp->p3 || iGen!=pOp->p4.i) ){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); /* If the schema-cookie from the database file matches the cookie @@ -85274,7 +86043,7 @@ case OP_SetCookie: { if( pOp->p1==1 ){ /* Invalidate all prepared statements whenever the TEMP database ** schema is changed. Ticket #1644 */ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); p->expired = 0; } if( rc ) goto abort_due_to_error; @@ -85292,59 +86061,78 @@ case OP_SetCookie: { ** values need not be contiguous but all P1 values should be small integers. ** It is an error for P1 to be negative. ** -** If P5!=0 then use the content of register P2 as the root page, not -** the value of P2 itself. -** -** There will be a read lock on the database whenever there is an -** open cursor. If the database was unlocked prior to this instruction -** then a read lock is acquired as part of this instruction. A read -** lock allows other processes to read the database but prohibits -** any other process from modifying the database. The read lock is -** released when all cursors are closed. If this instruction attempts -** to get a read lock but fails, the script terminates with an -** SQLITE_BUSY error code. +** Allowed P5 bits: +**
      +**
    • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxGT) +**
    ** ** The P4 value may be either an integer (P4_INT32) or a pointer to ** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table. +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. ** ** See also: OpenWrite, ReopenIdx */ /* Opcode: ReopenIdx P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** -** The ReopenIdx opcode works exactly like ReadOpen except that it first -** checks to see if the cursor on P1 is already open with a root page -** number of P2 and if it is this opcode becomes a no-op. In other words, +** The ReopenIdx opcode works like OP_OpenRead except that it first +** checks to see if the cursor on P1 is already open on the same +** b-tree and if it is this opcode becomes a no-op. In other words, ** if the cursor is already open, do not reopen it. ** -** The ReopenIdx opcode may only be used with P5==0 and with P4 being -** a P4_KEYINFO object. Furthermore, the P3 value must be the same as -** every other ReopenIdx or OpenRead for the same cursor number. +** The ReopenIdx opcode may only be used with P5==0 or P5==OPFLAG_SEEKEQ +** and with P4 being a P4_KEYINFO object. Furthermore, the P3 value must +** be the same as every other ReopenIdx or OpenRead for the same cursor +** number. ** -** See the OpenRead opcode documentation for additional information. +** Allowed P5 bits: +**
      +**
    • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxGT) +**
    +** +** See also: OP_OpenRead, OP_OpenWrite */ /* Opcode: OpenWrite P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** ** Open a read/write cursor named P1 on the table or index whose root -** page is P2. Or if P5!=0 use the content of register P2 to find the -** root page. +** page is P2 (or whose root page is held in register P2 if the +** OPFLAG_P2ISREG bit is set in P5 - see below). ** ** The P4 value may be either an integer (P4_INT32) or a pointer to ** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table, or to the -** largest index of any column of the table that is actually used. +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. ** -** This instruction works just like OpenRead except that it opens the cursor -** in read/write mode. For a given table, there can be one or more read-only -** cursors or a single read/write cursor but not both. +** Allowed P5 bits: +**
      +**
    • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxGT) +**
    • 0x08 OPFLAG_FORDELETE: This cursor is used only to seek +** and subsequently delete entries in an index btree. This is a +** hint to the storage engine that the storage engine is allowed to +** ignore. The hint is not used by the official SQLite b*tree storage +** engine, but is used by COMDB2. +**
    • 0x10 OPFLAG_P2ISREG: Use the content of register P2 +** as the root page, not the value of P2 itself. +**
    ** -** See also OpenRead. +** This instruction works like OpenRead except that it opens the cursor +** in read/write mode. +** +** See also: OP_OpenRead, OP_ReopenIdx */ case OP_ReopenIdx: { int nField; @@ -85373,7 +86161,7 @@ case OP_OpenWrite: assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx || p->readOnly==0 ); - if( p->expired ){ + if( p->expired==1 ){ rc = SQLITE_ABORT_ROLLBACK; goto abort_due_to_error; } @@ -85400,6 +86188,7 @@ case OP_OpenWrite: if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); assert( p2<=(p->nMem+1 - p->nCursor) ); + assert( pOp->opcode==OP_OpenWrite ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -85528,7 +86317,7 @@ case OP_OpenEphemeral: { rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBtx, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1); + rc = sqlite3BtreeBeginTrans(pCx->pBtx, 1, 0); } if( rc==SQLITE_OK ){ /* If a transient index is required, create it by calling @@ -85755,10 +86544,10 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3 */ -case OP_SeekLE: /* jump, in3 */ -case OP_SeekGE: /* jump, in3 */ -case OP_SeekGT: { /* jump, in3 */ +case OP_SeekLT: /* jump, in3, group */ +case OP_SeekLE: /* jump, in3, group */ +case OP_SeekGE: /* jump, in3, group */ +case OP_SeekGT: { /* jump, in3, group */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -85936,6 +86725,25 @@ seek_not_found: break; } +/* Opcode: SeekHit P1 P2 * * * +** Synopsis: seekHit=P2 +** +** Set the seekHit flag on cursor P1 to the value in P2. +** The seekHit flag is used by the IfNoHope opcode. +** +** P1 must be a valid b-tree cursor. P2 must be a boolean value, +** either 0 or 1. +*/ +case OP_SeekHit: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p2==0 || pOp->p2==1 ); + pC->seekHit = pOp->p2 & 1; + break; +} + /* Opcode: Found P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** @@ -85970,7 +86778,34 @@ seek_not_found: ** advanced in either direction. In other words, the Next and Prev ** opcodes do not work after this operation. ** -** See also: Found, NotExists, NoConflict +** See also: Found, NotExists, NoConflict, IfNoHope +*/ +/* Opcode: IfNoHope P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** Register P3 is the first of P4 registers that form an unpacked +** record. +** +** Cursor P1 is on an index btree. If the seekHit flag is set on P1, then +** this opcode is a no-op. But if the seekHit flag of P1 is clear, then +** check to see if there is any entry in P1 that matches the +** prefix identified by P3 and P4. If no entry matches the prefix, +** jump to P2. Otherwise fall through. +** +** This opcode behaves like OP_NotFound if the seekHit +** flag is clear and it behaves like OP_Noop if the seekHit flag is set. +** +** This opcode is used in IN clause processing for a multi-column key. +** If an IN clause is attached to an element of the key other than the +** left-most element, and if there are no matches on the most recent +** seek over the whole key, then it might be that one of the key element +** to the left is prohibiting a match, and hence there is "no hope" of +** any match regardless of how many IN clause elements are checked. +** In such a case, we abandon the IN clause search early, using this +** opcode. The opcode name comes from the fact that the +** jump is taken if there is "no hope" of achieving a match. +** +** See also: NotFound, SeekHit */ /* Opcode: NoConflict P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -85995,6 +86830,14 @@ seek_not_found: ** ** See also: NotFound, Found, NotExists */ +case OP_IfNoHope: { /* jump, in3 */ + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + if( pC->seekHit ) break; + /* Fall through into OP_NotFound */ +} case OP_NoConflict: /* jump, in3 */ case OP_NotFound: /* jump, in3 */ case OP_Found: { /* jump, in3 */ @@ -86132,18 +86975,26 @@ case OP_SeekRowid: { /* jump, in3 */ pIn3 = &aMem[pOp->p3]; if( (pIn3->flags & MEM_Int)==0 ){ + /* Make sure pIn3->u.i contains a valid integer representation of + ** the key value, but do not change the datatype of the register, as + ** other parts of the perpared statement might be depending on the + ** current datatype. */ + u16 origFlags = pIn3->flags; + int isNotInt; applyAffinity(pIn3, SQLITE_AFF_NUMERIC, encoding); - if( (pIn3->flags & MEM_Int)==0 ) goto jump_to_p2; + isNotInt = (pIn3->flags & MEM_Int)==0; + pIn3->flags = origFlags; + if( isNotInt ) goto jump_to_p2; } /* Fall through into OP_NotExists */ case OP_NotExists: /* jump, in3 */ pIn3 = &aMem[pOp->p3]; - assert( pIn3->flags & MEM_Int ); + assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); #ifdef SQLITE_DEBUG - pC->seekOp = 0; + pC->seekOp = OP_SeekRowid; #endif assert( pC->isTable ); assert( pC->eCurType==CURTYPE_BTREE ); @@ -86797,6 +87648,9 @@ case OP_NullRow: { assert( pC->uc.pCursor!=0 ); sqlite3BtreeClearCursor(pC->uc.pCursor); } +#ifdef SQLITE_DEBUG + if( pC->seekOp==0 ) pC->seekOp = OP_NullRow; +#endif break; } @@ -86915,7 +87769,7 @@ case OP_Sort: { /* jump */ p->aCounter[SQLITE_STMTSTATUS_SORT]++; /* Fall through into OP_Rewind */ } -/* Opcode: Rewind P1 P2 * * * +/* Opcode: Rewind P1 P2 * * P5 ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. @@ -86923,6 +87777,10 @@ case OP_Sort: { /* jump */ ** If the table or index is not empty, fall through to the following ** instruction. ** +** If P5 is non-zero and the table is not empty, then the "skip-next" +** flag is set on the cursor so that the next OP_Next instruction +** executed on it is a no-op. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. @@ -86947,6 +87805,9 @@ case OP_Rewind: { /* jump */ pCrsr = pC->uc.pCursor; assert( pCrsr ); rc = sqlite3BtreeFirst(pCrsr, &res); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p5 ) sqlite3BtreeSkipNext(pCrsr); +#endif pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -86983,12 +87844,7 @@ case OP_Rewind: { /* jump */ ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** -** See also: Prev, NextIfOpen -*/ -/* Opcode: NextIfOpen P1 P2 P3 P4 P5 -** -** This opcode works just like Next except that if cursor P1 is not -** open it behaves a no-op. +** See also: Prev */ /* Opcode: Prev P1 P2 P3 P4 P5 ** @@ -87016,11 +87872,6 @@ case OP_Rewind: { /* jump */ ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ -/* Opcode: PrevIfOpen P1 P2 P3 P4 P5 -** -** This opcode works just like Prev except that if cursor P1 is not -** open it behaves a no-op. -*/ /* Opcode: SorterNext P1 P2 * * P5 ** ** This opcode works just like OP_Next except that P1 must be a @@ -87035,10 +87886,6 @@ case OP_SorterNext: { /* jump */ assert( isSorter(pC) ); rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_PrevIfOpen: /* jump */ -case OP_NextIfOpen: /* jump */ - if( p->apCsr[pOp->p1]==0 ) break; - /* Fall through */ case OP_Prev: /* jump */ case OP_Next: /* jump */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -87049,17 +87896,17 @@ case OP_Next: /* jump */ assert( pC->eCurType==CURTYPE_BTREE ); assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); - assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); - assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); - /* The Next opcode is only used after SeekGT, SeekGE, and Rewind. + /* The Next opcode is only used after SeekGT, SeekGE, Rewind, and Found. ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ - assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen + assert( pOp->opcode!=OP_Next || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE - || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found); - assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found + || pC->seekOp==OP_NullRow); + assert( pOp->opcode!=OP_Prev || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE - || pC->seekOp==OP_Last ); + || pC->seekOp==OP_Last + || pC->seekOp==OP_NullRow); rc = pOp->p4.xAdvance(pC->uc.pCursor, pOp->p3); next_tail: @@ -87342,7 +88189,13 @@ case OP_IdxGE: { /* jump */ } r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; ip3+i, &aMem[pOp->p3+i]); + } + } #endif res = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); @@ -87529,7 +88382,8 @@ case OP_SqlExec: { /* Opcode: ParseSchema P1 * * P4 * ** ** Read and parse all entries from the SQLITE_MASTER table of database P1 -** that match the WHERE clause P4. +** that match the WHERE clause P4. If P4 is a NULL pointer, then the +** entire schema for P1 is reparsed. ** ** This opcode invokes the parser to create a new virtual machine, ** then runs the new virtual machine. It is thus a re-entrant opcode. @@ -87553,11 +88407,22 @@ case OP_ParseSchema: { iDb = pOp->p1; assert( iDb>=0 && iDbnDb ); assert( DbHasProperty(db, iDb, DB_SchemaLoaded) ); - /* Used to be a conditional */ { + +#ifndef SQLITE_OMIT_ALTERTABLE + if( pOp->p4.z==0 ){ + sqlite3SchemaClear(db->aDb[iDb].pSchema); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + rc = sqlite3InitOne(db, iDb, &p->zErrMsg, INITFLAG_AlterTable); + db->mDbFlags |= DBFLAG_SchemaChange; + p->expired = 0; + }else +#endif + { zMaster = MASTER_NAME; initData.db = db; initData.iDb = pOp->p1; initData.pzErrMsg = &p->zErrMsg; + initData.mInitFlags = 0; zSql = sqlite3MPrintf(db, "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", db->aDb[iDb].zDbSName, zMaster, pOp->p4.z); @@ -87710,11 +88575,11 @@ case OP_RowSetAdd: { /* in1, in2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; assert( (pIn2->flags & MEM_Int)!=0 ); - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; } - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); + assert( sqlite3VdbeMemIsRowSet(pIn1) ); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn2->u.i); break; } @@ -87730,8 +88595,9 @@ case OP_RowSetRead: { /* jump, in1, out3 */ i64 val; pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_RowSet)==0 - || sqlite3RowSetNext(pIn1->u.pRowSet, &val)==0 + assert( (pIn1->flags & MEM_Blob)==0 || sqlite3VdbeMemIsRowSet(pIn1) ); + if( (pIn1->flags & MEM_Blob)==0 + || sqlite3RowSetNext((RowSet*)pIn1->z, &val)==0 ){ /* The boolean index is empty */ sqlite3VdbeMemSetNull(pIn1); @@ -87780,20 +88646,19 @@ case OP_RowSetTest: { /* jump, in1, in3 */ /* If there is anything other than a rowset object in memory cell P1, ** delete it now and initialize P1 with an empty rowset */ - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; } - + assert( sqlite3VdbeMemIsRowSet(pIn1) ); assert( pOp->p4type==P4_INT32 ); assert( iSet==-1 || iSet>=0 ); if( iSet ){ - exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); + exists = sqlite3RowSetTest((RowSet*)pIn1->z, iSet, pIn3->u.i); VdbeBranchTaken(exists!=0,2); if( exists ) goto jump_to_p2; } if( iSet>=0 ){ - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn3->u.i); } break; } @@ -87857,7 +88722,7 @@ case OP_Program: { /* jump */ ** of the current program, and the memory required at runtime to execute ** the trigger program. If this trigger has been fired before, then pRt ** is already allocated. Otherwise, it must be initialized. */ - if( (pRt->flags&MEM_Frame)==0 ){ + if( (pRt->flags&MEM_Blob)==0 ){ /* SubProgram.nMem is set to the number of memory cells used by the ** program stored in SubProgram.aOp. As well as these, one memory ** cell is required for each cursor used by the program. Set local @@ -87875,8 +88740,10 @@ case OP_Program: { /* jump */ goto no_mem; } sqlite3VdbeMemRelease(pRt); - pRt->flags = MEM_Frame; - pRt->u.pFrame = pFrame; + pRt->flags = MEM_Blob|MEM_Dyn; + pRt->z = (char*)pFrame; + pRt->n = nByte; + pRt->xDel = sqlite3VdbeFrameMemDel; pFrame->v = p; pFrame->nChildMem = nMem; @@ -87892,6 +88759,9 @@ case OP_Program: { /* jump */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS pFrame->anExec = p->anExec; #endif +#ifdef SQLITE_DEBUG + pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -87899,7 +88769,8 @@ case OP_Program: { /* jump */ pMem->db = db; } }else{ - pFrame = pRt->u.pFrame; + pFrame = (VdbeFrame*)pRt->z; + assert( pRt->xDel==sqlite3VdbeFrameMemDel ); assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); assert( pProgram->nCsr==pFrame->nChildCsr ); @@ -88128,24 +88999,35 @@ case OP_DecrJumpZero: { /* jump, in1 */ } -/* Opcode: AggStep0 * P2 P3 P4 P5 +/* Opcode: AggStep * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Register P3 is the +** Execute the xStep function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the ** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. */ -/* Opcode: AggStep * P2 P3 P4 P5 +/* Opcode: AggInverse * P2 P3 P4 P5 +** Synopsis: accum=r[P3] inverse(r[P2@P5]) +** +** Execute the xInverse function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +*/ +/* Opcode: AggStep1 P1 P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to an sqlite3_context -** object that is used to run the function. Register P3 is -** as the accumulator. +** Execute the xStep (if P1==0) or xInverse (if P1!=0) function for an +** aggregate. The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. @@ -88156,7 +89038,8 @@ case OP_DecrJumpZero: { /* jump, in1 */ ** sqlite3_context only happens once, instead of on each call to the ** step function. */ -case OP_AggStep0: { +case OP_AggInverse: +case OP_AggStep: { int n; sqlite3_context *pCtx; @@ -88179,10 +89062,14 @@ case OP_AggStep0: { pCtx->argc = n; pOp->p4type = P4_FUNCCTX; pOp->p4.pCtx = pCtx; - pOp->opcode = OP_AggStep; + + /* OP_AggInverse must have P1==1 and OP_AggStep must have P1==0 */ + assert( pOp->p1==(pOp->opcode==OP_AggInverse) ); + + pOp->opcode = OP_AggStep1; /* Fall through into OP_AggStep */ } -case OP_AggStep: { +case OP_AggStep1: { int i; sqlite3_context *pCtx; Mem *pMem; @@ -88191,6 +89078,17 @@ case OP_AggStep: { pCtx = pOp->p4.pCtx; pMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + if( pOp->p1 ){ + /* This is an OP_AggInverse call. Verify that xStep has always + ** been called at least once prior to any xInverse call. */ + assert( pMem->uTemp==0x1122e0e3 ); + }else{ + /* This is an OP_AggStep call. Mark it as such. */ + pMem->uTemp = 0x1122e0e3; + } +#endif + /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it @@ -88211,7 +89109,13 @@ case OP_AggStep: { assert( pCtx->pOut->flags==MEM_Null ); assert( pCtx->isError==0 ); assert( pCtx->skipFlag==0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p1 ){ + (pCtx->pFunc->xInverse)(pCtx,pCtx->argc,pCtx->argv); + }else +#endif (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ + if( pCtx->isError ){ if( pCtx->isError>0 ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); @@ -88236,22 +89140,46 @@ case OP_AggStep: { /* Opcode: AggFinal P1 P2 * P4 * ** Synopsis: accum=r[P1] N=P2 ** -** Execute the finalizer function for an aggregate. P1 is -** the memory location that is the accumulator for the aggregate. +** P1 is the memory location that is the accumulator for an aggregate +** or window function. Execute the finalizer function +** for an aggregate and store the result in P1. ** ** P2 is the number of arguments that the step function takes and ** P4 is a pointer to the FuncDef for this function. The P2 ** argument is not used by this opcode. It is only there to disambiguate ** functions that can take varying numbers of arguments. The -** P4 argument is only needed for the degenerate case where +** P4 argument is only needed for the case where ** the step function was not previously called. */ +/* Opcode: AggValue * P2 P3 P4 * +** Synopsis: r[P3]=value N=P2 +** +** Invoke the xValue() function and store the result in register P3. +** +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the case where +** the step function was not previously called. +*/ +case OP_AggValue: case OP_AggFinal: { Mem *pMem; assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3==0 || pOp->opcode==OP_AggValue ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); - rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p3 ){ + rc = sqlite3VdbeMemAggValue(pMem, &aMem[pOp->p3], pOp->p4.pFunc); + pMem = &aMem[pOp->p3]; + }else +#endif + { + rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); + } + if( rc ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); goto abort_due_to_error; @@ -88446,7 +89374,7 @@ case OP_IncrVacuum: { /* jump */ } #endif -/* Opcode: Expire P1 * * * * +/* Opcode: Expire P1 P2 * * * ** ** Cause precompiled statements to expire. When an expired statement ** is executed using sqlite3_step() it will either automatically @@ -88455,12 +89383,19 @@ case OP_IncrVacuum: { /* jump */ ** ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, ** then only the currently executing statement is expired. +** +** If P2 is 0, then SQL statements are expired immediately. If P2 is 1, +** then running SQL statements are allowed to continue to run to completion. +** The P2==1 case occurs when a CREATE INDEX or similar schema change happens +** that might help the statement run faster but which does not affect the +** correctness of operation. */ case OP_Expire: { + assert( pOp->p2==0 || pOp->p2==1 ); if( !pOp->p1 ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, pOp->p2); }else{ - p->expired = 1; + p->expired = pOp->p2+1; } break; } @@ -88784,7 +89719,10 @@ case OP_VNext: { /* jump */ case OP_VRename: { sqlite3_vtab *pVtab; Mem *pName; - + int isLegacy; + + isLegacy = (db->flags & SQLITE_LegacyAlter); + db->flags |= SQLITE_LegacyAlter; pVtab = pOp->p4.pVtab->pVtab; pName = &aMem[pOp->p1]; assert( pVtab->pModule->xRename ); @@ -88798,6 +89736,7 @@ case OP_VRename: { rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); if( rc ) goto abort_due_to_error; rc = pVtab->pModule->xRename(pVtab, pName->z); + if( isLegacy==0 ) db->flags &= ~SQLITE_LegacyAlter; sqlite3VtabImportErrmsg(p, pVtab); p->expired = 0; if( rc ) goto abort_due_to_error; @@ -88846,6 +89785,7 @@ case OP_VUpdate: { || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace ); assert( p->readOnly==0 ); + if( db->mallocFailed ) goto no_mem; sqlite3VdbeIncrWriteCounter(p, 0); pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ @@ -88967,8 +89907,8 @@ case OP_MaxPgcnt: { /* out2 */ ** ** See also: Function0, AggStep, AggFinal */ -case OP_PureFunc0: -case OP_Function0: { +case OP_PureFunc0: /* group */ +case OP_Function0: { /* group */ int n; sqlite3_context *pCtx; @@ -88992,8 +89932,8 @@ case OP_Function0: { pOp->opcode += 2; /* Fall through into OP_Function */ } -case OP_PureFunc: -case OP_Function: { +case OP_PureFunc: /* group */ +case OP_Function: { /* group */ int i; sqlite3_context *pCtx; @@ -91918,7 +92858,11 @@ static int vdbeMergeEngineInit( ){ int rc = SQLITE_OK; /* Return code */ int i; /* For looping over PmaReader objects */ - int nTree = pMerger->nTree; + int nTree; /* Number of subtrees to merge */ + + /* Failure to allocate the merge would have been detected prior to + ** invoking this routine */ + assert( pMerger!=0 ); /* eMode is always INCRINIT_NORMAL in single-threaded mode */ assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); @@ -91927,6 +92871,7 @@ static int vdbeMergeEngineInit( assert( pMerger->pTask==0 ); pMerger->pTask = pTask; + nTree = pMerger->nTree; for(i=0; i0 && eMode==INCRINIT_ROOT ){ /* PmaReaders should be normally initialized in order, as if they are @@ -93055,6 +94000,14 @@ static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ }else if( pExpr->x.pList ){ if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( !ExprHasProperty(pExpr, EP_Reduced) && pExpr->pWin ){ + Window *pWin = pExpr->pWin; + if( sqlite3WalkExprList(pWalker, pWin->pPartition) ) return WRC_Abort; + if( sqlite3WalkExprList(pWalker, pWin->pOrderBy) ) return WRC_Abort; + if( sqlite3WalkExpr(pWalker, pWin->pFilter) ) return WRC_Abort; + } +#endif } break; } @@ -93427,6 +94380,9 @@ static int lookupName( if( sqlite3StrICmp(zTabName, zTab)!=0 ){ continue; } + if( IN_RENAME_OBJECT && pItem->zAlias ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->pTab); + } } if( 0==(cntTab++) ){ pMatch = pItem; @@ -93512,9 +94468,15 @@ static int lookupName( #ifndef SQLITE_OMIT_UPSERT if( pExpr->iTable==2 ){ testcase( iCol==(-1) ); - pExpr->iTable = pNC->uNC.pUpsert->regData + iCol; - eNewExprOp = TK_REGISTER; - ExprSetProperty(pExpr, EP_Alias); + if( IN_RENAME_OBJECT ){ + pExpr->iColumn = iCol; + pExpr->pTab = pTab; + eNewExprOp = TK_COLUMN; + }else{ + pExpr->iTable = pNC->uNC.pUpsert->regData + iCol; + eNewExprOp = TK_REGISTER; + ExprSetProperty(pExpr, EP_Alias); + } }else #endif /* SQLITE_OMIT_UPSERT */ { @@ -93599,6 +94561,9 @@ static int lookupName( cnt = 1; pMatch = 0; assert( zTab==0 && zDb==0 ); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + } goto lookupname_end; } } @@ -93826,17 +94791,24 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ zTable = 0; zColumn = pExpr->u.zToken; }else{ + Expr *pLeft = pExpr->pLeft; notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); pRight = pExpr->pRight; if( pRight->op==TK_ID ){ zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; }else{ assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; + zDb = pLeft->u.zToken; + pLeft = pRight->pLeft; + pRight = pRight->pRight; + } + zTable = pLeft->u.zToken; + zColumn = pRight->u.zToken; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, (void*)&pExpr->pTab, (void*)pLeft); } } return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); @@ -93919,40 +94891,95 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ NC_IdxExpr|NC_PartIdx); } } - if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ - sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); - pNC->nErr++; - is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 -#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION - && pParse->explain==0 + + if( 0==IN_RENAME_OBJECT ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( is_agg==0 || (pDef->funcFlags & SQLITE_FUNC_MINMAX) + || (pDef->xValue==0 && pDef->xInverse==0) + || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) + ); + if( pDef && pDef->xValue==0 && pExpr->pWin ){ + sqlite3ErrorMsg(pParse, + "%.*s() may not be used as a window function", nId, zId + ); + pNC->nErr++; + }else if( + (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) + || (is_agg && (pDef->funcFlags & SQLITE_FUNC_WINDOW) && !pExpr->pWin) + || (is_agg && pExpr->pWin && (pNC->ncFlags & NC_AllowWin)==0) + ){ + const char *zType; + if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pExpr->pWin ){ + zType = "window"; + }else{ + zType = "aggregate"; + } + sqlite3ErrorMsg(pParse, "misuse of %s function %.*s()",zType,nId,zId); + pNC->nErr++; + is_agg = 0; + } +#else + if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) ){ + sqlite3ErrorMsg(pParse,"misuse of aggregate function %.*s()",nId,zId); + pNC->nErr++; + is_agg = 0; + } #endif - ){ - sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); - pNC->nErr++; - }else if( wrong_num_args ){ - sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", - nId, zId); - pNC->nErr++; + else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); + pNC->nErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", + nId, zId); + pNC->nErr++; + } + if( is_agg ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + pNC->ncFlags &= ~(pExpr->pWin ? NC_AllowWin : NC_AllowAgg); +#else + pNC->ncFlags &= ~NC_AllowAgg; +#endif + } } - if( is_agg ) pNC->ncFlags &= ~NC_AllowAgg; sqlite3WalkExprList(pWalker, pList); if( is_agg ){ - NameContext *pNC2 = pNC; - pExpr->op = TK_AGG_FUNCTION; - pExpr->op2 = 0; - while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ - pExpr->op2++; - pNC2 = pNC2->pNext; - } - assert( pDef!=0 ); - if( pNC2 ){ - assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); - testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); - pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pExpr->pWin ){ + Select *pSel = pNC->pWinSelect; + sqlite3WalkExprList(pWalker, pExpr->pWin->pPartition); + sqlite3WalkExprList(pWalker, pExpr->pWin->pOrderBy); + sqlite3WalkExpr(pWalker, pExpr->pWin->pFilter); + sqlite3WindowUpdate(pParse, pSel->pWinDefn, pExpr->pWin, pDef); + if( 0==pSel->pWin + || 0==sqlite3WindowCompare(pParse, pSel->pWin, pExpr->pWin) + ){ + pExpr->pWin->pNextWin = pSel->pWin; + pSel->pWin = pExpr->pWin; + } + pNC->ncFlags |= NC_AllowWin; + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + NameContext *pNC2 = pNC; + pExpr->op = TK_AGG_FUNCTION; + pExpr->op2 = 0; + while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ + pExpr->op2++; + pNC2 = pNC2->pNext; + } + assert( pDef!=0 ); + if( pNC2 ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); + } + pNC->ncFlags |= NC_AllowAgg; } - pNC->ncFlags |= NC_AllowAgg; } /* FIX ME: Compute pExpr->affinity based on the expected return ** type of the function @@ -94353,6 +95380,19 @@ static int resolveOrderGroupBy( } for(j=0; jpEList->nExpr; j++){ if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pE->pWin ){ + /* Since this window function is being changed into a reference + ** to the same window function the result set, remove the instance + ** of this window function from the Select.pWin list. */ + Window **pp; + for(pp=&pSelect->pWin; *pp; pp=&(*pp)->pNextWin){ + if( *pp==pE->pWin ){ + *pp = (*pp)->pNextWin; + } + } + } +#endif pItem->u.x.iOrderByCol = j+1; } } @@ -94409,6 +95449,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; + sNC.pWinSelect = p; if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){ return WRC_Abort; } @@ -94457,12 +95498,13 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ /* Set up the local name-context to pass to sqlite3ResolveExprNames() to ** resolve the result-set expression list. */ - sNC.ncFlags = NC_AllowAgg; + sNC.ncFlags = NC_AllowAgg|NC_AllowWin; sNC.pSrcList = p->pSrc; sNC.pNext = pOuterNC; /* Resolve names in the result set. */ if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; + sNC.ncFlags &= ~NC_AllowWin; /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. @@ -94511,7 +95553,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** outer queries */ sNC.pNext = 0; - sNC.ncFlags |= NC_AllowAgg; + sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; /* If this is a converted compound query, move the ORDER BY clause from ** the sub-query back to the parent query. At this point each term @@ -94542,6 +95584,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( db->mallocFailed ){ return WRC_Abort; } + sNC.ncFlags &= ~NC_AllowWin; /* Resolve the GROUP BY clause. At the same time, make sure ** the GROUP BY clause does not contain aggregate functions. @@ -94890,14 +95933,6 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ while( p ){ int op = p->op; if( p->flags & EP_Generic ) break; - if( op==TK_CAST || op==TK_UPLUS ){ - p = p->pLeft; - continue; - } - if( op==TK_COLLATE || (op==TK_REGISTER && p->op2==TK_COLLATE) ){ - pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); - break; - } if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER || op==TK_TRIGGER) && p->pTab!=0 @@ -94911,6 +95946,14 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ } break; } + if( op==TK_CAST || op==TK_UPLUS ){ + p = p->pLeft; + continue; + } + if( op==TK_COLLATE || (op==TK_REGISTER && p->op2==TK_COLLATE) ){ + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); + break; + } if( p->flags & EP_Collate ){ if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){ p = p->pLeft; @@ -95330,7 +96373,6 @@ static void codeVectorCompare( Expr *pL, *pR; int r1, r2; assert( i>=0 && i0 ) sqlite3ExprCachePush(pParse); r1 = exprVectorRegister(pParse, pLeft, i, regLeft, &pL, ®Free1); r2 = exprVectorRegister(pParse, pRight, i, regRight, &pR, ®Free2); codeCompare(pParse, pL, pR, opx, r1, r2, dest, p5); @@ -95342,7 +96384,6 @@ static void codeVectorCompare( testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); sqlite3ReleaseTempReg(pParse, regFree1); sqlite3ReleaseTempReg(pParse, regFree2); - if( i>0 ) sqlite3ExprCachePop(pParse); if( i==nLeft-1 ){ break; } @@ -95690,7 +96731,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ ** Construct a new expression node for a function with multiple ** arguments. */ -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ +SQLITE_PRIVATE Expr *sqlite3ExprFunction( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Argument list */ + Token *pToken, /* Name of the function */ + int eDistinct /* SF_Distinct or SF_ALL or 0 */ +){ Expr *pNew; sqlite3 *db = pParse->db; assert( pToken ); @@ -95699,10 +96745,14 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ return 0; } + if( pList && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", pToken); + } pNew->x.pList = pList; ExprSetProperty(pNew, EP_HasFunc); assert( !ExprHasProperty(pNew, EP_xIsSelect) ); sqlite3ExprSetHeightAndFlags(pParse, pNew); + if( eDistinct==SF_Distinct ) ExprSetProperty(pNew, EP_Distinct); return pNew; } @@ -95812,6 +96862,9 @@ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ }else{ sqlite3ExprListDelete(db, p->x.pList); } + if( !ExprHasProperty(p, EP_Reduced) ){ + sqlite3WindowDelete(db, p->pWin); + } } if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); if( !ExprHasProperty(p, EP_Static) ){ @@ -95860,7 +96913,7 @@ static int exprStructSize(Expr *p){ ** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size ** (unreduced) Expr objects as they or originally constructed by the parser. ** During expression analysis, extra information is computed and moved into -** later parts of teh Expr object and that extra information might get chopped +** later parts of the Expr object and that extra information might get chopped ** off if the expression is reduced. Note also that it does not work to ** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal ** to reduce a pristine expression tree from the parser. The implementation @@ -95872,7 +96925,11 @@ static int dupedExprStructSize(Expr *p, int flags){ assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags || p->op==TK_SELECT_COLUMN ){ + if( 0==flags || p->op==TK_SELECT_COLUMN +#ifndef SQLITE_OMIT_WINDOWFUNC + || p->pWin +#endif + ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); @@ -96000,18 +97057,22 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ } /* Fill in pNew->pLeft and pNew->pRight. */ + zAlloc += dupedExprNodeSize(p, dupFlags); if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, dupFlags); if( !ExprHasProperty(pNew, EP_TokenOnly|EP_Leaf) ){ pNew->pLeft = p->pLeft ? exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; pNew->pRight = p->pRight ? exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; } - if( pzBuffer ){ - *pzBuffer = zAlloc; - } }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_Reduced|EP_TokenOnly) ){ + pNew->pWin = 0; + }else{ + pNew->pWin = sqlite3WindowDup(db, pNew, p->pWin); + } +#endif /* SQLITE_OMIT_WINDOWFUNC */ if( !ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ if( pNew->op==TK_SELECT_COLUMN ){ pNew->pLeft = p->pLeft; @@ -96023,6 +97084,9 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } } + if( pzBuffer ){ + *pzBuffer = zAlloc; + } } return pNew; } @@ -96218,7 +97282,11 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *pDup, int flags){ pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); - sqlite3SelectSetName(pNew, p->zSelName); +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); +#endif + pNew->selId = p->selId; *pp = pNew; pp = &pNew->pPrior; pNext = pNew; @@ -96390,6 +97458,9 @@ SQLITE_PRIVATE void sqlite3ExprListSetName( assert( pItem->zName==0 ); pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); if( dequote ) sqlite3Dequote(pItem->zName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)pItem->zName, pName); + } } } @@ -96570,6 +97641,9 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); + if( ExprHasProperty(pExpr, EP_FixedCol) && pWalker->eCode!=2 ){ + return WRC_Continue; + } if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ return WRC_Continue; } @@ -96625,10 +97699,17 @@ SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ } /* -** Walk an expression tree. Return non-zero if the expression is constant -** that does no originate from the ON or USING clauses of a join. -** Return 0 if it involves variables or function calls or terms from -** an ON or USING clause. +** Walk an expression tree. Return non-zero if +** +** (1) the expression is constant, and +** (2) the expression does originate in the ON or USING clause +** of a LEFT JOIN, and +** (3) the expression does not contain any EP_FixedCol TK_COLUMN +** operands created by the constant propagation optimization. +** +** When this routine returns true, it indicates that the expression +** can be added to the pParse->pConstExpr list and evaluated once when +** the prepared statement starts up. See sqlite3ExprCodeAtInit(). */ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ return exprIsConst(p, 2, 0); @@ -96658,7 +97739,7 @@ static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ Expr *p = pGroupBy->a[i].pExpr; if( sqlite3ExprCompare(0, pExpr, p, -1)<2 ){ CollSeq *pColl = sqlite3ExprNNCollSeq(pWalker->pParse, p); - if( sqlite3_stricmp("BINARY", pColl->zName)==0 ){ + if( sqlite3IsBinary(pColl) ){ return WRC_Prune; } } @@ -97080,7 +98161,8 @@ SQLITE_PRIVATE int sqlite3FindInIndex( sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); eType = IN_INDEX_ROWID; - + ExplainQueryPlan((pParse, 0, + "USING ROWID SEARCH ON TABLE %s FOR IN-OPERATOR",pTab->zName)); sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ @@ -97339,7 +98421,6 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( int rReg = 0; /* Register storing resulting */ Vdbe *v = sqlite3GetVdbe(pParse); if( NEVER(v==0) ) return 0; - sqlite3ExprCachePush(pParse); /* The evaluation of the IN/EXISTS/SELECT must be repeated every time it ** is encountered if any of the following is true: @@ -97475,7 +98556,6 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); }else{ sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, r3, 1); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pExpr->iTable, r2, r3, 1); } } @@ -97556,7 +98636,6 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( if( jmpIfDynamic>=0 ){ sqlite3VdbeJumpHere(v, jmpIfDynamic); } - sqlite3ExprCachePop(pParse); return rReg; } @@ -97675,7 +98754,6 @@ static void sqlite3ExprCodeIN( ** aiMap[] array contains a mapping from the original LHS field order to ** the field order that matches the RHS index. */ - sqlite3ExprCachePush(pParse); rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); for(i=0; idb, aiMap); @@ -97902,145 +98979,6 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ } } -/* -** Erase column-cache entry number i -*/ -static void cacheEntryClear(Parse *pParse, int i){ - if( pParse->aColCache[i].tempReg ){ - if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; - } - } - pParse->nColCache--; - if( inColCache ){ - pParse->aColCache[i] = pParse->aColCache[pParse->nColCache]; - } -} - - -/* -** Record in the column cache that a particular column from a -** particular table is stored in a particular register. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ - int i; - int minLru; - int idxLru; - struct yColCache *p; - - /* Unless an error has occurred, register numbers are always positive. */ - assert( iReg>0 || pParse->nErr || pParse->db->mallocFailed ); - assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ - - /* The SQLITE_ColumnCache flag disables the column cache. This is used - ** for testing only - to verify that SQLite always gets the same answer - ** with and without the column cache. - */ - if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; - - /* First replace any existing entry. - ** - ** Actually, the way the column cache is currently used, we are guaranteed - ** that the object will never already be in cache. Verify this guarantee. - */ -#ifndef NDEBUG - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - assert( p->iTable!=iTab || p->iColumn!=iCol ); - } -#endif - - /* If the cache is already full, delete the least recently used entry */ - if( pParse->nColCache>=SQLITE_N_COLCACHE ){ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - p = &pParse->aColCache[idxLru]; - }else{ - p = &pParse->aColCache[pParse->nColCache++]; - } - - /* Add the new entry to the end of the cache */ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; -} - -/* -** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. -** Purge the range of registers from the column cache. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - int i = 0; - while( inColCache ){ - struct yColCache *p = &pParse->aColCache[i]; - if( p->iReg >= iReg && p->iReg < iReg+nReg ){ - cacheEntryClear(pParse, i); - }else{ - i++; - } - } -} - -/* -** Remember the current column cache context. Any new entries added -** added to the column cache after this call are removed when the -** corresponding pop occurs. -*/ -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ - pParse->iCacheLevel++; -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("PUSH to %d\n", pParse->iCacheLevel); - } -#endif -} - -/* -** Remove from the column cache any entries that were added since the -** the previous sqlite3ExprCachePush operation. In other words, restore -** the cache to the state it was in prior the most recent Push. -*/ -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ - int i = 0; - assert( pParse->iCacheLevel>=1 ); - pParse->iCacheLevel--; -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("POP to %d\n", pParse->iCacheLevel); - } -#endif - while( inColCache ){ - if( pParse->aColCache[i].iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, i); - }else{ - i++; - } - } -} - -/* -** When a cached column is reused, make sure that its register is -** no longer available as a temp register. ticket #3879: that same -** register might be in the cache in multiple places, so be sure to -** get them all. -*/ -static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iReg==iReg ){ - p->tempReg = 0; - } - } -} /* Generate code that will load into register regOut a value that is ** appropriate for the iIdxCol-th column of index pIdx. @@ -98096,12 +99034,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. -** -** An effort is made to store the column value in register iReg. This -** is not garanteeed for GetColumn() - the result can be stored in -** any register. But the result is guaranteed to land in register iReg -** for GetColumnToReg(). +** table pTab and store the column value in register iReg. ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -98115,96 +99048,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( u8 p5 /* P5 value for OP_Column + FLAGS */ ){ Vdbe *v = pParse->pVdbe; - int i; - struct yColCache *p; - - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iTable==iTable && p->iColumn==iColumn ){ - p->lru = pParse->iCacheCnt++; - sqlite3ExprCachePinRegister(pParse, p->iReg); - return p->iReg; - } - } assert( v!=0 ); sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); if( p5 ){ sqlite3VdbeChangeP5(v, p5); - }else{ - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); } return iReg; } -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg( - Parse *pParse, /* Parsing and code generating context */ - Table *pTab, /* Description of the table we are reading from */ - int iColumn, /* Index of the table column */ - int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ -){ - int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); - if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); -} - - -/* -** Clear all column cache entries. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ - int i; - -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("CLEAR\n"); - } -#endif - for(i=0; inColCache; i++){ - if( pParse->aColCache[i].tempReg - && pParse->nTempRegaTempReg) - ){ - pParse->aTempReg[pParse->nTempReg++] = pParse->aColCache[i].iReg; - } - } - pParse->nColCache = 0; -} - -/* -** Record the fact that an affinity change has occurred on iCount -** registers starting with iStart. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ - sqlite3ExprCacheRemove(pParse, iStart, iCount); -} /* ** Generate code to move content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. +** over to iTo..iTo+nReg-1. */ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - sqlite3ExprCacheRemove(pParse, iFrom, nReg); } -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) -/* -** Return true if any register in the range iFrom..iTo (inclusive) -** is used as part of the column cache. -** -** This routine is used within assert() and testcase() macros only -** and does not appear in a normal build. -*/ -static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - int r = p->iReg; - if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ - } - return 0; -} -#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ - - /* ** Convert a scalar expression node to a TK_REGISTER referencing ** register iReg. The caller must ensure that iReg already contains @@ -98302,6 +99162,28 @@ expr_code_doover: } case TK_COLUMN: { int iTab = pExpr->iTable; + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + /* This COLUMN expression is really a constant due to WHERE clause + ** constraints, and that constant is coded by the pExpr->pLeft + ** expresssion. However, make sure the constant has the correct + ** datatype by applying the Affinity of the table column to the + ** constant. + */ + int iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); + int aff = sqlite3TableColumnAffinity(pExpr->pTab, pExpr->iColumn); + if( aff!=SQLITE_AFF_BLOB ){ + static const char zAff[] = "B\000C\000D\000E"; + assert( SQLITE_AFF_BLOB=='A' ); + assert( SQLITE_AFF_TEXT=='B' ); + if( iReg!=target ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iReg, target); + iReg = target; + } + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, + &zAff[(aff-'B')*2], P4_STATIC); + } + return iReg; + } if( iTab<0 ){ if( pParse->iSelfTab<0 ){ /* Generating CHECK constraints or inserting into partial index */ @@ -98382,8 +99264,6 @@ expr_code_doover: } sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); - testcase( usedAsColumnCache(pParse, inReg, inReg) ); - sqlite3ExprCacheAffinityChange(pParse, inReg, 1); return inReg; } #endif /* SQLITE_OMIT_CAST */ @@ -98527,6 +99407,12 @@ expr_code_doover: u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) && pExpr->pWin ){ + return pExpr->pWin->regResult; + } +#endif + if( ConstFactorOk(pParse) && sqlite3ExprIsConstantNotJoin(pExpr) ){ /* SQL functions can be expensive. So try to move constant functions ** out of the inner loop, even if that means an extra OP_Copy. */ @@ -98563,10 +99449,7 @@ expr_code_doover: for(i=1; ia[i].pExpr, target); - sqlite3ExprCachePop(pParse); } sqlite3VdbeResolveLabel(v, endCoalesce); break; @@ -98632,10 +99515,8 @@ expr_code_doover: } } - sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); - sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ }else{ r1 = 0; } @@ -98652,7 +99533,7 @@ expr_code_doover: ** "glob(B,A). We want to use the A in "A glob B" to test ** for function overloading. But we use the B term in "glob(B,A)". */ - if( nFarg>=2 && (pExpr->flags & EP_InfixFunc) ){ + if( nFarg>=2 && ExprHasProperty(pExpr, EP_InfixFunc) ){ pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); }else if( nFarg>0 ){ pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); @@ -98808,9 +99689,7 @@ expr_code_doover: case TK_IF_NULL_ROW: { int addrINR; addrINR = sqlite3VdbeAddOp1(v, OP_IfNullRow, pExpr->iTable); - sqlite3ExprCachePush(pParse); inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - sqlite3ExprCachePop(pParse); sqlite3VdbeJumpHere(v, addrINR); sqlite3VdbeChangeP3(v, addrINR, inReg); break; @@ -98847,7 +99726,6 @@ expr_code_doover: Expr opCompare; /* The X==Ei expression */ Expr *pX; /* The X expression */ Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ - VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); assert(pExpr->x.pList->nExpr > 0); @@ -98871,7 +99749,6 @@ expr_code_doover: regFree1 = 0; } for(i=0; iop==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); sqlite3VdbeGoto(v, endLabel); - sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } if( (nExpr&1)!=0 ){ - sqlite3ExprCachePush(pParse); sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); - sqlite3ExprCachePop(pParse); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( pParse->db->mallocFailed || pParse->nErr>0 - || pParse->iCacheLevel==iCacheLevel ); sqlite3VdbeResolveLabel(v, endLabel); break; } @@ -99045,7 +99917,7 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pExpr) ){ sqlite3ExprCodeAtInit(pParse, pExpr, target); }else{ sqlite3ExprCode(pParse, pExpr, target); @@ -99127,7 +99999,9 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( }else{ sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); } - }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 + && sqlite3ExprIsConstantNotJoin(pExpr) + ){ sqlite3ExprCodeAtInit(pParse, pExpr, target+i); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); @@ -99253,18 +100127,14 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse); break; } case TK_OR: { testcase( jumpIfNull==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -99423,19 +100293,15 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_AND: { testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse); break; } case TK_OR: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse); break; } case TK_NOT: { @@ -99657,7 +100523,8 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTa if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ if( combinedFlags & EP_xIsSelect ) return 2; - if( sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; + if( (combinedFlags & EP_FixedCol)==0 + && sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; assert( (combinedFlags & EP_Reduced)==0 ); @@ -99666,6 +100533,21 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Parse *pParse, Expr *pA, Expr *pB, int iTa if( pA->iTable!=pB->iTable && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; } +#ifndef SQLITE_OMIT_WINDOWFUNC + /* Justification for the assert(): + ** window functions have p->op==TK_FUNCTION but aggregate functions + ** have p->op==TK_AGG_FUNCTION. So any comparison between an aggregate + ** function and a window function should have failed before reaching + ** this point. And, it is not possible to have a window function and + ** a scalar function with the same name and number of arguments. So + ** if we reach this point, either A and B both window functions or + ** neither are a window functions. */ + assert( (pA->pWin==0)==(pB->pWin==0) ); + + if( pA->pWin!=0 ){ + if( sqlite3WindowCompare(pParse,pA->pWin,pB->pWin)!=0 ) return 2; + } +#endif } return 0; } @@ -99756,18 +100638,15 @@ SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Parse *pParse, Expr *pE1, Expr *pE2, i /* ** This is the Expr node callback for sqlite3ExprImpliesNotNullRow(). ** If the expression node requires that the table at pWalker->iCur -** have a non-NULL column, then set pWalker->eCode to 1 and abort. +** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. +** +** This routine controls an optimization. False positives (setting +** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives +** (never setting pWalker->eCode) is a harmless missed optimization. */ static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ - /* This routine is only called for WHERE clause expressions and so it - ** cannot have any TK_AGG_COLUMN entries because those are only found - ** in HAVING clauses. We can get a TK_AGG_FUNCTION in a WHERE clause, - ** but that is an illegal construct and the query will be rejected at - ** a later stage of processing, so the TK_AGG_FUNCTION case does not - ** need to be considered here. */ - assert( pExpr->op!=TK_AGG_COLUMN ); + testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); - if( ExprHasProperty(pExpr, EP_FromJoin) ) return WRC_Prune; switch( pExpr->op ){ case TK_ISNOT: @@ -100187,21 +101066,9 @@ SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ /* ** Deallocate a register, making available for reuse for some other ** purpose. -** -** If a register is currently being used by the column cache, then -** the deallocation is deferred until the column cache line that uses -** the register becomes stale. */ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ if( iReg && pParse->nTempRegaTempReg) ){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; inColCache; i++, p++){ - if( p->iReg==iReg ){ - p->tempReg = 1; - return; - } - } pParse->aTempReg[pParse->nTempReg++] = iReg; } } @@ -100215,7 +101082,6 @@ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ - assert( !usedAsColumnCache(pParse, i, i+n-1) ); pParse->iRangeReg += nReg; pParse->nRangeReg -= nReg; }else{ @@ -100229,7 +101095,6 @@ SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ sqlite3ReleaseTempReg(pParse, iReg); return; } - sqlite3ExprCacheRemove(pParse, iReg, nReg); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; pParse->iRangeReg = iReg; @@ -100291,352 +101156,6 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ */ #ifndef SQLITE_OMIT_ALTERTABLE - -/* -** This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TABLE or -** CREATE INDEX command. The second is a table name. The table name in -** the CREATE TABLE or CREATE INDEX statement is replaced with the third -** argument and the result returned. Examples: -** -** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def') -** -> 'CREATE TABLE def(a, b, c)' -** -** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def') -** -> 'CREATE INDEX i ON def(a, b, c)' -*/ -static void renameTableFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - - sqlite3 *db = sqlite3_context_db_handle(context); - - UNUSED_PARAMETER(NotUsed); - - /* The principle used to locate the table name in the CREATE TABLE - ** statement is that the table name is the first non-space token that - ** is immediately followed by a TK_LP or TK_USING token. - */ - if( zSql ){ - do { - if( !*zCsr ){ - /* Ran out of input before finding an opening bracket. Return NULL. */ - return; - } - - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; - - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - } while( token==TK_SPACE ); - assert( len>0 ); - } while( token!=TK_LP && token!=TK_USING ); - - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), - zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); - } -} - -/* -** This C function implements an SQL user function that is used by SQL code -** generated by the ALTER TABLE ... RENAME command to modify the definition -** of any foreign key constraints that use the table being renamed as the -** parent table. It is passed three arguments: -** -** 1) The complete text of the CREATE TABLE statement being modified, -** 2) The old name of the table being renamed, and -** 3) The new name of the table being renamed. -** -** It returns the new CREATE TABLE statement. For example: -** -** sqlite_rename_parent('CREATE TABLE t1(a REFERENCES t2)', 't2', 't3') -** -> 'CREATE TABLE t1(a REFERENCES t3)' -*/ -#ifndef SQLITE_OMIT_FOREIGN_KEY -static void renameParentFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - char *zOutput = 0; - char *zResult; - unsigned char const *zInput = sqlite3_value_text(argv[0]); - unsigned char const *zOld = sqlite3_value_text(argv[1]); - unsigned char const *zNew = sqlite3_value_text(argv[2]); - - unsigned const char *z; /* Pointer to token */ - int n; /* Length of token z */ - int token; /* Type of token */ - - UNUSED_PARAMETER(NotUsed); - if( zInput==0 || zOld==0 ) return; - for(z=zInput; *z; z=z+n){ - n = sqlite3GetToken(z, &token); - if( token==TK_REFERENCES ){ - char *zParent; - do { - z += n; - n = sqlite3GetToken(z, &token); - }while( token==TK_SPACE ); - - if( token==TK_ILLEGAL ) break; - zParent = sqlite3DbStrNDup(db, (const char *)z, n); - if( zParent==0 ) break; - sqlite3Dequote(zParent); - if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ - char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", - (zOutput?zOutput:""), (int)(z-zInput), zInput, (const char *)zNew - ); - sqlite3DbFree(db, zOutput); - zOutput = zOut; - zInput = &z[n]; - } - sqlite3DbFree(db, zParent); - } - } - - zResult = sqlite3MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite3_result_text(context, zResult, -1, SQLITE_DYNAMIC); - sqlite3DbFree(db, zOutput); -} -#endif - -#ifndef SQLITE_OMIT_TRIGGER -/* This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER -** statement. The second is a table name. The table name in the CREATE -** TRIGGER statement is replaced with the third argument and the result -** returned. This is analagous to renameTableFunc() above, except for CREATE -** TRIGGER, not CREATE INDEX and CREATE TABLE. -*/ -static void renameTriggerFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - int dist = 3; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - sqlite3 *db = sqlite3_context_db_handle(context); - - UNUSED_PARAMETER(NotUsed); - - /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediately - ** preceded by either TK_ON or TK_DOT and immediately followed by one - ** of TK_WHEN, TK_BEGIN or TK_FOR. - */ - if( zSql ){ - do { - - if( !*zCsr ){ - /* Ran out of input before finding the table name. Return NULL. */ - return; - } - - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; - - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - }while( token==TK_SPACE ); - assert( len>0 ); - - /* Variable 'dist' stores the number of tokens read since the most - ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN - ** token is read and 'dist' equals 2, the condition stated above - ** to be met. - ** - ** Note that ON cannot be a database, table or column name, so - ** there is no need to worry about syntax like - ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc. - */ - dist++; - if( token==TK_DOT || token==TK_ON ){ - dist = 0; - } - } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) ); - - /* Variable tname now contains the token that is the old table-name - ** in the CREATE TRIGGER statement. - */ - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), - zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); - } -} -#endif /* !SQLITE_OMIT_TRIGGER */ - -/* -** Register built-in functions used to help implement ALTER TABLE -*/ -SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static FuncDef aAlterTableFuncs[] = { - FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), -#ifndef SQLITE_OMIT_TRIGGER - FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), -#endif -#ifndef SQLITE_OMIT_FOREIGN_KEY - FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), -#endif - }; - sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); -} - -/* -** This function is used to create the text of expressions of the form: -** -** name= OR name= OR ... -** -** If argument zWhere is NULL, then a pointer string containing the text -** "name=" is returned, where is the quoted version -** of the string passed as argument zConstant. The returned buffer is -** allocated using sqlite3DbMalloc(). It is the responsibility of the -** caller to ensure that it is eventually freed. -** -** If argument zWhere is not NULL, then the string returned is -** " OR name=", where is the contents of zWhere. -** In this case zWhere is passed to sqlite3DbFree() before returning. -** -*/ -static char *whereOrName(sqlite3 *db, char *zWhere, char *zConstant){ - char *zNew; - if( !zWhere ){ - zNew = sqlite3MPrintf(db, "name=%Q", zConstant); - }else{ - zNew = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, zConstant); - sqlite3DbFree(db, zWhere); - } - return zNew; -} - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) -/* -** Generate the text of a WHERE expression which can be used to select all -** tables that have foreign key constraints that refer to table pTab (i.e. -** constraints for which pTab is the parent table) from the sqlite_master -** table. -*/ -static char *whereForeignKeys(Parse *pParse, Table *pTab){ - FKey *p; - char *zWhere = 0; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - zWhere = whereOrName(pParse->db, zWhere, p->pFrom->zName); - } - return zWhere; -} -#endif - -/* -** Generate the text of a WHERE expression which can be used to select all -** temporary triggers on table pTab from the sqlite_temp_master table. If -** table pTab has no temporary triggers, or is itself stored in the -** temporary database, NULL is returned. -*/ -static char *whereTempTriggers(Parse *pParse, Table *pTab){ - Trigger *pTrig; - char *zWhere = 0; - const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */ - - /* If the table is not located in the temp-db (in which case NULL is - ** returned, loop through the tables list of triggers. For each trigger - ** that is not part of the temp-db schema, add a clause to the WHERE - ** expression being built up in zWhere. - */ - if( pTab->pSchema!=pTempSchema ){ - sqlite3 *db = pParse->db; - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - if( pTrig->pSchema==pTempSchema ){ - zWhere = whereOrName(db, zWhere, pTrig->zName); - } - } - } - if( zWhere ){ - char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); - sqlite3DbFree(pParse->db, zWhere); - zWhere = zNew; - } - return zWhere; -} - -/* -** Generate code to drop and reload the internal representation of table -** pTab from the database, including triggers and temporary triggers. -** Argument zName is the name of the table in the database schema at -** the time the generated code is executed. This can be different from -** pTab->zName if this function is being called to code part of an -** "ALTER TABLE RENAME TO" statement. -*/ -static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ - Vdbe *v; - char *zWhere; - int iDb; /* Index of database containing pTab */ -#ifndef SQLITE_OMIT_TRIGGER - Trigger *pTrig; -#endif - - v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - assert( iDb>=0 ); - -#ifndef SQLITE_OMIT_TRIGGER - /* Drop any table triggers from the internal schema. */ - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); - assert( iTrigDb==iDb || iTrigDb==1 ); - sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->zName, 0); - } -#endif - - /* Drop the table and index from the internal schema. */ - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - - /* Reload the table, index and permanent trigger schemas. */ - zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); - if( !zWhere ) return; - sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); - -#ifndef SQLITE_OMIT_TRIGGER - /* Now, if the table is not stored in the temp database, reload any temp - ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); - } -#endif -} - /* ** Parameter zName is the name of a table that is about to be altered ** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). @@ -100653,6 +101172,49 @@ static int isSystemTable(Parse *pParse, const char *zName){ return 0; } +/* +** Generate code to verify that the schemas of database zDb and, if +** bTemp is not true, database "temp", can still be parsed. This is +** called at the end of the generation of an ALTER TABLE ... RENAME ... +** statement to ensure that the operation has not rendered any schema +** objects unusable. +*/ +static void renameTestSchema(Parse *pParse, const char *zDb, int bTemp){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM \"%w\".%s " + "WHERE name NOT LIKE 'sqlite_%%'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, %d)=NULL ", + zDb, MASTER_NAME, + zDb, bTemp + ); + + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM temp.%s " + "WHERE name NOT LIKE 'sqlite_%%'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, 1)=NULL ", + MASTER_NAME, zDb + ); + } +} + +/* +** Generate code to reload the schema for database iDb. And, if iDb!=1, for +** the temp database as well. +*/ +static void renameReloadSchema(Parse *pParse, int iDb){ + Vdbe *v = pParse->pVdbe; + if( v ){ + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0); + if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0); + } +} + /* ** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" ** command. @@ -100670,9 +101232,6 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( int nTabName; /* Number of UTF-8 characters in zTabName */ const char *zTabName; /* Original name of the table */ Vdbe *v; -#ifndef SQLITE_OMIT_TRIGGER - char *zWhere = 0; /* Where clause to locate temp triggers */ -#endif VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ u32 savedDbFlags; /* Saved value of db->mDbFlags */ @@ -100745,8 +101304,63 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( if( v==0 ){ goto exit_rename_table; } - sqlite3BeginWriteOperation(pParse, pVTab!=0, iDb); - sqlite3ChangeCookie(pParse, iDb); + + /* figure out how many UTF-8 characters are in zName */ + zTabName = pTab->zName; + nTabName = sqlite3Utf8CharLen(zTabName, -1); + + /* Rewrite all CREATE TABLE, INDEX, TRIGGER or VIEW statements in + ** the schema to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " + "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" + "AND name NOT LIKE 'sqlite_%%'" + , zDb, MASTER_NAME, zDb, zTabName, zName, (iDb==1), zTabName + ); + + /* Update the tbl_name and name columns of the sqlite_master table + ** as required. */ + sqlite3NestedParse(pParse, + "UPDATE %Q.%s SET " + "tbl_name = %Q, " + "name = CASE " + "WHEN type='table' THEN %Q " + "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " + "ELSE name END " + "WHERE tbl_name=%Q COLLATE nocase AND " + "(type='table' OR type='index' OR type='trigger');", + zDb, MASTER_NAME, + zName, zName, zName, + nTabName, zTabName + ); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* If the sqlite_sequence table exists in this database, then update + ** it with the new table name. + */ + if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", + zDb, zName, pTab->zName); + } +#endif + + /* If the table being renamed is not itself part of the temp database, + ** edit view and trigger definitions within the temp database + ** as required. */ + if( iDb!=1 ){ + sqlite3NestedParse(pParse, + "UPDATE sqlite_temp_master SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " + "tbl_name = " + "CASE WHEN tbl_name=%Q COLLATE nocase AND " + " sqlite_rename_test(%Q, sql, type, name, 1) " + "THEN %Q ELSE tbl_name END " + "WHERE type IN ('view', 'trigger')" + , zDb, zTabName, zName, zTabName, zDb, zName); + } /* If this is a virtual table, invoke the xRename() function if ** one is defined. The xRename() callback will modify the names @@ -100762,90 +101376,8 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif - /* figure out how many UTF-8 characters are in zName */ - zTabName = pTab->zName; - nTabName = sqlite3Utf8CharLen(zTabName, -1); - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - /* If foreign-key support is enabled, rewrite the CREATE TABLE - ** statements corresponding to all child tables of foreign key constraints - ** for which the renamed table is the parent table. */ - if( (zWhere=whereForeignKeys(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".%s SET " - "sql = sqlite_rename_parent(sql, %Q, %Q) " - "WHERE %s;", zDb, MASTER_NAME, zTabName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } - } -#endif - - /* Modify the sqlite_master table to use the new table name. */ - sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET " -#ifdef SQLITE_OMIT_TRIGGER - "sql = sqlite_rename_table(sql, %Q), " -#else - "sql = CASE " - "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)" - "ELSE sqlite_rename_table(sql, %Q) END, " -#endif - "tbl_name = %Q, " - "name = CASE " - "WHEN type='table' THEN %Q " - "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " - "'sqlite_autoindex_' || %Q || substr(name,%d+18) " - "ELSE name END " - "WHERE tbl_name=%Q COLLATE nocase AND " - "(type='table' OR type='index' OR type='trigger');", - zDb, MASTER_NAME, zName, zName, zName, -#ifndef SQLITE_OMIT_TRIGGER - zName, -#endif - zName, nTabName, zTabName - ); - -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* If the sqlite_sequence table exists in this database, then update - ** it with the new table name. - */ - if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", - zDb, zName, pTab->zName); - } -#endif - -#ifndef SQLITE_OMIT_TRIGGER - /* If there are TEMP triggers on this table, modify the sqlite_temp_master - ** table. Don't do this if the table being ALTERed is itself located in - ** the temp database. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE sqlite_temp_master SET " - "sql = sqlite_rename_trigger(sql, %Q), " - "tbl_name = %Q " - "WHERE %s;", zName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } -#endif - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - FKey *p; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - Table *pFrom = p->pFrom; - if( pFrom!=pTab ){ - reloadTableSchema(pParse, p->pFrom, pFrom->zName); - } - } - } -#endif - - /* Drop and reload the internal table schema. */ - reloadTableSchema(pParse, pTab, zName); + renameReloadSchema(pParse, iDb); + renameTestSchema(pParse, zDb, iDb==1); exit_rename_table: sqlite3SrcListDelete(db, pSrc); @@ -100871,12 +101403,11 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ - Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Vdbe *v; /* The prepared statement under construction */ int r1; /* Temporary registers */ db = pParse->db; if( pParse->nErr || db->mallocFailed ) return; - assert( v!=0 ); pNew = pParse->pNewTable; assert( pNew ); @@ -100971,17 +101502,20 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** from less than 3 to 4, as that will corrupt any preexisting DESC ** index. */ - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); - sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); - sqlite3ReleaseTempReg(pParse, r1); + v = sqlite3GetVdbe(pParse); + if( v ){ + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); + } - /* Reload the schema of the modified table. */ - reloadTableSchema(pParse, pTab, pTab->zName); + /* Reload the table definition */ + renameReloadSchema(pParse, iDb); } /* @@ -101002,7 +101536,6 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Table *pNew; Table *pTab; - Vdbe *v; int iDb; int i; int nAlloc; @@ -101066,16 +101599,1142 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew->addColOffset = pTab->addColOffset; pNew->nTabRef = 1; - /* Begin a transaction and increment the schema cookie. */ - sqlite3BeginWriteOperation(pParse, 0, iDb); - v = sqlite3GetVdbe(pParse); - if( !v ) goto exit_begin_add_column; - sqlite3ChangeCookie(pParse, iDb); - exit_begin_add_column: sqlite3SrcListDelete(db, pSrc); return; } + +/* +** Parameter pTab is the subject of an ALTER TABLE ... RENAME COLUMN +** command. This function checks if the table is a view or virtual +** table (columns of views or virtual tables may not be renamed). If so, +** it loads an error message into pParse and returns non-zero. +** +** Or, if pTab is not a view or virtual table, zero is returned. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +static int isRealTable(Parse *pParse, Table *pTab){ + const char *zType = 0; +#ifndef SQLITE_OMIT_VIEW + if( pTab->pSelect ){ + zType = "view"; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + zType = "virtual table"; + } +#endif + if( zType ){ + sqlite3ErrorMsg( + pParse, "cannot rename columns of %s \"%s\"", zType, pTab->zName + ); + return 1; + } + return 0; +} +#else /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ +# define isRealTable(x,y) (0) +#endif + +/* +** Handles the following parser reduction: +** +** cmd ::= ALTER TABLE pSrc RENAME COLUMN pOld TO pNew +*/ +SQLITE_PRIVATE void sqlite3AlterRenameColumn( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Table being altered. pSrc->nSrc==1 */ + Token *pOld, /* Name of column being changed */ + Token *pNew /* New column name */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table being updated */ + int iCol; /* Index of column being renamed */ + char *zOld = 0; /* Old column name */ + char *zNew = 0; /* New column name */ + const char *zDb; /* Name of schema containing the table */ + int iSchema; /* Index of the schema */ + int bQuote; /* True to quote the new name */ + + /* Locate the table to be altered */ + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_column; + + /* Cannot alter a system table */ + if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ) goto exit_rename_column; + if( SQLITE_OK!=isRealTable(pParse, pTab) ) goto exit_rename_column; + + /* Which schema holds the table to be altered */ + iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iSchema>=0 ); + zDb = db->aDb[iSchema].zDbSName; + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_column; + } +#endif + + /* Make sure the old name really is a column name in the table to be + ** altered. Set iCol to be the index of the column being renamed */ + zOld = sqlite3NameFromToken(db, pOld); + if( !zOld ) goto exit_rename_column; + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3StrICmp(pTab->aCol[iCol].zName, zOld) ) break; + } + if( iCol==pTab->nCol ){ + sqlite3ErrorMsg(pParse, "no such column: \"%s\"", zOld); + goto exit_rename_column; + } + + /* Do the rename operation using a recursive UPDATE statement that + ** uses the sqlite_rename_column() SQL function to compute the new + ** CREATE statement text for the sqlite_master table. + */ + zNew = sqlite3NameFromToken(db, pNew); + if( !zNew ) goto exit_rename_column; + assert( pNew->n>0 ); + bQuote = sqlite3Isquote(pNew->z[0]); + sqlite3NestedParse(pParse, + "UPDATE \"%w\".%s SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " + "WHERE name NOT LIKE 'sqlite_%%' AND (type != 'index' OR tbl_name = %Q)" + " AND sql NOT LIKE 'create virtual%%'", + zDb, MASTER_NAME, + zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, + pTab->zName + ); + + sqlite3NestedParse(pParse, + "UPDATE temp.%s SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " + "WHERE type IN ('trigger', 'view')", + MASTER_NAME, + zDb, pTab->zName, iCol, zNew, bQuote + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iSchema); + renameTestSchema(pParse, zDb, iSchema==1); + + exit_rename_column: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zOld); + sqlite3DbFree(db, zNew); + return; +} + +/* +** Each RenameToken object maps an element of the parse tree into +** the token that generated that element. The parse tree element +** might be one of: +** +** * A pointer to an Expr that represents an ID +** * The name of a table column in Column.zName +** +** A list of RenameToken objects can be constructed during parsing. +** Each new object is created by sqlite3RenameTokenMap(). +** As the parse tree is transformed, the sqlite3RenameTokenRemap() +** routine is used to keep the mapping current. +** +** After the parse finishes, renameTokenFind() routine can be used +** to look up the actual token value that created some element in +** the parse tree. +*/ +struct RenameToken { + void *p; /* Parse tree element created by token t */ + Token t; /* The token that created parse tree element p */ + RenameToken *pNext; /* Next is a list of all RenameToken objects */ +}; + +/* +** The context of an ALTER TABLE RENAME COLUMN operation that gets passed +** down into the Walker. +*/ +typedef struct RenameCtx RenameCtx; +struct RenameCtx { + RenameToken *pList; /* List of tokens to overwrite */ + int nList; /* Number of tokens in pList */ + int iCol; /* Index of column being renamed */ + Table *pTab; /* Table being ALTERed */ + const char *zOld; /* Old column name */ +}; + +#ifdef SQLITE_DEBUG +/* +** This function is only for debugging. It performs two tasks: +** +** 1. Checks that pointer pPtr does not already appear in the +** rename-token list. +** +** 2. Dereferences each pointer in the rename-token list. +** +** The second is most effective when debugging under valgrind or +** address-sanitizer or similar. If any of these pointers no longer +** point to valid objects, an exception is raised by the memory-checking +** tool. +** +** The point of this is to prevent comparisons of invalid pointer values. +** Even though this always seems to work, it is undefined according to the +** C standard. Example of undefined comparison: +** +** sqlite3_free(x); +** if( x==y ) ... +** +** Technically, as x no longer points into a valid object or to the byte +** following a valid object, it may not be used in comparison operations. +*/ +static void renameTokenCheckAll(Parse *pParse, void *pPtr){ + if( pParse->nErr==0 && pParse->db->mallocFailed==0 ){ + RenameToken *p; + u8 i = 0; + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p ){ + assert( p->p!=pPtr ); + i += *(u8*)(p->p); + } + } + } +} +#else +# define renameTokenCheckAll(x,y) +#endif + +/* +** Add a new RenameToken object mapping parse tree element pPtr into +** token *pToken to the Parse object currently under construction. +** +** Return a copy of pPtr. +*/ +SQLITE_PRIVATE void *sqlite3RenameTokenMap(Parse *pParse, void *pPtr, Token *pToken){ + RenameToken *pNew; + assert( pPtr || pParse->db->mallocFailed ); + renameTokenCheckAll(pParse, pPtr); + pNew = sqlite3DbMallocZero(pParse->db, sizeof(RenameToken)); + if( pNew ){ + pNew->p = pPtr; + pNew->t = *pToken; + pNew->pNext = pParse->pRename; + pParse->pRename = pNew; + } + + return pPtr; +} + +/* +** It is assumed that there is already a RenameToken object associated +** with parse tree element pFrom. This function remaps the associated token +** to parse tree element pTo. +*/ +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse *pParse, void *pTo, void *pFrom){ + RenameToken *p; + renameTokenCheckAll(pParse, pTo); + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p==pFrom ){ + p->p = pTo; + break; + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapExprCb(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + return WRC_Continue; +} + +/* +** Remove all nodes that are part of expression pExpr from the rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sqlite3WalkExpr(&sWalker, pExpr); +} + +/* +** Remove all nodes that are part of expression-list pEList from the +** rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ + if( pEList ){ + int i; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sqlite3WalkExprList(&sWalker, pEList); + for(i=0; inExpr; i++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zName); + } + } +} + +/* +** Free the list of RenameToken objects given in the second argument +*/ +static void renameTokenFree(sqlite3 *db, RenameToken *pToken){ + RenameToken *pNext; + RenameToken *p; + for(p=pToken; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Search the Parse object passed as the first argument for a RenameToken +** object associated with parse tree element pPtr. If found, remove it +** from the Parse object and add it to the list maintained by the +** RenameCtx object passed as the second argument. +*/ +static void renameTokenFind(Parse *pParse, struct RenameCtx *pCtx, void *pPtr){ + RenameToken **pp; + assert( pPtr!=0 ); + for(pp=&pParse->pRename; (*pp); pp=&(*pp)->pNext){ + if( (*pp)->p==pPtr ){ + RenameToken *pToken = *pp; + *pp = pToken->pNext; + pToken->pNext = pCtx->pList; + pCtx->pList = pToken; + pCtx->nList++; + break; + } + } +} + +/* +** This is a Walker select callback. It does nothing. It is only required +** because without a dummy callback, sqlite3WalkExpr() and similar do not +** descend into sub-select statements. +*/ +static int renameColumnSelectCb(Walker *pWalker, Select *p){ + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(p); + return WRC_Continue; +} + +/* +** This is a Walker expression callback. +** +** For every TK_COLUMN node in the expression tree, search to see +** if the column being references is the column being renamed by an +** ALTER TABLE statement. If it is, then attach its associated +** RenameToken object to the list of RenameToken objects being +** constructed in RenameCtx object at pWalker->u.pRename. +*/ +static int renameColumnExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_TRIGGER + && pExpr->iColumn==p->iCol + && pWalker->pParse->pTriggerTab==p->pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + }else if( pExpr->op==TK_COLUMN + && pExpr->iColumn==p->iCol + && p->pTab==pExpr->pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + } + return WRC_Continue; +} + +/* +** The RenameCtx contains a list of tokens that reference a column that +** is being renamed by an ALTER TABLE statement. Return the "last" +** RenameToken in the RenameCtx and remove that RenameToken from the +** RenameContext. "Last" means the last RenameToken encountered when +** the input SQL is parsed from left to right. Repeated calls to this routine +** return all column name tokens in the order that they are encountered +** in the SQL statement. +*/ +static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ + RenameToken *pBest = pCtx->pList; + RenameToken *pToken; + RenameToken **pp; + + for(pToken=pBest->pNext; pToken; pToken=pToken->pNext){ + if( pToken->t.z>pBest->t.z ) pBest = pToken; + } + for(pp=&pCtx->pList; *pp!=pBest; pp=&(*pp)->pNext); + *pp = pBest->pNext; + + return pBest; +} + +/* +** An error occured while parsing or otherwise processing a database +** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an +** ALTER TABLE RENAME COLUMN program. The error message emitted by the +** sub-routine is currently stored in pParse->zErrMsg. This function +** adds context to the error message and then stores it in pCtx. +*/ +static void renameColumnParseError( + sqlite3_context *pCtx, + int bPost, + sqlite3_value *pType, + sqlite3_value *pObject, + Parse *pParse +){ + const char *zT = (const char*)sqlite3_value_text(pType); + const char *zN = (const char*)sqlite3_value_text(pObject); + char *zErr; + + zErr = sqlite3_mprintf("error in %s %s%s: %s", + zT, zN, (bPost ? " after rename" : ""), + pParse->zErrMsg + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); +} + +/* +** For each name in the the expression-list pEList (i.e. each +** pEList->a[i].zName) that matches the string in zOld, extract the +** corresponding rename-token from Parse object pParse and add it +** to the RenameCtx pCtx. +*/ +static void renameColumnElistNames( + Parse *pParse, + RenameCtx *pCtx, + ExprList *pEList, + const char *zOld +){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + char *zName = pEList->a[i].zName; + if( 0==sqlite3_stricmp(zName, zOld) ){ + renameTokenFind(pParse, pCtx, (void*)zName); + } + } + } +} + +/* +** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) +** that matches the string in zOld, extract the corresponding rename-token +** from Parse object pParse and add it to the RenameCtx pCtx. +*/ +static void renameColumnIdlistNames( + Parse *pParse, + RenameCtx *pCtx, + IdList *pIdList, + const char *zOld +){ + if( pIdList ){ + int i; + for(i=0; inId; i++){ + char *zName = pIdList->a[i].zName; + if( 0==sqlite3_stricmp(zName, zOld) ){ + renameTokenFind(pParse, pCtx, (void*)zName); + } + } + } +} + +/* +** Parse the SQL statement zSql using Parse object (*p). The Parse object +** is initialized by this function before it is used. +*/ +static int renameParseSql( + Parse *p, /* Memory to use for Parse object */ + const char *zDb, /* Name of schema SQL belongs to */ + int bTable, /* 1 -> RENAME TABLE, 0 -> RENAME COLUMN */ + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL to parse */ + int bTemp /* True if SQL is from temp schema */ +){ + int rc; + char *zErr = 0; + + db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + + /* Parse the SQL statement passed as the first argument. If no error + ** occurs and the parse does not result in a new table, index or + ** trigger object, the database must be corrupt. */ + memset(p, 0, sizeof(Parse)); + p->eParseMode = (bTable ? PARSE_MODE_RENAME_TABLE : PARSE_MODE_RENAME_COLUMN); + p->db = db; + p->nQueryLoop = 1; + rc = sqlite3RunParser(p, zSql, &zErr); + assert( p->zErrMsg==0 ); + assert( rc!=SQLITE_OK || zErr==0 ); + assert( (0!=p->pNewTable) + (0!=p->pNewIndex) + (0!=p->pNewTrigger)<2 ); + p->zErrMsg = zErr; + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK + && p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0 + ){ + rc = SQLITE_CORRUPT_BKPT; + } + +#ifdef SQLITE_DEBUG + /* Ensure that all mappings in the Parse.pRename list really do map to + ** a part of the input string. */ + if( rc==SQLITE_OK ){ + int nSql = sqlite3Strlen30(zSql); + RenameToken *pToken; + for(pToken=p->pRename; pToken; pToken=pToken->pNext){ + assert( pToken->t.z>=zSql && &pToken->t.z[pToken->t.n]<=&zSql[nSql] ); + } + } +#endif + + db->init.iDb = 0; + return rc; +} + +/* +** This function edits SQL statement zSql, replacing each token identified +** by the linked list pRename with the text of zNew. If argument bQuote is +** true, then zNew is always quoted first. If no error occurs, the result +** is loaded into context object pCtx as the result. +** +** Or, if an error occurs (i.e. an OOM condition), an error is left in +** pCtx and an SQLite error code returned. +*/ +static int renameEditSql( + sqlite3_context *pCtx, /* Return result here */ + RenameCtx *pRename, /* Rename context */ + const char *zSql, /* SQL statement to edit */ + const char *zNew, /* New token text */ + int bQuote /* True to always quote token */ +){ + int nNew = sqlite3Strlen30(zNew); + int nSql = sqlite3Strlen30(zSql); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + int rc = SQLITE_OK; + char *zQuot; + char *zOut; + int nQuot; + + /* Set zQuot to point to a buffer containing a quoted copy of the + ** identifier zNew. If the corresponding identifier in the original + ** ALTER TABLE statement was quoted (bQuote==1), then set zNew to + ** point to zQuot so that all substitutions are made using the + ** quoted version of the new column name. */ + zQuot = sqlite3MPrintf(db, "\"%w\"", zNew); + if( zQuot==0 ){ + return SQLITE_NOMEM; + }else{ + nQuot = sqlite3Strlen30(zQuot); + } + if( bQuote ){ + zNew = zQuot; + nNew = nQuot; + } + + /* At this point pRename->pList contains a list of RenameToken objects + ** corresponding to all tokens in the input SQL that must be replaced + ** with the new column name. All that remains is to construct and + ** return the edited SQL string. */ + assert( nQuot>=nNew ); + zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + if( zOut ){ + int nOut = nSql; + memcpy(zOut, zSql, nSql); + while( pRename->pList ){ + int iOff; /* Offset of token to replace in zOut */ + RenameToken *pBest = renameColumnTokenNext(pRename); + + u32 nReplace; + const char *zReplace; + if( sqlite3IsIdChar(*pBest->t.z) ){ + nReplace = nNew; + zReplace = zNew; + }else{ + nReplace = nQuot; + zReplace = zQuot; + } + + iOff = pBest->t.z - zSql; + if( pBest->t.n!=nReplace ){ + memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], + nOut - (iOff + pBest->t.n) + ); + nOut += nReplace - pBest->t.n; + zOut[nOut] = '\0'; + } + memcpy(&zOut[iOff], zReplace, nReplace); + sqlite3DbFree(db, pBest); + } + + sqlite3_result_text(pCtx, zOut, -1, SQLITE_TRANSIENT); + sqlite3DbFree(db, zOut); + }else{ + rc = SQLITE_NOMEM; + } + + sqlite3_free(zQuot); + return rc; +} + +/* +** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming +** it was read from the schema of database zDb. Return SQLITE_OK if +** successful. Otherwise, return an SQLite error code and leave an error +** message in the Parse object. +*/ +static int renameResolveTrigger(Parse *pParse, const char *zDb){ + sqlite3 *db = pParse->db; + Trigger *pNew = pParse->pNewTrigger; + TriggerStep *pStep; + NameContext sNC; + int rc = SQLITE_OK; + + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + assert( pNew->pTabSchema ); + pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, + db->aDb[sqlite3SchemaToIndex(db, pNew->pTabSchema)].zDbSName + ); + pParse->eTriggerOp = pNew->op; + + /* Resolve symbols in WHEN clause */ + if( pNew->pWhen ){ + rc = sqlite3ResolveExprNames(&sNC, pNew->pWhen); + } + + for(pStep=pNew->step_list; rc==SQLITE_OK && pStep; pStep=pStep->pNext){ + if( pStep->pSelect ){ + sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); + if( pParse->nErr ) rc = pParse->rc; + } + if( rc==SQLITE_OK && pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(pParse, 0, pStep->zTarget, zDb); + if( pTarget==0 ){ + rc = SQLITE_ERROR; + }else if( SQLITE_OK==(rc = sqlite3ViewGetColumnNames(pParse, pTarget)) ){ + SrcList sSrc; + memset(&sSrc, 0, sizeof(sSrc)); + sSrc.nSrc = 1; + sSrc.a[0].zName = pStep->zTarget; + sSrc.a[0].pTab = pTarget; + sNC.pSrcList = &sSrc; + if( pStep->pWhere ){ + rc = sqlite3ResolveExprNames(&sNC, pStep->pWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprListNames(&sNC, pStep->pExprList); + } + assert( !pStep->pUpsert || (!pStep->pWhere && !pStep->pExprList) ); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + assert( rc==SQLITE_OK ); + pUpsert->pUpsertSrc = &sSrc; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc==SQLITE_OK ){ + ExprList *pUpsertSet = pUpsert->pUpsertSet; + rc = sqlite3ResolveExprListNames(&sNC, pUpsertSet); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + } + sNC.ncFlags = 0; + } + } + } + } + return rc; +} + +/* +** Invoke sqlite3WalkExpr() or sqlite3WalkSelect() on all Select or Expr +** objects that are part of the trigger passed as the second argument. +*/ +static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ + TriggerStep *pStep; + + /* Find tokens to edit in WHEN clause */ + sqlite3WalkExpr(pWalker, pTrigger->pWhen); + + /* Find tokens to edit in trigger steps */ + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + sqlite3WalkSelect(pWalker, pStep->pSelect); + sqlite3WalkExpr(pWalker, pStep->pWhere); + sqlite3WalkExprList(pWalker, pStep->pExprList); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + sqlite3WalkExprList(pWalker, pUpsert->pUpsertTarget); + sqlite3WalkExprList(pWalker, pUpsert->pUpsertSet); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); + } + } +} + +/* +** Free the contents of Parse object (*pParse). Do not free the memory +** occupied by the Parse object itself. +*/ +static void renameParseCleanup(Parse *pParse){ + sqlite3 *db = pParse->db; + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + if( pParse->pNewIndex ) sqlite3FreeIndex(db, pParse->pNewIndex); + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + sqlite3DbFree(db, pParse->zErrMsg); + renameTokenFree(db, pParse->pRename); + sqlite3ParserReset(pParse); +} + +/* +** SQL function: +** +** sqlite_rename_column(zSql, iCol, bQuote, zNew, zTable, zOld) +** +** 0. zSql: SQL statement to rewrite +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3. Database: Database name (e.g. "main") +** 4. Table: Table name +** 5. iCol: Index of column to rename +** 6. zNew: New column name +** 7. bQuote: Non-zero if the new column name should be quoted. +** 8. bTemp: True if zSql comes from temp schema +** +** Do a column rename operation on the CREATE statement given in zSql. +** The iCol-th column (left-most is 0) of table zTable is renamed from zCol +** into zNew. The name should be quoted if bQuote is true. +** +** This function is used internally by the ALTER TABLE RENAME COLUMN command. +** Though accessible to application code, it is not intended for use by +** applications. The existance of this function, and the way it works, +** is subject to change without notice. +** +** If any of the parameters are out-of-bounds, then simply return NULL. +** An out-of-bounds parameter can only occur when the application calls +** this function directly. The parameters will always be well-formed when +** this routine is invoked by the bytecode for a legitimate ALTER TABLE +** statement. +*/ +static void renameColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + RenameCtx sCtx; + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + const char *zDb = (const char*)sqlite3_value_text(argv[3]); + const char *zTable = (const char*)sqlite3_value_text(argv[4]); + int iCol = sqlite3_value_int(argv[5]); + const char *zNew = (const char*)sqlite3_value_text(argv[6]); + int bQuote = sqlite3_value_int(argv[7]); + int bTemp = sqlite3_value_int(argv[8]); + const char *zOld; + int rc; + Parse sParse; + Walker sWalker; + Index *pIdx; + int i; + Table *pTab; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zSql==0 ) return; + if( zTable==0 ) return; + if( zNew==0 ) return; + if( iCol<0 ) return; + sqlite3BtreeEnterAll(db); + pTab = sqlite3FindTable(db, zTable, zDb); + if( pTab==0 || iCol>=pTab->nCol ){ + sqlite3BtreeLeaveAll(db); + return; + } + zOld = pTab->aCol[iCol].zName; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.iCol = ((iCol==pTab->iPKey) ? -1 : iCol); + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + rc = renameParseSql(&sParse, zDb, 0, db, zSql, bTemp); + + /* Find tokens that need to be replaced. */ + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameColumnExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + sCtx.pTab = pTab; + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + if( sParse.pNewTable ){ + Select *pSelect = sParse.pNewTable->pSelect; + if( pSelect ){ + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, sParse.pNewTable->pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + }else{ + /* A regular table */ + int bFKOnly = sqlite3_stricmp(zTable, sParse.pNewTable->zName); + FKey *pFKey; + assert( sParse.pNewTable->pSelect==0 ); + sCtx.pTab = sParse.pNewTable; + if( bFKOnly==0 ){ + renameTokenFind( + &sParse, &sCtx, (void*)sParse.pNewTable->aCol[iCol].zName + ); + if( sCtx.iCol<0 ){ + renameTokenFind(&sParse, &sCtx, (void*)&sParse.pNewTable->iPKey); + } + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); + for(pIdx=sParse.pNewTable->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } + } + + for(pFKey=sParse.pNewTable->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + for(i=0; inCol; i++){ + if( bFKOnly==0 && pFKey->aCol[i].iFrom==iCol ){ + renameTokenFind(&sParse, &sCtx, (void*)&pFKey->aCol[i]); + } + if( 0==sqlite3_stricmp(pFKey->zTo, zTable) + && 0==sqlite3_stricmp(pFKey->aCol[i].zCol, zOld) + ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->aCol[i].zCol); + } + } + } + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ + /* A trigger */ + TriggerStep *pStep; + rc = renameResolveTrigger(&sParse, (bTemp ? 0 : zDb)); + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + + for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pTarget==pTab ){ + if( pStep->pUpsert ){ + ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; + renameColumnElistNames(&sParse, &sCtx, pUpsertSet, zOld); + } + renameColumnIdlistNames(&sParse, &sCtx, pStep->pIdList, zOld); + renameColumnElistNames(&sParse, &sCtx, pStep->pExprList, zOld); + } + } + } + + + /* Find tokens to edit in UPDATE OF clause */ + if( sParse.pTriggerTab==pTab ){ + renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); + } + + /* Find tokens to edit in various expressions and selects */ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } + + assert( rc==SQLITE_OK ); + rc = renameEditSql(context, &sCtx, zSql, zNew, bQuote); + +renameColumnFunc_done: + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, 0, argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + sqlite3BtreeLeaveAll(db); +} + +/* +** Walker expression callback used by "RENAME TABLE". +*/ +static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_COLUMN && p->pTab==pExpr->pTab ){ + renameTokenFind(pWalker->pParse, p, (void*)&pExpr->pTab); + } + return WRC_Continue; +} + +/* +** Walker select callback used by "RENAME TABLE". +*/ +static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ + int i; + RenameCtx *p = pWalker->u.pRename; + SrcList *pSrc = pSelect->pSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->pTab==p->pTab ){ + renameTokenFind(pWalker->pParse, p, pItem->zName); + } + } + + return WRC_Continue; +} + + +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 0: The database containing the table being renamed. +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3: The complete text of the schema statement being modified, +** 4: The old name of the table being renamed, and +** 5: The new name of the table being renamed. +** 6: True if the schema statement comes from the temp db. +** +** It returns the new schema statement. For example: +** +** sqlite_rename_table('main', 'CREATE TABLE t1(a REFERENCES t2)','t2','t3',0) +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDb = (const char*)sqlite3_value_text(argv[0]); + const char *zInput = (const char*)sqlite3_value_text(argv[3]); + const char *zOld = (const char*)sqlite3_value_text(argv[4]); + const char *zNew = (const char*)sqlite3_value_text(argv[5]); + int bTemp = sqlite3_value_int(argv[6]); + UNUSED_PARAMETER(NotUsed); + + if( zInput && zOld && zNew ){ + Parse sParse; + int rc; + int bQuote = 1; + RenameCtx sCtx; + Walker sWalker; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + memset(&sCtx, 0, sizeof(RenameCtx)); + sCtx.pTab = sqlite3FindTable(db, zOld, zDb); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameTableExprCb; + sWalker.xSelectCallback = renameTableSelectCb; + sWalker.u.pRename = &sCtx; + + rc = renameParseSql(&sParse, zDb, 1, db, zInput, bTemp); + + if( rc==SQLITE_OK ){ + int isLegacy = (db->flags & SQLITE_LegacyAlter); + if( sParse.pNewTable ){ + Table *pTab = sParse.pNewTable; + + if( pTab->pSelect ){ + if( isLegacy==0 ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + + sqlite3SelectPrep(&sParse, pTab->pSelect, &sNC); + if( sParse.nErr ) rc = sParse.rc; + sqlite3WalkSelect(&sWalker, pTab->pSelect); + } + }else{ + /* Modify any FK definitions to point to the new table. */ +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( db->flags & SQLITE_ForeignKeys ){ + FKey *pFKey; + for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + if( sqlite3_stricmp(pFKey->zTo, zOld)==0 ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->zTo); + } + } + } +#endif + + /* If this is the table being altered, fix any table refs in CHECK + ** expressions. Also update the name that appears right after the + ** "CREATE [VIRTUAL] TABLE" bit. */ + if( sqlite3_stricmp(zOld, pTab->zName)==0 ){ + sCtx.pTab = pTab; + if( isLegacy==0 ){ + sqlite3WalkExprList(&sWalker, pTab->pCheck); + } + renameTokenFind(&sParse, &sCtx, pTab->zName); + } + } + } + + else if( sParse.pNewIndex ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewIndex->zName); + if( isLegacy==0 ){ + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + } + } + +#ifndef SQLITE_OMIT_TRIGGER + else{ + Trigger *pTrigger = sParse.pNewTrigger; + TriggerStep *pStep; + if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) + && sCtx.pTab->pSchema==pTrigger->pTabSchema + ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewTrigger->table); + } + + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse, bTemp ? 0 : zDb); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, pTrigger); + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ + renameTokenFind(&sParse, &sCtx, pStep->zTarget); + } + } + } + } + } +#endif + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); + } + if( rc!=SQLITE_OK ){ + if( sParse.zErrMsg ){ + renameColumnParseError(context, 0, argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); + sqlite3BtreeLeaveAll(db); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + } + + return; +} + +/* +** An SQL user function that checks that there are no parse or symbol +** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement. +** After an ALTER TABLE .. RENAME operation is performed and the schema +** reloaded, this function is called on each SQL statement in the schema +** to ensure that it is still usable. +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement. +** 2: Object type ("view", "table", "trigger" or "index"). +** 3: Object name. +** 4: True if object is from temp schema. +** +** Unless it finds an error, this function normally returns NULL. However, it +** returns integer value 1 if: +** +** * the SQL argument creates a trigger, and +** * the table that the trigger is attached to is in database zDb. +*/ +static void renameTableTest( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + int bTemp = sqlite3_value_int(argv[4]); + int isLegacy = (db->flags & SQLITE_LegacyAlter); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zDb && zInput ){ + int rc; + Parse sParse; + rc = renameParseSql(&sParse, zDb, 1, db, zInput, bTemp); + if( rc==SQLITE_OK ){ + if( isLegacy==0 && sParse.pNewTable && sParse.pNewTable->pSelect ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + sqlite3SelectPrep(&sParse, sParse.pNewTable->pSelect, &sNC); + if( sParse.nErr ) rc = sParse.rc; + } + + else if( sParse.pNewTrigger ){ + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse, bTemp ? 0 : zDb); + } + if( rc==SQLITE_OK ){ + int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema); + int i2 = sqlite3FindDbName(db, zDb); + if( i1==i2 ) sqlite3_result_int(context, 1); + } + } + } + + if( rc!=SQLITE_OK ){ + renameColumnParseError(context, 1, argv[2], argv[3], &sParse); + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif +} + +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +SQLITE_PRIVATE void sqlite3AlterFunctions(void){ + static FuncDef aAlterTableFuncs[] = { + FUNCTION(sqlite_rename_column, 9, 0, 0, renameColumnFunc), + FUNCTION(sqlite_rename_table, 7, 0, 0, renameTableFunc), + FUNCTION(sqlite_rename_test, 5, 0, 0, renameTableTest), + }; + sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); +} #endif /* SQLITE_ALTER_TABLE */ /************** End of alter.c ***********************************************/ @@ -101567,6 +103226,7 @@ static const FuncDef statInitFuncdef = { 0, /* pNext */ statInit, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_init", /* zName */ {0} }; @@ -101883,6 +103543,7 @@ static const FuncDef statPushFuncdef = { 0, /* pNext */ statPush, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_push", /* zName */ {0} }; @@ -102034,6 +103695,7 @@ static const FuncDef statGetFuncdef = { 0, /* pNext */ statGet, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_get", /* zName */ {0} }; @@ -102353,10 +104015,7 @@ static void analyzeOneTable( callStatGet(v, regStat4, STAT_GET_NLT, regLt); callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); - /* We know that the regSampleRowid row exists because it was read by - ** the previous loop. Thus the not-found jump of seekOp will never - ** be taken */ - VdbeCoverageNeverTaken(v); + VdbeCoverage(v); #ifdef SQLITE_ENABLE_STAT3 sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); #else @@ -102996,7 +104655,7 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ + if( rc==SQLITE_OK ){ db->lookaside.bDisable++; rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bDisable--; @@ -103435,6 +105094,7 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ 0, /* pNext */ detachFunc, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "sqlite_detach", /* zName */ {0} }; @@ -103454,6 +105114,7 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p 0, /* pNext */ attachFunc, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "sqlite_attach", /* zName */ {0} }; @@ -103726,7 +105387,7 @@ SQLITE_API int sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -103855,7 +105516,7 @@ SQLITE_PRIVATE int sqlite3AuthCheck( /* Don't do any authorization checks if the database is initialising ** or if the parser is being invoked from within sqlite3_declare_vtab. */ - if( db->init.busy || IN_DECLARE_VTAB ){ + if( db->init.busy || IN_SPECIAL_PARSE ){ return SQLITE_OK; } @@ -104147,7 +105808,6 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ /* Get the VDBE program ready for execution */ if( v && pParse->nErr==0 && !db->mallocFailed ){ - assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; @@ -104362,7 +106022,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha /* ** Reclaim the memory used by an index */ -static void freeIndex(sqlite3 *db, Index *p){ +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif @@ -104402,7 +106062,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char p->pNext = pIndex->pNext; } } - freeIndex(db, pIndex); + sqlite3FreeIndex(db, pIndex); } db->mDbFlags |= DBFLAG_SchemaChange; } @@ -104548,7 +106208,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); } - freeIndex(db, pIndex); + sqlite3FreeIndex(db, pIndex); } /* Delete any foreign keys attached to this table. */ @@ -104706,7 +106366,7 @@ SQLITE_PRIVATE int sqlite3TwoPartName( return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy + assert( db->init.iDb==0 || db->init.busy || IN_RENAME_OBJECT || (db->mDbFlags & DBFLAG_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; @@ -104801,6 +106461,9 @@ SQLITE_PRIVATE void sqlite3StartTable( } if( !OMIT_TEMPDB && isTemp ) iDb = 1; zName = sqlite3NameFromToken(db, pName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)zName, pName); + } } pParse->sNameToken = *pName; if( zName==0 ) return; @@ -104836,7 +106499,7 @@ SQLITE_PRIVATE void sqlite3StartTable( ** and types will be used, so there is no need to test for namespace ** collisions. */ - if( !IN_DECLARE_VTAB ){ + if( !IN_SPECIAL_PARSE ){ char *zDb = db->aDb[iDb].zDbSName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; @@ -104995,6 +106658,7 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ } z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); if( z==0 ) return; + if( IN_RENAME_OBJECT ) sqlite3RenameTokenMap(pParse, (void*)z, pName); memcpy(z, pName->z, pName->n); z[pName->n] = 0; sqlite3Dequote(z); @@ -105201,6 +106865,9 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue( sqlite3DbFree(db, x.u.zToken); } } + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, pExpr); + } sqlite3ExprDelete(db, pExpr); } @@ -105292,6 +106959,9 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 && sortOrder!=SQLITE_SO_DESC ){ + if( IN_RENAME_OBJECT && pList ){ + sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pList->a[0].pExpr); + } pTab->iPKey = iCol; pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); @@ -105617,6 +107287,31 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ return 0; } +/* Recompute the colNotIdxed field of the Index. +** +** colNotIdxed is a bitmask that has a 0 bit representing each indexed +** columns that are within the first 63 columns of the table. The +** high-order bit of colNotIdxed is always 1. All unindexed columns +** of the table have a 1. +** +** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask +** to determine if the index is covering index. +*/ +static void recomputeColumnsNotIndexed(Index *pIdx){ + Bitmask m = 0; + int j; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xcolNotIdxed = ~m; + assert( (pIdx->colNotIdxed>>63)==1 ); +} + /* ** This routine runs at the end of parsing a CREATE TABLE statement that ** has a WITHOUT ROWID clause. The job of this routine is to convert both @@ -105659,10 +107354,6 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } } - /* The remaining transformations only apply to b-tree tables, not to - ** virtual tables */ - if( IN_DECLARE_VTAB ) return; - /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY ** into BTREE_BLOBKEY. */ @@ -105685,7 +107376,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ assert( pParse->pNewTable==pTab ); sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, SQLITE_IDXTYPE_PRIMARYKEY); - if( db->mallocFailed ) return; + if( db->mallocFailed || pParse->nErr ) return; pPk = sqlite3PrimaryKeyIndex(pTab); pTab->iPKey = -1; }else{ @@ -105765,6 +107456,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ }else{ pPk->nColumn = pTab->nCol; } + recomputeColumnsNotIndexed(pPk); } /* @@ -106068,7 +107760,12 @@ SQLITE_PRIVATE void sqlite3CreateView( ** allocated rather than point to the input string - which means that ** they will persist after the current sqlite3_exec() call returns. */ - p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + p->pSelect = pSelect; + pSelect = 0; + }else{ + p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); if( db->mallocFailed ) goto create_view_fail; @@ -106093,6 +107790,9 @@ SQLITE_PRIVATE void sqlite3CreateView( create_view_fail: sqlite3SelectDelete(db, pSelect); + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprlistUnmap(pParse, pCNames); + } sqlite3ExprListDelete(db, pCNames); return; } @@ -106166,6 +107866,10 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ assert( pTable->pSelect ); pSel = sqlite3SelectDup(db, pTable->pSelect, 0); if( pSel ){ +#ifndef SQLITE_OMIT_ALTERTABLE + u8 eParseMode = pParse->eParseMode; + pParse->eParseMode = PARSE_MODE_NORMAL; +#endif n = pParse->nTab; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; @@ -106211,10 +107915,18 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ sqlite3DeleteTable(db, pSelTab); sqlite3SelectDelete(db, pSel); db->lookaside.bDisable--; +#ifndef SQLITE_OMIT_ALTERTABLE + pParse->eParseMode = eParseMode; +#endif } else { nErr++; } pTable->pSchema->schemaFlags |= DB_UnresetViews; + if( db->mallocFailed ){ + sqlite3DeleteColumnNames(db, pTable); + pTable->aCol = 0; + pTable->nCol = 0; + } #endif /* SQLITE_OMIT_VIEW */ return nErr; } @@ -106553,8 +108265,10 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, v = sqlite3GetVdbe(pParse); if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); - sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); - sqlite3FkDropTable(pParse, pName, pTab); + if( !isView ){ + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); + sqlite3FkDropTable(pParse, pName, pTab); + } sqlite3CodeDropTable(pParse, pTab, iDb, isView); } @@ -106629,6 +108343,9 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( pFKey->pNextFrom = p->pFKey; z = (char*)&pFKey->aCol[nCol]; pFKey->zTo = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)z, pTo); + } memcpy(z, pTo->z, pTo->n); z[pTo->n] = 0; sqlite3Dequote(z); @@ -106651,12 +108368,18 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( pFromCol->a[i].zName); goto fk_end; } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, &pFKey->aCol[i], pFromCol->a[i].zName); + } } } if( pToCol ){ for(i=0; ia[i].zName); pFKey->aCol[i].zCol = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, z, pToCol->a[i].zName); + } memcpy(z, pToCol->a[i].zName, n); z[n] = 0; z += n+1; @@ -106989,21 +108712,23 @@ SQLITE_PRIVATE void sqlite3CreateIndex( if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto exit_create_index; } - if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ - sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + if( !IN_RENAME_OBJECT ){ + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, 0)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } + } + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } goto exit_create_index; } } - if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ - if( !ifNotExist ){ - sqlite3ErrorMsg(pParse, "index %s already exists", zName); - }else{ - assert( !db->init.busy ); - sqlite3CodeVerifySchema(pParse, iDb); - } - goto exit_create_index; - } }else{ int n; Index *pLoop; @@ -107018,13 +108743,13 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** The following statement converts "sqlite3_autoindex..." into ** "sqlite3_butoindex..." in order to make the names distinct. ** The "vtab_err.test" test demonstrates the need of this statement. */ - if( IN_DECLARE_VTAB ) zName[7]++; + if( IN_SPECIAL_PARSE ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( !IN_RENAME_OBJECT ){ const char *zDb = pDb->zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; @@ -107111,7 +108836,12 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** TODO: Issue a warning if the table primary key is used as part of the ** index key. */ - for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ + pListItem = pList->a; + if( IN_RENAME_OBJECT ){ + pIndex->aColExpr = pList; + pList = 0; + } + for(i=0; inKeyCol; i++, pListItem++){ Expr *pCExpr; /* The i-th index expression */ int requestedSortOrder; /* ASC or DESC on the i-th expression */ const char *zColl; /* Collation sequence name */ @@ -107127,12 +108857,8 @@ SQLITE_PRIVATE void sqlite3CreateIndex( goto exit_create_index; } if( pIndex->aColExpr==0 ){ - ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); - pIndex->aColExpr = pCopy; - if( !db->mallocFailed ){ - assert( pCopy!=0 ); - pListItem = &pCopy->a[i]; - } + pIndex->aColExpr = pList; + pList = 0; } j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; @@ -107198,6 +108924,7 @@ SQLITE_PRIVATE void sqlite3CreateIndex( ** it as a covering index */ assert( HasRowid(pTab) || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + recomputeColumnsNotIndexed(pIndex); if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ pIndex->isCovering = 1; for(j=0; jnCol; j++){ @@ -107270,98 +108997,101 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } } - /* Link the new Index structure to its table and to the other - ** in-memory database structures. - */ - assert( pParse->nErr==0 ); - if( db->init.busy ){ - Index *p; - assert( !IN_DECLARE_VTAB ); - assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); - p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, pIndex); - if( p ){ - assert( p==pIndex ); /* Malloc must have failed */ - sqlite3OomFault(db); - goto exit_create_index; - } - db->mDbFlags |= DBFLAG_SchemaChange; - if( pTblName!=0 ){ - pIndex->tnum = db->init.newTnum; - } - } + if( !IN_RENAME_OBJECT ){ - /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the - ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then - ** emit code to allocate the index rootpage on disk and make an entry for - ** the index in the sqlite_master table and populate the index with - ** content. But, do not do this if we are simply reading the sqlite_master - ** table to parse the schema, or if this index is the PRIMARY KEY index - ** of a WITHOUT ROWID table. - ** - ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY - ** or UNIQUE index in a CREATE TABLE statement. Since the table - ** has just been created, it contains no data and the index initialization - ** step can be skipped. - */ - else if( HasRowid(pTab) || pTblName!=0 ){ - Vdbe *v; - char *zStmt; - int iMem = ++pParse->nMem; - - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto exit_create_index; - - sqlite3BeginWriteOperation(pParse, 1, iDb); - - /* Create the rootpage for the index using CreateIndex. But before - ** doing so, code a Noop instruction and store its address in - ** Index.tnum. This is required in case this index is actually a - ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In - ** that case the convertToWithoutRowidTable() routine will replace - ** the Noop with a Goto to jump over the VDBE code generated below. */ - pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); - sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); - - /* Gather the complete text of the CREATE INDEX statement into - ** the zStmt variable + /* Link the new Index structure to its table and to the other + ** in-memory database structures. */ - if( pStart ){ - int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; - if( pName->z[n-1]==';' ) n--; - /* A named index with an explicit CREATE INDEX statement */ - zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", - onError==OE_None ? "" : " UNIQUE", n, pName->z); - }else{ - /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ - /* zStmt = sqlite3MPrintf(""); */ - zStmt = 0; + assert( pParse->nErr==0 ); + if( db->init.busy ){ + Index *p; + assert( !IN_SPECIAL_PARSE ); + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + sqlite3OomFault(db); + goto exit_create_index; + } + db->mDbFlags |= DBFLAG_SchemaChange; + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; + } } - /* Add an entry in sqlite_master for this index + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_master table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_master + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. + ** + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. */ - sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zDbSName, MASTER_NAME, - pIndex->zName, - pTab->zName, - iMem, - zStmt - ); - sqlite3DbFree(db, zStmt); + else if( HasRowid(pTab) || pTblName!=0 ){ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; - /* Fill the index with data and reparse the schema. Code an OP_Expire - ** to invalidate all pre-compiled statements. - */ - if( pTblName ){ - sqlite3RefillIndex(pParse, pIndex, iMem); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp0(v, OP_Expire); + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + if( pStart ){ + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_master for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zDbSName, MASTER_NAME, + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 1); + } + + sqlite3VdbeJumpHere(v, pIndex->tnum); } - - sqlite3VdbeJumpHere(v, pIndex->tnum); } /* When adding an index to the list of indices for a table, make @@ -107385,10 +109115,15 @@ SQLITE_PRIVATE void sqlite3CreateIndex( } pIndex = 0; } + else if( IN_RENAME_OBJECT ){ + assert( pParse->pNewIndex==0 ); + pParse->pNewIndex = pIndex; + pIndex = 0; + } /* Clean up before exiting */ exit_create_index: - if( pIndex ) freeIndex(db, pIndex); + if( pIndex ) sqlite3FreeIndex(db, pIndex); sqlite3ExprDelete(db, pPIWhere); sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); @@ -107557,7 +109292,8 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( ** ** A new IdList is returned, or NULL if malloc() fails. */ -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ + sqlite3 *db = pParse->db; int i; if( pList==0 ){ pList = sqlite3DbMallocZero(db, sizeof(IdList) ); @@ -107575,6 +109311,9 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pT return 0; } pList->a[i].zName = sqlite3NameFromToken(db, pToken); + if( IN_RENAME_OBJECT && pList->a[i].zName ){ + sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken); + } return pList; } @@ -107821,6 +109560,12 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( } assert( p->nSrc>0 ); pItem = &p->a[p->nSrc-1]; + assert( (pTable==0)==(pDatabase==0) ); + assert( pItem->zName==0 || pDatabase!=0 ); + if( IN_RENAME_OBJECT && pItem->zName ){ + Token *pToken = (ALWAYS(pDatabase) && pDatabase->z) ? pDatabase : pTable; + sqlite3RenameTokenMap(pParse, pItem->zName, pToken); + } assert( pAlias!=0 ); if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); @@ -109374,9 +111119,8 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( } iKey = iPk; }else{ - iKey = pParse->nMem + 1; - iKey = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iTabCur, iKey, 0); - if( iKey>pParse->nMem ) pParse->nMem = iKey; + iKey = ++pParse->nMem; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, -1, iKey); } if( eOnePass!=ONEPASS_OFF ){ @@ -109809,7 +111553,6 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( if( pIdx->pPartIdxWhere ){ *piPartIdxLabel = sqlite3VdbeMakeLabel(v); pParse->iSelfTab = iDataCur + 1; - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); pParse->iSelfTab = 0; @@ -109856,7 +111599,6 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ if( iLabel ){ sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); - sqlite3ExprCachePop(pParse); } } @@ -111369,7 +113111,7 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ i64 v = sqlite3_value_int64(argv[0]); p->rSum += v; if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ - p->overflow = 1; + p->approx = p->overflow = 1; } }else{ p->rSum += sqlite3_value_double(argv[0]); @@ -111377,6 +113119,32 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ } } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + /* p is always non-NULL because sumStep() will have been called first + ** to initialize it */ + if( ALWAYS(p) && type!=SQLITE_NULL ){ + assert( p->cnt>0 ); + p->cnt--; + assert( type==SQLITE_INTEGER || p->approx ); + if( type==SQLITE_INTEGER && p->approx==0 ){ + i64 v = sqlite3_value_int64(argv[0]); + p->rSum -= v; + p->iSum -= v; + }else{ + p->rSum -= sqlite3_value_double(argv[0]); + } + } +} +#else +# define sumInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ static void sumFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); @@ -111411,6 +113179,9 @@ static void totalFinalize(sqlite3_context *context){ typedef struct CountCtx CountCtx; struct CountCtx { i64 n; +#ifdef SQLITE_DEBUG + int bInverse; /* True if xInverse() ever called */ +#endif }; /* @@ -111428,7 +113199,7 @@ static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ ** sure it still operates correctly, verify that its count agrees with our ** internal count when using count(*) and when the total count can be ** expressed as a 32-bit integer. */ - assert( argc==1 || p==0 || p->n>0x7fffffff + assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse || p->n==sqlite3_aggregate_count(context) ); #endif } @@ -111437,6 +113208,21 @@ static void countFinalize(sqlite3_context *context){ p = sqlite3_aggregate_context(context, 0); sqlite3_result_int64(context, p ? p->n : 0); } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(ctx, sizeof(*p)); + /* p is always non-NULL since countStep() will have been called first */ + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && ALWAYS(p) ){ + p->n--; +#ifdef SQLITE_DEBUG + p->bInverse = 1; +#endif + } +} +#else +# define countInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** Routines to implement min() and max() aggregate functions. @@ -111453,7 +113239,7 @@ static void minmaxStep( pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); if( !pBest ) return; - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + if( sqlite3_value_type(pArg)==SQLITE_NULL ){ if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); }else if( pBest->flags ){ int max; @@ -111479,16 +113265,26 @@ static void minmaxStep( sqlite3VdbeMemCopy(pBest, pArg); } } -static void minMaxFinalize(sqlite3_context *context){ +static void minMaxValueFinalize(sqlite3_context *context, int bValue){ sqlite3_value *pRes; pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); if( pRes ){ if( pRes->flags ){ sqlite3_result_value(context, pRes); } - sqlite3VdbeMemRelease(pRes); + if( bValue==0 ) sqlite3VdbeMemRelease(pRes); } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void minMaxValue(sqlite3_context *context){ + minMaxValueFinalize(context, 1); +} +#else +# define minMaxValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void minMaxFinalize(sqlite3_context *context){ + minMaxValueFinalize(context, 0); +} /* ** group_concat(EXPR, ?SEPARATOR?) @@ -111525,6 +113321,38 @@ static void groupConcatStep( if( zVal ) sqlite3_str_append(pAccum, zVal, nVal); } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n; + StrAccum *pAccum; + assert( argc==1 || argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); + /* pAccum is always non-NULL since groupConcatStep() will have always + ** run frist to initialize it */ + if( ALWAYS(pAccum) ){ + n = sqlite3_value_bytes(argv[0]); + if( argc==2 ){ + n += sqlite3_value_bytes(argv[1]); + }else{ + n++; + } + if( n>=(int)pAccum->nChar ){ + pAccum->nChar = 0; + }else{ + pAccum->nChar -= n; + memmove(pAccum->zText, &pAccum->zText[n], pAccum->nChar); + } + if( pAccum->nChar==0 ) pAccum->mxAlloc = 0; + } +} +#else +# define groupConcatInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ static void groupConcatFinalize(sqlite3_context *context){ StrAccum *pAccum; pAccum = sqlite3_aggregate_context(context, 0); @@ -111539,6 +113367,24 @@ static void groupConcatFinalize(sqlite3_context *context){ } } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatValue(sqlite3_context *context){ + sqlite3_str *pAccum; + pAccum = (sqlite3_str*)sqlite3_aggregate_context(context, 0); + if( pAccum ){ + if( pAccum->accError==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + const char *zText = sqlite3_str_value(pAccum); + sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); + } + } +} +#else +# define groupConcatValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** This routine does per-connection function registration. Most @@ -111576,10 +113422,10 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) }else{ pInfo = (struct compareInfo*)&likeInfoNorm; } - sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); + sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); + sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0, 0, 0); sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, - (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); + (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0, 0, 0); setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); setLikeOptFlag(db, "like", caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); @@ -111688,11 +113534,11 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(trim, 2, 3, 0, trimFunc ), FUNCTION(min, -1, 0, 1, minmaxFunc ), FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE2(min, 1, 0, 1, minmaxStep, minMaxFinalize, + WAGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, SQLITE_FUNC_MINMAX ), FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE2(max, 1, 1, 1, minmaxStep, minMaxFinalize, + WAGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize, minMaxValue, 0, SQLITE_FUNC_MINMAX ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), @@ -111723,14 +113569,17 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ FUNCTION(zeroblob, 1, 0, 0, zeroblobFunc ), FUNCTION(substr, 2, 0, 0, substrFunc ), FUNCTION(substr, 3, 0, 0, substrFunc ), - AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), - AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), - AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - AGGREGATE2(count, 0, 0, 0, countStep, countFinalize, - SQLITE_FUNC_COUNT ), - AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), - AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), - AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), + WAGGREGATE(sum, 1,0,0, sumStep, sumFinalize, sumFinalize, sumInverse, 0), + WAGGREGATE(total, 1,0,0, sumStep,totalFinalize,totalFinalize,sumInverse, 0), + WAGGREGATE(avg, 1,0,0, sumStep, avgFinalize, avgFinalize, sumInverse, 0), + WAGGREGATE(count, 0,0,0, countStep, + countFinalize, countFinalize, countInverse, SQLITE_FUNC_COUNT ), + WAGGREGATE(count, 1,0,0, countStep, + countFinalize, countFinalize, countInverse, 0 ), + WAGGREGATE(group_concat, 1, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), + WAGGREGATE(group_concat, 2, 0, 0, groupConcatStep, + groupConcatFinalize, groupConcatValue, groupConcatInverse, 0), LIKEFUNC(glob, 2, &globInfo, SQLITE_FUNC_LIKE|SQLITE_FUNC_CASE), #ifdef SQLITE_CASE_SENSITIVE_LIKE @@ -111750,6 +113599,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(); #endif + sqlite3WindowFunctions(); #if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) sqlite3AnalyzeFunctions(); #endif @@ -112487,11 +114337,12 @@ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ */ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ sqlite3 *db = pParse->db; - if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) && !pTab->pSelect ){ + if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) ){ int iSkip = 0; Vdbe *v = sqlite3GetVdbe(pParse); assert( v ); /* VDBE has already been allocated */ + assert( pTab->pSelect==0 ); /* Not a view */ if( sqlite3FkReferences(pTab)==0 ){ /* Search for a deferred foreign key constraint for which this table ** is the child table. If one cannot be found, return without @@ -114386,44 +116237,6 @@ static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ return !w.eCode; } -/* -** An instance of the ConstraintAddr object remembers the byte-code addresses -** for sections of the constraint checks that deal with uniqueness constraints -** on the rowid and on the upsert constraint. -** -** This information is passed into checkReorderConstraintChecks() to insert -** some OP_Goto operations so that the rowid and upsert constraints occur -** in the correct order relative to other constraints. -*/ -typedef struct ConstraintAddr ConstraintAddr; -struct ConstraintAddr { - int ipkTop; /* Subroutine for rowid constraint check */ - int upsertTop; /* Label for upsert constraint check subroutine */ - int upsertTop2; /* Copy of upsertTop not cleared by the call */ - int upsertBtm; /* upsert constraint returns to this label */ - int ipkBtm; /* Return opcode rowid constraint check */ -}; - -/* -** Generate any OP_Goto operations needed to cause constraints to be -** run that haven't already been run. -*/ -static void reorderConstraintChecks(Vdbe *v, ConstraintAddr *p){ - if( p->upsertTop ){ - testcase( sqlite3VdbeLabelHasBeenResolved(v, p->upsertTop) ); - sqlite3VdbeGoto(v, p->upsertTop); - VdbeComment((v, "call upsert subroutine")); - sqlite3VdbeResolveLabel(v, p->upsertBtm); - p->upsertTop = 0; - } - if( p->ipkTop ){ - sqlite3VdbeGoto(v, p->ipkTop); - VdbeComment((v, "call rowid unique-check subroutine")); - sqlite3VdbeJumpHere(v, p->ipkBtm); - p->ipkTop = 0; - } -} - /* ** Generate code to do constraint checks prior to an INSERT or an UPDATE ** on table pTab. @@ -114533,11 +116346,13 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ - ConstraintAddr sAddr;/* Address information for constraint reordering */ Index *pUpIdx = 0; /* Index to which to apply the upsert */ u8 isUpdate; /* True if this is an UPDATE operation */ u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ int upsertBypass = 0; /* Address of Goto to bypass upsert subroutine */ + int upsertJump = 0; /* Address of Goto that jumps into upsert subroutine */ + int ipkTop = 0; /* Top of the IPK uniqueness check */ + int ipkBottom = 0; /* OP_Goto at the end of the IPK uniqueness check */ isUpdate = regOldData!=0; db = pParse->db; @@ -114545,7 +116360,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( assert( v!=0 ); assert( pTab->pSelect==0 ); /* This table is not a VIEW */ nCol = pTab->nCol; - memset(&sAddr, 0, sizeof(sAddr)); /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for ** normal rowid tables. nPkField is the number of key fields in the @@ -114649,8 +116463,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* UNIQUE and PRIMARY KEY constraints should be handled in the following ** order: ** - ** (1) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore - ** (2) OE_Update + ** (1) OE_Update + ** (2) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore ** (3) OE_Replace ** ** OE_Fail and OE_Ignore must happen before any changes are made. @@ -114659,6 +116473,11 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** could happen in any order, but they are grouped up front for ** convenience. ** + ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** The order of constraints used to have OE_Update as (2) and OE_Abort + ** and so forth as (1). But apparently PostgreSQL checks the OE_Update + ** constraint before any others, so it had to be moved. + ** ** Constraint checking code is generated in this order: ** (A) The rowid constraint ** (B) Unique index constraints that do not have OE_Replace as their @@ -114678,11 +116497,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( overrideError = OE_Ignore; pUpsert = 0; }else if( (pUpIdx = pUpsert->pUpsertIdx)!=0 ){ - /* If the constraint-target is on some column other than - ** then ROWID, then we might need to move the UPSERT around - ** so that it occurs in the correct order. */ - sAddr.upsertTop = sAddr.upsertTop2 = sqlite3VdbeMakeLabel(v); - sAddr.upsertBtm = sqlite3VdbeMakeLabel(v); + /* If the constraint-target uniqueness check must be run first. + ** Jump to that uniqueness check now */ + upsertJump = sqlite3VdbeAddOp0(v, OP_Goto); + VdbeComment((v, "UPSERT constraint goes first")); } } @@ -114714,16 +116532,12 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** to defer the running of the rowid conflict checking until after ** the UNIQUE constraints have run. */ - assert( OE_Update>OE_Replace ); - assert( OE_Ignore=OE_Replace - && (pUpsert || onError!=overrideError) - && pTab->pIndex + if( onError==OE_Replace /* IPK rule is REPLACE */ + && onError!=overrideError /* Rules for other contraints are different */ + && pTab->pIndex /* There exist other constraints */ ){ - sAddr.ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + VdbeComment((v, "defer IPK REPLACE until last")); } if( isUpdate ){ @@ -114818,9 +116632,9 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } } sqlite3VdbeResolveLabel(v, addrRowidOk); - if( sAddr.ipkTop ){ - sAddr.ipkBtm = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, sAddr.ipkTop-1); + if( ipkTop ){ + ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeJumpHere(v, ipkTop-1); } } @@ -114839,18 +116653,18 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ if( pUpIdx==pIdx ){ - addrUniqueOk = sAddr.upsertBtm; + addrUniqueOk = upsertJump+1; upsertBypass = sqlite3VdbeGoto(v, 0); VdbeComment((v, "Skip upsert subroutine")); - sqlite3VdbeResolveLabel(v, sAddr.upsertTop2); + sqlite3VdbeJumpHere(v, upsertJump); }else{ addrUniqueOk = sqlite3VdbeMakeLabel(v); } - VdbeNoopComment((v, "uniqueness check for %s", pIdx->zName)); - if( bAffinityDone==0 ){ + if( bAffinityDone==0 && (pUpIdx==0 || pUpIdx==pIdx) ){ sqlite3TableAffinity(v, pTab, regNewData+1); bAffinityDone = 1; } + VdbeNoopComment((v, "uniqueness check for %s", pIdx->zName)); iThisCur = iIdxCur+ix; @@ -114921,15 +116735,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } } - /* Invoke subroutines to handle IPK replace and upsert prior to running - ** the first REPLACE constraint check. */ - if( onError==OE_Replace ){ - testcase( sAddr.ipkTop ); - testcase( sAddr.upsertTop - && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) ); - reorderConstraintChecks(v, &sAddr); - } - /* Collision detection may be omitted if all of the following are true: ** (1) The conflict resolution algorithm is REPLACE ** (2) The table is a WITHOUT ROWID table @@ -114950,7 +116755,6 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } /* Check to see if the new index entry will be unique */ - sqlite3ExprCachePush(pParse); sqlite3VdbeVerifyAbortable(v, onError); sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, regIdx, pIdx->nKeyCol); VdbeCoverage(v); @@ -115052,19 +116856,21 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( } } if( pUpIdx==pIdx ){ + sqlite3VdbeGoto(v, upsertJump+1); sqlite3VdbeJumpHere(v, upsertBypass); }else{ sqlite3VdbeResolveLabel(v, addrUniqueOk); } - sqlite3ExprCachePop(pParse); if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); - } - testcase( sAddr.ipkTop!=0 ); - testcase( sAddr.upsertTop - && sqlite3VdbeLabelHasBeenResolved(v,sAddr.upsertTop) ); - reorderConstraintChecks(v, &sAddr); - + + /* If the IPK constraint is a REPLACE, run it last */ + if( ipkTop ){ + sqlite3VdbeGoto(v, ipkTop+1); + VdbeComment((v, "Do IPK REPLACE")); + sqlite3VdbeJumpHere(v, ipkBottom); + } + *pbMayReplace = seenReplace; VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } @@ -115160,7 +116966,6 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( sqlite3SetMakeRecordP5(v, pTab); if( !bAffinityDone ){ sqlite3TableAffinity(v, pTab, 0); - sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); } if( pParse->nested ){ pik_flags = 0; @@ -116136,6 +117941,12 @@ struct sqlite3_api_routines { int (*str_errcode)(sqlite3_str*); int (*str_length)(sqlite3_str*); char *(*str_value)(sqlite3_str*); + int (*create_window_function)(sqlite3*,const char*,int,int,void*, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInv)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*)); }; /* @@ -116421,6 +118232,8 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_str_errcode sqlite3_api->str_errcode #define sqlite3_str_length sqlite3_api->str_length #define sqlite3_str_value sqlite3_api->str_value +/* Version 3.25.0 and later */ +#define sqlite3_create_window_function sqlite3_api->create_window_function #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) @@ -116874,7 +118687,9 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_str_reset, sqlite3_str_errcode, sqlite3_str_length, - sqlite3_str_value + sqlite3_str_value, + /* Version 3.25.0 and later */ + sqlite3_create_window_function }; /* @@ -117669,6 +119484,11 @@ static const PragmaName aPragmaName[] = { /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "legacy_alter_table", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_LegacyAlter }, {/* zName: */ "legacy_file_format", /* ePragTyp: */ PragTyp_FLAG, /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, @@ -117922,7 +119742,7 @@ static const PragmaName aPragmaName[] = { /* iArg: */ SQLITE_WriteSchema }, #endif }; -/* Number of pragmas: 60 on by default, 77 total. */ +/* Number of pragmas: 61 on by default, 78 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ @@ -119447,7 +121267,6 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pTab->tnum<1 ) continue; /* Skip VIEWs or VIRTUAL TABLEs */ pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); - sqlite3ExprCacheClear(pParse); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); /* reg[7] counts the number of entries in the table. @@ -119461,6 +121280,11 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + if( !isQuick ){ + /* Sanity check on record header decoding */ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + } /* Verify that all NOT NULL columns really are NOT NULL */ for(j=0; jnCol; j++){ char *zErr; @@ -119485,7 +121309,6 @@ SQLITE_PRIVATE void sqlite3Pragma( char *zErr; int k; pParse->iSelfTab = iDataCur + 1; - sqlite3ExprCachePush(pParse); for(k=pCheck->nExpr-1; k>0; k--){ sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); } @@ -119498,14 +121321,10 @@ SQLITE_PRIVATE void sqlite3Pragma( sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); integrityCheckResultRow(v); sqlite3VdbeResolveLabel(v, addrCkOk); - sqlite3ExprCachePop(pParse); } sqlite3ExprListDelete(db, pCheck); } if( !isQuick ){ /* Omit the remaining tests for quick_check */ - /* Sanity check on record header decoding */ - sqlite3VdbeAddOp3(v, OP_Column, iDataCur, pTab->nCol-1, 3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4, jmp5; @@ -120120,7 +121939,6 @@ static int pragmaVtabConnect( } if( i==0 ){ sqlite3_str_appendf(&acc, "(\"%s\"", pPragma->zName); - cSep = ','; i++; } j = 0; @@ -120413,15 +122231,23 @@ static void corruptSchema( const char *zExtra /* Error information */ ){ sqlite3 *db = pData->db; - if( !db->mallocFailed && (db->flags & SQLITE_WriteSchema)==0 ){ + if( db->mallocFailed ){ + pData->rc = SQLITE_NOMEM_BKPT; + }else if( pData->pzErrMsg[0]!=0 ){ + /* A error message has already been generated. Do not overwrite it */ + }else if( pData->mInitFlags & INITFLAG_AlterTable ){ + *pData->pzErrMsg = sqlite3DbStrDup(db, zExtra); + pData->rc = SQLITE_ERROR; + }else if( db->flags & SQLITE_WriteSchema ){ + pData->rc = SQLITE_CORRUPT_BKPT; + }else{ char *z; if( zObj==0 ) zObj = "?"; z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); if( zExtra && zExtra[0] ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); - sqlite3DbFree(db, *pData->pzErrMsg); *pData->pzErrMsg = z; + pData->rc = SQLITE_CORRUPT_BKPT; } - pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; } /* @@ -120473,7 +122299,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); db->init.iDb = saved_iDb; - assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); + /* assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); */ if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); @@ -120520,7 +122346,7 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** auxiliary databases. Return one of the SQLITE_ error codes to ** indicate success or failure. */ -static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ +SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ int rc; int i; #ifndef SQLITE_OMIT_DEPRECATED @@ -120555,6 +122381,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; + initData.mInitFlags = mFlags; sqlite3InitCallback(&initData, 3, (char **)azArg, 0); if( initData.rc ){ rc = initData.rc; @@ -120576,7 +122403,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** will be closed before this function returns. */ sqlite3BtreeEnter(pDb->pBt); if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ - rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0, 0); if( rc!=SQLITE_OK ){ sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); goto initone_error_out; @@ -120761,14 +122588,14 @@ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ assert( db->nDb>0 ); /* Do the main schema first */ if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, 0, pzErrMsg); + rc = sqlite3InitOne(db, 0, pzErrMsg, 0); if( rc ) return rc; } /* All other schemas after the main schema. The "temp" schema must be last */ for(i=db->nDb-1; i>0; i--){ assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) ); if( !DbHasProperty(db, i, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, i, pzErrMsg); + rc = sqlite3InitOne(db, i, pzErrMsg, 0); if( rc ) return rc; } } @@ -120821,7 +122648,7 @@ static void schemaIsValid(Parse *pParse){ ** on the b-tree database, open one now. If a transaction is opened, it ** will be closed immediately after reading the meta-value. */ if( !sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ sqlite3OomFault(db); } @@ -121318,7 +123145,7 @@ SQLITE_API int sqlite3_prepare16_v3( /***/ int sqlite3SelectTrace = 0; # define SELECTTRACE(K,P,S,X) \ if(sqlite3SelectTrace&(K)) \ - sqlite3DebugPrintf("%s/%d/%p: ",(S)->zSelName,(P)->addrExplain,(S)),\ + sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ sqlite3DebugPrintf X #else # define SELECTTRACE(K,P,S,X) @@ -121365,8 +123192,8 @@ struct SortCtx { int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ int labelDone; /* Jump here when done, ex: LIMIT reached */ + int labelOBLopt; /* Jump here when sorter is full */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ - u8 bOrderedInnerLoop; /* ORDER BY correctly sorts the inner loop */ #ifdef SQLITE_ENABLE_SORTER_REFERENCES u8 nDefer; /* Number of valid entries in aDefer[] */ struct DeferredCsr { @@ -121393,6 +123220,11 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprDelete(db, p->pHaving); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ + sqlite3WindowListDelete(db, p->pWinDefn); + } +#endif if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); if( bFree ) sqlite3DbFreeNN(db, p); p = pPrior; @@ -121443,9 +123275,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; -#if SELECTTRACE_ENABLED - pNew->zSelName[0] = 0; -#endif + pNew->selId = ++pParse->nSelect; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; @@ -121459,6 +123289,10 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->pNext = 0; pNew->pLimit = pLimit; pNew->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = 0; +#endif if( pParse->db->mallocFailed ) { clearSelect(pParse->db, pNew, pNew!=&standin); pNew = 0; @@ -121469,17 +123303,6 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( return pNew; } -#if SELECTTRACE_ENABLED -/* -** Set the name of a Select object -*/ -SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ - if( p && zName ){ - sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); - } -} -#endif - /* ** Delete the given Select structure and all of its substructures. @@ -121826,14 +123649,6 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ return 0; } -/* Forward reference */ -static KeyInfo *keyInfoFromExprList( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* Form the KeyInfo object from this ExprList */ - int iStart, /* Begin with this column of pList */ - int nExtra /* Add this many extra columns to the end */ -); - /* ** An instance of this object holds information (beyond pParse and pSelect) ** needed to load the next result row that is to be added to the sorter. @@ -121975,7 +123790,7 @@ static void pushOntoSorter( memset(pKI->aSortOrder, 0, pKI->nKeyField); /* Makes OP_Jump testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); testcase( pKI->nAllField > pKI->nKeyField+2 ); - pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, + pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, pKI->nAllField-pKI->nKeyField-1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); @@ -122002,10 +123817,10 @@ static void pushOntoSorter( ** than LIMIT+OFFSET items in the sorter. ** ** If the new record does not need to be inserted into the sorter, - ** jump to the next iteration of the loop. Or, if the - ** pSort->bOrderedInnerLoop flag is set to indicate that the inner - ** loop delivers items in sorted order, jump to the next iteration - ** of the outer loop. + ** jump to the next iteration of the loop. If the pSort->labelOBLopt + ** value is not zero, then it is a label of where to jump. Otherwise, + ** just bypass the row insert logic. See the header comment on the + ** sqlite3WhereOrderByLimitOptLabel() function for additional info. */ int iCsr = pSort->iECursor; sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4); @@ -122027,9 +123842,8 @@ static void pushOntoSorter( sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, regBase+nOBSat, nBase-nOBSat); if( iSkip ){ - assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 ); sqlite3VdbeChangeP2(v, iSkip, - sqlite3VdbeCurrentAddr(v) + pSort->bOrderedInnerLoop); + pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } } @@ -122458,7 +124272,6 @@ static void selectInnerLoop( assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, r1, pDest->zAffSdst, nResultCol); - sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); sqlite3ReleaseTempReg(pParse, r1); } @@ -122502,7 +124315,6 @@ static void selectInnerLoop( sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); - sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } @@ -122645,7 +124457,7 @@ SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList( +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* Form the KeyInfo object from this ExprList */ int iStart, /* Begin with this column of pList */ @@ -122859,7 +124671,6 @@ static void generateSortTail( assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, pDest->zAffSdst, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regRow, nColumn); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn); break; } @@ -122874,7 +124685,6 @@ static void generateSortTail( testcase( eDest==SRT_Coroutine ); if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); - sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); }else{ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); } @@ -123475,7 +125285,6 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ ** The current implementation interprets "LIMIT 0" to mean ** no rows. */ - sqlite3ExprCacheClear(pParse); if( pLimit ){ assert( pLimit->op==TK_LIMIT ); assert( pLimit->pLeft!=0 ); @@ -124261,7 +126070,6 @@ static int generateOutputSubroutine( r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, r1, pDest->zAffSdst, pIn->nSdst); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, pIn->iSdst, pIn->nSdst); sqlite3ReleaseTempReg(pParse, r1); @@ -124304,7 +126112,6 @@ static int generateOutputSubroutine( default: { assert( pDest->eDest==SRT_Output ); sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); break; } } @@ -124759,7 +126566,7 @@ static Expr *substExpr( Expr *pCopy = pSubst->pEList->a[pExpr->iColumn].pExpr; Expr ifNullRow; assert( pSubst->pEList!=0 && pExpr->iColumnpEList->nExpr ); - assert( pExpr->pLeft==0 && pExpr->pRight==0 ); + assert( pExpr->pRight==0 ); if( sqlite3ExprIsVector(pCopy) ){ sqlite3VectorErrorMsg(pSubst->pParse, pCopy); }else{ @@ -124973,6 +126780,10 @@ static void substSelect( ** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily ** return the value X for which Y was maximal.) ** +** (25) If either the subquery or the parent query contains a window +** function in the select list or ORDER BY clause, flattening +** is not attempted. +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -125016,6 +126827,10 @@ static int flattenSubquery( pSub = pSubitem->pSelect; assert( pSub!=0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */ +#endif + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, @@ -125126,8 +126941,8 @@ static int flattenSubquery( assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 ); /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", - pSub->zSelName, pSub, iFrom)); + SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n", + pSub->selId, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; @@ -125178,7 +126993,6 @@ static int flattenSubquery( p->pPrior = 0; p->pLimit = 0; pNew = sqlite3SelectDup(db, p, 0); - sqlite3SelectSetName(pNew, pSub->zSelName); p->pLimit = pLimit; p->pOrderBy = pOrderBy; p->pSrc = pSrc; @@ -125191,7 +127005,7 @@ static int flattenSubquery( pNew->pNext = p; p->pPrior = pNew; SELECTTRACE(2,pParse,p,("compound-subquery flattener" - " creates %s.%p as peer\n",pNew->zSelName, pNew)); + " creates %u as peer\n",pNew->selId)); } if( db->mallocFailed ) return 1; } @@ -125376,7 +127190,168 @@ static int flattenSubquery( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** A structure to keep track of all of the column values that fixed to +** a known value due to WHERE clause constraints of the form COLUMN=VALUE. +*/ +typedef struct WhereConst WhereConst; +struct WhereConst { + Parse *pParse; /* Parsing context */ + int nConst; /* Number for COLUMN=CONSTANT terms */ + int nChng; /* Number of times a constant is propagated */ + Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */ +}; +/* +** Add a new entry to the pConst object +*/ +static void constInsert( + WhereConst *pConst, + Expr *pColumn, + Expr *pValue +){ + + pConst->nConst++; + pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr, + pConst->nConst*2*sizeof(Expr*)); + if( pConst->apExpr==0 ){ + pConst->nConst = 0; + }else{ + if( ExprHasProperty(pValue, EP_FixedCol) ) pValue = pValue->pLeft; + pConst->apExpr[pConst->nConst*2-2] = pColumn; + pConst->apExpr[pConst->nConst*2-1] = pValue; + } +} + +/* +** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE +** is a constant expression and where the term must be true because it +** is part of the AND-connected terms of the expression. For each term +** found, add it to the pConst structure. +*/ +static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ + Expr *pRight, *pLeft; + if( pExpr==0 ) return; + if( ExprHasProperty(pExpr, EP_FromJoin) ) return; + if( pExpr->op==TK_AND ){ + findConstInWhere(pConst, pExpr->pRight); + findConstInWhere(pConst, pExpr->pLeft); + return; + } + if( pExpr->op!=TK_EQ ) return; + pRight = pExpr->pRight; + pLeft = pExpr->pLeft; + assert( pRight!=0 ); + assert( pLeft!=0 ); + if( pRight->op==TK_COLUMN + && !ExprHasProperty(pRight, EP_FixedCol) + && sqlite3ExprIsConstant(pLeft) + && sqlite3IsBinary(sqlite3BinaryCompareCollSeq(pConst->pParse,pLeft,pRight)) + ){ + constInsert(pConst, pRight, pLeft); + }else + if( pLeft->op==TK_COLUMN + && !ExprHasProperty(pLeft, EP_FixedCol) + && sqlite3ExprIsConstant(pRight) + && sqlite3IsBinary(sqlite3BinaryCompareCollSeq(pConst->pParse,pLeft,pRight)) + ){ + constInsert(pConst, pLeft, pRight); + } +} + +/* +** This is a Walker expression callback. pExpr is a candidate expression +** to be replaced by a value. If pExpr is equivalent to one of the +** columns named in pWalker->u.pConst, then overwrite it with its +** corresponding value. +*/ +static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ + int i; + WhereConst *pConst; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + if( ExprHasProperty(pExpr, EP_FixedCol) ) return WRC_Continue; + pConst = pWalker->u.pConst; + for(i=0; inConst; i++){ + Expr *pColumn = pConst->apExpr[i*2]; + if( pColumn==pExpr ) continue; + if( pColumn->iTable!=pExpr->iTable ) continue; + if( pColumn->iColumn!=pExpr->iColumn ) continue; + /* A match is found. Add the EP_FixedCol property */ + pConst->nChng++; + ExprClearProperty(pExpr, EP_Leaf); + ExprSetProperty(pExpr, EP_FixedCol); + assert( pExpr->pLeft==0 ); + pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0); + break; + } + return WRC_Prune; +} + +/* +** The WHERE-clause constant propagation optimization. +** +** If the WHERE clause contains terms of the form COLUMN=CONSTANT or +** CONSTANT=COLUMN that must be tree (in other words, if the terms top-level +** AND-connected terms that are not part of a ON clause from a LEFT JOIN) +** then throughout the query replace all other occurrences of COLUMN +** with CONSTANT within the WHERE clause. +** +** For example, the query: +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b +** +** Is transformed into +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39 +** +** Return true if any transformations where made and false if not. +** +** Implementation note: Constant propagation is tricky due to affinity +** and collating sequence interactions. Consider this example: +** +** CREATE TABLE t1(a INT,b TEXT); +** INSERT INTO t1 VALUES(123,'0123'); +** SELECT * FROM t1 WHERE a=123 AND b=a; +** SELECT * FROM t1 WHERE a=123 AND b=123; +** +** The two SELECT statements above should return different answers. b=a +** is alway true because the comparison uses numeric affinity, but b=123 +** is false because it uses text affinity and '0123' is not the same as '123'. +** To work around this, the expression tree is not actually changed from +** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol +** and the "123" value is hung off of the pLeft pointer. Code generator +** routines know to generate the constant "123" instead of looking up the +** column value. Also, to avoid collation problems, this optimization is +** only attempted if the "a=123" term uses the default BINARY collation. +*/ +static int propagateConstants( + Parse *pParse, /* The parsing context */ + Select *p /* The query in which to propagate constants */ +){ + WhereConst x; + Walker w; + int nChng = 0; + x.pParse = pParse; + do{ + x.nConst = 0; + x.nChng = 0; + x.apExpr = 0; + findConstInWhere(&x, p->pWhere); + if( x.nConst ){ + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = propagateConstantExprRewrite; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.u.pConst = &x; + sqlite3WalkExpr(&w, p->pWhere); + sqlite3DbFree(x.pParse->db, x.apExpr); + nChng += x.nChng; + } + }while( x.nChng ); + return nChng; +} #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* @@ -125406,7 +127381,7 @@ static int flattenSubquery( ** (2) The inner query is the recursive part of a common table expression. ** ** (3) The inner query has a LIMIT clause (since the changes to the WHERE -** close would change the meaning of the LIMIT). +** clause would change the meaning of the LIMIT). ** ** (4) The inner query is the right operand of a LEFT JOIN and the ** expression to be pushed down does not come from the ON clause @@ -125425,6 +127400,10 @@ static int flattenSubquery( ** But if the (b2=2) term were to be pushed down into the bb subquery, ** then the (1,1,NULL) row would be suppressed. ** +** (6) The inner query features one or more window-functions (since +** changes to the WHERE clause of the inner query could change the +** window over which window functions are calculated). +** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ @@ -125440,6 +127419,10 @@ static int pushDownWhereTerms( if( pWhere==0 ) return 0; if( pSubq->selFlags & SF_Recursive ) return 0; /* restriction (2) */ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pWin ) return 0; /* restriction (6) */ +#endif + #ifdef SQLITE_DEBUG /* Only the first term of a compound can have a WITH clause. But make ** sure no other terms are marked SF_Recursive in case something changes @@ -125885,6 +127868,35 @@ static void selectPopWith(Walker *pWalker, Select *p){ #define selectPopWith 0 #endif +/* +** The SrcList_item structure passed as the second argument represents a +** sub-query in the FROM clause of a SELECT statement. This function +** allocates and populates the SrcList_item.pTab object. If successful, +** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, +** SQLITE_NOMEM. +*/ +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, struct SrcList_item *pFrom){ + Select *pSel = pFrom->pSelect; + Table *pTab; + + assert( pSel ); + pFrom->pTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + if( pTab==0 ) return SQLITE_NOMEM; + pTab->nTabRef = 1; + if( pFrom->zAlias ){ + pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias); + }else{ + pTab->zName = sqlite3MPrintf(pParse->db, "subquery_%u", pSel->selId); + } + while( pSel->pPrior ){ pSel = pSel->pPrior; } + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); + pTab->iPKey = -1; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); + pTab->tabFlags |= TF_Ephemeral; + + return SQLITE_OK; +} + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: @@ -125957,19 +127969,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pSel!=0 ); assert( pFrom->pTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; - pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTab==0 ) return WRC_Abort; - pTab->nTabRef = 1; - if( pFrom->zAlias ){ - pTab->zName = sqlite3DbStrDup(db, pFrom->zAlias); - }else{ - pTab->zName = sqlite3MPrintf(db, "subquery_%p", (void*)pTab); - } - while( pSel->pPrior ){ pSel = pSel->pPrior; } - sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); - pTab->iPKey = -1; - pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral; + if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; #endif }else{ /* An ordinary table or view name in the FROM clause */ @@ -125992,7 +127992,6 @@ static int selectExpander(Walker *pWalker, Select *p){ if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); - sqlite3SelectSetName(pFrom->pSelect, pTab->zName); nCol = pTab->nCol; pTab->nCol = -1; sqlite3WalkSelect(pWalker, pFrom->pSelect); @@ -126270,7 +128269,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ struct SrcList_item *pFrom; assert( p->selFlags & SF_Resolved ); - assert( (p->selFlags & SF_HasTypeInfo)==0 ); + if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; pTabList = p->pSrc; @@ -126373,7 +128372,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -126397,11 +128396,17 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ } } + /* ** Update the accumulator memory cells for an aggregate based on ** the current cursor position. +** +** If regAcc is non-zero and there are no min() or max() aggregates +** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator +** registers i register regAcc contains 0. The caller will take care +** of setting and clearing regAcc. */ -static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ +static void updateAccumulator(Parse *pParse, int regAcc, AggInfo *pAggInfo){ Vdbe *v = pParse->pVdbe; int i; int regHit = 0; @@ -126444,36 +128449,24 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp3(v, OP_AggStep0, 0, regAgg, pF->iMem); + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, pF->iMem); sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); - sqlite3ExprCacheClear(pParse); } } - - /* Before populating the accumulator registers, clear the column cache. - ** Otherwise, if any of the required column values are already present - ** in registers, sqlite3ExprCode() may use OP_SCopy to copy the value - ** to pC->iMem. But by the time the value is used, the original register - ** may have been used, invalidating the underlying buffer holding the - ** text or blob value. See ticket [883034dcb5]. - ** - ** Another solution would be to change the OP_SCopy used to copy cached - ** values to an OP_Copy. - */ + if( regHit==0 && pAggInfo->nAccumulator ){ + regHit = regAcc; + } if( regHit ){ addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } - sqlite3ExprCacheClear(pParse); for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); } pAggInfo->directMode = 0; - sqlite3ExprCacheClear(pParse); if( addrHitTest ){ sqlite3VdbeJumpHere(v, addrHitTest); } @@ -126603,6 +128596,7 @@ static struct SrcList_item *isSelfJoinView( ** The transformation only works if all of the following are true: ** ** * The subquery is a UNION ALL of two or more terms +** * The subquery does not have a LIMIT clause ** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries ** * The outer query is a simple count(*) ** @@ -126626,6 +128620,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){ do{ if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ if( pSub->pWhere ) return 0; /* No WHERE clause */ + if( pSub->pLimit ) return 0; /* No LIMIT clause */ if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ pSub = pSub->pPrior; /* Repeat over compound */ }while( pSub ); @@ -126738,14 +128733,10 @@ SQLITE_PRIVATE int sqlite3Select( p->selFlags &= ~SF_Distinct; } sqlite3SelectPrep(pParse, p, 0); - memset(&sSort, 0, sizeof(sSort)); - sSort.pOrderBy = p->pOrderBy; - pTabList = p->pSrc; if( pParse->nErr || db->mallocFailed ){ goto select_end; } assert( p->pEList!=0 ); - isAgg = (p->selFlags & SF_Aggregate)!=0; #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x104 ){ SELECTTRACE(0x104,pParse,p, ("after name resolution:\n")); @@ -126757,6 +128748,22 @@ SQLITE_PRIVATE int sqlite3Select( generateColumnNames(pParse, p); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( sqlite3WindowRewrite(pParse, p) ){ + goto select_end; + } +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x108 ){ + SELECTTRACE(0x104,pParse,p, ("after window rewrite:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +#endif /* SQLITE_OMIT_WINDOWFUNC */ + pTabList = p->pSrc; + isAgg = (p->selFlags & SF_Aggregate)!=0; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; + /* Try to various optimizations (flattening subqueries, and strength ** reduction of join operators) in the FROM clause up into the main query */ @@ -126856,6 +128863,35 @@ SQLITE_PRIVATE int sqlite3Select( } #endif + /* Do the WHERE-clause constant propagation optimization if this is + ** a join. No need to speed time on this operation for non-join queries + ** as the equivalent optimization will be handled by query planner in + ** sqlite3WhereBegin(). + */ + if( pTabList->nSrc>1 + && OptimizationEnabled(db, SQLITE_PropagateConst) + && propagateConstants(pParse, p) + ){ +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + SELECTTRACE(0x100,pParse,p,("After constant propagation:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + }else{ + SELECTTRACE(0x100,pParse,p,("Constant propagation not helpful\n")); + } + +#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION + if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) + && countOfViewOptimization(pParse, p) + ){ + if( db->mallocFailed ) goto select_end; + pEList = p->pEList; + pTabList = p->pSrc; + } +#endif + /* For each term in the FROM clause, do two things: ** (1) Authorized unreferenced tables ** (2) Generate code for all sub-queries @@ -126929,7 +128965,8 @@ SQLITE_PRIVATE int sqlite3Select( ){ #if SELECTTRACE_ENABLED if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); + SELECTTRACE(0x100,pParse,p, + ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -126963,7 +129000,7 @@ SQLITE_PRIVATE int sqlite3Select( VdbeComment((v, "%s", pItem->pTab->zName)); pItem->addrFillSub = addrTop; sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); - ExplainQueryPlan((pParse, 1, "CO-ROUTINE 0x%p", pSub)); + ExplainQueryPlan((pParse, 1, "CO-ROUTINE %u", pSub->selId)); sqlite3Select(pParse, pSub, &dest); pItem->pTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; @@ -127002,7 +129039,7 @@ SQLITE_PRIVATE int sqlite3Select( pSub->nSelectRow = pPrior->pSelect->nSelectRow; }else{ sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - ExplainQueryPlan((pParse, 1, "MATERIALIZE 0x%p", pSub)); + ExplainQueryPlan((pParse, 1, "MATERIALIZE %u", pSub->selId)); sqlite3Select(pParse, pSub, &dest); } pItem->pTab->nRowLogEst = pSub->nSelectRow; @@ -127033,16 +129070,6 @@ SQLITE_PRIVATE int sqlite3Select( } #endif -#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION - if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) - && countOfViewOptimization(pParse, p) - ){ - if( db->mallocFailed ) goto select_end; - pEList = p->pEList; - pTabList = p->pSrc; - } -#endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: @@ -127086,7 +129113,8 @@ SQLITE_PRIVATE int sqlite3Select( */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, pEList->nExpr); + pKeyInfo = sqlite3KeyInfoFromExprList( + pParse, sSort.pOrderBy, 0, pEList->nExpr); sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, @@ -127120,9 +129148,9 @@ SQLITE_PRIVATE int sqlite3Select( if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -127131,9 +129159,16 @@ SQLITE_PRIVATE int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ - u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) + | (p->selFlags & SF_FixedLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = p->pWin; /* Master window object (or NULL) */ + if( pWin ){ + sqlite3WindowCodeInit(pParse, pWin); + } +#endif assert( WHERE_USE_LIMIT==SF_FixedLimit ); - wctrlFlags |= p->selFlags & SF_FixedLimit; + /* Begin the database scan. */ SELECTTRACE(1,pParse,p,("WhereBegin\n")); @@ -127148,7 +129183,7 @@ SQLITE_PRIVATE int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); - sSort.bOrderedInnerLoop = sqlite3WhereOrderedInnerLoop(pWInfo); + sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } @@ -127162,15 +129197,37 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } - /* Use the standard inner loop. */ assert( p->pEList==pEList ); - selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, - sqlite3WhereContinueLabel(pWInfo), - sqlite3WhereBreakLabel(pWInfo)); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + int addrGosub = sqlite3VdbeMakeLabel(v); + int iCont = sqlite3VdbeMakeLabel(v); + int iBreak = sqlite3VdbeMakeLabel(v); + int regGosub = ++pParse->nMem; - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); + sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub); + + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeResolveLabel(v, addrGosub); + VdbeNoopComment((v, "inner-loop subroutine")); + sSort.labelOBLopt = 0; + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp1(v, OP_Return, regGosub); + VdbeComment((v, "end inner-loop subroutine")); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); + + /* End the database scan loop. + */ + sqlite3WhereEnd(pWInfo); + } }else{ /* This case when there exist aggregate functions or a GROUP BY clause ** or both */ @@ -127299,7 +129356,7 @@ SQLITE_PRIVATE int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, sAggInfo.nColumn); + pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pGroupBy,0,sAggInfo.nColumn); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -127318,8 +129375,6 @@ SQLITE_PRIVATE int sqlite3Select( pParse->nMem += pGroupBy->nExpr; sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); /* Begin a loop that will extract all source rows in GROUP BY order. @@ -127365,15 +129420,14 @@ SQLITE_PRIVATE int sqlite3Select( } } regBase = sqlite3GetTempRange(pParse, nCol); - sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; for(i=0; iiSorterColumn>=j ){ int r1 = j + regBase; - sqlite3ExprCodeGetColumnToReg(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + sqlite3ExprCodeGetColumnOfTable(v, + pCol->pTab, pCol->iTable, pCol->iColumn, r1); j++; } } @@ -127389,8 +129443,6 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); sAggInfo.useSortingIdx = 1; - sqlite3ExprCacheClear(pParse); - } /* If the index or temporary table used by the GROUP BY sort @@ -127413,7 +129465,6 @@ SQLITE_PRIVATE int sqlite3Select( ** from the previous row currently stored in a0, a1, a2... */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); - sqlite3ExprCacheClear(pParse); if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, sortOut, sortPTab); @@ -127452,7 +129503,7 @@ SQLITE_PRIVATE int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, &sAggInfo); + updateAccumulator(pParse, iUseFlag, &sAggInfo); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); @@ -127504,6 +129555,8 @@ SQLITE_PRIVATE int sqlite3Select( */ sqlite3VdbeResolveLabel(v, addrReset); resetAccumulator(pParse, &sAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ @@ -127569,6 +129622,23 @@ SQLITE_PRIVATE int sqlite3Select( }else #endif /* SQLITE_OMIT_BTREECOUNT */ { + int regAcc = 0; /* "populate accumulators" flag */ + + /* If there are accumulator registers but no min() or max() functions, + ** allocate register regAcc. Register regAcc will contain 0 the first + ** time the inner loop runs, and 1 thereafter. The code generated + ** by updateAccumulator() only updates the accumulator registers if + ** regAcc contains 0. */ + if( sAggInfo.nAccumulator ){ + for(i=0; ifuncFlags&SQLITE_FUNC_NEEDCOLL ) break; + } + if( i==sAggInfo.nFunc ){ + regAcc = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); + } + } + /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. @@ -127590,7 +129660,8 @@ SQLITE_PRIVATE int sqlite3Select( if( pWInfo==0 ){ goto select_end; } - updateAccumulator(pParse, &sAggInfo); + updateAccumulator(pParse, regAcc, &sAggInfo); + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", @@ -128034,14 +130105,16 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ - if( !noErr ){ - sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); - }else{ - assert( !db->init.busy ); - sqlite3CodeVerifySchema(pParse, iDb); + if( !IN_RENAME_OBJECT ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + } + goto trigger_cleanup; } - goto trigger_cleanup; } /* Do not create a trigger on a system table */ @@ -128065,7 +130138,7 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( } #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( !IN_RENAME_OBJECT ){ int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int code = SQLITE_CREATE_TRIGGER; const char *zDb = db->aDb[iTabDb].zDbSName; @@ -128099,8 +130172,15 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( pTrigger->pTabSchema = pTab->pSchema; pTrigger->op = (u8)op; pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; - pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); - pTrigger->pColumns = sqlite3IdListDup(db, pColumns); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pTrigger->table, pTableName->a[0].zName); + pTrigger->pWhen = pWhen; + pWhen = 0; + }else{ + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + pTrigger->pColumns = pColumns; + pColumns = 0; assert( pParse->pNewTrigger==0 ); pParse->pNewTrigger = pTrigger; @@ -128149,6 +130229,14 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( goto triggerfinish_cleanup; } +#ifndef SQLITE_OMIT_ALTERTABLE + if( IN_RENAME_OBJECT ){ + assert( !db->init.busy ); + pParse->pNewTrigger = pTrig; + pTrig = 0; + }else +#endif + /* if we are not initializing, ** build the sqlite_master entry */ @@ -128190,7 +130278,7 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( triggerfinish_cleanup: sqlite3DeleteTrigger(db, pTrig); - assert( !pParse->pNewTrigger ); + assert( IN_RENAME_OBJECT || !pParse->pNewTrigger ); sqlite3DeleteTriggerStep(db, pStepList); } @@ -128237,12 +130325,13 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep( ** If an OOM error occurs, NULL is returned and db->mallocFailed is set. */ static TriggerStep *triggerStepAllocate( - sqlite3 *db, /* Database connection */ + Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ Token *pName, /* The target name */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); @@ -128253,6 +130342,9 @@ static TriggerStep *triggerStepAllocate( pTriggerStep->zTarget = z; pTriggerStep->op = op; pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + } } return pTriggerStep; } @@ -128265,7 +130357,7 @@ static TriggerStep *triggerStepAllocate( ** body of a trigger. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( - sqlite3 *db, /* The database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ @@ -128274,13 +130366,19 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pSelect = pSelect; + pSelect = 0; + }else{ + pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } pTriggerStep->pIdList = pColumn; pTriggerStep->pUpsert = pUpsert; pTriggerStep->orconf = orconf; @@ -128301,7 +130399,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( ** sees an UPDATE statement inside the body of a CREATE TRIGGER. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( - sqlite3 *db, /* The database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table to be updated */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ @@ -128309,12 +130407,20 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pExprList = pEList; + pTriggerStep->pWhere = pWhere; + pEList = 0; + pWhere = 0; + }else{ + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + } pTriggerStep->orconf = orconf; } sqlite3ExprListDelete(db, pEList); @@ -128328,17 +130434,23 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( ** sees a DELETE statement inside the body of a CREATE TRIGGER. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( - sqlite3 *db, /* Database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* The table from which rows are deleted */ Expr *pWhere, /* The WHERE clause */ const char *zStart, /* Start of SQL text */ const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName, zStart, zEnd); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pWhere = pWhere; + pWhere = 0; + }else{ + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + } pTriggerStep->orconf = OE_Default; } sqlite3ExprDelete(db, pWhere); @@ -129523,7 +131635,7 @@ SQLITE_PRIVATE void sqlite3Update( if( !isView && aiCurOnePass[0]!=iDataCur && aiCurOnePass[1]!=iDataCur ){ assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey); - VdbeCoverageNeverTaken(v); + VdbeCoverage(v); } if( eOnePass!=ONEPASS_SINGLE ){ labelContinue = sqlite3VdbeMakeLabel(v); @@ -129610,13 +131722,7 @@ SQLITE_PRIVATE void sqlite3Update( */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); - if( tmask & TRIGGER_BEFORE ){ - /* This value will be recomputed in After-BEFORE-trigger-reload-loop - ** below, so make sure that it is not cached and reused. - ** Ticket d85fffd6ffe856092ed8daefa811b1e399706b28. */ - sqlite3ExprCacheRemove(pParse, regNew+i, 1); - } + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); } @@ -130153,10 +132259,12 @@ SQLITE_PRIVATE void sqlite3UpsertDoUpdate( Vdbe *v = pParse->pVdbe; sqlite3 *db = pParse->db; SrcList *pSrc; /* FROM clause for the UPDATE */ - int iDataCur = pUpsert->iDataCur; + int iDataCur; assert( v!=0 ); + assert( pUpsert!=0 ); VdbeNoopComment((v, "Begin DO UPDATE of UPSERT")); + iDataCur = pUpsert->iDataCur; if( pIdx && iCur!=iDataCur ){ if( HasRowid(pTab) ){ int regRowid = sqlite3GetTempReg(pParse); @@ -130426,7 +132534,7 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db, int iDb){ */ rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeBeginTrans(pMain, 2); + rc = sqlite3BtreeBeginTrans(pMain, 2, 0); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Do not attempt to change the page size for a WAL database */ @@ -130844,7 +132952,7 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ assert( sqlite3_mutex_held(db->mutex) ); if( p ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); do { VTable *pNext = p->pNext; sqlite3VtabUnlock(p); @@ -131340,7 +133448,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ assert( IsVirtual(pTab) ); memset(&sParse, 0, sizeof(sParse)); - sParse.declareVtab = 1; + sParse.eParseMode = PARSE_MODE_DECLARE_VTAB; sParse.db = db; sParse.nQueryLoop = 1; if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable, &zErr) @@ -131381,7 +133489,7 @@ SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } - sParse.declareVtab = 0; + sParse.eParseMode = PARSE_MODE_NORMAL; if( sParse.pVdbe ){ sqlite3VdbeFinalize(sParse.pVdbe); @@ -131935,6 +134043,8 @@ struct WhereLevel { struct InLoop { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ + int iBase; /* Base register of multi-key index record */ + int nPrefix; /* Number of prior entires in the key */ u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ @@ -132173,6 +134283,7 @@ struct WhereClause { WhereInfo *pWInfo; /* WHERE clause processing context */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ + u8 hasOr; /* True if any a[].eOperator is WO_OR */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ @@ -132346,6 +134457,7 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet*, Expr*); SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); @@ -132408,6 +134520,7 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ +#define WHERE_IN_EARLYOUT 0x00040000 /* Perhaps quit IN loops early */ /************** End of whereInt.h ********************************************/ /************** Continuing where we left off in wherecode.c ******************/ @@ -132542,7 +134655,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); sqlite3_str_appendall(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ - sqlite3_str_appendf(&str, " SUBQUERY 0x%p", pItem->pSelect); + sqlite3_str_appendf(&str, " SUBQUERY %u", pItem->pSelect->selId); }else{ sqlite3_str_appendf(&str, " TABLE %s", pItem->zName); } @@ -132739,7 +134852,6 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ /* Code the OP_Affinity opcode if there is anything left to do. */ if( n>0 ){ sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); } } @@ -132983,7 +135095,14 @@ static int codeEqualityTerm( sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); if( i==iEq ){ pIn->iCur = iTab; - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; + if( iEq>0 && (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ){ + pIn->iBase = iReg - i; + pIn->nPrefix = i; + pLoop->wsFlags |= WHERE_IN_EARLYOUT; + }else{ + pIn->nPrefix = 0; + } }else{ pIn->eEndLoopOp = OP_Noop; } @@ -133270,11 +135389,8 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - Vdbe *v = pWalker->pParse->pVdbe; int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCodeGetColumnOfTable( - v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg - ); + sqlite3ExprCode(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ @@ -133627,7 +135743,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); pLevel->op = OP_Goto; }else @@ -133641,7 +135757,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int nConstraint = pLoop->nLTerm; int iIn; /* Counter for IN constraints */ - sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; for(j=0; jaddrNxt; sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); pLevel->op = OP_Noop; }else if( (pLoop->wsFlags & WHERE_IPK)!=0 && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 @@ -133810,7 +135921,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, pX->op==TK_LE); VdbeCoverageIf(v, pX->op==TK_LT); VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); }else{ sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); @@ -133845,7 +135955,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( testOp!=OP_Noop ){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); VdbeCoverageIf(v, testOp==OP_Le); VdbeCoverageIf(v, testOp==OP_Lt); @@ -134050,6 +136159,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** above has already left the cursor sitting on the correct row, ** so no further seeking is needed */ }else{ + if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ + sqlite3VdbeAddOp1(v, OP_SeekHit, iIdxCur); + } op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); @@ -134068,7 +136180,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( nConstraint = nEq; if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 @@ -134093,7 +136204,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( } }else if( bStopAtNull ){ sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); endEq = 0; nConstraint++; } @@ -134113,6 +136223,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); } + if( pLoop->wsFlags & WHERE_IN_EARLYOUT ){ + sqlite3VdbeAddOp2(v, OP_SeekHit, iIdxCur, 1); + } + /* Seek the table cursor, if required */ if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ @@ -134123,7 +136237,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( )){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); VdbeCoverage(v); }else{ @@ -134358,23 +136471,23 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** row will be skipped in subsequent sub-WHERE clauses. */ if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, regRowid); jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, - r,iSet); + regRowid, iSet); VdbeCoverage(v); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); int nPk = pPk->nKeyCol; int iPk; + int r; /* Read the PK into an array of temp registers. */ r = sqlite3GetTempRange(pParse, nPk); for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol, r+iPk); } /* Check if the temp table already contains this key. If so, @@ -134607,7 +136720,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); @@ -134823,18 +136935,18 @@ static int isLikeOrGlob( int *pisComplete, /* True if the only wildcard is % in the last character */ int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - const u8 *z = 0; /* String on RHS of LIKE operator */ + const u8 *z = 0; /* String on RHS of LIKE operator */ Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ + u8 c; /* One character in z[] */ int cnt; /* Number of non-wildcard prefix characters */ - char wc[4]; /* Wildcard characters */ + u8 wc[4]; /* Wildcard characters */ sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ int rc; /* Result code to return */ - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, (char*)wc) ){ return 0; } #ifdef SQLITE_EBCDIC @@ -134859,23 +136971,6 @@ static int isLikeOrGlob( } if( z ){ - /* If the RHS begins with a digit or a minus sign, then the LHS must - ** be an ordinary column (not a virtual table column) with TEXT affinity. - ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false - ** even though "lhs LIKE rhs" is true. But if the RHS does not start - ** with a digit or '-', then "lhs LIKE rhs" will always be false if - ** the LHS is numeric and so the optimization still works. - */ - if( sqlite3Isdigit(z[0]) || z[0]=='-' ){ - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - sqlite3ValueFree(pVal); - return 0; - } - } - /* Count the number of prefix characters prior to the first wildcard */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ @@ -134885,11 +136980,13 @@ static int isLikeOrGlob( /* The optimization is possible only if (1) the pattern does not begin ** with a wildcard and if (2) the non-wildcard prefix does not end with - ** an (illegal 0xff) character. The second condition is necessary so + ** an (illegal 0xff) character, or (3) the pattern does not consist of + ** a single escape character. The second condition is necessary so ** that we can increment the prefix key to find an upper bound for the - ** range search. - */ - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + ** range search. The third is because the caller assumes that the pattern + ** consists of at least one character after all escapes have been + ** removed. */ + if( cnt!=0 && 255!=(u8)z[cnt-1] && (cnt>1 || z[0]!=wc[3]) ){ Expr *pPrefix; /* A "complete" match if the pattern ends with "*" or "%" */ @@ -134906,6 +137003,32 @@ static int isLikeOrGlob( zNew[iTo++] = zNew[iFrom]; } zNew[iTo] = 0; + + /* If the RHS begins with a digit or a minus sign, then the LHS must be + ** an ordinary column (not a virtual table column) with TEXT affinity. + ** Otherwise the LHS might be numeric and "lhs >= rhs" would be false + ** even though "lhs LIKE rhs" is true. But if the RHS does not start + ** with a digit or '-', then "lhs LIKE rhs" will always be false if + ** the LHS is numeric and so the optimization still works. + ** + ** 2018-09-10 ticket c94369cae9b561b1f996d0054bfab11389f9d033 + ** The RHS pattern must not be '/%' because the termination condition + ** will then become "x<'0'" and if the affinity is numeric, will then + ** be converted into "x<0", which is incorrect. + */ + if( sqlite3Isdigit(zNew[0]) + || zNew[0]=='-' + || (zNew[0]+1=='0' && iTo==1) + ){ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || IsVirtual(pLeft->pTab) /* Value might be numeric */ + ){ + sqlite3ExprDelete(db, pPrefix); + sqlite3ValueFree(pVal); + return 0; + } + } } *ppPrefix = pPrefix; @@ -134967,6 +137090,7 @@ static int isLikeOrGlob( ** If the expression matches none of the patterns above, return 0. */ static int isAuxiliaryVtabOperator( + sqlite3 *db, /* Parsing context */ Expr *pExpr, /* Test this expression */ unsigned char *peOp2, /* OUT: 0 for MATCH, or else an op2 value */ Expr **ppLeft, /* Column expression to left of MATCH/op2 */ @@ -134990,16 +137114,54 @@ static int isAuxiliaryVtabOperator( if( pList==0 || pList->nExpr!=2 ){ return 0; } + + /* Built-in operators MATCH, GLOB, LIKE, and REGEXP attach to a + ** virtual table on their second argument, which is the same as + ** the left-hand side operand in their in-fix form. + ** + ** vtab_column MATCH expression + ** MATCH(expression,vtab_column) + */ pCol = pList->a[1].pExpr; - if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){ - return 0; + if( pCol->op==TK_COLUMN && IsVirtual(pCol->pTab) ){ + for(i=0; iu.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; + } + } } - for(i=0; iu.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - *ppRight = pList->a[0].pExpr; - *ppLeft = pCol; - return 1; + + /* We can also match against the first column of overloaded + ** functions where xFindFunction returns a value of at least + ** SQLITE_INDEX_CONSTRAINT_FUNCTION. + ** + ** OVERLOADED(vtab_column,expression) + ** + ** Historically, xFindFunction expected to see lower-case function + ** names. But for this use case, xFindFunction is expected to deal + ** with function names in an arbitrary case. + */ + pCol = pList->a[0].pExpr; + if( pCol->op==TK_COLUMN && IsVirtual(pCol->pTab) ){ + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xNotUsed)(sqlite3_context*,int,sqlite3_value**); + void *pNotUsed; + pVtab = sqlite3GetVTable(db, pCol->pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction!=0 ){ + i = pMod->xFindFunction(pVtab,2, pExpr->u.zToken, &xNotUsed, &pNotUsed); + if( i>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + *peOp2 = i; + *ppRight = pList->a[1].pExpr; + *ppLeft = pCol; + return 1; + } } } }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){ @@ -135301,7 +137463,12 @@ static void exprAnalyzeOrTerm( ** empty. */ pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; + if( indexable ){ + pTerm->eOperator = WO_OR; + pWC->hasOr = 1; + }else{ + pTerm->eOperator = WO_OR; + } /* For a two-way OR, attempt to implementation case 2. */ @@ -135442,7 +137609,7 @@ static void exprAnalyzeOrTerm( idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; + /* pTerm = &pWC->a[idxTerm]; // would be needed if pTerm where used again */ markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); @@ -135481,7 +137648,7 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){ return 0; } pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); - if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; + if( sqlite3IsBinary(pColl) ) return 1; return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight); } @@ -135640,7 +137807,7 @@ static void exprAnalyze( pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); } pMaskSet->bVarSelect = 0; - prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); + prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); if( pMaskSet->bVarSelect ) pTerm->wtFlags |= TERM_VARSELECT; if( ExprHasProperty(pExpr, EP_FromJoin) ){ Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); @@ -135822,7 +137989,7 @@ static void exprAnalyze( } *pC = c + 1; } - zCollSeqName = noCase ? "NOCASE" : "BINARY"; + zCollSeqName = noCase ? "NOCASE" : sqlite3StrBINARY; pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), @@ -135859,7 +138026,7 @@ static void exprAnalyze( */ if( pWC->op==TK_AND ){ Expr *pRight = 0, *pLeft = 0; - int res = isAuxiliaryVtabOperator(pExpr, &eOp2, &pLeft, &pRight); + int res = isAuxiliaryVtabOperator(db, pExpr, &eOp2, &pLeft, &pRight); while( res-- > 0 ){ int idxNew; WhereTerm *pNewTerm; @@ -136033,6 +138200,7 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( WhereInfo *pWInfo /* The WHERE processing context */ ){ pWC->pWInfo = pWInfo; + pWC->hasOr = 0; pWC->pOuter = 0; pWC->nTerm = 0; pWC->nSlot = ArraySize(pWC->aStatic); @@ -136069,17 +138237,18 @@ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ ** a bitmask indicating which tables are used in that expression ** tree. */ -SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ Bitmask mask; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ + if( p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ return sqlite3WhereGetMask(pMaskSet, p->iTable); + }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + assert( p->op!=TK_IF_NULL_ROW ); + return 0; } mask = (p->op==TK_IF_NULL_ROW) ? sqlite3WhereGetMask(pMaskSet, p->iTable) : 0; - assert( !ExprHasProperty(p, EP_TokenOnly) ); - if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); + if( p->pLeft ) mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pLeft); if( p->pRight ){ - mask |= sqlite3WhereExprUsage(pMaskSet, p->pRight); + mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pRight); assert( p->x.pList==0 ); }else if( ExprHasProperty(p, EP_xIsSelect) ){ if( ExprHasProperty(p, EP_VarSelect) ) pMaskSet->bVarSelect = 1; @@ -136089,6 +138258,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ } return mask; } +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + return p ? sqlite3WhereExprUsageNN(pMaskSet,p) : 0; +} SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ int i; Bitmask mask = 0; @@ -136230,15 +138402,38 @@ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ } /* -** Return TRUE if the innermost loop of the WHERE clause implementation -** returns rows in ORDER BY order for complete run of the inner loop. +** In the ORDER BY LIMIT optimization, if the inner-most loop is known +** to emit rows in increasing order, and if the last row emitted by the +** inner-most loop did not fit within the sorter, then we can skip all +** subsequent rows for the current iteration of the inner loop (because they +** will not fit in the sorter either) and continue with the second inner +** loop - the loop immediately outside the inner-most. ** -** Across multiple iterations of outer loops, the output rows need not be -** sorted. As long as rows are sorted for just the innermost loop, this -** routine can return TRUE. +** When a row does not fit in the sorter (because the sorter already +** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the +** label returned by this function. +** +** If the ORDER BY LIMIT optimization applies, the jump destination should +** be the continuation for the second-inner-most loop. If the ORDER BY +** LIMIT optimization does not apply, then the jump destination should +** be the continuation for the inner-most loop. +** +** It is always safe for this routine to return the continuation of the +** inner-most loop, in the sense that a correct answer will result. +** Returning the continuation the second inner loop is an optimization +** that might make the code run a little faster, but should not change +** the final answer. */ -SQLITE_PRIVATE int sqlite3WhereOrderedInnerLoop(WhereInfo *pWInfo){ - return pWInfo->bOrderedInnerLoop; +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ + WhereLevel *pInner; + if( !pWInfo->bOrderedInnerLoop ){ + /* The ORDER BY LIMIT optimization does not apply. Jump to the + ** continuation of the inner-most loop. */ + return pWInfo->iContinue; + } + pInner = &pWInfo->a[pWInfo->nLevel-1]; + assert( pInner->addrNxt!=0 ); + return pInner->addrNxt; } /* @@ -136965,7 +139160,6 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ - sqlite3ExprCachePush(pParse); pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; if( pTabItem->fg.viaCoroutine ){ int regYield = pTabItem->regReturn; @@ -136973,7 +139167,7 @@ static void constructAutomaticIndex( sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } @@ -137002,7 +139196,6 @@ static void constructAutomaticIndex( sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); @@ -137108,6 +139301,20 @@ static sqlite3_index_info *allocateIndexInfo( testcase( pTerm->eOperator & WO_ALL ); if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; + if( (pSrc->fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) + && (pTerm->eOperator & (WO_IS|WO_ISNULL)) + ){ + /* An "IS" term in the WHERE clause where the virtual table is the rhs + ** of a LEFT JOIN. Do not pass this term to the virtual table + ** implementation, as this can lead to incorrect results from SQL such + ** as: + ** + ** "LEFT JOIN vtab WHERE vtab.col IS NULL" */ + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + continue; + } assert( pTerm->u.leftColumn>=(-1) ); pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; @@ -137599,7 +139806,9 @@ static int whereRangeScanEst( Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 && nEqnSampleCol ){ + if( p->nSample>0 && nEqnSampleCol + && OptimizationEnabled(pParse->db, SQLITE_Stat34) + ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; tRowcnt a[2]; @@ -138614,7 +140823,6 @@ static int whereLoopAddBtreeIndex( if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; if( ExprHasProperty(pExpr, EP_xIsSelect) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ int i; @@ -138634,6 +140842,42 @@ static int whereLoopAddBtreeIndex( assert( nIn>0 ); /* RHS always has 2 or more terms... The parser ** changes "x IN (?)" into "x=?". */ } + if( pProbe->hasStat1 ){ + LogEst M, logK, safetyMargin; + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the RHS of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + ** If we do not have real sqlite_stat1 data, always prefer to use + ** the index. + */ + M = pProbe->aiRowLogEst[saved_nEq]; + logK = estLog(nIn); + safetyMargin = 10; /* TUNING: extra weight for indexed IN */ + if( M + logK + safetyMargin < nIn + rLogSize ){ + WHERETRACE(0x40, + ("Scan preferred over IN operator on column %d of \"%s\" (%d<%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + continue; + }else{ + WHERETRACE(0x40, + ("IN operator preferred on column %d of \"%s\" (%d>=%d)\n", + saved_nEq, pProbe->zName, M+logK+10, nIn+rLogSize)); + } + } + pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; @@ -138712,6 +140956,7 @@ static int whereLoopAddBtreeIndex( && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pExpr = pTerm->pExpr; if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ @@ -138800,6 +141045,7 @@ static int whereLoopAddBtreeIndex( if( saved_nEq==saved_nSkip && saved_nEq+1nKeyCol && pProbe->noSkipScan==0 + && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ @@ -138863,24 +141109,6 @@ static int indexMightHelpWithOrderBy( return 0; } -/* -** Return a bitmask where 1s indicate that the corresponding column of -** the table is used by an index. Only the first 63 columns are considered. -*/ -static Bitmask columnsInIndex(Index *pIdx){ - Bitmask m = 0; - int j; - for(j=pIdx->nColumn-1; j>=0; j--){ - int x = pIdx->aiColumn[j]; - if( x>=0 ){ - testcase( x==BMS-1 ); - testcase( x==BMS-2 ); - if( xwsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; }else{ - m = pSrc->colUsed & ~columnsInIndex(pProbe); + m = pSrc->colUsed & pProbe->colNotIdxed; pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; } @@ -139347,7 +141575,7 @@ SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int if( pX->pLeft ){ pC = sqlite3BinaryCompareCollSeq(pHidden->pParse, pX->pLeft, pX->pRight); } - zRet = (pC ? pC->zName : "BINARY"); + zRet = (pC ? pC->zName : sqlite3StrBINARY); } return zRet; } @@ -139663,7 +141891,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ { rc = whereLoopAddBtree(pBuilder, mPrereq); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); } mPrior |= pNew->maskSelf; @@ -140198,7 +142426,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo, nRowEst, nOrderBy, isOrdered ); } - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]); + /* TUNING: Add a small extra penalty (5) to sorting as an + ** extra encouragment to the query planner to select a plan + ** where the rows emerge in the correct order without any sorting + ** required. */ + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 5; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", @@ -140388,6 +142620,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } + pWInfo->bOrderedInnerLoop = 0; if( pWInfo->pOrderBy ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ @@ -140499,7 +142732,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ } if( j!=pIdx->nKeyCol ) continue; pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; - if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ pLoop->wsFlags |= WHERE_IDX_ONLY; } pLoop->nLTerm = j; @@ -141179,6 +143412,26 @@ whereBeginError: return 0; } +/* +** Part of sqlite3WhereEnd() will rewrite opcodes to reference the +** index rather than the main table. In SQLITE_DEBUG mode, we want +** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine +** does that. +*/ +#ifndef SQLITE_DEBUG +# define OpcodeRewriteTrace(D,K,P) /* no-op */ +#else +# define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P) + static void sqlite3WhereOpcodeRewriteTrace( + sqlite3 *db, + int pc, + VdbeOp *pOp + ){ + if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; + sqlite3VdbePrintOp(0, pc, pOp); + } +#endif + /* ** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. @@ -141195,7 +143448,6 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ /* Generate loop termination code. */ VdbeModuleComment((v, "End WHERE-core")); - sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; @@ -141246,10 +143498,17 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ sqlite3VdbeJumpHere(v, pIn->addrInTop+1); if( pIn->eEndLoopOp!=OP_Noop ){ + if( pIn->nPrefix ){ + assert( pLoop->wsFlags & WHERE_IN_EARLYOUT ); + sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur, + sqlite3VdbeCurrentAddr(v)+2, + pIn->iBase, pIn->nPrefix); + VdbeCoverage(v); + } sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); VdbeCoverage(v); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next); } sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } @@ -141340,6 +143599,11 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ){ last = sqlite3VdbeCurrentAddr(v); k = pLevel->addrBody; +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE opcodes in range %d..%d\n", k, last-1); + } +#endif pOp = sqlite3VdbeGetOp(v, k); for(; kp1!=pLevel->iTabCur ) continue; @@ -141359,16 +143623,22 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( x>=0 ){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); } assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 || pWInfo->eOnePass ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; + OpcodeRewriteTrace(db, k, pOp); }else if( pOp->opcode==OP_IfNullRow ){ pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); } } +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n"); +#endif } } @@ -141380,6 +143650,2261 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } /************** End of where.c ***********************************************/ +/************** Begin file window.c ******************************************/ +/* +** 2018 May 08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + +/* +** SELECT REWRITING +** +** Any SELECT statement that contains one or more window functions in +** either the select list or ORDER BY clause (the only two places window +** functions may be used) is transformed by function sqlite3WindowRewrite() +** in order to support window function processing. For example, with the +** schema: +** +** CREATE TABLE t1(a, b, c, d, e, f, g); +** +** the statement: +** +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM t1 ORDER BY e; +** +** is transformed to: +** +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT a, e, c, d, b FROM t1 ORDER BY c, d +** ) ORDER BY e; +** +** The flattening optimization is disabled when processing this transformed +** SELECT statement. This allows the implementation of the window function +** (in this case max()) to process rows sorted in order of (c, d), which +** makes things easier for obvious reasons. More generally: +** +** * FROM, WHERE, GROUP BY and HAVING clauses are all moved to +** the sub-query. +** +** * ORDER BY, LIMIT and OFFSET remain part of the parent query. +** +** * Terminals from each of the expression trees that make up the +** select-list and ORDER BY expressions in the parent query are +** selected by the sub-query. For the purposes of the transformation, +** terminals are column references and aggregate functions. +** +** If there is more than one window function in the SELECT that uses +** the same window declaration (the OVER bit), then a single scan may +** be used to process more than one window function. For example: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY c ORDER BY d) +** FROM t1; +** +** is transformed in the same way as the example above. However: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY a ORDER BY b) +** FROM t1; +** +** Must be transformed to: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT e, min(e) OVER (PARTITION BY a ORDER BY b), c, d, b FROM +** SELECT a, e, c, d, b FROM t1 ORDER BY a, b +** ) ORDER BY c, d +** ) ORDER BY e; +** +** so that both min() and max() may process rows in the order defined by +** their respective window declarations. +** +** INTERFACE WITH SELECT.C +** +** When processing the rewritten SELECT statement, code in select.c calls +** sqlite3WhereBegin() to begin iterating through the results of the +** sub-query, which is always implemented as a co-routine. It then calls +** sqlite3WindowCodeStep() to process rows and finish the scan by calling +** sqlite3WhereEnd(). +** +** sqlite3WindowCodeStep() generates VM code so that, for each row returned +** by the sub-query a sub-routine (OP_Gosub) coded by select.c is invoked. +** When the sub-routine is invoked: +** +** * The results of all window-functions for the row are stored +** in the associated Window.regResult registers. +** +** * The required terminal values are stored in the current row of +** temp table Window.iEphCsr. +** +** In some cases, depending on the window frame and the specific window +** functions invoked, sqlite3WindowCodeStep() caches each entire partition +** in a temp table before returning any rows. In other cases it does not. +** This detail is encapsulated within this file, the code generated by +** select.c is the same in either case. +** +** BUILT-IN WINDOW FUNCTIONS +** +** This implementation features the following built-in window functions: +** +** row_number() +** rank() +** dense_rank() +** percent_rank() +** cume_dist() +** ntile(N) +** lead(expr [, offset [, default]]) +** lag(expr [, offset [, default]]) +** first_value(expr) +** last_value(expr) +** nth_value(expr, N) +** +** These are the same built-in window functions supported by Postgres. +** Although the behaviour of aggregate window functions (functions that +** can be used as either aggregates or window funtions) allows them to +** be implemented using an API, built-in window functions are much more +** esoteric. Additionally, some window functions (e.g. nth_value()) +** may only be implemented by caching the entire partition in memory. +** As such, some built-in window functions use the same API as aggregate +** window functions and some are implemented directly using VDBE +** instructions. Additionally, for those functions that use the API, the +** window frame is sometimes modified before the SELECT statement is +** rewritten. For example, regardless of the specified window frame, the +** row_number() function always uses: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** See sqlite3WindowUpdate() for details. +** +** As well as some of the built-in window functions, aggregate window +** functions min() and max() are implemented using VDBE instructions if +** the start of the window frame is declared as anything other than +** UNBOUNDED PRECEDING. +*/ + +/* +** Implementation of built-in window function row_number(). Assumes that the +** window frame has been coerced to: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void row_numberStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) (*p)++; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void row_numberValueFunc(sqlite3_context *pCtx){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + sqlite3_result_int64(pCtx, (p ? *p : 0)); +} + +/* +** Context object type used by rank(), dense_rank(), percent_rank() and +** cume_dist(). +*/ +struct CallCount { + i64 nValue; + i64 nStep; + i64 nTotal; +}; + +/* +** Implementation of built-in window function dense_rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void dense_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) p->nStep = 1; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void dense_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nStep ){ + p->nValue++; + p->nStep = 0; + } + sqlite3_result_int64(pCtx, p->nValue); + } +} + +/* +** Implementation of built-in window function rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nStep++; + if( p->nValue==0 ){ + p->nValue = p->nStep; + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + sqlite3_result_int64(pCtx, p->nValue); + p->nValue = 0; + } +} + +/* +** Implementation of built-in window function percent_rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void percent_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==1 ); + + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal==0 ){ + p->nTotal = sqlite3_value_int64(apArg[0]); + } + p->nStep++; + if( p->nValue==0 ){ + p->nValue = p->nStep; + } + } +} +static void percent_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal>1 ){ + double r = (double)(p->nValue-1) / (double)(p->nTotal-1); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_double(pCtx, 0.0); + } + p->nValue = 0; + } +} + +/* +** Implementation of built-in window function cume_dist(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void cume_distStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal==0 ){ + p->nTotal = sqlite3_value_int64(apArg[0]); + } + p->nStep++; + } +} +static void cume_distValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->nTotal ){ + double r = (double)(p->nStep) / (double)(p->nTotal); + sqlite3_result_double(pCtx, r); + } +} + +/* +** Context object for ntile() window function. +*/ +struct NtileCtx { + i64 nTotal; /* Total rows in partition */ + i64 nParam; /* Parameter passed to ntile(N) */ + i64 iRow; /* Current row */ +}; + +/* +** Implementation of ntile(). This assumes that the window frame has +** been coerced to: +** +** ROWS UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void ntileStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==2 ); UNUSED_PARAMETER(nArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal==0 ){ + p->nParam = sqlite3_value_int64(apArg[0]); + p->nTotal = sqlite3_value_int64(apArg[1]); + if( p->nParam<=0 ){ + sqlite3_result_error( + pCtx, "argument of ntile must be a positive integer", -1 + ); + } + } + p->iRow++; + } +} +static void ntileValueFunc(sqlite3_context *pCtx){ + struct NtileCtx *p; + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->nParam>0 ){ + int nSize = (p->nTotal / p->nParam); + if( nSize==0 ){ + sqlite3_result_int64(pCtx, p->iRow); + }else{ + i64 nLarge = p->nTotal - p->nParam*nSize; + i64 iSmall = nLarge*(nSize+1); + i64 iRow = p->iRow-1; + + assert( (nLarge*(nSize+1) + (p->nParam-nLarge)*nSize)==p->nTotal ); + + if( iRowpVal); + p->pVal = sqlite3_value_dup(apArg[0]); + if( p->pVal==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + p->nVal++; + } + } +} +static void last_valueInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct LastValueCtx *p; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( ALWAYS(p) ){ + p->nVal--; + if( p->nVal==0 ){ + sqlite3_value_free(p->pVal); + p->pVal = 0; + } + } +} +static void last_valueValueFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + } +} +static void last_valueFinalizeFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + sqlite3_value_free(p->pVal); + p->pVal = 0; + } +} + +/* +** Static names for the built-in window function names. These static +** names are used, rather than string literals, so that FuncDef objects +** can be associated with a particular window function by direct +** comparison of the zName pointer. Example: +** +** if( pFuncDef->zName==row_valueName ){ ... } +*/ +static const char row_numberName[] = "row_number"; +static const char dense_rankName[] = "dense_rank"; +static const char rankName[] = "rank"; +static const char percent_rankName[] = "percent_rank"; +static const char cume_distName[] = "cume_dist"; +static const char ntileName[] = "ntile"; +static const char last_valueName[] = "last_value"; +static const char nth_valueName[] = "nth_value"; +static const char first_valueName[] = "first_value"; +static const char leadName[] = "lead"; +static const char lagName[] = "lag"; + +/* +** No-op implementations of xStep() and xFinalize(). Used as place-holders +** for built-in window functions that never call those interfaces. +** +** The noopValueFunc() is called but is expected to do nothing. The +** noopStepFunc() is never called, and so it is marked with NO_TEST to +** let the test coverage routine know not to expect this function to be +** invoked. +*/ +static void noopStepFunc( /*NO_TEST*/ + sqlite3_context *p, /*NO_TEST*/ + int n, /*NO_TEST*/ + sqlite3_value **a /*NO_TEST*/ +){ /*NO_TEST*/ + UNUSED_PARAMETER(p); /*NO_TEST*/ + UNUSED_PARAMETER(n); /*NO_TEST*/ + UNUSED_PARAMETER(a); /*NO_TEST*/ + assert(0); /*NO_TEST*/ +} /*NO_TEST*/ +static void noopValueFunc(sqlite3_context *p){ UNUSED_PARAMETER(p); /*no-op*/ } + +/* Window functions that use all window interfaces: xStep, xFinal, +** xValue, and xInverse */ +#define WINDOWFUNCALL(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## FinalizeFunc, name ## ValueFunc, \ + name ## InvFunc, name ## Name, {0} \ +} + +/* Window functions that are implemented using bytecode and thus have +** no-op routines for their methods */ +#define WINDOWFUNCNOOP(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + noopStepFunc, noopValueFunc, noopValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + +/* Window functions that use all window interfaces: xStep, the +** same routine for xFinalize and xValue and which never call +** xInverse. */ +#define WINDOWFUNCX(name,nArg,extra) { \ + nArg, (SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## ValueFunc, name ## ValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + + +/* +** Register those built-in window functions that are not also aggregates. +*/ +SQLITE_PRIVATE void sqlite3WindowFunctions(void){ + static FuncDef aWindowFuncs[] = { + WINDOWFUNCX(row_number, 0, 0), + WINDOWFUNCX(dense_rank, 0, 0), + WINDOWFUNCX(rank, 0, 0), + WINDOWFUNCX(percent_rank, 0, SQLITE_FUNC_WINDOW_SIZE), + WINDOWFUNCX(cume_dist, 0, SQLITE_FUNC_WINDOW_SIZE), + WINDOWFUNCX(ntile, 1, SQLITE_FUNC_WINDOW_SIZE), + WINDOWFUNCALL(last_value, 1, 0), + WINDOWFUNCNOOP(nth_value, 2, 0), + WINDOWFUNCNOOP(first_value, 1, 0), + WINDOWFUNCNOOP(lead, 1, 0), + WINDOWFUNCNOOP(lead, 2, 0), + WINDOWFUNCNOOP(lead, 3, 0), + WINDOWFUNCNOOP(lag, 1, 0), + WINDOWFUNCNOOP(lag, 2, 0), + WINDOWFUNCNOOP(lag, 3, 0), + }; + sqlite3InsertBuiltinFuncs(aWindowFuncs, ArraySize(aWindowFuncs)); +} + +/* +** This function is called immediately after resolving the function name +** for a window function within a SELECT statement. Argument pList is a +** linked list of WINDOW definitions for the current SELECT statement. +** Argument pFunc is the function definition just resolved and pWin +** is the Window object representing the associated OVER clause. This +** function updates the contents of pWin as follows: +** +** * If the OVER clause refered to a named window (as in "max(x) OVER win"), +** search list pList for a matching WINDOW definition, and update pWin +** accordingly. If no such WINDOW clause can be found, leave an error +** in pParse. +** +** * If the function is a built-in window function that requires the +** window to be coerced (see "BUILT-IN WINDOW FUNCTIONS" at the top +** of this file), pWin is updated here. +*/ +SQLITE_PRIVATE void sqlite3WindowUpdate( + Parse *pParse, + Window *pList, /* List of named windows for this SELECT */ + Window *pWin, /* Window frame to update */ + FuncDef *pFunc /* Window function definition */ +){ + if( pWin->zName && pWin->eType==0 ){ + Window *p; + for(p=pList; p; p=p->pNextWin){ + if( sqlite3StrICmp(p->zName, pWin->zName)==0 ) break; + } + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such window: %s", pWin->zName); + return; + } + pWin->pPartition = sqlite3ExprListDup(pParse->db, p->pPartition, 0); + pWin->pOrderBy = sqlite3ExprListDup(pParse->db, p->pOrderBy, 0); + pWin->pStart = sqlite3ExprDup(pParse->db, p->pStart, 0); + pWin->pEnd = sqlite3ExprDup(pParse->db, p->pEnd, 0); + pWin->eStart = p->eStart; + pWin->eEnd = p->eEnd; + pWin->eType = p->eType; + } + if( pFunc->funcFlags & SQLITE_FUNC_WINDOW ){ + sqlite3 *db = pParse->db; + if( pWin->pFilter ){ + sqlite3ErrorMsg(pParse, + "FILTER clause may only be used with aggregate window functions" + ); + }else + if( pFunc->zName==row_numberName || pFunc->zName==ntileName ){ + sqlite3ExprDelete(db, pWin->pStart); + sqlite3ExprDelete(db, pWin->pEnd); + pWin->pStart = pWin->pEnd = 0; + pWin->eType = TK_ROWS; + pWin->eStart = TK_UNBOUNDED; + pWin->eEnd = TK_CURRENT; + }else + + if( pFunc->zName==dense_rankName || pFunc->zName==rankName + || pFunc->zName==percent_rankName || pFunc->zName==cume_distName + ){ + sqlite3ExprDelete(db, pWin->pStart); + sqlite3ExprDelete(db, pWin->pEnd); + pWin->pStart = pWin->pEnd = 0; + pWin->eType = TK_RANGE; + pWin->eStart = TK_UNBOUNDED; + pWin->eEnd = TK_CURRENT; + } + } + pWin->pFunc = pFunc; +} + +/* +** Context object passed through sqlite3WalkExprList() to +** selectWindowRewriteExprCb() by selectWindowRewriteEList(). +*/ +typedef struct WindowRewrite WindowRewrite; +struct WindowRewrite { + Window *pWin; + SrcList *pSrc; + ExprList *pSub; + Select *pSubSelect; /* Current sub-select, if any */ +}; + +/* +** Callback function used by selectWindowRewriteEList(). If necessary, +** this function appends to the output expression-list and updates +** expression (*ppExpr) in place. +*/ +static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Parse *pParse = pWalker->pParse; + + /* If this function is being called from within a scalar sub-select + ** that used by the SELECT statement being processed, only process + ** TK_COLUMN expressions that refer to it (the outer SELECT). Do + ** not process aggregates or window functions at all, as they belong + ** to the scalar sub-select. */ + if( p->pSubSelect ){ + if( pExpr->op!=TK_COLUMN ){ + return WRC_Continue; + }else{ + int nSrc = p->pSrc->nSrc; + int i; + for(i=0; iiTable==p->pSrc->a[i].iCursor ) break; + } + if( i==nSrc ) return WRC_Continue; + } + } + + switch( pExpr->op ){ + + case TK_FUNCTION: + if( pExpr->pWin==0 ){ + break; + }else{ + Window *pWin; + for(pWin=p->pWin; pWin; pWin=pWin->pNextWin){ + if( pExpr->pWin==pWin ){ + assert( pWin->pOwner==pExpr ); + return WRC_Prune; + } + } + } + /* Fall through. */ + + case TK_AGG_FUNCTION: + case TK_COLUMN: { + Expr *pDup = sqlite3ExprDup(pParse->db, pExpr, 0); + p->pSub = sqlite3ExprListAppend(pParse, p->pSub, pDup); + if( p->pSub ){ + assert( ExprHasProperty(pExpr, EP_Static)==0 ); + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(pParse->db, pExpr); + ExprClearProperty(pExpr, EP_Static); + memset(pExpr, 0, sizeof(Expr)); + + pExpr->op = TK_COLUMN; + pExpr->iColumn = p->pSub->nExpr-1; + pExpr->iTable = p->pWin->iEphCsr; + } + + break; + } + + default: /* no-op */ + break; + } + + return WRC_Continue; +} +static int selectWindowRewriteSelectCb(Walker *pWalker, Select *pSelect){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Select *pSave = p->pSubSelect; + if( pSave==pSelect ){ + return WRC_Continue; + }else{ + p->pSubSelect = pSelect; + sqlite3WalkSelect(pWalker, pSelect); + p->pSubSelect = pSave; + } + return WRC_Prune; +} + + +/* +** Iterate through each expression in expression-list pEList. For each: +** +** * TK_COLUMN, +** * aggregate function, or +** * window function with a Window object that is not a member of the +** Window list passed as the second argument (pWin). +** +** Append the node to output expression-list (*ppSub). And replace it +** with a TK_COLUMN that reads the (N-1)th element of table +** pWin->iEphCsr, where N is the number of elements in (*ppSub) after +** appending the new one. +*/ +static void selectWindowRewriteEList( + Parse *pParse, + Window *pWin, + SrcList *pSrc, + ExprList *pEList, /* Rewrite expressions in this list */ + ExprList **ppSub /* IN/OUT: Sub-select expression-list */ +){ + Walker sWalker; + WindowRewrite sRewrite; + + memset(&sWalker, 0, sizeof(Walker)); + memset(&sRewrite, 0, sizeof(WindowRewrite)); + + sRewrite.pSub = *ppSub; + sRewrite.pWin = pWin; + sRewrite.pSrc = pSrc; + + sWalker.pParse = pParse; + sWalker.xExprCallback = selectWindowRewriteExprCb; + sWalker.xSelectCallback = selectWindowRewriteSelectCb; + sWalker.u.pRewrite = &sRewrite; + + (void)sqlite3WalkExprList(&sWalker, pEList); + + *ppSub = sRewrite.pSub; +} + +/* +** Append a copy of each expression in expression-list pAppend to +** expression list pList. Return a pointer to the result list. +*/ +static ExprList *exprListAppendList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + ExprList *pAppend /* List of values to append. Might be NULL */ +){ + if( pAppend ){ + int i; + int nInit = pList ? pList->nExpr : 0; + for(i=0; inExpr; i++){ + Expr *pDup = sqlite3ExprDup(pParse->db, pAppend->a[i].pExpr, 0); + pList = sqlite3ExprListAppend(pParse, pList, pDup); + if( pList ) pList->a[nInit+i].sortOrder = pAppend->a[i].sortOrder; + } + } + return pList; +} + +/* +** If the SELECT statement passed as the second argument does not invoke +** any SQL window functions, this function is a no-op. Otherwise, it +** rewrites the SELECT statement so that window function xStep functions +** are invoked in the correct order as described under "SELECT REWRITING" +** at the top of this file. +*/ +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ + int rc = SQLITE_OK; + if( p->pWin ){ + Vdbe *v = sqlite3GetVdbe(pParse); + sqlite3 *db = pParse->db; + Select *pSub = 0; /* The subquery */ + SrcList *pSrc = p->pSrc; + Expr *pWhere = p->pWhere; + ExprList *pGroupBy = p->pGroupBy; + Expr *pHaving = p->pHaving; + ExprList *pSort = 0; + + ExprList *pSublist = 0; /* Expression list for sub-query */ + Window *pMWin = p->pWin; /* Master window object */ + Window *pWin; /* Window object iterator */ + + p->pSrc = 0; + p->pWhere = 0; + p->pGroupBy = 0; + p->pHaving = 0; + + /* Create the ORDER BY clause for the sub-select. This is the concatenation + ** of the window PARTITION and ORDER BY clauses. Then, if this makes it + ** redundant, remove the ORDER BY from the parent SELECT. */ + pSort = sqlite3ExprListDup(db, pMWin->pPartition, 0); + pSort = exprListAppendList(pParse, pSort, pMWin->pOrderBy); + if( pSort && p->pOrderBy ){ + if( sqlite3ExprListCompare(pSort, p->pOrderBy, -1)==0 ){ + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + } + } + + /* Assign a cursor number for the ephemeral table used to buffer rows. + ** The OpenEphemeral instruction is coded later, after it is known how + ** many columns the table will have. */ + pMWin->iEphCsr = pParse->nTab++; + + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pEList, &pSublist); + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, &pSublist); + pMWin->nBufferCol = (pSublist ? pSublist->nExpr : 0); + + /* Append the PARTITION BY and ORDER BY expressions to the to the + ** sub-select expression list. They are required to figure out where + ** boundaries for partitions and sets of peer rows lie. */ + pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition); + pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy); + + /* Append the arguments passed to each window function to the + ** sub-select expression list. Also allocate two registers for each + ** window function - one for the accumulator, another for interim + ** results. */ + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pSublist = exprListAppendList(pParse, pSublist, pWin->pOwner->x.pList); + if( pWin->pFilter ){ + Expr *pFilter = sqlite3ExprDup(db, pWin->pFilter, 0); + pSublist = sqlite3ExprListAppend(pParse, pSublist, pFilter); + } + pWin->regAccum = ++pParse->nMem; + pWin->regResult = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + /* If there is no ORDER BY or PARTITION BY clause, and the window + ** function accepts zero arguments, and there are no other columns + ** selected (e.g. "SELECT row_number() OVER () FROM t1"), it is possible + ** that pSublist is still NULL here. Add a constant expression here to + ** keep everything legal in this case. + */ + if( pSublist==0 ){ + pSublist = sqlite3ExprListAppend(pParse, 0, + sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0) + ); + } + + pSub = sqlite3SelectNew( + pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 + ); + p->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + assert( p->pSrc || db->mallocFailed ); + if( p->pSrc ){ + p->pSrc->a[0].pSelect = pSub; + sqlite3SrcListAssignCursors(pParse, p->pSrc); + if( sqlite3ExpandSubquery(pParse, &p->pSrc->a[0]) ){ + rc = SQLITE_NOMEM; + }else{ + pSub->selFlags |= SF_Expanded; + p->selFlags &= ~SF_Aggregate; + sqlite3SelectPrep(pParse, pSub, 0); + } + + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, pSublist->nExpr); + }else{ + sqlite3SelectDelete(db, pSub); + } + if( db->mallocFailed ) rc = SQLITE_NOMEM; + } + + return rc; +} + +/* +** Free the Window object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3 *db, Window *p){ + if( p ){ + sqlite3ExprDelete(db, p->pFilter); + sqlite3ExprListDelete(db, p->pPartition); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pEnd); + sqlite3ExprDelete(db, p->pStart); + sqlite3DbFree(db, p->zName); + sqlite3DbFree(db, p); + } +} + +/* +** Free the linked list of Window objects starting at the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ + while( p ){ + Window *pNext = p->pNextWin; + sqlite3WindowDelete(db, p); + p = pNext; + } +} + +/* +** The argument expression is an PRECEDING or FOLLOWING offset. The +** value should be a non-negative integer. If the value is not a +** constant, change it to NULL. The fact that it is then a non-negative +** integer will be caught later. But it is important not to leave +** variable values in the expression tree. +*/ +static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ + if( 0==sqlite3ExprIsConstant(pExpr) ){ + sqlite3ExprDelete(pParse->db, pExpr); + pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); + } + return pExpr; +} + +/* +** Allocate and return a new Window object describing a Window Definition. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAlloc( + Parse *pParse, /* Parsing context */ + int eType, /* Frame type. TK_RANGE or TK_ROWS */ + int eStart, /* Start type: CURRENT, PRECEDING, FOLLOWING, UNBOUNDED */ + Expr *pStart, /* Start window size if TK_PRECEDING or FOLLOWING */ + int eEnd, /* End type: CURRENT, FOLLOWING, TK_UNBOUNDED, PRECEDING */ + Expr *pEnd /* End window size if TK_FOLLOWING or PRECEDING */ +){ + Window *pWin = 0; + + /* Parser assures the following: */ + assert( eType==TK_RANGE || eType==TK_ROWS ); + assert( eStart==TK_CURRENT || eStart==TK_PRECEDING + || eStart==TK_UNBOUNDED || eStart==TK_FOLLOWING ); + assert( eEnd==TK_CURRENT || eEnd==TK_FOLLOWING + || eEnd==TK_UNBOUNDED || eEnd==TK_PRECEDING ); + assert( (eStart==TK_PRECEDING || eStart==TK_FOLLOWING)==(pStart!=0) ); + assert( (eEnd==TK_FOLLOWING || eEnd==TK_PRECEDING)==(pEnd!=0) ); + + + /* If a frame is declared "RANGE" (not "ROWS"), then it may not use + ** either " PRECEDING" or " FOLLOWING". + */ + if( eType==TK_RANGE && (pStart!=0 || pEnd!=0) ){ + sqlite3ErrorMsg(pParse, "RANGE must use only UNBOUNDED or CURRENT ROW"); + goto windowAllocErr; + } + + /* Additionally, the + ** starting boundary type may not occur earlier in the following list than + ** the ending boundary type: + ** + ** UNBOUNDED PRECEDING + ** PRECEDING + ** CURRENT ROW + ** FOLLOWING + ** UNBOUNDED FOLLOWING + ** + ** The parser ensures that "UNBOUNDED PRECEDING" cannot be used as an ending + ** boundary, and than "UNBOUNDED FOLLOWING" cannot be used as a starting + ** frame boundary. + */ + if( (eStart==TK_CURRENT && eEnd==TK_PRECEDING) + || (eStart==TK_FOLLOWING && (eEnd==TK_PRECEDING || eEnd==TK_CURRENT)) + ){ + sqlite3ErrorMsg(pParse, "unsupported frame delimiter for ROWS"); + goto windowAllocErr; + } + + pWin = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( pWin==0 ) goto windowAllocErr; + pWin->eType = eType; + pWin->eStart = eStart; + pWin->eEnd = eEnd; + pWin->pEnd = sqlite3WindowOffsetExpr(pParse, pEnd); + pWin->pStart = sqlite3WindowOffsetExpr(pParse, pStart); + return pWin; + +windowAllocErr: + sqlite3ExprDelete(pParse->db, pEnd); + sqlite3ExprDelete(pParse->db, pStart); + return 0; +} + +/* +** Attach window object pWin to expression p. +*/ +SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ + if( p ){ + /* This routine is only called for the parser. If pWin was not + ** allocated due to an OOM, then the parser would fail before ever + ** invoking this routine */ + if( ALWAYS(pWin) ){ + p->pWin = pWin; + pWin->pOwner = p; + if( p->flags & EP_Distinct ){ + sqlite3ErrorMsg(pParse, + "DISTINCT is not supported for window functions"); + } + } + }else{ + sqlite3WindowDelete(pParse->db, pWin); + } +} + +/* +** Return 0 if the two window objects are identical, or non-zero otherwise. +** Identical window objects can be processed in a single scan. +*/ +SQLITE_PRIVATE int sqlite3WindowCompare(Parse *pParse, Window *p1, Window *p2){ + if( p1->eType!=p2->eType ) return 1; + if( p1->eStart!=p2->eStart ) return 1; + if( p1->eEnd!=p2->eEnd ) return 1; + if( sqlite3ExprCompare(pParse, p1->pStart, p2->pStart, -1) ) return 1; + if( sqlite3ExprCompare(pParse, p1->pEnd, p2->pEnd, -1) ) return 1; + if( sqlite3ExprListCompare(p1->pPartition, p2->pPartition, -1) ) return 1; + if( sqlite3ExprListCompare(p1->pOrderBy, p2->pOrderBy, -1) ) return 1; + return 0; +} + + +/* +** This is called by code in select.c before it calls sqlite3WhereBegin() +** to begin iterating through the sub-query results. It is used to allocate +** and initialize registers and cursors used by sqlite3WindowCodeStep(). +*/ +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Window *pMWin){ + Window *pWin; + Vdbe *v = sqlite3GetVdbe(pParse); + int nPart = (pMWin->pPartition ? pMWin->pPartition->nExpr : 0); + nPart += (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + if( nPart ){ + pMWin->regPart = pParse->nMem+1; + pParse->nMem += nPart; + sqlite3VdbeAddOp3(v, OP_Null, 0, pMWin->regPart, pMWin->regPart+nPart-1); + } + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *p = pWin->pFunc; + if( (p->funcFlags & SQLITE_FUNC_MINMAX) && pWin->eStart!=TK_UNBOUNDED ){ + /* The inline versions of min() and max() require a single ephemeral + ** table and 3 registers. The registers are used as follows: + ** + ** regApp+0: slot to copy min()/max() argument to for MakeRecord + ** regApp+1: integer value used to ensure keys are unique + ** regApp+2: output of MakeRecord + */ + ExprList *pList = pWin->pOwner->x.pList; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pList, 0, 0); + pWin->csrApp = pParse->nTab++; + pWin->regApp = pParse->nMem+1; + pParse->nMem += 3; + if( pKeyInfo && pWin->pFunc->zName[1]=='i' ){ + assert( pKeyInfo->aSortOrder[0]==0 ); + pKeyInfo->aSortOrder[0] = 1; + } + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pWin->csrApp, 2); + sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + else if( p->zName==nth_valueName || p->zName==first_valueName ){ + /* Allocate two registers at pWin->regApp. These will be used to + ** store the start and end index of the current frame. */ + assert( pMWin->iEphCsr ); + pWin->regApp = pParse->nMem+1; + pWin->csrApp = pParse->nTab++; + pParse->nMem += 2; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + else if( p->zName==leadName || p->zName==lagName ){ + assert( pMWin->iEphCsr ); + pWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + } +} + +/* +** A "PRECEDING " (eCond==0) or "FOLLOWING " (eCond==1) or the +** value of the second argument to nth_value() (eCond==2) has just been +** evaluated and the result left in register reg. This function generates VM +** code to check that the value is a non-negative integer and throws an +** exception if it is not. +*/ +static void windowCheckIntValue(Parse *pParse, int reg, int eCond){ + static const char *azErr[] = { + "frame starting offset must be a non-negative integer", + "frame ending offset must be a non-negative integer", + "second argument to nth_value must be a positive integer" + }; + static int aOp[] = { OP_Ge, OP_Ge, OP_Gt }; + Vdbe *v = sqlite3GetVdbe(pParse); + int regZero = sqlite3GetTempReg(pParse); + assert( eCond==0 || eCond==1 || eCond==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regZero); + sqlite3VdbeAddOp2(v, OP_MustBeInt, reg, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverageIf(v, eCond==0); + VdbeCoverageIf(v, eCond==1); + VdbeCoverageIf(v, eCond==2); + sqlite3VdbeAddOp3(v, aOp[eCond], regZero, sqlite3VdbeCurrentAddr(v)+2, reg); + VdbeCoverageNeverNullIf(v, eCond==0); + VdbeCoverageNeverNullIf(v, eCond==1); + VdbeCoverageNeverNullIf(v, eCond==2); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_ERROR, OE_Abort); + sqlite3VdbeAppendP4(v, (void*)azErr[eCond], P4_STATIC); + sqlite3ReleaseTempReg(pParse, regZero); +} + +/* +** Return the number of arguments passed to the window-function associated +** with the object passed as the only argument to this function. +*/ +static int windowArgCount(Window *pWin){ + ExprList *pList = pWin->pOwner->x.pList; + return (pList ? pList->nExpr : 0); +} + +/* +** Generate VM code to invoke either xStep() (if bInverse is 0) or +** xInverse (if bInverse is non-zero) for each window function in the +** linked list starting at pMWin. Or, for built-in window functions +** that do not use the standard function API, generate the required +** inline VM code. +** +** If argument csr is greater than or equal to 0, then argument reg is +** the first register in an array of registers guaranteed to be large +** enough to hold the array of arguments for each function. In this case +** the arguments are extracted from the current row of csr into the +** array of registers before invoking OP_AggStep or OP_AggInverse +** +** Or, if csr is less than zero, then the array of registers at reg is +** already populated with all columns from the current row of the sub-query. +** +** If argument regPartSize is non-zero, then it is a register containing the +** number of rows in the current partition. +*/ +static void windowAggStep( + Parse *pParse, + Window *pMWin, /* Linked list of window functions */ + int csr, /* Read arguments from this cursor */ + int bInverse, /* True to invoke xInverse instead of xStep */ + int reg, /* Array of registers */ + int regPartSize /* Register containing size of partition */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + int flags = pWin->pFunc->funcFlags; + int regArg; + int nArg = windowArgCount(pWin); + + if( csr>=0 ){ + int i; + for(i=0; iiArgCol+i, reg+i); + } + regArg = reg; + if( flags & SQLITE_FUNC_WINDOW_SIZE ){ + if( nArg==0 ){ + regArg = regPartSize; + }else{ + sqlite3VdbeAddOp2(v, OP_SCopy, regPartSize, reg+nArg); + } + nArg++; + } + }else{ + assert( !(flags & SQLITE_FUNC_WINDOW_SIZE) ); + regArg = reg + pWin->iArgCol; + } + + if( (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && pWin->eStart!=TK_UNBOUNDED + ){ + int addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regArg); + VdbeCoverage(v); + if( bInverse==0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg, pWin->regApp); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pWin->regApp, 2, pWin->regApp+2); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pWin->csrApp, pWin->regApp+2); + }else{ + sqlite3VdbeAddOp4Int(v, OP_SeekGE, pWin->csrApp, 0, regArg, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Delete, pWin->csrApp); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + } + sqlite3VdbeJumpHere(v, addrIsNull); + }else if( pWin->regApp ){ + assert( pWin->pFunc->zName==nth_valueName + || pWin->pFunc->zName==first_valueName + ); + assert( bInverse==0 || bInverse==1 ); + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1); + }else if( pWin->pFunc->zName==leadName + || pWin->pFunc->zName==lagName + ){ + /* no-op */ + }else{ + int addrIf = 0; + if( pWin->pFilter ){ + int regTmp; + assert( nArg==0 || nArg==pWin->pOwner->x.pList->nExpr ); + assert( nArg || pWin->pOwner->x.pList==0 ); + if( csr>0 ){ + regTmp = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); + }else{ + regTmp = regArg + nArg; + } + addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); + VdbeCoverage(v); + if( csr>0 ){ + sqlite3ReleaseTempReg(pParse, regTmp); + } + } + if( pWin->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl; + assert( nArg>0 ); + pColl = sqlite3ExprNNCollSeq(pParse, pWin->pOwner->x.pList->a[0].pExpr); + sqlite3VdbeAddOp4(v, OP_CollSeq, 0,0,0, (const char*)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, + bInverse, regArg, pWin->regAccum); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); + } + } +} + +/* +** Generate VM code to invoke either xValue() (bFinal==0) or xFinalize() +** (bFinal==1) for each window function in the linked list starting at +** pMWin. Or, for built-in window-functions that do not use the standard +** API, generate the equivalent VM code. +*/ +static void windowAggFinal(Parse *pParse, Window *pMWin, int bFinal){ + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + if( (pWin->pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && pWin->eStart!=TK_UNBOUNDED + ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + sqlite3VdbeAddOp1(v, OP_Last, pWin->csrApp); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, pWin->csrApp, 0, pWin->regResult); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + if( bFinal ){ + sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); + } + }else if( pWin->regApp ){ + }else{ + if( bFinal ){ + sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, windowArgCount(pWin)); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + sqlite3VdbeAddOp2(v, OP_Copy, pWin->regAccum, pWin->regResult); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + }else{ + sqlite3VdbeAddOp3(v, OP_AggValue, pWin->regAccum, windowArgCount(pWin), + pWin->regResult); + sqlite3VdbeAppendP4(v, pWin->pFunc, P4_FUNCDEF); + } + } + } +} + +/* +** This function generates VM code to invoke the sub-routine at address +** lblFlushPart once for each partition with the entire partition cached in +** the Window.iEphCsr temp table. +*/ +static void windowPartitionCache( + Parse *pParse, + Select *p, /* The rewritten SELECT statement */ + WhereInfo *pWInfo, /* WhereInfo to call WhereEnd() on */ + int regFlushPart, /* Register to use with Gosub lblFlushPart */ + int lblFlushPart, /* Subroutine to Gosub to */ + int *pRegSize /* OUT: Register containing partition size */ +){ + Window *pMWin = p->pWin; + Vdbe *v = sqlite3GetVdbe(pParse); + int iSubCsr = p->pSrc->a[0].iCursor; + int nSub = p->pSrc->a[0].pTab->nCol; + int k; + + int reg = pParse->nMem+1; + int regRecord = reg+nSub; + int regRowid = regRecord+1; + + *pRegSize = regRowid; + pParse->nMem += nSub + 2; + + /* Load the column values for the row returned by the sub-select + ** into an array of registers starting at reg. */ + for(k=0; kpPartition ){ + int addr; + ExprList *pPart = pMWin->pPartition; + int nPart = pPart->nExpr; + int regNewPart = reg + pMWin->nBufferCol; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart,nPart); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); + VdbeCoverageEqNe(v); + sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); + sqlite3VdbeAddOp2(v, OP_Gosub, regFlushPart, lblFlushPart); + VdbeComment((v, "call flush_partition")); + } + + /* Buffer the current row in the ephemeral table. */ + sqlite3VdbeAddOp2(v, OP_NewRowid, pMWin->iEphCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, pMWin->iEphCsr, regRecord, regRowid); + + /* End of the input loop */ + sqlite3WhereEnd(pWInfo); + + /* Invoke "flush_partition" to deal with the final (or only) partition */ + sqlite3VdbeAddOp2(v, OP_Gosub, regFlushPart, lblFlushPart); + VdbeComment((v, "call flush_partition")); +} + +/* +** Invoke the sub-routine at regGosub (generated by code in select.c) to +** return the current row of Window.iEphCsr. If all window functions are +** aggregate window functions that use the standard API, a single +** OP_Gosub instruction is all that this routine generates. Extra VM code +** for per-row processing is only generated for the following built-in window +** functions: +** +** nth_value() +** first_value() +** lag() +** lead() +*/ +static void windowReturnOneRow( + Parse *pParse, + Window *pMWin, + int regGosub, + int addrGosub +){ + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ){ + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(v); + int tmpReg = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + + if( pFunc->zName==nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column, pMWin->iEphCsr, pWin->iArgCol+1,tmpReg); + windowCheckIntValue(pParse, tmpReg, 2); + }else{ + sqlite3VdbeAddOp2(v, OP_Integer, 1, tmpReg); + } + sqlite3VdbeAddOp3(v, OP_Add, tmpReg, pWin->regApp, tmpReg); + sqlite3VdbeAddOp3(v, OP_Gt, pWin->regApp+1, lbl, tmpReg); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, 0, tmpReg); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + else if( pFunc->zName==leadName || pFunc->zName==lagName ){ + int nArg = pWin->pOwner->x.pList->nExpr; + int iEph = pMWin->iEphCsr; + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(v); + int tmpReg = sqlite3GetTempReg(pParse); + + if( nArg<3 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+2, pWin->regResult); + } + sqlite3VdbeAddOp2(v, OP_Rowid, iEph, tmpReg); + if( nArg<2 ){ + int val = (pFunc->zName==leadName ? 1 : -1); + sqlite3VdbeAddOp2(v, OP_AddImm, tmpReg, val); + }else{ + int op = (pFunc->zName==leadName ? OP_Add : OP_Subtract); + int tmpReg2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+1, tmpReg2); + sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg); + sqlite3ReleaseTempReg(pParse, tmpReg2); + } + + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + } + sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrGosub); +} + +/* +** Invoke the code generated by windowReturnOneRow() and, optionally, the +** xInverse() function for each window function, for one or more rows +** from the Window.iEphCsr temp table. This routine generates VM code +** similar to: +** +** while( regCtr>0 ){ +** regCtr--; +** windowReturnOneRow() +** if( bInverse ){ +** AggInverse +** } +** Next (Window.iEphCsr) +** } +*/ +static void windowReturnRows( + Parse *pParse, + Window *pMWin, /* List of window functions */ + int regCtr, /* Register containing number of rows */ + int regGosub, /* Register for Gosub addrGosub */ + int addrGosub, /* Address of sub-routine for ReturnOneRow */ + int regInvArg, /* Array of registers for xInverse args */ + int regInvSize /* Register containing size of partition */ +){ + int addr; + Vdbe *v = sqlite3GetVdbe(pParse); + windowAggFinal(pParse, pMWin, 0); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, regCtr, sqlite3VdbeCurrentAddr(v)+2 ,1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, 0); + windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); + if( regInvArg ){ + windowAggStep(pParse, pMWin, pMWin->iEphCsr, 1, regInvArg, regInvSize); + } + sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, addr); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr+1); /* The OP_Goto */ +} + +/* +** Generate code to set the accumulator register for each window function +** in the linked list passed as the second argument to NULL. And perform +** any equivalent initialization required by any built-in window functions +** in the list. +*/ +static int windowInitAccum(Parse *pParse, Window *pMWin){ + Vdbe *v = sqlite3GetVdbe(pParse); + int regArg; + int nArg = 0; + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + nArg = MAX(nArg, windowArgCount(pWin)); + if( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + + if( (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && pWin->csrApp ){ + assert( pWin->eStart!=TK_UNBOUNDED ); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + } + regArg = pParse->nMem+1; + pParse->nMem += nArg; + return regArg; +} + + +/* +** This function does the work of sqlite3WindowCodeStep() for all "ROWS" +** window frame types except for "BETWEEN UNBOUNDED PRECEDING AND CURRENT +** ROW". Pseudo-code for each follows. +** +** ROWS BETWEEN PRECEDING AND FOLLOWING +** +** ... +** if( new partition ){ +** Gosub flush_partition +** } +** Insert (record in eph-table) +** sqlite3WhereEnd() +** Gosub flush_partition +** +** flush_partition: +** Once { +** OpenDup (iEphCsr -> csrStart) +** OpenDup (iEphCsr -> csrEnd) +** } +** regStart = // PRECEDING expression +** regEnd = // FOLLOWING expression +** if( regStart<0 || regEnd<0 ){ error! } +** Rewind (csr,csrStart,csrEnd) // if EOF goto flush_partition_done +** Next(csrEnd) // if EOF skip Aggstep +** Aggstep (csrEnd) +** if( (regEnd--)<=0 ){ +** AggFinal (xValue) +** Gosub addrGosub +** Next(csr) // if EOF goto flush_partition_done +** if( (regStart--)<=0 ){ +** AggInverse (csrStart) +** Next(csrStart) +** } +** } +** flush_partition_done: +** ResetSorter (csr) +** Return +** +** ROWS BETWEEN PRECEDING AND CURRENT ROW +** ROWS BETWEEN CURRENT ROW AND FOLLOWING +** ROWS BETWEEN UNBOUNDED PRECEDING AND FOLLOWING +** +** These are similar to the above. For "CURRENT ROW", intialize the +** register to 0. For "UNBOUNDED PRECEDING" to infinity. +** +** ROWS BETWEEN PRECEDING AND UNBOUNDED FOLLOWING +** ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +** +** Rewind (csr,csrStart,csrEnd) // if EOF goto flush_partition_done +** while( 1 ){ +** Next(csrEnd) // Exit while(1) at EOF +** Aggstep (csrEnd) +** } +** while( 1 ){ +** AggFinal (xValue) +** Gosub addrGosub +** Next(csr) // if EOF goto flush_partition_done +** if( (regStart--)<=0 ){ +** AggInverse (csrStart) +** Next(csrStart) +** } +** } +** +** For the "CURRENT ROW AND UNBOUNDED FOLLOWING" case, the final if() +** condition is always true (as if regStart were initialized to 0). +** +** RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +** +** This is the only RANGE case handled by this routine. It modifies the +** second while( 1 ) loop in "ROWS BETWEEN CURRENT ... UNBOUNDED..." to +** be: +** +** while( 1 ){ +** AggFinal (xValue) +** while( 1 ){ +** regPeer++ +** Gosub addrGosub +** Next(csr) // if EOF goto flush_partition_done +** if( new peer ) break; +** } +** while( (regPeer--)>0 ){ +** AggInverse (csrStart) +** Next(csrStart) +** } +** } +** +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** regEnd = regEnd - regStart +** Rewind (csr,csrStart,csrEnd) // if EOF goto flush_partition_done +** Aggstep (csrEnd) +** Next(csrEnd) // if EOF fall-through +** if( (regEnd--)<=0 ){ +** if( (regStart--)<=0 ){ +** AggFinal (xValue) +** Gosub addrGosub +** Next(csr) // if EOF goto flush_partition_done +** } +** AggInverse (csrStart) +** Next (csrStart) +** } +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** +** Replace the bit after "Rewind" in the above with: +** +** if( (regEnd--)<=0 ){ +** AggStep (csrEnd) +** Next (csrEnd) +** } +** AggFinal (xValue) +** Gosub addrGosub +** Next(csr) // if EOF goto flush_partition_done +** if( (regStart--)<=0 ){ +** AggInverse (csr2) +** Next (csr2) +** } +** +*/ +static void windowCodeRowExprStep( + Parse *pParse, + Select *p, + WhereInfo *pWInfo, + int regGosub, + int addrGosub +){ + Window *pMWin = p->pWin; + Vdbe *v = sqlite3GetVdbe(pParse); + int regFlushPart; /* Register for "Gosub flush_partition" */ + int lblFlushPart; /* Label for "Gosub flush_partition" */ + int lblFlushDone; /* Label for "Gosub flush_partition_done" */ + + int regArg; + int addr; + int csrStart = pParse->nTab++; + int csrEnd = pParse->nTab++; + int regStart; /* Value of PRECEDING */ + int regEnd; /* Value of FOLLOWING */ + int addrGoto; + int addrTop; + int addrIfPos1 = 0; + int addrIfPos2 = 0; + int regSize = 0; + + assert( pMWin->eStart==TK_PRECEDING + || pMWin->eStart==TK_CURRENT + || pMWin->eStart==TK_FOLLOWING + || pMWin->eStart==TK_UNBOUNDED + ); + assert( pMWin->eEnd==TK_FOLLOWING + || pMWin->eEnd==TK_CURRENT + || pMWin->eEnd==TK_UNBOUNDED + || pMWin->eEnd==TK_PRECEDING + ); + + /* Allocate register and label for the "flush_partition" sub-routine. */ + regFlushPart = ++pParse->nMem; + lblFlushPart = sqlite3VdbeMakeLabel(v); + lblFlushDone = sqlite3VdbeMakeLabel(v); + + regStart = ++pParse->nMem; + regEnd = ++pParse->nMem; + + windowPartitionCache(pParse, p, pWInfo, regFlushPart, lblFlushPart, ®Size); + + addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); + + /* Start of "flush_partition" */ + sqlite3VdbeResolveLabel(v, lblFlushPart); + sqlite3VdbeAddOp2(v, OP_Once, 0, sqlite3VdbeCurrentAddr(v)+3); + VdbeCoverage(v); + VdbeComment((v, "Flush_partition subroutine")); + sqlite3VdbeAddOp2(v, OP_OpenDup, csrStart, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, csrEnd, pMWin->iEphCsr); + + /* If either regStart or regEnd are not non-negative integers, throw + ** an exception. */ + if( pMWin->pStart ){ + sqlite3ExprCode(pParse, pMWin->pStart, regStart); + windowCheckIntValue(pParse, regStart, 0); + } + if( pMWin->pEnd ){ + sqlite3ExprCode(pParse, pMWin->pEnd, regEnd); + windowCheckIntValue(pParse, regEnd, 1); + } + + /* If this is "ROWS FOLLOWING AND ROWS FOLLOWING", do: + ** + ** if( regEndpEnd && pMWin->eStart==TK_FOLLOWING ){ + assert( pMWin->pStart!=0 ); + assert( pMWin->eEnd==TK_FOLLOWING ); + sqlite3VdbeAddOp3(v, OP_Ge, regStart, sqlite3VdbeCurrentAddr(v)+2, regEnd); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp2(v, OP_Copy, regSize, regStart); + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regEnd); + } + + if( pMWin->pStart && pMWin->eEnd==TK_PRECEDING ){ + assert( pMWin->pEnd!=0 ); + assert( pMWin->eStart==TK_PRECEDING ); + sqlite3VdbeAddOp3(v, OP_Le, regStart, sqlite3VdbeCurrentAddr(v)+3, regEnd); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp2(v, OP_Copy, regSize, regStart); + sqlite3VdbeAddOp2(v, OP_Copy, regSize, regEnd); + } + + /* Initialize the accumulator register for each window function to NULL */ + regArg = windowInitAccum(pParse, pMWin); + + sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr, lblFlushDone); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, csrStart, lblFlushDone); + VdbeCoverageNeverTaken(v); + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeAddOp2(v, OP_Rewind, csrEnd, lblFlushDone); + VdbeCoverageNeverTaken(v); + sqlite3VdbeChangeP5(v, 1); + + /* Invoke AggStep function for each window function using the row that + ** csrEnd currently points to. Or, if csrEnd is already at EOF, + ** do nothing. */ + addrTop = sqlite3VdbeCurrentAddr(v); + if( pMWin->eEnd==TK_PRECEDING ){ + addrIfPos1 = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0 , 1); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Next, csrEnd, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + addr = sqlite3VdbeAddOp0(v, OP_Goto); + windowAggStep(pParse, pMWin, csrEnd, 0, regArg, regSize); + if( pMWin->eEnd==TK_UNBOUNDED ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeJumpHere(v, addr); + addrTop = sqlite3VdbeCurrentAddr(v); + }else{ + sqlite3VdbeJumpHere(v, addr); + if( pMWin->eEnd==TK_PRECEDING ){ + sqlite3VdbeJumpHere(v, addrIfPos1); + } + } + + if( pMWin->eEnd==TK_FOLLOWING ){ + addrIfPos1 = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0 , 1); + VdbeCoverage(v); + } + if( pMWin->eStart==TK_FOLLOWING ){ + addrIfPos2 = sqlite3VdbeAddOp3(v, OP_IfPos, regStart, 0 , 1); + VdbeCoverage(v); + } + windowAggFinal(pParse, pMWin, 0); + windowReturnOneRow(pParse, pMWin, regGosub, addrGosub); + sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblFlushDone); + if( pMWin->eStart==TK_FOLLOWING ){ + sqlite3VdbeJumpHere(v, addrIfPos2); + } + + if( pMWin->eStart==TK_CURRENT + || pMWin->eStart==TK_PRECEDING + || pMWin->eStart==TK_FOLLOWING + ){ + int lblSkipInverse = sqlite3VdbeMakeLabel(v);; + if( pMWin->eStart==TK_PRECEDING ){ + sqlite3VdbeAddOp3(v, OP_IfPos, regStart, lblSkipInverse, 1); + VdbeCoverage(v); + } + if( pMWin->eStart==TK_FOLLOWING ){ + sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblSkipInverse); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, csrStart, sqlite3VdbeCurrentAddr(v)+1); + VdbeCoverageAlwaysTaken(v); + } + windowAggStep(pParse, pMWin, csrStart, 1, regArg, regSize); + sqlite3VdbeResolveLabel(v, lblSkipInverse); + } + if( pMWin->eEnd==TK_FOLLOWING ){ + sqlite3VdbeJumpHere(v, addrIfPos1); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + + /* flush_partition_done: */ + sqlite3VdbeResolveLabel(v, lblFlushDone); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pMWin->iEphCsr); + sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); + VdbeComment((v, "end flush_partition subroutine")); + + /* Jump to here to skip over flush_partition */ + sqlite3VdbeJumpHere(v, addrGoto); +} + +/* +** This function does the work of sqlite3WindowCodeStep() for cases that +** would normally be handled by windowCodeDefaultStep() when there are +** one or more built-in window-functions that require the entire partition +** to be cached in a temp table before any rows can be returned. Additionally. +** "RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING" is always handled by +** this function. +** +** Pseudo-code corresponding to the VM code generated by this function +** for each type of window follows. +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** flush_partition: +** Once { +** OpenDup (iEphCsr -> csrLead) +** } +** Integer ctr 0 +** foreach row (csrLead){ +** if( new peer ){ +** AggFinal (xValue) +** for(i=0; i csrLead) +** } +** foreach row (csrLead) { +** AggStep (csrLead) +** } +** foreach row (iEphCsr) { +** Gosub addrGosub +** } +** +** RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +** +** flush_partition: +** Once { +** OpenDup (iEphCsr -> csrLead) +** } +** foreach row (csrLead){ +** AggStep (csrLead) +** } +** Rewind (csrLead) +** Integer ctr 0 +** foreach row (csrLead){ +** if( new peer ){ +** AggFinal (xValue) +** for(i=0; ipWin; + Vdbe *v = sqlite3GetVdbe(pParse); + int k; + int addr; + ExprList *pPart = pMWin->pPartition; + ExprList *pOrderBy = pMWin->pOrderBy; + int nPeer = pOrderBy ? pOrderBy->nExpr : 0; + int regNewPeer; + + int addrGoto; /* Address of Goto used to jump flush_par.. */ + int addrNext; /* Jump here for next iteration of loop */ + int regFlushPart; + int lblFlushPart; + int csrLead; + int regCtr; + int regArg; /* Register array to martial function args */ + int regSize; + int lblEmpty; + int bReverse = pMWin->pOrderBy && pMWin->eStart==TK_CURRENT + && pMWin->eEnd==TK_UNBOUNDED; + + assert( (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_CURRENT) + || (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_UNBOUNDED) + || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_CURRENT) + || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_UNBOUNDED) + ); + + lblEmpty = sqlite3VdbeMakeLabel(v); + regNewPeer = pParse->nMem+1; + pParse->nMem += nPeer; + + /* Allocate register and label for the "flush_partition" sub-routine. */ + regFlushPart = ++pParse->nMem; + lblFlushPart = sqlite3VdbeMakeLabel(v); + + csrLead = pParse->nTab++; + regCtr = ++pParse->nMem; + + windowPartitionCache(pParse, p, pWInfo, regFlushPart, lblFlushPart, ®Size); + addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); + + /* Start of "flush_partition" */ + sqlite3VdbeResolveLabel(v, lblFlushPart); + sqlite3VdbeAddOp2(v, OP_Once, 0, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_OpenDup, csrLead, pMWin->iEphCsr); + + /* Initialize the accumulator register for each window function to NULL */ + regArg = windowInitAccum(pParse, pMWin); + + sqlite3VdbeAddOp2(v, OP_Integer, 0, regCtr); + sqlite3VdbeAddOp2(v, OP_Rewind, csrLead, lblEmpty); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr, lblEmpty); + VdbeCoverageNeverTaken(v); + + if( bReverse ){ + int addr2 = sqlite3VdbeCurrentAddr(v); + windowAggStep(pParse, pMWin, csrLead, 0, regArg, regSize); + sqlite3VdbeAddOp2(v, OP_Next, csrLead, addr2); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, csrLead, lblEmpty); + VdbeCoverageNeverTaken(v); + } + addrNext = sqlite3VdbeCurrentAddr(v); + + if( pOrderBy && (pMWin->eEnd==TK_CURRENT || pMWin->eStart==TK_CURRENT) ){ + int bCurrent = (pMWin->eStart==TK_CURRENT); + int addrJump = 0; /* Address of OP_Jump below */ + if( pMWin->eType==TK_RANGE ){ + int iOff = pMWin->nBufferCol + (pPart ? pPart->nExpr : 0); + int regPeer = pMWin->regPart + (pPart ? pPart->nExpr : 0); + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); + for(k=0; kiEphCsr); + sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); + + /* Jump to here to skip over flush_partition */ + sqlite3VdbeJumpHere(v, addrGoto); +} + + +/* +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** ... +** if( new partition ){ +** AggFinal (xFinalize) +** Gosub addrGosub +** ResetSorter eph-table +** } +** else if( new peer ){ +** AggFinal (xValue) +** Gosub addrGosub +** ResetSorter eph-table +** } +** AggStep +** Insert (record into eph-table) +** sqlite3WhereEnd() +** AggFinal (xFinalize) +** Gosub addrGosub +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING +** +** As above, except take no action for a "new peer". Invoke +** the sub-routine once only for each partition. +** +** RANGE BETWEEN CURRENT ROW AND CURRENT ROW +** +** As above, except that the "new peer" condition is handled in the +** same way as "new partition" (so there is no "else if" block). +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** As above, except assume every row is a "new peer". +*/ +static void windowCodeDefaultStep( + Parse *pParse, + Select *p, + WhereInfo *pWInfo, + int regGosub, + int addrGosub +){ + Window *pMWin = p->pWin; + Vdbe *v = sqlite3GetVdbe(pParse); + int k; + int iSubCsr = p->pSrc->a[0].iCursor; + int nSub = p->pSrc->a[0].pTab->nCol; + int reg = pParse->nMem+1; + int regRecord = reg+nSub; + int regRowid = regRecord+1; + int addr; + ExprList *pPart = pMWin->pPartition; + ExprList *pOrderBy = pMWin->pOrderBy; + + assert( pMWin->eType==TK_RANGE + || (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_CURRENT) + ); + + assert( (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_CURRENT) + || (pMWin->eStart==TK_UNBOUNDED && pMWin->eEnd==TK_UNBOUNDED) + || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_CURRENT) + || (pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_UNBOUNDED && !pOrderBy) + ); + + if( pMWin->eEnd==TK_UNBOUNDED ){ + pOrderBy = 0; + } + + pParse->nMem += nSub + 2; + + /* Load the individual column values of the row returned by + ** the sub-select into an array of registers. */ + for(k=0; knExpr : 0); + int addrGoto = 0; + int addrJump = 0; + int nPeer = (pOrderBy ? pOrderBy->nExpr : 0); + + if( pPart ){ + int regNewPart = reg + pMWin->nBufferCol; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart,nPart); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + addrJump = sqlite3VdbeAddOp3(v, OP_Jump, addr+2, 0, addr+2); + VdbeCoverageEqNe(v); + windowAggFinal(pParse, pMWin, 1); + if( pOrderBy ){ + addrGoto = sqlite3VdbeAddOp0(v, OP_Goto); + } + } + + if( pOrderBy ){ + int regNewPeer = reg + pMWin->nBufferCol + nPart; + int regPeer = pMWin->regPart + nPart; + + if( addrJump ) sqlite3VdbeJumpHere(v, addrJump); + if( pMWin->eType==TK_RANGE ){ + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPeer, regPeer, nPeer); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + addrJump = sqlite3VdbeAddOp3(v, OP_Jump, addr+2, 0, addr+2); + VdbeCoverage(v); + }else{ + addrJump = 0; + } + windowAggFinal(pParse, pMWin, pMWin->eStart==TK_CURRENT); + if( addrGoto ) sqlite3VdbeJumpHere(v, addrGoto); + } + + sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr,sqlite3VdbeCurrentAddr(v)+3); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrGosub); + sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, sqlite3VdbeCurrentAddr(v)-1); + VdbeCoverage(v); + + sqlite3VdbeAddOp1(v, OP_ResetSorter, pMWin->iEphCsr); + sqlite3VdbeAddOp3( + v, OP_Copy, reg+pMWin->nBufferCol, pMWin->regPart, nPart+nPeer-1 + ); + + if( addrJump ) sqlite3VdbeJumpHere(v, addrJump); + } + + /* Invoke step function for window functions */ + windowAggStep(pParse, pMWin, -1, 0, reg, 0); + + /* Buffer the current row in the ephemeral table. */ + if( pMWin->nBufferCol>0 ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, pMWin->nBufferCol, regRecord); + }else{ + sqlite3VdbeAddOp2(v, OP_Blob, 0, regRecord); + sqlite3VdbeAppendP4(v, (void*)"", 0); + } + sqlite3VdbeAddOp2(v, OP_NewRowid, pMWin->iEphCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, pMWin->iEphCsr, regRecord, regRowid); + + /* End the database scan loop. */ + sqlite3WhereEnd(pWInfo); + + windowAggFinal(pParse, pMWin, 1); + sqlite3VdbeAddOp2(v, OP_Rewind, pMWin->iEphCsr,sqlite3VdbeCurrentAddr(v)+3); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Gosub, regGosub, addrGosub); + sqlite3VdbeAddOp2(v, OP_Next, pMWin->iEphCsr, sqlite3VdbeCurrentAddr(v)-1); + VdbeCoverage(v); +} + +/* +** Allocate and return a duplicate of the Window object indicated by the +** third argument. Set the Window.pOwner field of the new object to +** pOwner. +*/ +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ + Window *pNew = 0; + if( p ){ + pNew = sqlite3DbMallocZero(db, sizeof(Window)); + if( pNew ){ + pNew->zName = sqlite3DbStrDup(db, p->zName); + pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); + pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); + pNew->eType = p->eType; + pNew->eEnd = p->eEnd; + pNew->eStart = p->eStart; + pNew->pStart = sqlite3ExprDup(db, p->pStart, 0); + pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0); + pNew->pOwner = pOwner; + } + } + return pNew; +} + +/* +** Return a copy of the linked list of Window objects passed as the +** second argument. +*/ +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p){ + Window *pWin; + Window *pRet = 0; + Window **pp = &pRet; + + for(pWin=p; pWin; pWin=pWin->pNextWin){ + *pp = sqlite3WindowDup(db, 0, pWin); + if( *pp==0 ) break; + pp = &((*pp)->pNextWin); + } + + return pRet; +} + +/* +** sqlite3WhereBegin() has already been called for the SELECT statement +** passed as the second argument when this function is invoked. It generates +** code to populate the Window.regResult register for each window function and +** invoke the sub-routine at instruction addrGosub once for each row. +** This function calls sqlite3WhereEnd() before returning. +*/ +SQLITE_PRIVATE void sqlite3WindowCodeStep( + Parse *pParse, /* Parse context */ + Select *p, /* Rewritten SELECT statement */ + WhereInfo *pWInfo, /* Context returned by sqlite3WhereBegin() */ + int regGosub, /* Register for OP_Gosub */ + int addrGosub /* OP_Gosub here to return each row */ +){ + Window *pMWin = p->pWin; + + /* There are three different functions that may be used to do the work + ** of this one, depending on the window frame and the specific built-in + ** window functions used (if any). + ** + ** windowCodeRowExprStep() handles all "ROWS" window frames, except for: + ** + ** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ** + ** The exception is because windowCodeRowExprStep() implements all window + ** frame types by caching the entire partition in a temp table, and + ** "ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW" is easy enough to + ** implement without such a cache. + ** + ** windowCodeCacheStep() is used for: + ** + ** RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + ** + ** It is also used for anything not handled by windowCodeRowExprStep() + ** that invokes a built-in window function that requires the entire + ** partition to be cached in a temp table before any rows are returned + ** (e.g. nth_value() or percent_rank()). + ** + ** Finally, assuming there is no built-in window function that requires + ** the partition to be cached, windowCodeDefaultStep() is used for: + ** + ** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ** RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING + ** RANGE BETWEEN CURRENT ROW AND CURRENT ROW + ** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + ** + ** windowCodeDefaultStep() is the only one of the three functions that + ** does not cache each partition in a temp table before beginning to + ** return rows. + */ + if( pMWin->eType==TK_ROWS + && (pMWin->eStart!=TK_UNBOUNDED||pMWin->eEnd!=TK_CURRENT||!pMWin->pOrderBy) + ){ + VdbeModuleComment((pParse->pVdbe, "Begin RowExprStep()")); + windowCodeRowExprStep(pParse, p, pWInfo, regGosub, addrGosub); + }else{ + Window *pWin; + int bCache = 0; /* True to use CacheStep() */ + + if( pMWin->eStart==TK_CURRENT && pMWin->eEnd==TK_UNBOUNDED ){ + bCache = 1; + }else{ + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pFunc; + if( (pFunc->funcFlags & SQLITE_FUNC_WINDOW_SIZE) + || (pFunc->zName==nth_valueName) + || (pFunc->zName==first_valueName) + || (pFunc->zName==leadName) + || (pFunc->zName==lagName) + ){ + bCache = 1; + break; + } + } + } + + /* Otherwise, call windowCodeDefaultStep(). */ + if( bCache ){ + VdbeModuleComment((pParse->pVdbe, "Begin CacheStep()")); + windowCodeCacheStep(pParse, p, pWInfo, regGosub, addrGosub); + }else{ + VdbeModuleComment((pParse->pVdbe, "Begin DefaultStep()")); + windowCodeDefaultStep(pParse, p, pWInfo, regGosub, addrGosub); + } + } +} + +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/************** End of window.c **********************************************/ /************** Begin file parse.c *******************************************/ /* ** 2000-05-29 @@ -141457,6 +145982,8 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ struct TrigEvent { int a; IdList * b; }; +struct FrameBound { int eType; Expr *pExpr; }; + /* ** Disable lookaside memory allocation for objects that might be ** shared across database connections. @@ -141497,10 +146024,21 @@ static void disableLookaside(Parse *pParse){ static Expr *tokenExpr(Parse *pParse, int op, Token t){ Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); if( p ){ - memset(p, 0, sizeof(Expr)); + /* memset(p, 0, sizeof(Expr)); */ p->op = (u8)op; + p->affinity = 0; p->flags = EP_Leaf; p->iAgg = -1; + p->pLeft = p->pRight = 0; + p->x.pList = 0; + p->pAggInfo = 0; + p->pTab = 0; + p->op2 = 0; + p->iTable = 0; + p->iColumn = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + p->pWin = 0; +#endif p->u.zToken = (char*)&p[1]; memcpy(p->u.zToken, t.z, t.n); p->u.zToken[t.n] = 0; @@ -141511,15 +146049,19 @@ static void disableLookaside(Parse *pParse){ #if SQLITE_MAX_EXPR_DEPTH>0 p->nHeight = 1; #endif + if( IN_RENAME_OBJECT ){ + return (Expr*)sqlite3RenameTokenMap(pParse, (void*)p, &t); + } } return p; } + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL ){ + if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; @@ -141610,26 +146152,28 @@ static void disableLookaside(Parse *pParse){ # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 255 +#define YYCODETYPE unsigned short int +#define YYNOCODE 277 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 84 +#define YYWILDCARD 91 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - const char* yy36; - TriggerStep* yy47; - With* yy91; - struct {int value; int mask;} yy107; - Expr* yy182; - Upsert* yy198; - ExprList* yy232; - struct TrigEvent yy300; - Select* yy399; - SrcList* yy427; - int yy502; - IdList* yy510; + Expr* yy18; + struct TrigEvent yy34; + IdList* yy48; + int yy70; + struct {int value; int mask;} yy111; + struct FrameBound yy119; + SrcList* yy135; + TriggerStep* yy207; + Window* yy327; + Upsert* yy340; + const char* yy392; + ExprList* yy420; + With* yy449; + Select* yy489; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -141645,18 +146189,19 @@ typedef union { #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; #define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 490 -#define YYNRULE 341 -#define YYNTOKEN 145 -#define YY_MAX_SHIFT 489 -#define YY_MIN_SHIFTREDUCE 705 -#define YY_MAX_SHIFTREDUCE 1045 -#define YY_ERROR_ACTION 1046 -#define YY_ACCEPT_ACTION 1047 -#define YY_NO_ACTION 1048 -#define YY_MIN_REDUCE 1049 -#define YY_MAX_REDUCE 1389 +#define YYNSTATE 521 +#define YYNRULE 367 +#define YYNTOKEN 155 +#define YY_MAX_SHIFT 520 +#define YY_MIN_SHIFTREDUCE 756 +#define YY_MAX_SHIFTREDUCE 1122 +#define YY_ERROR_ACTION 1123 +#define YY_ACCEPT_ACTION 1124 +#define YY_NO_ACTION 1125 +#define YY_MIN_REDUCE 1126 +#define YY_MAX_REDUCE 1492 /************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -141721,503 +146266,568 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1657) +#define YY_ACTTAB_COUNT (2009) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 349, 99, 96, 185, 99, 96, 185, 233, 1047, 1, - /* 10 */ 1, 489, 2, 1051, 484, 477, 477, 477, 260, 351, - /* 20 */ 121, 1310, 1120, 1120, 1178, 1115, 1094, 1128, 380, 380, - /* 30 */ 380, 835, 454, 410, 1115, 59, 59, 1357, 425, 836, - /* 40 */ 710, 711, 712, 106, 107, 97, 1023, 1023, 900, 903, - /* 50 */ 892, 892, 104, 104, 105, 105, 105, 105, 346, 238, - /* 60 */ 238, 99, 96, 185, 238, 238, 889, 889, 901, 904, - /* 70 */ 460, 481, 351, 99, 96, 185, 481, 347, 1177, 82, - /* 80 */ 388, 214, 182, 23, 194, 103, 103, 103, 103, 102, - /* 90 */ 102, 101, 101, 101, 100, 381, 106, 107, 97, 1023, - /* 100 */ 1023, 900, 903, 892, 892, 104, 104, 105, 105, 105, - /* 110 */ 105, 10, 385, 484, 24, 484, 1333, 489, 2, 1051, - /* 120 */ 335, 1043, 108, 893, 260, 351, 121, 99, 96, 185, - /* 130 */ 100, 381, 386, 1128, 59, 59, 59, 59, 103, 103, - /* 140 */ 103, 103, 102, 102, 101, 101, 101, 100, 381, 106, - /* 150 */ 107, 97, 1023, 1023, 900, 903, 892, 892, 104, 104, - /* 160 */ 105, 105, 105, 105, 360, 238, 238, 170, 170, 467, - /* 170 */ 455, 467, 464, 67, 381, 329, 169, 481, 351, 343, - /* 180 */ 338, 400, 1044, 68, 101, 101, 101, 100, 381, 393, - /* 190 */ 194, 103, 103, 103, 103, 102, 102, 101, 101, 101, - /* 200 */ 100, 381, 106, 107, 97, 1023, 1023, 900, 903, 892, - /* 210 */ 892, 104, 104, 105, 105, 105, 105, 483, 385, 103, - /* 220 */ 103, 103, 103, 102, 102, 101, 101, 101, 100, 381, - /* 230 */ 268, 351, 946, 946, 422, 296, 102, 102, 101, 101, - /* 240 */ 101, 100, 381, 861, 103, 103, 103, 103, 102, 102, - /* 250 */ 101, 101, 101, 100, 381, 106, 107, 97, 1023, 1023, - /* 260 */ 900, 903, 892, 892, 104, 104, 105, 105, 105, 105, - /* 270 */ 484, 983, 1383, 206, 1353, 1383, 438, 435, 434, 281, - /* 280 */ 396, 269, 1089, 941, 351, 1002, 433, 861, 743, 401, - /* 290 */ 282, 57, 57, 482, 145, 791, 791, 103, 103, 103, - /* 300 */ 103, 102, 102, 101, 101, 101, 100, 381, 106, 107, - /* 310 */ 97, 1023, 1023, 900, 903, 892, 892, 104, 104, 105, - /* 320 */ 105, 105, 105, 281, 1002, 1003, 1004, 206, 879, 319, - /* 330 */ 438, 435, 434, 981, 259, 474, 360, 351, 1118, 1118, - /* 340 */ 433, 736, 379, 378, 872, 1002, 1356, 322, 871, 766, - /* 350 */ 103, 103, 103, 103, 102, 102, 101, 101, 101, 100, - /* 360 */ 381, 106, 107, 97, 1023, 1023, 900, 903, 892, 892, - /* 370 */ 104, 104, 105, 105, 105, 105, 484, 801, 484, 871, - /* 380 */ 871, 873, 401, 282, 1002, 1003, 1004, 1030, 360, 1030, - /* 390 */ 351, 983, 1384, 213, 880, 1384, 145, 59, 59, 59, - /* 400 */ 59, 1002, 244, 103, 103, 103, 103, 102, 102, 101, - /* 410 */ 101, 101, 100, 381, 106, 107, 97, 1023, 1023, 900, - /* 420 */ 903, 892, 892, 104, 104, 105, 105, 105, 105, 274, - /* 430 */ 484, 110, 467, 479, 467, 444, 259, 474, 232, 232, - /* 440 */ 1002, 1003, 1004, 351, 210, 335, 982, 866, 1385, 336, - /* 450 */ 481, 59, 59, 981, 245, 307, 103, 103, 103, 103, - /* 460 */ 102, 102, 101, 101, 101, 100, 381, 106, 107, 97, - /* 470 */ 1023, 1023, 900, 903, 892, 892, 104, 104, 105, 105, - /* 480 */ 105, 105, 453, 459, 484, 408, 377, 259, 474, 271, - /* 490 */ 183, 273, 209, 208, 207, 356, 351, 307, 178, 177, - /* 500 */ 127, 1006, 1098, 14, 14, 43, 43, 1044, 425, 103, - /* 510 */ 103, 103, 103, 102, 102, 101, 101, 101, 100, 381, - /* 520 */ 106, 107, 97, 1023, 1023, 900, 903, 892, 892, 104, - /* 530 */ 104, 105, 105, 105, 105, 294, 1132, 408, 160, 484, - /* 540 */ 408, 1006, 129, 962, 1209, 239, 239, 481, 307, 425, - /* 550 */ 1309, 1097, 351, 235, 243, 272, 820, 481, 963, 425, - /* 560 */ 11, 11, 103, 103, 103, 103, 102, 102, 101, 101, - /* 570 */ 101, 100, 381, 964, 362, 1002, 106, 107, 97, 1023, - /* 580 */ 1023, 900, 903, 892, 892, 104, 104, 105, 105, 105, - /* 590 */ 105, 1275, 161, 126, 777, 289, 1209, 292, 1072, 357, - /* 600 */ 1209, 1127, 476, 357, 778, 425, 247, 425, 351, 248, - /* 610 */ 414, 364, 414, 171, 1002, 1003, 1004, 84, 103, 103, - /* 620 */ 103, 103, 102, 102, 101, 101, 101, 100, 381, 1002, - /* 630 */ 184, 484, 106, 107, 97, 1023, 1023, 900, 903, 892, - /* 640 */ 892, 104, 104, 105, 105, 105, 105, 1123, 1209, 287, - /* 650 */ 484, 1209, 11, 11, 179, 820, 259, 474, 307, 237, - /* 660 */ 182, 351, 321, 365, 414, 308, 367, 366, 1002, 1003, - /* 670 */ 1004, 44, 44, 87, 103, 103, 103, 103, 102, 102, - /* 680 */ 101, 101, 101, 100, 381, 106, 107, 97, 1023, 1023, - /* 690 */ 900, 903, 892, 892, 104, 104, 105, 105, 105, 105, - /* 700 */ 246, 368, 280, 128, 10, 358, 146, 796, 835, 258, - /* 710 */ 1020, 88, 795, 86, 351, 421, 836, 943, 376, 348, - /* 720 */ 191, 943, 1318, 267, 308, 279, 456, 103, 103, 103, - /* 730 */ 103, 102, 102, 101, 101, 101, 100, 381, 106, 95, - /* 740 */ 97, 1023, 1023, 900, 903, 892, 892, 104, 104, 105, - /* 750 */ 105, 105, 105, 420, 249, 238, 238, 238, 238, 79, - /* 760 */ 375, 125, 305, 29, 262, 978, 351, 481, 337, 481, - /* 770 */ 756, 755, 304, 278, 415, 15, 81, 940, 1126, 940, - /* 780 */ 103, 103, 103, 103, 102, 102, 101, 101, 101, 100, - /* 790 */ 381, 107, 97, 1023, 1023, 900, 903, 892, 892, 104, - /* 800 */ 104, 105, 105, 105, 105, 457, 263, 484, 174, 484, - /* 810 */ 238, 238, 863, 407, 402, 216, 216, 351, 409, 193, - /* 820 */ 283, 216, 481, 81, 763, 764, 266, 5, 13, 13, - /* 830 */ 34, 34, 103, 103, 103, 103, 102, 102, 101, 101, - /* 840 */ 101, 100, 381, 97, 1023, 1023, 900, 903, 892, 892, - /* 850 */ 104, 104, 105, 105, 105, 105, 93, 475, 1002, 4, - /* 860 */ 403, 1002, 340, 431, 1002, 297, 212, 1277, 81, 746, - /* 870 */ 1163, 152, 926, 478, 166, 212, 757, 829, 930, 939, - /* 880 */ 216, 939, 858, 103, 103, 103, 103, 102, 102, 101, - /* 890 */ 101, 101, 100, 381, 238, 238, 382, 1002, 1003, 1004, - /* 900 */ 1002, 1003, 1004, 1002, 1003, 1004, 481, 439, 472, 746, - /* 910 */ 105, 105, 105, 105, 98, 758, 1162, 145, 930, 412, - /* 920 */ 879, 406, 793, 81, 395, 89, 90, 91, 105, 105, - /* 930 */ 105, 105, 1323, 92, 484, 382, 486, 485, 240, 275, - /* 940 */ 871, 103, 103, 103, 103, 102, 102, 101, 101, 101, - /* 950 */ 100, 381, 1096, 371, 355, 45, 45, 259, 474, 103, - /* 960 */ 103, 103, 103, 102, 102, 101, 101, 101, 100, 381, - /* 970 */ 1150, 871, 871, 873, 874, 21, 1332, 991, 384, 730, - /* 980 */ 722, 242, 123, 1298, 124, 875, 333, 333, 332, 227, - /* 990 */ 330, 991, 384, 719, 256, 242, 484, 391, 413, 1297, - /* 1000 */ 333, 333, 332, 227, 330, 748, 187, 719, 265, 470, - /* 1010 */ 1279, 1002, 484, 417, 391, 390, 264, 11, 11, 284, - /* 1020 */ 187, 732, 265, 93, 475, 875, 4, 1279, 1281, 419, - /* 1030 */ 264, 369, 416, 11, 11, 1159, 288, 484, 399, 1346, - /* 1040 */ 478, 379, 378, 291, 484, 293, 189, 250, 295, 1027, - /* 1050 */ 1002, 1003, 1004, 190, 1029, 1111, 140, 188, 11, 11, - /* 1060 */ 189, 732, 1028, 382, 923, 46, 46, 190, 1095, 230, - /* 1070 */ 140, 188, 462, 93, 475, 472, 4, 300, 309, 391, - /* 1080 */ 373, 6, 1069, 217, 739, 310, 1030, 879, 1030, 1171, - /* 1090 */ 478, 352, 1279, 90, 91, 800, 259, 474, 1208, 484, - /* 1100 */ 92, 1268, 382, 486, 485, 352, 1002, 871, 879, 426, - /* 1110 */ 259, 474, 172, 382, 238, 238, 1146, 170, 1021, 389, - /* 1120 */ 47, 47, 1157, 739, 872, 472, 481, 469, 871, 350, - /* 1130 */ 1214, 83, 475, 389, 4, 1078, 1071, 879, 871, 871, - /* 1140 */ 873, 874, 21, 90, 91, 1002, 1003, 1004, 478, 251, - /* 1150 */ 92, 251, 382, 486, 485, 443, 370, 871, 1021, 871, - /* 1160 */ 871, 873, 224, 241, 306, 441, 301, 440, 211, 1060, - /* 1170 */ 820, 382, 822, 447, 299, 1059, 484, 1061, 1143, 962, - /* 1180 */ 430, 796, 484, 472, 1340, 312, 795, 465, 871, 871, - /* 1190 */ 873, 874, 21, 314, 963, 879, 316, 59, 59, 1002, - /* 1200 */ 9, 90, 91, 48, 48, 238, 238, 210, 92, 964, - /* 1210 */ 382, 486, 485, 176, 334, 871, 242, 481, 1193, 238, - /* 1220 */ 238, 333, 333, 332, 227, 330, 394, 270, 719, 277, - /* 1230 */ 471, 481, 467, 466, 484, 145, 217, 1201, 1002, 1003, - /* 1240 */ 1004, 187, 3, 265, 184, 445, 871, 871, 873, 874, - /* 1250 */ 21, 264, 1337, 450, 1051, 39, 39, 392, 356, 260, - /* 1260 */ 342, 121, 468, 411, 436, 821, 180, 1094, 1128, 820, - /* 1270 */ 303, 1021, 1272, 1271, 299, 259, 474, 238, 238, 1002, - /* 1280 */ 473, 189, 484, 318, 327, 238, 238, 484, 190, 481, - /* 1290 */ 446, 140, 188, 1343, 238, 238, 1038, 481, 148, 175, - /* 1300 */ 238, 238, 484, 49, 49, 219, 481, 484, 35, 35, - /* 1310 */ 1317, 1021, 481, 484, 1035, 484, 1315, 484, 1002, 1003, - /* 1320 */ 1004, 484, 66, 36, 36, 194, 352, 484, 38, 38, - /* 1330 */ 484, 259, 474, 69, 50, 50, 51, 51, 52, 52, - /* 1340 */ 359, 484, 12, 12, 484, 1198, 484, 158, 53, 53, - /* 1350 */ 405, 112, 112, 385, 389, 484, 26, 484, 143, 484, - /* 1360 */ 150, 484, 54, 54, 397, 40, 40, 55, 55, 484, - /* 1370 */ 79, 484, 153, 1190, 484, 154, 56, 56, 41, 41, - /* 1380 */ 58, 58, 133, 133, 484, 398, 484, 429, 484, 155, - /* 1390 */ 134, 134, 135, 135, 484, 63, 63, 484, 341, 484, - /* 1400 */ 339, 484, 196, 484, 156, 42, 42, 113, 113, 60, - /* 1410 */ 60, 484, 404, 484, 27, 114, 114, 1204, 115, 115, - /* 1420 */ 111, 111, 132, 132, 131, 131, 1266, 418, 484, 162, - /* 1430 */ 484, 200, 119, 119, 118, 118, 484, 74, 424, 484, - /* 1440 */ 1286, 484, 231, 484, 202, 484, 167, 286, 427, 116, - /* 1450 */ 116, 117, 117, 290, 203, 442, 1062, 62, 62, 204, - /* 1460 */ 64, 64, 61, 61, 33, 33, 37, 37, 344, 372, - /* 1470 */ 1114, 1105, 748, 1113, 374, 1112, 254, 458, 1086, 255, - /* 1480 */ 345, 1085, 302, 1084, 1355, 78, 1154, 311, 1104, 449, - /* 1490 */ 452, 1155, 1153, 218, 7, 313, 315, 320, 1152, 85, - /* 1500 */ 1252, 317, 109, 80, 463, 225, 461, 1068, 25, 487, - /* 1510 */ 997, 323, 257, 226, 229, 228, 1136, 324, 325, 326, - /* 1520 */ 488, 136, 1057, 1052, 1302, 1303, 1301, 706, 1300, 137, - /* 1530 */ 122, 138, 383, 173, 1082, 261, 186, 252, 1081, 65, - /* 1540 */ 387, 120, 938, 936, 855, 353, 149, 1079, 139, 151, - /* 1550 */ 192, 780, 195, 276, 952, 157, 141, 361, 70, 363, - /* 1560 */ 859, 159, 71, 72, 142, 73, 955, 354, 147, 197, - /* 1570 */ 198, 951, 130, 16, 199, 285, 216, 1032, 201, 423, - /* 1580 */ 164, 944, 163, 28, 721, 428, 304, 165, 205, 759, - /* 1590 */ 75, 432, 298, 17, 18, 437, 76, 253, 878, 144, - /* 1600 */ 877, 906, 77, 986, 30, 448, 987, 31, 451, 181, - /* 1610 */ 234, 236, 168, 828, 823, 89, 910, 921, 81, 907, - /* 1620 */ 215, 905, 909, 961, 960, 19, 221, 20, 220, 22, - /* 1630 */ 32, 331, 876, 731, 94, 790, 794, 8, 992, 222, - /* 1640 */ 480, 328, 1048, 1048, 1048, 1048, 1048, 1048, 1048, 1048, - /* 1650 */ 223, 1048, 1048, 1048, 1048, 1348, 1347, + /* 0 */ 368, 105, 102, 197, 105, 102, 197, 515, 1124, 1, + /* 10 */ 1, 520, 2, 1128, 515, 1192, 1171, 1456, 275, 370, + /* 20 */ 127, 1389, 1197, 1197, 1192, 1166, 178, 1205, 64, 64, + /* 30 */ 477, 887, 322, 428, 348, 37, 37, 808, 362, 888, + /* 40 */ 509, 509, 509, 112, 113, 103, 1100, 1100, 953, 956, + /* 50 */ 946, 946, 110, 110, 111, 111, 111, 111, 365, 252, + /* 60 */ 252, 515, 252, 252, 497, 515, 309, 515, 459, 515, + /* 70 */ 1079, 491, 512, 478, 6, 512, 809, 134, 498, 228, + /* 80 */ 194, 428, 37, 37, 515, 208, 64, 64, 64, 64, + /* 90 */ 13, 13, 109, 109, 109, 109, 108, 108, 107, 107, + /* 100 */ 107, 106, 401, 258, 381, 13, 13, 398, 397, 428, + /* 110 */ 252, 252, 370, 476, 405, 1104, 1079, 1080, 1081, 386, + /* 120 */ 1106, 390, 497, 512, 497, 1423, 1419, 304, 1105, 307, + /* 130 */ 1256, 496, 370, 499, 16, 16, 112, 113, 103, 1100, + /* 140 */ 1100, 953, 956, 946, 946, 110, 110, 111, 111, 111, + /* 150 */ 111, 262, 1107, 495, 1107, 401, 112, 113, 103, 1100, + /* 160 */ 1100, 953, 956, 946, 946, 110, 110, 111, 111, 111, + /* 170 */ 111, 129, 1425, 343, 1420, 339, 1059, 492, 1057, 263, + /* 180 */ 73, 105, 102, 197, 994, 109, 109, 109, 109, 108, + /* 190 */ 108, 107, 107, 107, 106, 401, 370, 111, 111, 111, + /* 200 */ 111, 104, 492, 89, 1432, 109, 109, 109, 109, 108, + /* 210 */ 108, 107, 107, 107, 106, 401, 111, 111, 111, 111, + /* 220 */ 112, 113, 103, 1100, 1100, 953, 956, 946, 946, 110, + /* 230 */ 110, 111, 111, 111, 111, 109, 109, 109, 109, 108, + /* 240 */ 108, 107, 107, 107, 106, 401, 114, 108, 108, 107, + /* 250 */ 107, 107, 106, 401, 109, 109, 109, 109, 108, 108, + /* 260 */ 107, 107, 107, 106, 401, 152, 399, 399, 399, 109, + /* 270 */ 109, 109, 109, 108, 108, 107, 107, 107, 106, 401, + /* 280 */ 178, 493, 1412, 434, 1037, 1486, 1079, 515, 1486, 370, + /* 290 */ 421, 297, 357, 412, 74, 1079, 109, 109, 109, 109, + /* 300 */ 108, 108, 107, 107, 107, 106, 401, 1413, 37, 37, + /* 310 */ 1431, 274, 506, 112, 113, 103, 1100, 1100, 953, 956, + /* 320 */ 946, 946, 110, 110, 111, 111, 111, 111, 1436, 520, + /* 330 */ 2, 1128, 1079, 1080, 1081, 430, 275, 1079, 127, 366, + /* 340 */ 933, 1079, 1080, 1081, 220, 1205, 913, 458, 455, 454, + /* 350 */ 392, 167, 515, 1035, 152, 445, 924, 453, 152, 874, + /* 360 */ 923, 289, 109, 109, 109, 109, 108, 108, 107, 107, + /* 370 */ 107, 106, 401, 13, 13, 261, 853, 252, 252, 227, + /* 380 */ 106, 401, 370, 1079, 1080, 1081, 311, 388, 1079, 296, + /* 390 */ 512, 923, 923, 925, 231, 323, 1255, 1388, 1423, 490, + /* 400 */ 274, 506, 12, 208, 274, 506, 112, 113, 103, 1100, + /* 410 */ 1100, 953, 956, 946, 946, 110, 110, 111, 111, 111, + /* 420 */ 111, 1440, 286, 1128, 288, 1079, 1097, 247, 275, 1098, + /* 430 */ 127, 387, 405, 389, 1079, 1080, 1081, 1205, 159, 238, + /* 440 */ 255, 321, 461, 316, 460, 225, 790, 105, 102, 197, + /* 450 */ 513, 314, 842, 842, 445, 109, 109, 109, 109, 108, + /* 460 */ 108, 107, 107, 107, 106, 401, 515, 514, 515, 252, + /* 470 */ 252, 1079, 1080, 1081, 435, 370, 1098, 933, 1460, 794, + /* 480 */ 274, 506, 512, 105, 102, 197, 336, 63, 63, 64, + /* 490 */ 64, 27, 790, 924, 287, 208, 1354, 923, 515, 112, + /* 500 */ 113, 103, 1100, 1100, 953, 956, 946, 946, 110, 110, + /* 510 */ 111, 111, 111, 111, 107, 107, 107, 106, 401, 49, + /* 520 */ 49, 515, 28, 1079, 405, 497, 421, 297, 923, 923, + /* 530 */ 925, 186, 468, 1079, 467, 999, 999, 442, 515, 1079, + /* 540 */ 334, 515, 45, 45, 1083, 342, 173, 168, 109, 109, + /* 550 */ 109, 109, 108, 108, 107, 107, 107, 106, 401, 13, + /* 560 */ 13, 205, 13, 13, 252, 252, 1195, 1195, 370, 1079, + /* 570 */ 1080, 1081, 787, 265, 5, 359, 494, 512, 469, 1079, + /* 580 */ 1080, 1081, 398, 397, 1079, 1079, 1080, 1081, 3, 282, + /* 590 */ 1079, 1083, 112, 113, 103, 1100, 1100, 953, 956, 946, + /* 600 */ 946, 110, 110, 111, 111, 111, 111, 252, 252, 1015, + /* 610 */ 220, 1079, 873, 458, 455, 454, 943, 943, 954, 957, + /* 620 */ 512, 252, 252, 453, 1016, 1079, 445, 1107, 1209, 1107, + /* 630 */ 1079, 1080, 1081, 515, 512, 426, 1079, 1080, 1081, 1017, + /* 640 */ 512, 109, 109, 109, 109, 108, 108, 107, 107, 107, + /* 650 */ 106, 401, 1052, 515, 50, 50, 515, 1079, 1080, 1081, + /* 660 */ 828, 370, 1051, 379, 411, 1064, 1358, 207, 408, 773, + /* 670 */ 829, 1079, 1080, 1081, 64, 64, 322, 64, 64, 1302, + /* 680 */ 947, 411, 410, 1358, 1360, 112, 113, 103, 1100, 1100, + /* 690 */ 953, 956, 946, 946, 110, 110, 111, 111, 111, 111, + /* 700 */ 294, 482, 515, 1037, 1487, 515, 434, 1487, 354, 1120, + /* 710 */ 483, 996, 913, 485, 466, 996, 132, 178, 33, 450, + /* 720 */ 1203, 136, 406, 64, 64, 479, 64, 64, 419, 369, + /* 730 */ 283, 1146, 252, 252, 109, 109, 109, 109, 108, 108, + /* 740 */ 107, 107, 107, 106, 401, 512, 224, 440, 411, 266, + /* 750 */ 1358, 266, 252, 252, 370, 296, 416, 284, 934, 396, + /* 760 */ 976, 470, 400, 252, 252, 512, 9, 473, 231, 500, + /* 770 */ 354, 1036, 1035, 1488, 355, 374, 512, 1121, 112, 113, + /* 780 */ 103, 1100, 1100, 953, 956, 946, 946, 110, 110, 111, + /* 790 */ 111, 111, 111, 252, 252, 1015, 515, 1347, 295, 252, + /* 800 */ 252, 252, 252, 1098, 375, 249, 512, 445, 872, 322, + /* 810 */ 1016, 480, 512, 195, 512, 434, 273, 15, 15, 515, + /* 820 */ 314, 515, 95, 515, 93, 1017, 367, 109, 109, 109, + /* 830 */ 109, 108, 108, 107, 107, 107, 106, 401, 515, 1121, + /* 840 */ 39, 39, 51, 51, 52, 52, 503, 370, 515, 1204, + /* 850 */ 1098, 918, 439, 341, 133, 436, 223, 222, 221, 53, + /* 860 */ 53, 322, 1400, 761, 762, 763, 515, 370, 88, 54, + /* 870 */ 54, 112, 113, 103, 1100, 1100, 953, 956, 946, 946, + /* 880 */ 110, 110, 111, 111, 111, 111, 407, 55, 55, 196, + /* 890 */ 515, 112, 113, 103, 1100, 1100, 953, 956, 946, 946, + /* 900 */ 110, 110, 111, 111, 111, 111, 135, 264, 1149, 376, + /* 910 */ 515, 40, 40, 515, 872, 515, 993, 515, 993, 116, + /* 920 */ 109, 109, 109, 109, 108, 108, 107, 107, 107, 106, + /* 930 */ 401, 41, 41, 515, 43, 43, 44, 44, 56, 56, + /* 940 */ 109, 109, 109, 109, 108, 108, 107, 107, 107, 106, + /* 950 */ 401, 515, 379, 515, 57, 57, 515, 799, 515, 379, + /* 960 */ 515, 445, 200, 515, 323, 515, 1397, 515, 1459, 515, + /* 970 */ 1287, 817, 58, 58, 14, 14, 515, 59, 59, 118, + /* 980 */ 118, 60, 60, 515, 46, 46, 61, 61, 62, 62, + /* 990 */ 47, 47, 515, 190, 189, 91, 515, 140, 140, 515, + /* 1000 */ 394, 515, 277, 1200, 141, 141, 515, 1115, 515, 992, + /* 1010 */ 515, 992, 515, 69, 69, 370, 278, 48, 48, 259, + /* 1020 */ 65, 65, 119, 119, 246, 246, 260, 66, 66, 120, + /* 1030 */ 120, 121, 121, 117, 117, 370, 515, 512, 383, 112, + /* 1040 */ 113, 103, 1100, 1100, 953, 956, 946, 946, 110, 110, + /* 1050 */ 111, 111, 111, 111, 515, 872, 515, 139, 139, 112, + /* 1060 */ 113, 103, 1100, 1100, 953, 956, 946, 946, 110, 110, + /* 1070 */ 111, 111, 111, 111, 1287, 138, 138, 125, 125, 515, + /* 1080 */ 12, 515, 281, 1287, 515, 445, 131, 1287, 109, 109, + /* 1090 */ 109, 109, 108, 108, 107, 107, 107, 106, 401, 515, + /* 1100 */ 124, 124, 122, 122, 515, 123, 123, 515, 109, 109, + /* 1110 */ 109, 109, 108, 108, 107, 107, 107, 106, 401, 515, + /* 1120 */ 68, 68, 463, 783, 515, 70, 70, 302, 67, 67, + /* 1130 */ 1032, 253, 253, 356, 1287, 191, 196, 1433, 465, 1301, + /* 1140 */ 38, 38, 384, 94, 512, 42, 42, 177, 848, 274, + /* 1150 */ 506, 385, 420, 847, 1356, 441, 508, 376, 377, 153, + /* 1160 */ 423, 872, 432, 370, 224, 251, 194, 887, 182, 293, + /* 1170 */ 783, 848, 88, 254, 466, 888, 847, 915, 807, 806, + /* 1180 */ 230, 1241, 910, 370, 17, 413, 797, 112, 113, 103, + /* 1190 */ 1100, 1100, 953, 956, 946, 946, 110, 110, 111, 111, + /* 1200 */ 111, 111, 395, 814, 815, 1175, 983, 112, 101, 103, + /* 1210 */ 1100, 1100, 953, 956, 946, 946, 110, 110, 111, 111, + /* 1220 */ 111, 111, 375, 422, 427, 429, 298, 230, 230, 88, + /* 1230 */ 1240, 451, 312, 797, 226, 88, 109, 109, 109, 109, + /* 1240 */ 108, 108, 107, 107, 107, 106, 401, 86, 433, 979, + /* 1250 */ 927, 881, 226, 983, 230, 415, 109, 109, 109, 109, + /* 1260 */ 108, 108, 107, 107, 107, 106, 401, 320, 845, 781, + /* 1270 */ 846, 100, 130, 100, 1403, 290, 370, 319, 1377, 1376, + /* 1280 */ 437, 1449, 299, 1237, 303, 306, 308, 310, 1188, 1174, + /* 1290 */ 1173, 1172, 315, 324, 325, 1228, 370, 927, 1249, 271, + /* 1300 */ 1286, 113, 103, 1100, 1100, 953, 956, 946, 946, 110, + /* 1310 */ 110, 111, 111, 111, 111, 1224, 1235, 502, 501, 1292, + /* 1320 */ 1221, 1155, 103, 1100, 1100, 953, 956, 946, 946, 110, + /* 1330 */ 110, 111, 111, 111, 111, 1148, 1137, 1136, 1138, 1443, + /* 1340 */ 446, 244, 184, 98, 507, 188, 4, 353, 327, 109, + /* 1350 */ 109, 109, 109, 108, 108, 107, 107, 107, 106, 401, + /* 1360 */ 510, 329, 331, 199, 414, 456, 292, 285, 318, 109, + /* 1370 */ 109, 109, 109, 108, 108, 107, 107, 107, 106, 401, + /* 1380 */ 11, 1271, 1279, 402, 361, 192, 1171, 1351, 431, 505, + /* 1390 */ 346, 1350, 333, 98, 507, 504, 4, 187, 1446, 1115, + /* 1400 */ 233, 1396, 155, 1394, 1112, 152, 72, 75, 378, 425, + /* 1410 */ 510, 165, 149, 157, 933, 1276, 86, 30, 1268, 417, + /* 1420 */ 96, 96, 8, 160, 161, 162, 163, 97, 418, 402, + /* 1430 */ 517, 516, 449, 402, 923, 210, 358, 424, 1282, 438, + /* 1440 */ 169, 214, 360, 1345, 80, 504, 31, 444, 1365, 301, + /* 1450 */ 245, 274, 506, 216, 174, 305, 488, 447, 217, 462, + /* 1460 */ 1139, 487, 218, 363, 933, 923, 923, 925, 926, 24, + /* 1470 */ 96, 96, 1191, 1190, 1189, 391, 1182, 97, 1163, 402, + /* 1480 */ 517, 516, 799, 364, 923, 1162, 317, 1161, 98, 507, + /* 1490 */ 1181, 4, 1458, 472, 393, 269, 270, 475, 481, 1232, + /* 1500 */ 85, 1233, 326, 328, 232, 510, 495, 1231, 330, 98, + /* 1510 */ 507, 1230, 4, 486, 335, 923, 923, 925, 926, 24, + /* 1520 */ 1435, 1068, 404, 181, 336, 256, 510, 115, 402, 332, + /* 1530 */ 352, 352, 351, 241, 349, 1214, 1414, 770, 338, 10, + /* 1540 */ 504, 340, 272, 92, 1331, 1213, 87, 183, 484, 402, + /* 1550 */ 201, 488, 280, 239, 344, 345, 489, 1145, 29, 933, + /* 1560 */ 279, 504, 1074, 518, 240, 96, 96, 242, 243, 519, + /* 1570 */ 1134, 1129, 97, 154, 402, 517, 516, 372, 373, 923, + /* 1580 */ 933, 142, 143, 128, 1381, 267, 96, 96, 852, 757, + /* 1590 */ 203, 144, 403, 97, 1382, 402, 517, 516, 204, 1380, + /* 1600 */ 923, 146, 1379, 1159, 1158, 71, 1156, 276, 202, 185, + /* 1610 */ 923, 923, 925, 926, 24, 198, 257, 126, 991, 989, + /* 1620 */ 907, 98, 507, 156, 4, 145, 158, 206, 831, 209, + /* 1630 */ 291, 923, 923, 925, 926, 24, 1005, 911, 510, 164, + /* 1640 */ 147, 380, 371, 382, 166, 76, 77, 274, 506, 148, + /* 1650 */ 78, 79, 1008, 211, 212, 1004, 137, 213, 18, 300, + /* 1660 */ 230, 402, 997, 1109, 443, 215, 32, 170, 171, 772, + /* 1670 */ 409, 448, 319, 504, 219, 172, 452, 81, 19, 457, + /* 1680 */ 313, 20, 82, 268, 488, 150, 810, 179, 83, 487, + /* 1690 */ 464, 151, 933, 180, 959, 84, 1040, 34, 96, 96, + /* 1700 */ 471, 1041, 35, 474, 193, 97, 248, 402, 517, 516, + /* 1710 */ 1068, 404, 923, 250, 256, 880, 229, 175, 875, 352, + /* 1720 */ 352, 351, 241, 349, 100, 21, 770, 22, 1054, 1056, + /* 1730 */ 7, 98, 507, 1045, 4, 337, 1058, 23, 974, 201, + /* 1740 */ 176, 280, 88, 923, 923, 925, 926, 24, 510, 279, + /* 1750 */ 960, 958, 962, 1014, 963, 1013, 235, 234, 25, 36, + /* 1760 */ 99, 90, 507, 928, 4, 511, 350, 782, 26, 841, + /* 1770 */ 236, 402, 347, 1069, 237, 1125, 1125, 1451, 510, 203, + /* 1780 */ 1450, 1125, 1125, 504, 1125, 1125, 1125, 204, 1125, 1125, + /* 1790 */ 146, 1125, 1125, 1125, 1125, 1125, 1125, 202, 1125, 1125, + /* 1800 */ 1125, 402, 933, 1125, 1125, 1125, 1125, 1125, 96, 96, + /* 1810 */ 1125, 1125, 1125, 504, 1125, 97, 1125, 402, 517, 516, + /* 1820 */ 1125, 1125, 923, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 1830 */ 1125, 371, 933, 1125, 1125, 1125, 274, 506, 96, 96, + /* 1840 */ 1125, 1125, 1125, 1125, 1125, 97, 1125, 402, 517, 516, + /* 1850 */ 1125, 1125, 923, 923, 923, 925, 926, 24, 1125, 409, + /* 1860 */ 1125, 1125, 1125, 256, 1125, 1125, 1125, 1125, 352, 352, + /* 1870 */ 351, 241, 349, 1125, 1125, 770, 1125, 1125, 1125, 1125, + /* 1880 */ 1125, 1125, 1125, 923, 923, 925, 926, 24, 201, 1125, + /* 1890 */ 280, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 279, 1125, + /* 1900 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 1910 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 1920 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 203, 1125, + /* 1930 */ 1125, 1125, 1125, 1125, 1125, 1125, 204, 1125, 1125, 146, + /* 1940 */ 1125, 1125, 1125, 1125, 1125, 1125, 202, 1125, 1125, 1125, + /* 1950 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 1960 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 1970 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 1980 */ 371, 1125, 1125, 1125, 1125, 274, 506, 1125, 1125, 1125, + /* 1990 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, + /* 2000 */ 1125, 1125, 1125, 1125, 1125, 1125, 1125, 1125, 409, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 174, 226, 227, 228, 226, 227, 228, 172, 145, 146, - /* 10 */ 147, 148, 149, 150, 153, 169, 170, 171, 155, 19, - /* 20 */ 157, 246, 192, 193, 177, 181, 182, 164, 169, 170, - /* 30 */ 171, 31, 164, 153, 190, 174, 175, 187, 153, 39, - /* 40 */ 7, 8, 9, 43, 44, 45, 46, 47, 48, 49, - /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 174, 196, - /* 60 */ 197, 226, 227, 228, 196, 197, 46, 47, 48, 49, - /* 70 */ 209, 208, 19, 226, 227, 228, 208, 174, 177, 26, - /* 80 */ 195, 213, 214, 22, 221, 85, 86, 87, 88, 89, - /* 90 */ 90, 91, 92, 93, 94, 95, 43, 44, 45, 46, - /* 100 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 110 */ 57, 172, 249, 153, 53, 153, 147, 148, 149, 150, - /* 120 */ 22, 23, 69, 103, 155, 19, 157, 226, 227, 228, - /* 130 */ 94, 95, 247, 164, 174, 175, 174, 175, 85, 86, - /* 140 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 43, - /* 150 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 160 */ 54, 55, 56, 57, 153, 196, 197, 153, 153, 209, - /* 170 */ 210, 209, 210, 67, 95, 161, 237, 208, 19, 165, - /* 180 */ 165, 242, 84, 24, 91, 92, 93, 94, 95, 223, - /* 190 */ 221, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 200 */ 94, 95, 43, 44, 45, 46, 47, 48, 49, 50, - /* 210 */ 51, 52, 53, 54, 55, 56, 57, 153, 249, 85, - /* 220 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 230 */ 219, 19, 109, 110, 111, 23, 89, 90, 91, 92, - /* 240 */ 93, 94, 95, 73, 85, 86, 87, 88, 89, 90, - /* 250 */ 91, 92, 93, 94, 95, 43, 44, 45, 46, 47, - /* 260 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 270 */ 153, 22, 23, 101, 173, 26, 104, 105, 106, 109, - /* 280 */ 110, 111, 181, 11, 19, 59, 114, 73, 23, 110, - /* 290 */ 111, 174, 175, 116, 80, 118, 119, 85, 86, 87, - /* 300 */ 88, 89, 90, 91, 92, 93, 94, 95, 43, 44, - /* 310 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 320 */ 55, 56, 57, 109, 98, 99, 100, 101, 83, 153, - /* 330 */ 104, 105, 106, 84, 120, 121, 153, 19, 192, 193, - /* 340 */ 114, 23, 89, 90, 99, 59, 23, 230, 103, 26, - /* 350 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 360 */ 95, 43, 44, 45, 46, 47, 48, 49, 50, 51, - /* 370 */ 52, 53, 54, 55, 56, 57, 153, 91, 153, 134, - /* 380 */ 135, 136, 110, 111, 98, 99, 100, 134, 153, 136, - /* 390 */ 19, 22, 23, 26, 23, 26, 80, 174, 175, 174, - /* 400 */ 175, 59, 219, 85, 86, 87, 88, 89, 90, 91, - /* 410 */ 92, 93, 94, 95, 43, 44, 45, 46, 47, 48, - /* 420 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 16, - /* 430 */ 153, 22, 209, 210, 209, 210, 120, 121, 196, 197, - /* 440 */ 98, 99, 100, 19, 46, 22, 23, 23, 252, 253, - /* 450 */ 208, 174, 175, 84, 219, 153, 85, 86, 87, 88, - /* 460 */ 89, 90, 91, 92, 93, 94, 95, 43, 44, 45, - /* 470 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, - /* 480 */ 56, 57, 153, 153, 153, 153, 209, 120, 121, 76, - /* 490 */ 153, 78, 109, 110, 111, 97, 19, 153, 89, 90, - /* 500 */ 198, 59, 183, 174, 175, 174, 175, 84, 153, 85, - /* 510 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 520 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 530 */ 53, 54, 55, 56, 57, 16, 197, 153, 22, 153, - /* 540 */ 153, 99, 198, 12, 153, 196, 197, 208, 153, 153, - /* 550 */ 195, 183, 19, 23, 222, 142, 26, 208, 27, 153, - /* 560 */ 174, 175, 85, 86, 87, 88, 89, 90, 91, 92, - /* 570 */ 93, 94, 95, 42, 188, 59, 43, 44, 45, 46, - /* 580 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, - /* 590 */ 57, 195, 22, 198, 63, 76, 153, 78, 167, 168, - /* 600 */ 153, 195, 167, 168, 73, 153, 222, 153, 19, 222, - /* 610 */ 153, 220, 153, 24, 98, 99, 100, 140, 85, 86, - /* 620 */ 87, 88, 89, 90, 91, 92, 93, 94, 95, 59, - /* 630 */ 100, 153, 43, 44, 45, 46, 47, 48, 49, 50, - /* 640 */ 51, 52, 53, 54, 55, 56, 57, 195, 153, 195, - /* 650 */ 153, 153, 174, 175, 26, 125, 120, 121, 153, 213, - /* 660 */ 214, 19, 153, 220, 153, 153, 188, 220, 98, 99, - /* 670 */ 100, 174, 175, 140, 85, 86, 87, 88, 89, 90, - /* 680 */ 91, 92, 93, 94, 95, 43, 44, 45, 46, 47, + /* 0 */ 184, 238, 239, 240, 238, 239, 240, 163, 155, 156, + /* 10 */ 157, 158, 159, 160, 163, 191, 192, 183, 165, 19, + /* 20 */ 167, 258, 202, 203, 200, 191, 163, 174, 184, 185, + /* 30 */ 174, 31, 163, 163, 171, 184, 185, 35, 175, 39, + /* 40 */ 179, 180, 181, 43, 44, 45, 46, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 184, 206, + /* 60 */ 207, 163, 206, 207, 220, 163, 16, 163, 66, 163, + /* 70 */ 59, 270, 219, 229, 273, 219, 74, 208, 174, 223, + /* 80 */ 224, 163, 184, 185, 163, 232, 184, 185, 184, 185, + /* 90 */ 184, 185, 92, 93, 94, 95, 96, 97, 98, 99, + /* 100 */ 100, 101, 102, 233, 198, 184, 185, 96, 97, 163, + /* 110 */ 206, 207, 19, 163, 261, 104, 105, 106, 107, 198, + /* 120 */ 109, 119, 220, 219, 220, 274, 275, 77, 117, 79, + /* 130 */ 187, 229, 19, 229, 184, 185, 43, 44, 45, 46, + /* 140 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 150 */ 57, 233, 141, 134, 143, 102, 43, 44, 45, 46, + /* 160 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 170 */ 57, 152, 274, 216, 276, 218, 83, 163, 85, 233, + /* 180 */ 67, 238, 239, 240, 11, 92, 93, 94, 95, 96, + /* 190 */ 97, 98, 99, 100, 101, 102, 19, 54, 55, 56, + /* 200 */ 57, 58, 163, 26, 163, 92, 93, 94, 95, 96, + /* 210 */ 97, 98, 99, 100, 101, 102, 54, 55, 56, 57, + /* 220 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 230 */ 53, 54, 55, 56, 57, 92, 93, 94, 95, 96, + /* 240 */ 97, 98, 99, 100, 101, 102, 69, 96, 97, 98, + /* 250 */ 99, 100, 101, 102, 92, 93, 94, 95, 96, 97, + /* 260 */ 98, 99, 100, 101, 102, 81, 179, 180, 181, 92, + /* 270 */ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, + /* 280 */ 163, 267, 268, 163, 22, 23, 59, 163, 26, 19, + /* 290 */ 117, 118, 175, 109, 24, 59, 92, 93, 94, 95, + /* 300 */ 96, 97, 98, 99, 100, 101, 102, 268, 184, 185, + /* 310 */ 269, 127, 128, 43, 44, 45, 46, 47, 48, 49, + /* 320 */ 50, 51, 52, 53, 54, 55, 56, 57, 157, 158, + /* 330 */ 159, 160, 105, 106, 107, 163, 165, 59, 167, 184, + /* 340 */ 90, 105, 106, 107, 108, 174, 73, 111, 112, 113, + /* 350 */ 19, 22, 163, 91, 81, 163, 106, 121, 81, 132, + /* 360 */ 110, 16, 92, 93, 94, 95, 96, 97, 98, 99, + /* 370 */ 100, 101, 102, 184, 185, 255, 98, 206, 207, 26, + /* 380 */ 101, 102, 19, 105, 106, 107, 23, 198, 59, 116, + /* 390 */ 219, 141, 142, 143, 24, 163, 187, 205, 274, 275, + /* 400 */ 127, 128, 182, 232, 127, 128, 43, 44, 45, 46, + /* 410 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 420 */ 57, 158, 77, 160, 79, 59, 26, 182, 165, 59, + /* 430 */ 167, 199, 261, 102, 105, 106, 107, 174, 72, 108, + /* 440 */ 109, 110, 111, 112, 113, 114, 59, 238, 239, 240, + /* 450 */ 123, 120, 125, 126, 163, 92, 93, 94, 95, 96, + /* 460 */ 97, 98, 99, 100, 101, 102, 163, 163, 163, 206, + /* 470 */ 207, 105, 106, 107, 254, 19, 106, 90, 197, 23, + /* 480 */ 127, 128, 219, 238, 239, 240, 22, 184, 185, 184, + /* 490 */ 185, 22, 105, 106, 149, 232, 205, 110, 163, 43, + /* 500 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 510 */ 54, 55, 56, 57, 98, 99, 100, 101, 102, 184, + /* 520 */ 185, 163, 53, 59, 261, 220, 117, 118, 141, 142, + /* 530 */ 143, 131, 174, 59, 229, 116, 117, 118, 163, 59, + /* 540 */ 163, 163, 184, 185, 59, 242, 72, 22, 92, 93, + /* 550 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 184, + /* 560 */ 185, 24, 184, 185, 206, 207, 202, 203, 19, 105, + /* 570 */ 106, 107, 23, 198, 22, 174, 198, 219, 220, 105, + /* 580 */ 106, 107, 96, 97, 59, 105, 106, 107, 22, 174, + /* 590 */ 59, 106, 43, 44, 45, 46, 47, 48, 49, 50, + /* 600 */ 51, 52, 53, 54, 55, 56, 57, 206, 207, 12, + /* 610 */ 108, 59, 132, 111, 112, 113, 46, 47, 48, 49, + /* 620 */ 219, 206, 207, 121, 27, 59, 163, 141, 207, 143, + /* 630 */ 105, 106, 107, 163, 219, 234, 105, 106, 107, 42, + /* 640 */ 219, 92, 93, 94, 95, 96, 97, 98, 99, 100, + /* 650 */ 101, 102, 76, 163, 184, 185, 163, 105, 106, 107, + /* 660 */ 63, 19, 86, 163, 163, 23, 163, 130, 205, 21, + /* 670 */ 73, 105, 106, 107, 184, 185, 163, 184, 185, 237, + /* 680 */ 110, 180, 181, 180, 181, 43, 44, 45, 46, 47, /* 690 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, - /* 700 */ 243, 189, 243, 198, 172, 250, 251, 117, 31, 201, - /* 710 */ 26, 139, 122, 141, 19, 220, 39, 29, 220, 211, - /* 720 */ 24, 33, 153, 164, 153, 164, 19, 85, 86, 87, - /* 730 */ 88, 89, 90, 91, 92, 93, 94, 95, 43, 44, - /* 740 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, - /* 750 */ 55, 56, 57, 65, 243, 196, 197, 196, 197, 131, - /* 760 */ 189, 22, 103, 24, 153, 23, 19, 208, 26, 208, - /* 770 */ 102, 103, 113, 23, 242, 22, 26, 134, 164, 136, - /* 780 */ 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, - /* 790 */ 95, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 800 */ 53, 54, 55, 56, 57, 98, 153, 153, 124, 153, - /* 810 */ 196, 197, 23, 23, 61, 26, 26, 19, 23, 123, - /* 820 */ 23, 26, 208, 26, 7, 8, 153, 22, 174, 175, - /* 830 */ 174, 175, 85, 86, 87, 88, 89, 90, 91, 92, - /* 840 */ 93, 94, 95, 45, 46, 47, 48, 49, 50, 51, - /* 850 */ 52, 53, 54, 55, 56, 57, 19, 20, 59, 22, - /* 860 */ 111, 59, 164, 23, 59, 23, 26, 153, 26, 59, - /* 870 */ 153, 72, 23, 36, 72, 26, 35, 23, 59, 134, - /* 880 */ 26, 136, 133, 85, 86, 87, 88, 89, 90, 91, - /* 890 */ 92, 93, 94, 95, 196, 197, 59, 98, 99, 100, - /* 900 */ 98, 99, 100, 98, 99, 100, 208, 66, 71, 99, - /* 910 */ 54, 55, 56, 57, 58, 74, 153, 80, 99, 19, - /* 920 */ 83, 223, 23, 26, 153, 26, 89, 90, 54, 55, - /* 930 */ 56, 57, 153, 96, 153, 98, 99, 100, 22, 153, - /* 940 */ 103, 85, 86, 87, 88, 89, 90, 91, 92, 93, - /* 950 */ 94, 95, 183, 112, 158, 174, 175, 120, 121, 85, - /* 960 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 970 */ 215, 134, 135, 136, 137, 138, 0, 1, 2, 23, - /* 980 */ 21, 5, 26, 153, 22, 59, 10, 11, 12, 13, - /* 990 */ 14, 1, 2, 17, 212, 5, 153, 153, 98, 153, - /* 1000 */ 10, 11, 12, 13, 14, 108, 30, 17, 32, 193, - /* 1010 */ 153, 59, 153, 153, 170, 171, 40, 174, 175, 153, - /* 1020 */ 30, 59, 32, 19, 20, 99, 22, 170, 171, 233, - /* 1030 */ 40, 188, 236, 174, 175, 153, 153, 153, 79, 123, - /* 1040 */ 36, 89, 90, 153, 153, 153, 70, 188, 153, 97, - /* 1050 */ 98, 99, 100, 77, 102, 153, 80, 81, 174, 175, - /* 1060 */ 70, 99, 110, 59, 105, 174, 175, 77, 153, 238, - /* 1070 */ 80, 81, 188, 19, 20, 71, 22, 153, 153, 235, - /* 1080 */ 19, 22, 164, 24, 59, 153, 134, 83, 136, 153, - /* 1090 */ 36, 115, 235, 89, 90, 91, 120, 121, 153, 153, - /* 1100 */ 96, 142, 98, 99, 100, 115, 59, 103, 83, 239, - /* 1110 */ 120, 121, 199, 59, 196, 197, 153, 153, 59, 143, - /* 1120 */ 174, 175, 153, 98, 99, 71, 208, 153, 103, 165, - /* 1130 */ 153, 19, 20, 143, 22, 153, 153, 83, 134, 135, - /* 1140 */ 136, 137, 138, 89, 90, 98, 99, 100, 36, 185, - /* 1150 */ 96, 187, 98, 99, 100, 91, 95, 103, 99, 134, - /* 1160 */ 135, 136, 101, 102, 103, 104, 105, 106, 107, 153, - /* 1170 */ 26, 59, 125, 164, 113, 153, 153, 153, 212, 12, - /* 1180 */ 19, 117, 153, 71, 153, 212, 122, 164, 134, 135, - /* 1190 */ 136, 137, 138, 212, 27, 83, 212, 174, 175, 59, - /* 1200 */ 200, 89, 90, 174, 175, 196, 197, 46, 96, 42, - /* 1210 */ 98, 99, 100, 172, 151, 103, 5, 208, 203, 196, - /* 1220 */ 197, 10, 11, 12, 13, 14, 216, 216, 17, 244, - /* 1230 */ 63, 208, 209, 210, 153, 80, 24, 203, 98, 99, - /* 1240 */ 100, 30, 22, 32, 100, 164, 134, 135, 136, 137, - /* 1250 */ 138, 40, 148, 164, 150, 174, 175, 102, 97, 155, - /* 1260 */ 203, 157, 164, 244, 178, 125, 186, 182, 164, 125, - /* 1270 */ 177, 59, 177, 177, 113, 120, 121, 196, 197, 59, - /* 1280 */ 232, 70, 153, 216, 202, 196, 197, 153, 77, 208, - /* 1290 */ 209, 80, 81, 156, 196, 197, 60, 208, 248, 200, - /* 1300 */ 196, 197, 153, 174, 175, 123, 208, 153, 174, 175, - /* 1310 */ 160, 99, 208, 153, 38, 153, 160, 153, 98, 99, - /* 1320 */ 100, 153, 245, 174, 175, 221, 115, 153, 174, 175, - /* 1330 */ 153, 120, 121, 245, 174, 175, 174, 175, 174, 175, - /* 1340 */ 160, 153, 174, 175, 153, 225, 153, 22, 174, 175, - /* 1350 */ 97, 174, 175, 249, 143, 153, 224, 153, 43, 153, - /* 1360 */ 191, 153, 174, 175, 18, 174, 175, 174, 175, 153, - /* 1370 */ 131, 153, 194, 203, 153, 194, 174, 175, 174, 175, - /* 1380 */ 174, 175, 174, 175, 153, 160, 153, 18, 153, 194, - /* 1390 */ 174, 175, 174, 175, 153, 174, 175, 153, 225, 153, - /* 1400 */ 203, 153, 159, 153, 194, 174, 175, 174, 175, 174, - /* 1410 */ 175, 153, 203, 153, 224, 174, 175, 191, 174, 175, - /* 1420 */ 174, 175, 174, 175, 174, 175, 203, 160, 153, 191, - /* 1430 */ 153, 159, 174, 175, 174, 175, 153, 139, 62, 153, - /* 1440 */ 241, 153, 160, 153, 159, 153, 22, 240, 179, 174, - /* 1450 */ 175, 174, 175, 160, 159, 97, 160, 174, 175, 159, - /* 1460 */ 174, 175, 174, 175, 174, 175, 174, 175, 179, 64, - /* 1470 */ 176, 184, 108, 176, 95, 176, 234, 126, 176, 234, - /* 1480 */ 179, 178, 176, 176, 176, 97, 218, 217, 184, 179, - /* 1490 */ 179, 218, 218, 160, 22, 217, 217, 160, 218, 139, - /* 1500 */ 229, 217, 130, 129, 127, 25, 128, 163, 26, 162, - /* 1510 */ 13, 206, 231, 154, 6, 154, 207, 205, 204, 203, - /* 1520 */ 152, 166, 152, 152, 172, 172, 172, 4, 172, 166, - /* 1530 */ 180, 166, 3, 22, 172, 144, 15, 180, 172, 172, - /* 1540 */ 82, 16, 23, 23, 121, 254, 132, 172, 112, 124, - /* 1550 */ 24, 20, 126, 16, 1, 124, 112, 61, 53, 37, - /* 1560 */ 133, 132, 53, 53, 112, 53, 98, 254, 251, 34, - /* 1570 */ 123, 1, 5, 22, 97, 142, 26, 75, 123, 41, - /* 1580 */ 97, 68, 68, 24, 20, 19, 113, 22, 107, 28, - /* 1590 */ 22, 67, 23, 22, 22, 67, 22, 67, 23, 37, - /* 1600 */ 23, 23, 26, 23, 22, 24, 23, 22, 24, 123, - /* 1610 */ 23, 23, 22, 98, 125, 26, 11, 23, 26, 23, - /* 1620 */ 34, 23, 23, 23, 23, 34, 22, 34, 26, 22, - /* 1630 */ 22, 15, 23, 23, 22, 117, 23, 22, 1, 123, - /* 1640 */ 26, 23, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1650 */ 123, 255, 255, 255, 255, 123, 123, 255, 255, 255, - /* 1660 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1670 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1680 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1690 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1700 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1710 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1720 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1730 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1740 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1750 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1760 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1770 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1780 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1790 */ 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - /* 1800 */ 255, 255, + /* 700 */ 174, 163, 163, 22, 23, 163, 163, 26, 22, 23, + /* 710 */ 220, 29, 73, 220, 272, 33, 22, 163, 24, 19, + /* 720 */ 174, 208, 259, 184, 185, 19, 184, 185, 80, 175, + /* 730 */ 230, 174, 206, 207, 92, 93, 94, 95, 96, 97, + /* 740 */ 98, 99, 100, 101, 102, 219, 46, 65, 247, 195, + /* 750 */ 247, 197, 206, 207, 19, 116, 117, 118, 23, 220, + /* 760 */ 112, 174, 220, 206, 207, 219, 22, 174, 24, 174, + /* 770 */ 22, 23, 91, 264, 265, 168, 219, 91, 43, 44, + /* 780 */ 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, + /* 790 */ 55, 56, 57, 206, 207, 12, 163, 149, 255, 206, + /* 800 */ 207, 206, 207, 59, 104, 23, 219, 163, 26, 163, + /* 810 */ 27, 105, 219, 163, 219, 163, 211, 184, 185, 163, + /* 820 */ 120, 163, 146, 163, 148, 42, 221, 92, 93, 94, + /* 830 */ 95, 96, 97, 98, 99, 100, 101, 102, 163, 91, + /* 840 */ 184, 185, 184, 185, 184, 185, 63, 19, 163, 205, + /* 850 */ 106, 23, 245, 163, 208, 248, 116, 117, 118, 184, + /* 860 */ 185, 163, 163, 7, 8, 9, 163, 19, 26, 184, + /* 870 */ 185, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 880 */ 52, 53, 54, 55, 56, 57, 163, 184, 185, 107, + /* 890 */ 163, 43, 44, 45, 46, 47, 48, 49, 50, 51, + /* 900 */ 52, 53, 54, 55, 56, 57, 208, 255, 177, 178, + /* 910 */ 163, 184, 185, 163, 132, 163, 141, 163, 143, 22, + /* 920 */ 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + /* 930 */ 102, 184, 185, 163, 184, 185, 184, 185, 184, 185, + /* 940 */ 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, + /* 950 */ 102, 163, 163, 163, 184, 185, 163, 115, 163, 163, + /* 960 */ 163, 163, 15, 163, 163, 163, 163, 163, 23, 163, + /* 970 */ 163, 26, 184, 185, 184, 185, 163, 184, 185, 184, + /* 980 */ 185, 184, 185, 163, 184, 185, 184, 185, 184, 185, + /* 990 */ 184, 185, 163, 96, 97, 147, 163, 184, 185, 163, + /* 1000 */ 199, 163, 163, 205, 184, 185, 163, 60, 163, 141, + /* 1010 */ 163, 143, 163, 184, 185, 19, 163, 184, 185, 230, + /* 1020 */ 184, 185, 184, 185, 206, 207, 230, 184, 185, 184, + /* 1030 */ 185, 184, 185, 184, 185, 19, 163, 219, 231, 43, + /* 1040 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 1050 */ 54, 55, 56, 57, 163, 26, 163, 184, 185, 43, + /* 1060 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + /* 1070 */ 54, 55, 56, 57, 163, 184, 185, 184, 185, 163, + /* 1080 */ 182, 163, 163, 163, 163, 163, 22, 163, 92, 93, + /* 1090 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 163, + /* 1100 */ 184, 185, 184, 185, 163, 184, 185, 163, 92, 93, + /* 1110 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 163, + /* 1120 */ 184, 185, 98, 59, 163, 184, 185, 205, 184, 185, + /* 1130 */ 23, 206, 207, 26, 163, 26, 107, 153, 154, 237, + /* 1140 */ 184, 185, 231, 147, 219, 184, 185, 249, 124, 127, + /* 1150 */ 128, 231, 254, 129, 163, 231, 177, 178, 262, 263, + /* 1160 */ 118, 132, 19, 19, 46, 223, 224, 31, 24, 23, + /* 1170 */ 106, 124, 26, 22, 272, 39, 129, 23, 109, 110, + /* 1180 */ 26, 163, 140, 19, 22, 234, 59, 43, 44, 45, + /* 1190 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + /* 1200 */ 56, 57, 231, 7, 8, 193, 59, 43, 44, 45, + /* 1210 */ 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, + /* 1220 */ 56, 57, 104, 61, 23, 23, 23, 26, 26, 26, + /* 1230 */ 163, 23, 23, 106, 26, 26, 92, 93, 94, 95, + /* 1240 */ 96, 97, 98, 99, 100, 101, 102, 138, 105, 23, + /* 1250 */ 59, 23, 26, 106, 26, 163, 92, 93, 94, 95, + /* 1260 */ 96, 97, 98, 99, 100, 101, 102, 110, 23, 23, + /* 1270 */ 23, 26, 26, 26, 163, 163, 19, 120, 163, 163, + /* 1280 */ 163, 130, 163, 163, 163, 163, 163, 163, 163, 193, + /* 1290 */ 193, 163, 163, 163, 163, 225, 19, 106, 163, 222, + /* 1300 */ 163, 44, 45, 46, 47, 48, 49, 50, 51, 52, + /* 1310 */ 53, 54, 55, 56, 57, 163, 163, 203, 163, 163, + /* 1320 */ 222, 163, 45, 46, 47, 48, 49, 50, 51, 52, + /* 1330 */ 53, 54, 55, 56, 57, 163, 163, 163, 163, 163, + /* 1340 */ 251, 250, 209, 19, 20, 182, 22, 161, 222, 92, + /* 1350 */ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, + /* 1360 */ 36, 222, 222, 260, 226, 188, 256, 226, 187, 92, + /* 1370 */ 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, + /* 1380 */ 210, 213, 213, 59, 213, 196, 192, 187, 256, 244, + /* 1390 */ 212, 187, 226, 19, 20, 71, 22, 210, 166, 60, + /* 1400 */ 130, 170, 260, 170, 38, 81, 257, 257, 170, 104, + /* 1410 */ 36, 22, 43, 201, 90, 236, 138, 235, 213, 18, + /* 1420 */ 96, 97, 48, 204, 204, 204, 204, 103, 170, 105, + /* 1430 */ 106, 107, 18, 59, 110, 169, 213, 213, 201, 170, + /* 1440 */ 201, 169, 236, 213, 146, 71, 235, 62, 253, 252, + /* 1450 */ 170, 127, 128, 169, 22, 170, 82, 189, 169, 104, + /* 1460 */ 170, 87, 169, 189, 90, 141, 142, 143, 144, 145, + /* 1470 */ 96, 97, 186, 186, 186, 64, 194, 103, 186, 105, + /* 1480 */ 106, 107, 115, 189, 110, 188, 186, 186, 19, 20, + /* 1490 */ 194, 22, 186, 189, 102, 246, 246, 189, 133, 228, + /* 1500 */ 104, 228, 227, 227, 170, 36, 134, 228, 227, 19, + /* 1510 */ 20, 228, 22, 84, 271, 141, 142, 143, 144, 145, + /* 1520 */ 0, 1, 2, 216, 22, 5, 36, 137, 59, 227, + /* 1530 */ 10, 11, 12, 13, 14, 217, 269, 17, 216, 22, + /* 1540 */ 71, 170, 243, 146, 241, 217, 136, 215, 135, 59, + /* 1550 */ 30, 82, 32, 25, 214, 213, 87, 173, 26, 90, + /* 1560 */ 40, 71, 13, 172, 164, 96, 97, 164, 6, 162, + /* 1570 */ 162, 162, 103, 263, 105, 106, 107, 266, 266, 110, + /* 1580 */ 90, 176, 176, 190, 182, 190, 96, 97, 98, 4, + /* 1590 */ 70, 176, 3, 103, 182, 105, 106, 107, 78, 182, + /* 1600 */ 110, 81, 182, 182, 182, 182, 182, 151, 88, 22, + /* 1610 */ 141, 142, 143, 144, 145, 15, 89, 16, 23, 23, + /* 1620 */ 128, 19, 20, 139, 22, 119, 131, 24, 20, 133, + /* 1630 */ 16, 141, 142, 143, 144, 145, 1, 140, 36, 131, + /* 1640 */ 119, 61, 122, 37, 139, 53, 53, 127, 128, 119, + /* 1650 */ 53, 53, 105, 34, 130, 1, 5, 104, 22, 149, + /* 1660 */ 26, 59, 68, 75, 41, 130, 24, 68, 104, 20, + /* 1670 */ 150, 19, 120, 71, 114, 22, 67, 22, 22, 67, + /* 1680 */ 23, 22, 22, 67, 82, 37, 28, 23, 138, 87, + /* 1690 */ 22, 153, 90, 23, 23, 26, 23, 22, 96, 97, + /* 1700 */ 24, 23, 22, 24, 130, 103, 23, 105, 106, 107, + /* 1710 */ 1, 2, 110, 23, 5, 105, 34, 22, 132, 10, + /* 1720 */ 11, 12, 13, 14, 26, 34, 17, 34, 85, 83, + /* 1730 */ 44, 19, 20, 23, 22, 24, 75, 34, 23, 30, + /* 1740 */ 26, 32, 26, 141, 142, 143, 144, 145, 36, 40, + /* 1750 */ 23, 23, 23, 23, 11, 23, 22, 26, 22, 22, + /* 1760 */ 22, 19, 20, 23, 22, 26, 15, 23, 22, 124, + /* 1770 */ 130, 59, 23, 1, 130, 277, 277, 130, 36, 70, + /* 1780 */ 130, 277, 277, 71, 277, 277, 277, 78, 277, 277, + /* 1790 */ 81, 277, 277, 277, 277, 277, 277, 88, 277, 277, + /* 1800 */ 277, 59, 90, 277, 277, 277, 277, 277, 96, 97, + /* 1810 */ 277, 277, 277, 71, 277, 103, 277, 105, 106, 107, + /* 1820 */ 277, 277, 110, 277, 277, 277, 277, 277, 277, 277, + /* 1830 */ 277, 122, 90, 277, 277, 277, 127, 128, 96, 97, + /* 1840 */ 277, 277, 277, 277, 277, 103, 277, 105, 106, 107, + /* 1850 */ 277, 277, 110, 141, 142, 143, 144, 145, 277, 150, + /* 1860 */ 277, 277, 277, 5, 277, 277, 277, 277, 10, 11, + /* 1870 */ 12, 13, 14, 277, 277, 17, 277, 277, 277, 277, + /* 1880 */ 277, 277, 277, 141, 142, 143, 144, 145, 30, 277, + /* 1890 */ 32, 277, 277, 277, 277, 277, 277, 277, 40, 277, + /* 1900 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 1910 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 1920 */ 277, 277, 277, 277, 277, 277, 277, 277, 70, 277, + /* 1930 */ 277, 277, 277, 277, 277, 277, 78, 277, 277, 81, + /* 1940 */ 277, 277, 277, 277, 277, 277, 88, 277, 277, 277, + /* 1950 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 1960 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 1970 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 1980 */ 122, 277, 277, 277, 277, 127, 128, 277, 277, 277, + /* 1990 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, 277, + /* 2000 */ 277, 277, 277, 277, 277, 277, 277, 277, 150, 277, + /* 2010 */ 277, 277, 277, 277, 277, 277, 277, 277, 277, }; -#define YY_SHIFT_COUNT (489) +#define YY_SHIFT_COUNT (520) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1637) +#define YY_SHIFT_MAX (1858) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 990, 976, 1211, 837, 837, 316, 1054, 1054, 1054, 1054, - /* 10 */ 214, 0, 0, 106, 642, 1054, 1054, 1054, 1054, 1054, - /* 20 */ 1054, 1054, 1054, 952, 952, 226, 1155, 316, 316, 316, - /* 30 */ 316, 316, 316, 53, 159, 212, 265, 318, 371, 424, - /* 40 */ 477, 533, 589, 642, 642, 642, 642, 642, 642, 642, - /* 50 */ 642, 642, 642, 642, 642, 642, 642, 642, 642, 642, - /* 60 */ 695, 642, 747, 798, 798, 1004, 1054, 1054, 1054, 1054, - /* 70 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, - /* 80 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, - /* 90 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1112, 1054, 1054, - /* 100 */ 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, 1054, - /* 110 */ 1054, 856, 874, 874, 874, 874, 874, 134, 147, 93, - /* 120 */ 342, 959, 1161, 253, 253, 342, 367, 367, 367, 367, - /* 130 */ 179, 36, 79, 1657, 1657, 1657, 1061, 1061, 1061, 516, - /* 140 */ 799, 516, 516, 531, 531, 802, 249, 369, 342, 342, - /* 150 */ 342, 342, 342, 342, 342, 342, 342, 342, 342, 342, - /* 160 */ 342, 342, 342, 342, 342, 342, 342, 342, 342, 272, - /* 170 */ 442, 442, 536, 1657, 1657, 1657, 1025, 245, 245, 570, - /* 180 */ 172, 286, 805, 1047, 1140, 1220, 342, 342, 342, 342, - /* 190 */ 342, 342, 342, 342, 170, 342, 342, 342, 342, 342, - /* 200 */ 342, 342, 342, 342, 342, 342, 342, 841, 841, 841, - /* 210 */ 342, 342, 342, 342, 530, 342, 342, 342, 1059, 342, - /* 220 */ 342, 1167, 342, 342, 342, 342, 342, 342, 342, 342, - /* 230 */ 123, 688, 177, 1212, 1212, 1212, 1212, 1144, 177, 177, - /* 240 */ 1064, 409, 33, 628, 707, 707, 900, 628, 628, 900, - /* 250 */ 897, 323, 398, 677, 677, 677, 707, 572, 684, 590, - /* 260 */ 739, 1236, 1182, 1182, 1276, 1276, 1182, 1253, 1325, 1315, - /* 270 */ 1239, 1346, 1346, 1346, 1346, 1182, 1369, 1239, 1239, 1253, - /* 280 */ 1325, 1315, 1315, 1239, 1182, 1369, 1298, 1376, 1182, 1369, - /* 290 */ 1424, 1182, 1369, 1182, 1369, 1424, 1358, 1358, 1358, 1405, - /* 300 */ 1424, 1358, 1364, 1358, 1405, 1358, 1358, 1424, 1379, 1379, - /* 310 */ 1424, 1351, 1388, 1351, 1388, 1351, 1388, 1351, 1388, 1182, - /* 320 */ 1472, 1182, 1360, 1372, 1377, 1374, 1378, 1239, 1480, 1482, - /* 330 */ 1497, 1497, 1508, 1508, 1508, 1657, 1657, 1657, 1657, 1657, - /* 340 */ 1657, 1657, 1657, 1657, 1657, 1657, 1657, 1657, 1657, 1657, - /* 350 */ 1657, 20, 413, 98, 423, 519, 383, 962, 742, 61, - /* 360 */ 696, 749, 750, 753, 789, 790, 795, 797, 840, 842, - /* 370 */ 810, 668, 817, 659, 819, 849, 854, 899, 643, 745, - /* 380 */ 956, 926, 916, 1523, 1529, 1511, 1391, 1521, 1458, 1525, - /* 390 */ 1519, 1520, 1423, 1414, 1436, 1526, 1425, 1531, 1426, 1537, - /* 400 */ 1553, 1431, 1427, 1444, 1496, 1522, 1429, 1505, 1509, 1510, - /* 410 */ 1512, 1452, 1468, 1535, 1447, 1570, 1567, 1551, 1477, 1433, - /* 420 */ 1513, 1550, 1514, 1502, 1538, 1455, 1483, 1559, 1564, 1566, - /* 430 */ 1473, 1481, 1565, 1524, 1568, 1571, 1569, 1572, 1528, 1561, - /* 440 */ 1574, 1530, 1562, 1575, 1577, 1578, 1576, 1580, 1582, 1581, - /* 450 */ 1583, 1585, 1584, 1486, 1587, 1588, 1515, 1586, 1590, 1489, - /* 460 */ 1589, 1591, 1592, 1593, 1594, 1596, 1598, 1589, 1599, 1600, - /* 470 */ 1602, 1601, 1604, 1605, 1607, 1608, 1609, 1610, 1612, 1613, - /* 480 */ 1615, 1614, 1518, 1516, 1527, 1532, 1533, 1618, 1616, 1637, + /* 0 */ 1709, 1520, 1858, 1324, 1324, 277, 1374, 1469, 1602, 1712, + /* 10 */ 1712, 1712, 273, 0, 0, 113, 1016, 1712, 1712, 1712, + /* 20 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 11, 11, 236, + /* 30 */ 184, 277, 277, 277, 277, 277, 277, 93, 177, 270, + /* 40 */ 363, 456, 549, 642, 735, 828, 848, 996, 1144, 1016, + /* 50 */ 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, 1016, + /* 60 */ 1016, 1016, 1016, 1016, 1016, 1016, 1164, 1016, 1257, 1277, + /* 70 */ 1277, 1490, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, + /* 80 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, + /* 90 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, 1712, + /* 100 */ 1712, 1712, 1712, 1742, 1712, 1712, 1712, 1712, 1712, 1712, + /* 110 */ 1712, 1712, 1712, 1712, 1712, 1712, 1712, 143, 162, 162, + /* 120 */ 162, 162, 162, 204, 151, 416, 531, 648, 700, 531, + /* 130 */ 486, 486, 531, 353, 353, 353, 353, 409, 279, 53, + /* 140 */ 2009, 2009, 331, 331, 331, 329, 366, 329, 329, 597, + /* 150 */ 597, 464, 474, 262, 681, 531, 531, 531, 531, 531, + /* 160 */ 531, 531, 531, 531, 531, 531, 531, 531, 531, 531, + /* 170 */ 531, 531, 531, 531, 531, 531, 531, 173, 485, 984, + /* 180 */ 984, 576, 485, 19, 1022, 2009, 2009, 2009, 387, 250, + /* 190 */ 250, 525, 502, 278, 552, 227, 480, 566, 531, 531, + /* 200 */ 531, 531, 531, 531, 531, 531, 531, 531, 639, 531, + /* 210 */ 531, 531, 531, 531, 531, 531, 531, 531, 531, 531, + /* 220 */ 531, 2, 2, 2, 531, 531, 531, 531, 782, 531, + /* 230 */ 531, 531, 744, 531, 531, 783, 531, 531, 531, 531, + /* 240 */ 531, 531, 531, 531, 419, 682, 327, 370, 370, 370, + /* 250 */ 370, 1029, 327, 327, 1024, 897, 856, 947, 1109, 706, + /* 260 */ 706, 1143, 1109, 1109, 1143, 842, 945, 1118, 1136, 1136, + /* 270 */ 1136, 706, 676, 400, 1047, 694, 1339, 1270, 1270, 1366, + /* 280 */ 1366, 1270, 1305, 1389, 1369, 1278, 1401, 1401, 1401, 1401, + /* 290 */ 1270, 1414, 1278, 1278, 1305, 1389, 1369, 1369, 1278, 1270, + /* 300 */ 1414, 1298, 1385, 1270, 1414, 1432, 1270, 1414, 1270, 1414, + /* 310 */ 1432, 1355, 1355, 1355, 1411, 1432, 1355, 1367, 1355, 1411, + /* 320 */ 1355, 1355, 1432, 1392, 1392, 1432, 1365, 1396, 1365, 1396, + /* 330 */ 1365, 1396, 1365, 1396, 1270, 1372, 1429, 1502, 1390, 1372, + /* 340 */ 1517, 1270, 1397, 1390, 1410, 1413, 1278, 1528, 1532, 1549, + /* 350 */ 1549, 1562, 1562, 1562, 2009, 2009, 2009, 2009, 2009, 2009, + /* 360 */ 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, 2009, + /* 370 */ 570, 345, 686, 748, 50, 740, 1064, 1107, 469, 537, + /* 380 */ 1042, 1146, 1162, 1154, 1201, 1202, 1203, 1208, 1209, 1127, + /* 390 */ 1069, 1196, 1157, 1147, 1226, 1228, 1245, 775, 868, 1246, + /* 400 */ 1247, 1191, 1151, 1585, 1589, 1587, 1456, 1600, 1527, 1601, + /* 410 */ 1595, 1596, 1492, 1484, 1506, 1603, 1495, 1608, 1496, 1614, + /* 420 */ 1635, 1508, 1497, 1521, 1580, 1606, 1505, 1592, 1593, 1597, + /* 430 */ 1598, 1530, 1547, 1619, 1524, 1654, 1651, 1636, 1553, 1510, + /* 440 */ 1594, 1634, 1599, 1588, 1623, 1535, 1564, 1642, 1649, 1652, + /* 450 */ 1552, 1560, 1653, 1609, 1655, 1656, 1657, 1659, 1612, 1658, + /* 460 */ 1660, 1616, 1648, 1664, 1550, 1668, 1538, 1670, 1671, 1669, + /* 470 */ 1673, 1675, 1676, 1678, 1680, 1679, 1574, 1683, 1690, 1610, + /* 480 */ 1682, 1695, 1586, 1698, 1691, 1698, 1693, 1643, 1661, 1646, + /* 490 */ 1686, 1710, 1711, 1714, 1716, 1703, 1715, 1698, 1727, 1728, + /* 500 */ 1729, 1730, 1731, 1732, 1734, 1743, 1736, 1737, 1740, 1744, + /* 510 */ 1738, 1746, 1739, 1645, 1640, 1644, 1647, 1650, 1749, 1751, + /* 520 */ 1772, }; -#define YY_REDUCE_COUNT (350) -#define YY_REDUCE_MIN (-225) -#define YY_REDUCE_MAX (1375) +#define YY_REDUCE_COUNT (369) +#define YY_REDUCE_MIN (-237) +#define YY_REDUCE_MAX (1424) static const short yy_reduce_ofst[] = { - /* 0 */ -137, -31, 1104, 1023, 1081, -132, -40, -38, 223, 225, - /* 10 */ 698, -153, -99, -225, -165, 386, 478, 843, 859, -139, - /* 20 */ 884, 117, 277, 844, 857, 964, 559, 561, 614, 918, - /* 30 */ 1009, 1089, 1098, -222, -222, -222, -222, -222, -222, -222, - /* 40 */ -222, -222, -222, -222, -222, -222, -222, -222, -222, -222, - /* 50 */ -222, -222, -222, -222, -222, -222, -222, -222, -222, -222, - /* 60 */ -222, -222, -222, -222, -222, 329, 331, 497, 654, 656, - /* 70 */ 781, 891, 946, 1029, 1129, 1134, 1149, 1154, 1160, 1162, - /* 80 */ 1164, 1168, 1174, 1177, 1188, 1191, 1193, 1202, 1204, 1206, - /* 90 */ 1208, 1216, 1218, 1221, 1231, 1233, 1235, 1241, 1244, 1246, - /* 100 */ 1248, 1250, 1258, 1260, 1275, 1277, 1283, 1286, 1288, 1290, - /* 110 */ 1292, -222, -222, -222, -222, -222, -222, -222, -222, -222, - /* 120 */ -115, 796, -156, -154, -141, 14, 242, 349, 242, 349, - /* 130 */ -61, -222, -222, -222, -222, -222, 101, 101, 101, 332, - /* 140 */ 302, 384, 387, -170, 146, 344, 196, 196, 15, 11, - /* 150 */ 183, 235, 395, 355, 396, 406, 452, 457, 391, 459, - /* 160 */ 443, 447, 511, 495, 454, 512, 505, 571, 498, 532, - /* 170 */ 431, 435, 339, 455, 446, 508, -174, -116, -97, -120, - /* 180 */ -150, 64, 176, 330, 337, 509, 569, 611, 653, 673, - /* 190 */ 714, 717, 763, 771, -34, 779, 786, 830, 846, 860, - /* 200 */ 866, 882, 883, 890, 892, 895, 902, 319, 368, 769, - /* 210 */ 915, 924, 925, 932, 755, 936, 945, 963, 782, 969, - /* 220 */ 974, 816, 977, 64, 982, 983, 1016, 1022, 1024, 1031, - /* 230 */ 870, 831, 913, 966, 973, 981, 984, 755, 913, 913, - /* 240 */ 1000, 1041, 1063, 1015, 1010, 1011, 985, 1034, 1057, 1019, - /* 250 */ 1086, 1080, 1085, 1093, 1095, 1096, 1067, 1048, 1082, 1099, - /* 260 */ 1137, 1050, 1150, 1156, 1077, 1088, 1180, 1120, 1132, 1169, - /* 270 */ 1170, 1178, 1181, 1195, 1210, 1225, 1243, 1197, 1209, 1173, - /* 280 */ 1190, 1226, 1238, 1223, 1267, 1272, 1199, 1207, 1282, 1285, - /* 290 */ 1269, 1293, 1295, 1296, 1300, 1289, 1294, 1297, 1299, 1287, - /* 300 */ 1301, 1302, 1303, 1306, 1304, 1307, 1308, 1310, 1242, 1245, - /* 310 */ 1311, 1268, 1270, 1273, 1278, 1274, 1279, 1280, 1284, 1333, - /* 320 */ 1271, 1337, 1281, 1309, 1305, 1312, 1314, 1316, 1344, 1347, - /* 330 */ 1359, 1361, 1368, 1370, 1371, 1291, 1313, 1317, 1355, 1352, - /* 340 */ 1353, 1354, 1356, 1363, 1350, 1357, 1362, 1366, 1367, 1375, - /* 350 */ 1365, + /* 0 */ -147, 171, 263, -96, 358, -144, -149, -102, 124, -156, + /* 10 */ -98, 305, 401, -57, 209, -237, 245, -94, -79, 189, + /* 20 */ 375, 490, 493, 378, 303, 539, 542, 501, 503, 554, + /* 30 */ 415, 526, 546, 557, 587, 593, 595, -234, -234, -234, + /* 40 */ -234, -234, -234, -234, -234, -234, -234, -234, -234, -234, + /* 50 */ -234, -234, -234, -234, -234, -234, -234, -234, -234, -234, + /* 60 */ -234, -234, -234, -234, -234, -234, -234, -234, -234, -234, + /* 70 */ -234, -50, 335, 470, 633, 656, 658, 660, 675, 685, + /* 80 */ 703, 727, 747, 750, 752, 754, 770, 788, 790, 793, + /* 90 */ 795, 797, 800, 802, 804, 806, 813, 820, 829, 833, + /* 100 */ 836, 838, 843, 845, 847, 849, 873, 891, 893, 916, + /* 110 */ 918, 921, 936, 941, 944, 956, 961, -234, -234, -234, + /* 120 */ -234, -234, -234, -234, -234, -234, 463, 607, -176, 14, + /* 130 */ -139, 87, -137, 818, 925, 818, 925, 898, -234, -234, + /* 140 */ -234, -234, -166, -166, -166, -130, -131, -82, -54, -180, + /* 150 */ 364, 41, 513, 509, 509, 117, 500, 789, 796, 646, + /* 160 */ 192, 291, 644, 798, 120, 807, 543, 911, 920, 652, + /* 170 */ 924, 922, 232, 698, 801, 971, 39, 220, 731, 442, + /* 180 */ 902, -199, 979, -43, 421, 896, 942, 605, -184, -126, + /* 190 */ 155, 172, 281, 304, 377, 538, 650, 690, 699, 723, + /* 200 */ 803, 839, 853, 919, 991, 1018, 1067, 1092, 951, 1111, + /* 210 */ 1112, 1115, 1116, 1117, 1119, 1120, 1121, 1122, 1123, 1124, + /* 220 */ 1125, 1012, 1096, 1097, 1128, 1129, 1130, 1131, 1070, 1135, + /* 230 */ 1137, 1152, 1077, 1153, 1155, 1114, 1156, 304, 1158, 1172, + /* 240 */ 1173, 1174, 1175, 1176, 1089, 1091, 1133, 1098, 1126, 1139, + /* 250 */ 1140, 1070, 1133, 1133, 1170, 1163, 1186, 1103, 1168, 1138, + /* 260 */ 1141, 1110, 1169, 1171, 1132, 1177, 1189, 1194, 1181, 1200, + /* 270 */ 1204, 1166, 1145, 1178, 1187, 1232, 1142, 1231, 1233, 1149, + /* 280 */ 1150, 1238, 1179, 1182, 1212, 1205, 1219, 1220, 1221, 1222, + /* 290 */ 1258, 1266, 1223, 1224, 1206, 1211, 1237, 1239, 1230, 1269, + /* 300 */ 1272, 1195, 1197, 1280, 1284, 1268, 1285, 1289, 1290, 1293, + /* 310 */ 1274, 1286, 1287, 1288, 1282, 1294, 1292, 1297, 1300, 1296, + /* 320 */ 1301, 1306, 1304, 1249, 1250, 1308, 1271, 1275, 1273, 1276, + /* 330 */ 1279, 1281, 1283, 1302, 1334, 1307, 1243, 1267, 1318, 1322, + /* 340 */ 1303, 1371, 1299, 1328, 1332, 1340, 1342, 1384, 1391, 1400, + /* 350 */ 1403, 1407, 1408, 1409, 1311, 1312, 1310, 1405, 1402, 1412, + /* 360 */ 1417, 1420, 1406, 1393, 1395, 1421, 1422, 1423, 1424, 1415, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1389, 1389, 1389, 1261, 1046, 1151, 1261, 1261, 1261, 1261, - /* 10 */ 1046, 1181, 1181, 1312, 1077, 1046, 1046, 1046, 1046, 1046, - /* 20 */ 1046, 1260, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 30 */ 1046, 1046, 1046, 1187, 1046, 1046, 1046, 1046, 1262, 1263, - /* 40 */ 1046, 1046, 1046, 1311, 1313, 1197, 1196, 1195, 1194, 1294, - /* 50 */ 1168, 1192, 1185, 1189, 1256, 1257, 1255, 1259, 1262, 1263, - /* 60 */ 1046, 1188, 1226, 1240, 1225, 1046, 1046, 1046, 1046, 1046, - /* 70 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 80 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 90 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 100 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 110 */ 1046, 1234, 1239, 1246, 1238, 1235, 1228, 1227, 1229, 1230, - /* 120 */ 1046, 1067, 1116, 1046, 1046, 1046, 1329, 1328, 1046, 1046, - /* 130 */ 1077, 1231, 1232, 1243, 1242, 1241, 1319, 1345, 1344, 1046, - /* 140 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 150 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 160 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1077, - /* 170 */ 1073, 1073, 1046, 1324, 1151, 1142, 1046, 1046, 1046, 1046, - /* 180 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1316, 1314, 1046, - /* 190 */ 1276, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 200 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 210 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1147, 1046, - /* 220 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1339, - /* 230 */ 1046, 1289, 1130, 1147, 1147, 1147, 1147, 1149, 1131, 1129, - /* 240 */ 1141, 1077, 1053, 1191, 1170, 1170, 1378, 1191, 1191, 1378, - /* 250 */ 1091, 1359, 1088, 1181, 1181, 1181, 1170, 1258, 1148, 1141, - /* 260 */ 1046, 1381, 1156, 1156, 1380, 1380, 1156, 1200, 1206, 1119, - /* 270 */ 1191, 1125, 1125, 1125, 1125, 1156, 1064, 1191, 1191, 1200, - /* 280 */ 1206, 1119, 1119, 1191, 1156, 1064, 1293, 1375, 1156, 1064, - /* 290 */ 1269, 1156, 1064, 1156, 1064, 1269, 1117, 1117, 1117, 1106, - /* 300 */ 1269, 1117, 1091, 1117, 1106, 1117, 1117, 1269, 1273, 1273, - /* 310 */ 1269, 1174, 1169, 1174, 1169, 1174, 1169, 1174, 1169, 1156, - /* 320 */ 1264, 1156, 1046, 1186, 1175, 1184, 1182, 1191, 1070, 1109, - /* 330 */ 1342, 1342, 1338, 1338, 1338, 1386, 1386, 1324, 1354, 1077, - /* 340 */ 1077, 1077, 1077, 1354, 1093, 1093, 1077, 1077, 1077, 1077, - /* 350 */ 1354, 1046, 1046, 1046, 1046, 1046, 1046, 1349, 1046, 1278, - /* 360 */ 1160, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 370 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 380 */ 1046, 1046, 1211, 1046, 1049, 1321, 1046, 1046, 1320, 1046, - /* 390 */ 1046, 1046, 1046, 1046, 1046, 1161, 1046, 1046, 1046, 1046, - /* 400 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 410 */ 1046, 1046, 1046, 1046, 1377, 1046, 1046, 1046, 1046, 1046, - /* 420 */ 1046, 1292, 1291, 1046, 1046, 1158, 1046, 1046, 1046, 1046, - /* 430 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 440 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 450 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 460 */ 1183, 1046, 1176, 1046, 1046, 1046, 1046, 1368, 1046, 1046, - /* 470 */ 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, 1046, - /* 480 */ 1046, 1363, 1133, 1213, 1046, 1212, 1216, 1046, 1058, 1046, + /* 0 */ 1492, 1492, 1492, 1340, 1123, 1229, 1123, 1123, 1123, 1340, + /* 10 */ 1340, 1340, 1123, 1259, 1259, 1391, 1154, 1123, 1123, 1123, + /* 20 */ 1123, 1123, 1123, 1123, 1339, 1123, 1123, 1123, 1123, 1123, + /* 30 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1265, 1123, + /* 40 */ 1123, 1123, 1123, 1123, 1341, 1342, 1123, 1123, 1123, 1390, + /* 50 */ 1392, 1275, 1274, 1273, 1272, 1373, 1246, 1270, 1263, 1267, + /* 60 */ 1335, 1336, 1334, 1338, 1342, 1341, 1123, 1266, 1306, 1320, + /* 70 */ 1305, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 80 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 90 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 100 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 110 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1314, 1319, 1325, + /* 120 */ 1318, 1315, 1308, 1307, 1309, 1310, 1123, 1144, 1193, 1123, + /* 130 */ 1123, 1123, 1123, 1409, 1408, 1123, 1123, 1154, 1311, 1312, + /* 140 */ 1322, 1321, 1398, 1448, 1447, 1123, 1123, 1123, 1123, 1123, + /* 150 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 160 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 170 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1154, 1150, 1300, + /* 180 */ 1299, 1418, 1150, 1253, 1123, 1404, 1229, 1220, 1123, 1123, + /* 190 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 200 */ 1123, 1395, 1393, 1123, 1355, 1123, 1123, 1123, 1123, 1123, + /* 210 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 220 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 230 */ 1123, 1123, 1225, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 240 */ 1123, 1123, 1123, 1442, 1123, 1368, 1207, 1225, 1225, 1225, + /* 250 */ 1225, 1227, 1208, 1206, 1219, 1154, 1130, 1484, 1269, 1248, + /* 260 */ 1248, 1481, 1269, 1269, 1481, 1168, 1462, 1165, 1259, 1259, + /* 270 */ 1259, 1248, 1337, 1226, 1219, 1123, 1484, 1234, 1234, 1483, + /* 280 */ 1483, 1234, 1278, 1284, 1196, 1269, 1202, 1202, 1202, 1202, + /* 290 */ 1234, 1141, 1269, 1269, 1278, 1284, 1196, 1196, 1269, 1234, + /* 300 */ 1141, 1372, 1478, 1234, 1141, 1348, 1234, 1141, 1234, 1141, + /* 310 */ 1348, 1194, 1194, 1194, 1183, 1348, 1194, 1168, 1194, 1183, + /* 320 */ 1194, 1194, 1348, 1352, 1352, 1348, 1252, 1247, 1252, 1247, + /* 330 */ 1252, 1247, 1252, 1247, 1234, 1253, 1417, 1123, 1264, 1253, + /* 340 */ 1343, 1234, 1123, 1264, 1262, 1260, 1269, 1147, 1186, 1445, + /* 350 */ 1445, 1441, 1441, 1441, 1489, 1489, 1404, 1457, 1154, 1154, + /* 360 */ 1154, 1154, 1457, 1170, 1170, 1154, 1154, 1154, 1154, 1457, + /* 370 */ 1123, 1123, 1123, 1123, 1123, 1123, 1452, 1123, 1357, 1238, + /* 380 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 390 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 400 */ 1123, 1123, 1289, 1123, 1126, 1401, 1123, 1123, 1399, 1123, + /* 410 */ 1123, 1123, 1123, 1123, 1123, 1239, 1123, 1123, 1123, 1123, + /* 420 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 430 */ 1123, 1123, 1123, 1123, 1480, 1123, 1123, 1123, 1123, 1123, + /* 440 */ 1123, 1371, 1370, 1123, 1123, 1236, 1123, 1123, 1123, 1123, + /* 450 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 460 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 470 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 480 */ 1123, 1123, 1123, 1261, 1123, 1416, 1123, 1123, 1123, 1123, + /* 490 */ 1123, 1123, 1123, 1430, 1254, 1123, 1123, 1471, 1123, 1123, + /* 500 */ 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, 1123, + /* 510 */ 1123, 1123, 1466, 1210, 1291, 1123, 1290, 1294, 1123, 1135, + /* 520 */ 1123, }; /********** End of lemon-generated parsing tables *****************************/ @@ -142313,11 +146923,18 @@ static const YYCODETYPE yyFallback[] = { 59, /* REPLACE => ID */ 59, /* RESTRICT => ID */ 59, /* ROW => ID */ + 59, /* ROWS => ID */ 59, /* TRIGGER => ID */ 59, /* VACUUM => ID */ 59, /* VIEW => ID */ 59, /* VIRTUAL => ID */ 59, /* WITH => ID */ + 59, /* CURRENT => ID */ + 59, /* FOLLOWING => ID */ + 59, /* PARTITION => ID */ + 59, /* PRECEDING => ID */ + 59, /* RANGE => ID */ + 59, /* UNBOUNDED => ID */ 59, /* REINDEX => ID */ 59, /* RENAME => ID */ 59, /* CTIME_KW => ID */ @@ -142484,185 +147101,207 @@ static const char *const yyTokenName[] = { /* 73 */ "REPLACE", /* 74 */ "RESTRICT", /* 75 */ "ROW", - /* 76 */ "TRIGGER", - /* 77 */ "VACUUM", - /* 78 */ "VIEW", - /* 79 */ "VIRTUAL", - /* 80 */ "WITH", - /* 81 */ "REINDEX", - /* 82 */ "RENAME", - /* 83 */ "CTIME_KW", - /* 84 */ "ANY", - /* 85 */ "BITAND", - /* 86 */ "BITOR", - /* 87 */ "LSHIFT", - /* 88 */ "RSHIFT", - /* 89 */ "PLUS", - /* 90 */ "MINUS", - /* 91 */ "STAR", - /* 92 */ "SLASH", - /* 93 */ "REM", - /* 94 */ "CONCAT", - /* 95 */ "COLLATE", - /* 96 */ "BITNOT", - /* 97 */ "ON", - /* 98 */ "INDEXED", - /* 99 */ "STRING", - /* 100 */ "JOIN_KW", - /* 101 */ "CONSTRAINT", - /* 102 */ "DEFAULT", - /* 103 */ "NULL", - /* 104 */ "PRIMARY", - /* 105 */ "UNIQUE", - /* 106 */ "CHECK", - /* 107 */ "REFERENCES", - /* 108 */ "AUTOINCR", - /* 109 */ "INSERT", - /* 110 */ "DELETE", - /* 111 */ "UPDATE", - /* 112 */ "SET", - /* 113 */ "DEFERRABLE", - /* 114 */ "FOREIGN", - /* 115 */ "DROP", - /* 116 */ "UNION", - /* 117 */ "ALL", - /* 118 */ "EXCEPT", - /* 119 */ "INTERSECT", - /* 120 */ "SELECT", - /* 121 */ "VALUES", - /* 122 */ "DISTINCT", - /* 123 */ "DOT", - /* 124 */ "FROM", - /* 125 */ "JOIN", - /* 126 */ "USING", - /* 127 */ "ORDER", - /* 128 */ "GROUP", - /* 129 */ "HAVING", - /* 130 */ "LIMIT", - /* 131 */ "WHERE", - /* 132 */ "INTO", - /* 133 */ "NOTHING", - /* 134 */ "FLOAT", - /* 135 */ "BLOB", - /* 136 */ "INTEGER", - /* 137 */ "VARIABLE", - /* 138 */ "CASE", - /* 139 */ "WHEN", - /* 140 */ "THEN", - /* 141 */ "ELSE", - /* 142 */ "INDEX", - /* 143 */ "ALTER", - /* 144 */ "ADD", - /* 145 */ "input", - /* 146 */ "cmdlist", - /* 147 */ "ecmd", - /* 148 */ "cmdx", - /* 149 */ "explain", - /* 150 */ "cmd", - /* 151 */ "transtype", - /* 152 */ "trans_opt", - /* 153 */ "nm", - /* 154 */ "savepoint_opt", - /* 155 */ "create_table", - /* 156 */ "create_table_args", - /* 157 */ "createkw", - /* 158 */ "temp", - /* 159 */ "ifnotexists", - /* 160 */ "dbnm", - /* 161 */ "columnlist", - /* 162 */ "conslist_opt", - /* 163 */ "table_options", - /* 164 */ "select", - /* 165 */ "columnname", - /* 166 */ "carglist", - /* 167 */ "typetoken", - /* 168 */ "typename", - /* 169 */ "signed", - /* 170 */ "plus_num", - /* 171 */ "minus_num", - /* 172 */ "scanpt", - /* 173 */ "ccons", - /* 174 */ "term", - /* 175 */ "expr", - /* 176 */ "onconf", - /* 177 */ "sortorder", - /* 178 */ "autoinc", - /* 179 */ "eidlist_opt", - /* 180 */ "refargs", - /* 181 */ "defer_subclause", - /* 182 */ "refarg", - /* 183 */ "refact", - /* 184 */ "init_deferred_pred_opt", - /* 185 */ "conslist", - /* 186 */ "tconscomma", - /* 187 */ "tcons", - /* 188 */ "sortlist", - /* 189 */ "eidlist", - /* 190 */ "defer_subclause_opt", - /* 191 */ "orconf", - /* 192 */ "resolvetype", - /* 193 */ "raisetype", - /* 194 */ "ifexists", - /* 195 */ "fullname", - /* 196 */ "selectnowith", - /* 197 */ "oneselect", - /* 198 */ "wqlist", - /* 199 */ "multiselect_op", - /* 200 */ "distinct", - /* 201 */ "selcollist", - /* 202 */ "from", - /* 203 */ "where_opt", - /* 204 */ "groupby_opt", - /* 205 */ "having_opt", - /* 206 */ "orderby_opt", - /* 207 */ "limit_opt", - /* 208 */ "values", - /* 209 */ "nexprlist", - /* 210 */ "exprlist", - /* 211 */ "sclp", - /* 212 */ "as", - /* 213 */ "seltablist", - /* 214 */ "stl_prefix", - /* 215 */ "joinop", - /* 216 */ "indexed_opt", - /* 217 */ "on_opt", - /* 218 */ "using_opt", - /* 219 */ "xfullname", - /* 220 */ "idlist", - /* 221 */ "with", - /* 222 */ "setlist", - /* 223 */ "insert_cmd", - /* 224 */ "idlist_opt", - /* 225 */ "upsert", - /* 226 */ "likeop", - /* 227 */ "between_op", - /* 228 */ "in_op", - /* 229 */ "paren_exprlist", - /* 230 */ "case_operand", - /* 231 */ "case_exprlist", - /* 232 */ "case_else", - /* 233 */ "uniqueflag", - /* 234 */ "collate", - /* 235 */ "nmnum", - /* 236 */ "trigger_decl", - /* 237 */ "trigger_cmd_list", - /* 238 */ "trigger_time", - /* 239 */ "trigger_event", - /* 240 */ "foreach_clause", - /* 241 */ "when_clause", - /* 242 */ "trigger_cmd", - /* 243 */ "trnm", - /* 244 */ "tridxby", - /* 245 */ "database_kw_opt", - /* 246 */ "key_opt", - /* 247 */ "add_column_fullname", - /* 248 */ "kwcolumn_opt", - /* 249 */ "create_vtab", - /* 250 */ "vtabarglist", - /* 251 */ "vtabarg", - /* 252 */ "vtabargtoken", - /* 253 */ "lp", - /* 254 */ "anylist", + /* 76 */ "ROWS", + /* 77 */ "TRIGGER", + /* 78 */ "VACUUM", + /* 79 */ "VIEW", + /* 80 */ "VIRTUAL", + /* 81 */ "WITH", + /* 82 */ "CURRENT", + /* 83 */ "FOLLOWING", + /* 84 */ "PARTITION", + /* 85 */ "PRECEDING", + /* 86 */ "RANGE", + /* 87 */ "UNBOUNDED", + /* 88 */ "REINDEX", + /* 89 */ "RENAME", + /* 90 */ "CTIME_KW", + /* 91 */ "ANY", + /* 92 */ "BITAND", + /* 93 */ "BITOR", + /* 94 */ "LSHIFT", + /* 95 */ "RSHIFT", + /* 96 */ "PLUS", + /* 97 */ "MINUS", + /* 98 */ "STAR", + /* 99 */ "SLASH", + /* 100 */ "REM", + /* 101 */ "CONCAT", + /* 102 */ "COLLATE", + /* 103 */ "BITNOT", + /* 104 */ "ON", + /* 105 */ "INDEXED", + /* 106 */ "STRING", + /* 107 */ "JOIN_KW", + /* 108 */ "CONSTRAINT", + /* 109 */ "DEFAULT", + /* 110 */ "NULL", + /* 111 */ "PRIMARY", + /* 112 */ "UNIQUE", + /* 113 */ "CHECK", + /* 114 */ "REFERENCES", + /* 115 */ "AUTOINCR", + /* 116 */ "INSERT", + /* 117 */ "DELETE", + /* 118 */ "UPDATE", + /* 119 */ "SET", + /* 120 */ "DEFERRABLE", + /* 121 */ "FOREIGN", + /* 122 */ "DROP", + /* 123 */ "UNION", + /* 124 */ "ALL", + /* 125 */ "EXCEPT", + /* 126 */ "INTERSECT", + /* 127 */ "SELECT", + /* 128 */ "VALUES", + /* 129 */ "DISTINCT", + /* 130 */ "DOT", + /* 131 */ "FROM", + /* 132 */ "JOIN", + /* 133 */ "USING", + /* 134 */ "ORDER", + /* 135 */ "GROUP", + /* 136 */ "HAVING", + /* 137 */ "LIMIT", + /* 138 */ "WHERE", + /* 139 */ "INTO", + /* 140 */ "NOTHING", + /* 141 */ "FLOAT", + /* 142 */ "BLOB", + /* 143 */ "INTEGER", + /* 144 */ "VARIABLE", + /* 145 */ "CASE", + /* 146 */ "WHEN", + /* 147 */ "THEN", + /* 148 */ "ELSE", + /* 149 */ "INDEX", + /* 150 */ "ALTER", + /* 151 */ "ADD", + /* 152 */ "WINDOW", + /* 153 */ "OVER", + /* 154 */ "FILTER", + /* 155 */ "input", + /* 156 */ "cmdlist", + /* 157 */ "ecmd", + /* 158 */ "cmdx", + /* 159 */ "explain", + /* 160 */ "cmd", + /* 161 */ "transtype", + /* 162 */ "trans_opt", + /* 163 */ "nm", + /* 164 */ "savepoint_opt", + /* 165 */ "create_table", + /* 166 */ "create_table_args", + /* 167 */ "createkw", + /* 168 */ "temp", + /* 169 */ "ifnotexists", + /* 170 */ "dbnm", + /* 171 */ "columnlist", + /* 172 */ "conslist_opt", + /* 173 */ "table_options", + /* 174 */ "select", + /* 175 */ "columnname", + /* 176 */ "carglist", + /* 177 */ "typetoken", + /* 178 */ "typename", + /* 179 */ "signed", + /* 180 */ "plus_num", + /* 181 */ "minus_num", + /* 182 */ "scanpt", + /* 183 */ "ccons", + /* 184 */ "term", + /* 185 */ "expr", + /* 186 */ "onconf", + /* 187 */ "sortorder", + /* 188 */ "autoinc", + /* 189 */ "eidlist_opt", + /* 190 */ "refargs", + /* 191 */ "defer_subclause", + /* 192 */ "refarg", + /* 193 */ "refact", + /* 194 */ "init_deferred_pred_opt", + /* 195 */ "conslist", + /* 196 */ "tconscomma", + /* 197 */ "tcons", + /* 198 */ "sortlist", + /* 199 */ "eidlist", + /* 200 */ "defer_subclause_opt", + /* 201 */ "orconf", + /* 202 */ "resolvetype", + /* 203 */ "raisetype", + /* 204 */ "ifexists", + /* 205 */ "fullname", + /* 206 */ "selectnowith", + /* 207 */ "oneselect", + /* 208 */ "wqlist", + /* 209 */ "multiselect_op", + /* 210 */ "distinct", + /* 211 */ "selcollist", + /* 212 */ "from", + /* 213 */ "where_opt", + /* 214 */ "groupby_opt", + /* 215 */ "having_opt", + /* 216 */ "orderby_opt", + /* 217 */ "limit_opt", + /* 218 */ "window_clause", + /* 219 */ "values", + /* 220 */ "nexprlist", + /* 221 */ "sclp", + /* 222 */ "as", + /* 223 */ "seltablist", + /* 224 */ "stl_prefix", + /* 225 */ "joinop", + /* 226 */ "indexed_opt", + /* 227 */ "on_opt", + /* 228 */ "using_opt", + /* 229 */ "exprlist", + /* 230 */ "xfullname", + /* 231 */ "idlist", + /* 232 */ "with", + /* 233 */ "setlist", + /* 234 */ "insert_cmd", + /* 235 */ "idlist_opt", + /* 236 */ "upsert", + /* 237 */ "over_clause", + /* 238 */ "likeop", + /* 239 */ "between_op", + /* 240 */ "in_op", + /* 241 */ "paren_exprlist", + /* 242 */ "case_operand", + /* 243 */ "case_exprlist", + /* 244 */ "case_else", + /* 245 */ "uniqueflag", + /* 246 */ "collate", + /* 247 */ "nmnum", + /* 248 */ "trigger_decl", + /* 249 */ "trigger_cmd_list", + /* 250 */ "trigger_time", + /* 251 */ "trigger_event", + /* 252 */ "foreach_clause", + /* 253 */ "when_clause", + /* 254 */ "trigger_cmd", + /* 255 */ "trnm", + /* 256 */ "tridxby", + /* 257 */ "database_kw_opt", + /* 258 */ "key_opt", + /* 259 */ "add_column_fullname", + /* 260 */ "kwcolumn_opt", + /* 261 */ "create_vtab", + /* 262 */ "vtabarglist", + /* 263 */ "vtabarg", + /* 264 */ "vtabargtoken", + /* 265 */ "lp", + /* 266 */ "anylist", + /* 267 */ "windowdefn_list", + /* 268 */ "windowdefn", + /* 269 */ "window", + /* 270 */ "frame_opt", + /* 271 */ "part_opt", + /* 272 */ "filter_opt", + /* 273 */ "range_or_rows", + /* 274 */ "frame_bound", + /* 275 */ "frame_bound_s", + /* 276 */ "frame_bound_e", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -142758,259 +147397,285 @@ static const char *const yyRuleName[] = { /* 85 */ "multiselect_op ::= UNION ALL", /* 86 */ "multiselect_op ::= EXCEPT|INTERSECT", /* 87 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 88 */ "values ::= VALUES LP nexprlist RP", - /* 89 */ "values ::= values COMMA LP exprlist RP", - /* 90 */ "distinct ::= DISTINCT", - /* 91 */ "distinct ::= ALL", - /* 92 */ "distinct ::=", - /* 93 */ "sclp ::=", - /* 94 */ "selcollist ::= sclp scanpt expr scanpt as", - /* 95 */ "selcollist ::= sclp scanpt STAR", - /* 96 */ "selcollist ::= sclp scanpt nm DOT STAR", - /* 97 */ "as ::= AS nm", - /* 98 */ "as ::=", - /* 99 */ "from ::=", - /* 100 */ "from ::= FROM seltablist", - /* 101 */ "stl_prefix ::= seltablist joinop", - /* 102 */ "stl_prefix ::=", - /* 103 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 104 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", - /* 105 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 106 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 107 */ "dbnm ::=", - /* 108 */ "dbnm ::= DOT nm", - /* 109 */ "fullname ::= nm", - /* 110 */ "fullname ::= nm DOT nm", - /* 111 */ "xfullname ::= nm", - /* 112 */ "xfullname ::= nm DOT nm", - /* 113 */ "xfullname ::= nm DOT nm AS nm", - /* 114 */ "xfullname ::= nm AS nm", - /* 115 */ "joinop ::= COMMA|JOIN", - /* 116 */ "joinop ::= JOIN_KW JOIN", - /* 117 */ "joinop ::= JOIN_KW nm JOIN", - /* 118 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 119 */ "on_opt ::= ON expr", - /* 120 */ "on_opt ::=", - /* 121 */ "indexed_opt ::=", - /* 122 */ "indexed_opt ::= INDEXED BY nm", - /* 123 */ "indexed_opt ::= NOT INDEXED", - /* 124 */ "using_opt ::= USING LP idlist RP", - /* 125 */ "using_opt ::=", - /* 126 */ "orderby_opt ::=", - /* 127 */ "orderby_opt ::= ORDER BY sortlist", - /* 128 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 129 */ "sortlist ::= expr sortorder", - /* 130 */ "sortorder ::= ASC", - /* 131 */ "sortorder ::= DESC", - /* 132 */ "sortorder ::=", - /* 133 */ "groupby_opt ::=", - /* 134 */ "groupby_opt ::= GROUP BY nexprlist", - /* 135 */ "having_opt ::=", - /* 136 */ "having_opt ::= HAVING expr", - /* 137 */ "limit_opt ::=", - /* 138 */ "limit_opt ::= LIMIT expr", - /* 139 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 140 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 141 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt", - /* 142 */ "where_opt ::=", - /* 143 */ "where_opt ::= WHERE expr", - /* 144 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt", - /* 145 */ "setlist ::= setlist COMMA nm EQ expr", - /* 146 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", - /* 147 */ "setlist ::= nm EQ expr", - /* 148 */ "setlist ::= LP idlist RP EQ expr", - /* 149 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", - /* 150 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES", - /* 151 */ "upsert ::=", - /* 152 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt", - /* 153 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING", - /* 154 */ "upsert ::= ON CONFLICT DO NOTHING", - /* 155 */ "insert_cmd ::= INSERT orconf", - /* 156 */ "insert_cmd ::= REPLACE", - /* 157 */ "idlist_opt ::=", - /* 158 */ "idlist_opt ::= LP idlist RP", - /* 159 */ "idlist ::= idlist COMMA nm", - /* 160 */ "idlist ::= nm", - /* 161 */ "expr ::= LP expr RP", - /* 162 */ "expr ::= ID|INDEXED", - /* 163 */ "expr ::= JOIN_KW", - /* 164 */ "expr ::= nm DOT nm", - /* 165 */ "expr ::= nm DOT nm DOT nm", - /* 166 */ "term ::= NULL|FLOAT|BLOB", - /* 167 */ "term ::= STRING", - /* 168 */ "term ::= INTEGER", - /* 169 */ "expr ::= VARIABLE", - /* 170 */ "expr ::= expr COLLATE ID|STRING", - /* 171 */ "expr ::= CAST LP expr AS typetoken RP", - /* 172 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 173 */ "expr ::= ID|INDEXED LP STAR RP", - /* 174 */ "term ::= CTIME_KW", - /* 175 */ "expr ::= LP nexprlist COMMA expr RP", - /* 176 */ "expr ::= expr AND expr", - /* 177 */ "expr ::= expr OR expr", - /* 178 */ "expr ::= expr LT|GT|GE|LE expr", - /* 179 */ "expr ::= expr EQ|NE expr", - /* 180 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 181 */ "expr ::= expr PLUS|MINUS expr", - /* 182 */ "expr ::= expr STAR|SLASH|REM expr", - /* 183 */ "expr ::= expr CONCAT expr", - /* 184 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 185 */ "expr ::= expr likeop expr", - /* 186 */ "expr ::= expr likeop expr ESCAPE expr", - /* 187 */ "expr ::= expr ISNULL|NOTNULL", - /* 188 */ "expr ::= expr NOT NULL", - /* 189 */ "expr ::= expr IS expr", - /* 190 */ "expr ::= expr IS NOT expr", - /* 191 */ "expr ::= NOT expr", - /* 192 */ "expr ::= BITNOT expr", - /* 193 */ "expr ::= MINUS expr", - /* 194 */ "expr ::= PLUS expr", - /* 195 */ "between_op ::= BETWEEN", - /* 196 */ "between_op ::= NOT BETWEEN", - /* 197 */ "expr ::= expr between_op expr AND expr", - /* 198 */ "in_op ::= IN", - /* 199 */ "in_op ::= NOT IN", - /* 200 */ "expr ::= expr in_op LP exprlist RP", - /* 201 */ "expr ::= LP select RP", - /* 202 */ "expr ::= expr in_op LP select RP", - /* 203 */ "expr ::= expr in_op nm dbnm paren_exprlist", - /* 204 */ "expr ::= EXISTS LP select RP", - /* 205 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 206 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 207 */ "case_exprlist ::= WHEN expr THEN expr", - /* 208 */ "case_else ::= ELSE expr", - /* 209 */ "case_else ::=", - /* 210 */ "case_operand ::= expr", - /* 211 */ "case_operand ::=", - /* 212 */ "exprlist ::=", - /* 213 */ "nexprlist ::= nexprlist COMMA expr", - /* 214 */ "nexprlist ::= expr", - /* 215 */ "paren_exprlist ::=", - /* 216 */ "paren_exprlist ::= LP exprlist RP", - /* 217 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 218 */ "uniqueflag ::= UNIQUE", - /* 219 */ "uniqueflag ::=", - /* 220 */ "eidlist_opt ::=", - /* 221 */ "eidlist_opt ::= LP eidlist RP", - /* 222 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 223 */ "eidlist ::= nm collate sortorder", - /* 224 */ "collate ::=", - /* 225 */ "collate ::= COLLATE ID|STRING", - /* 226 */ "cmd ::= DROP INDEX ifexists fullname", - /* 227 */ "cmd ::= VACUUM", - /* 228 */ "cmd ::= VACUUM nm", - /* 229 */ "cmd ::= PRAGMA nm dbnm", - /* 230 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 231 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 232 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 233 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 234 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 235 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 236 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 237 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 238 */ "trigger_time ::= BEFORE|AFTER", - /* 239 */ "trigger_time ::= INSTEAD OF", - /* 240 */ "trigger_time ::=", - /* 241 */ "trigger_event ::= DELETE|INSERT", - /* 242 */ "trigger_event ::= UPDATE", - /* 243 */ "trigger_event ::= UPDATE OF idlist", - /* 244 */ "when_clause ::=", - /* 245 */ "when_clause ::= WHEN expr", - /* 246 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 247 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 248 */ "trnm ::= nm DOT nm", - /* 249 */ "tridxby ::= INDEXED BY nm", - /* 250 */ "tridxby ::= NOT INDEXED", - /* 251 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt", - /* 252 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", - /* 253 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", - /* 254 */ "trigger_cmd ::= scanpt select scanpt", - /* 255 */ "expr ::= RAISE LP IGNORE RP", - /* 256 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 257 */ "raisetype ::= ROLLBACK", - /* 258 */ "raisetype ::= ABORT", - /* 259 */ "raisetype ::= FAIL", - /* 260 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 261 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 262 */ "cmd ::= DETACH database_kw_opt expr", - /* 263 */ "key_opt ::=", - /* 264 */ "key_opt ::= KEY expr", - /* 265 */ "cmd ::= REINDEX", - /* 266 */ "cmd ::= REINDEX nm dbnm", - /* 267 */ "cmd ::= ANALYZE", - /* 268 */ "cmd ::= ANALYZE nm dbnm", - /* 269 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 270 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 271 */ "add_column_fullname ::= fullname", - /* 272 */ "cmd ::= create_vtab", - /* 273 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 274 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 275 */ "vtabarg ::=", - /* 276 */ "vtabargtoken ::= ANY", - /* 277 */ "vtabargtoken ::= lp anylist RP", - /* 278 */ "lp ::= LP", - /* 279 */ "with ::= WITH wqlist", - /* 280 */ "with ::= WITH RECURSIVE wqlist", - /* 281 */ "wqlist ::= nm eidlist_opt AS LP select RP", - /* 282 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", - /* 283 */ "input ::= cmdlist", - /* 284 */ "cmdlist ::= cmdlist ecmd", - /* 285 */ "cmdlist ::= ecmd", - /* 286 */ "ecmd ::= SEMI", - /* 287 */ "ecmd ::= cmdx SEMI", - /* 288 */ "ecmd ::= explain cmdx", - /* 289 */ "trans_opt ::=", - /* 290 */ "trans_opt ::= TRANSACTION", - /* 291 */ "trans_opt ::= TRANSACTION nm", - /* 292 */ "savepoint_opt ::= SAVEPOINT", - /* 293 */ "savepoint_opt ::=", - /* 294 */ "cmd ::= create_table create_table_args", - /* 295 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 296 */ "columnlist ::= columnname carglist", - /* 297 */ "nm ::= ID|INDEXED", - /* 298 */ "nm ::= STRING", - /* 299 */ "nm ::= JOIN_KW", - /* 300 */ "typetoken ::= typename", - /* 301 */ "typename ::= ID|STRING", - /* 302 */ "signed ::= plus_num", - /* 303 */ "signed ::= minus_num", - /* 304 */ "carglist ::= carglist ccons", - /* 305 */ "carglist ::=", - /* 306 */ "ccons ::= NULL onconf", - /* 307 */ "conslist_opt ::= COMMA conslist", - /* 308 */ "conslist ::= conslist tconscomma tcons", - /* 309 */ "conslist ::= tcons", - /* 310 */ "tconscomma ::=", - /* 311 */ "defer_subclause_opt ::= defer_subclause", - /* 312 */ "resolvetype ::= raisetype", - /* 313 */ "selectnowith ::= oneselect", - /* 314 */ "oneselect ::= values", - /* 315 */ "sclp ::= selcollist COMMA", - /* 316 */ "as ::= ID|STRING", - /* 317 */ "expr ::= term", - /* 318 */ "likeop ::= LIKE_KW|MATCH", - /* 319 */ "exprlist ::= nexprlist", - /* 320 */ "nmnum ::= plus_num", - /* 321 */ "nmnum ::= nm", - /* 322 */ "nmnum ::= ON", - /* 323 */ "nmnum ::= DELETE", - /* 324 */ "nmnum ::= DEFAULT", - /* 325 */ "plus_num ::= INTEGER|FLOAT", - /* 326 */ "foreach_clause ::=", - /* 327 */ "foreach_clause ::= FOR EACH ROW", - /* 328 */ "trnm ::= nm", - /* 329 */ "tridxby ::=", - /* 330 */ "database_kw_opt ::= DATABASE", - /* 331 */ "database_kw_opt ::=", - /* 332 */ "kwcolumn_opt ::=", - /* 333 */ "kwcolumn_opt ::= COLUMNKW", - /* 334 */ "vtabarglist ::= vtabarg", - /* 335 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 336 */ "vtabarg ::= vtabarg vtabargtoken", - /* 337 */ "anylist ::=", - /* 338 */ "anylist ::= anylist LP anylist RP", - /* 339 */ "anylist ::= anylist ANY", - /* 340 */ "with ::=", + /* 88 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", + /* 89 */ "values ::= VALUES LP nexprlist RP", + /* 90 */ "values ::= values COMMA LP nexprlist RP", + /* 91 */ "distinct ::= DISTINCT", + /* 92 */ "distinct ::= ALL", + /* 93 */ "distinct ::=", + /* 94 */ "sclp ::=", + /* 95 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 96 */ "selcollist ::= sclp scanpt STAR", + /* 97 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 98 */ "as ::= AS nm", + /* 99 */ "as ::=", + /* 100 */ "from ::=", + /* 101 */ "from ::= FROM seltablist", + /* 102 */ "stl_prefix ::= seltablist joinop", + /* 103 */ "stl_prefix ::=", + /* 104 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", + /* 105 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", + /* 106 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", + /* 107 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", + /* 108 */ "dbnm ::=", + /* 109 */ "dbnm ::= DOT nm", + /* 110 */ "fullname ::= nm", + /* 111 */ "fullname ::= nm DOT nm", + /* 112 */ "xfullname ::= nm", + /* 113 */ "xfullname ::= nm DOT nm", + /* 114 */ "xfullname ::= nm DOT nm AS nm", + /* 115 */ "xfullname ::= nm AS nm", + /* 116 */ "joinop ::= COMMA|JOIN", + /* 117 */ "joinop ::= JOIN_KW JOIN", + /* 118 */ "joinop ::= JOIN_KW nm JOIN", + /* 119 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 120 */ "on_opt ::= ON expr", + /* 121 */ "on_opt ::=", + /* 122 */ "indexed_opt ::=", + /* 123 */ "indexed_opt ::= INDEXED BY nm", + /* 124 */ "indexed_opt ::= NOT INDEXED", + /* 125 */ "using_opt ::= USING LP idlist RP", + /* 126 */ "using_opt ::=", + /* 127 */ "orderby_opt ::=", + /* 128 */ "orderby_opt ::= ORDER BY sortlist", + /* 129 */ "sortlist ::= sortlist COMMA expr sortorder", + /* 130 */ "sortlist ::= expr sortorder", + /* 131 */ "sortorder ::= ASC", + /* 132 */ "sortorder ::= DESC", + /* 133 */ "sortorder ::=", + /* 134 */ "groupby_opt ::=", + /* 135 */ "groupby_opt ::= GROUP BY nexprlist", + /* 136 */ "having_opt ::=", + /* 137 */ "having_opt ::= HAVING expr", + /* 138 */ "limit_opt ::=", + /* 139 */ "limit_opt ::= LIMIT expr", + /* 140 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 141 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 142 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt", + /* 143 */ "where_opt ::=", + /* 144 */ "where_opt ::= WHERE expr", + /* 145 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt", + /* 146 */ "setlist ::= setlist COMMA nm EQ expr", + /* 147 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 148 */ "setlist ::= nm EQ expr", + /* 149 */ "setlist ::= LP idlist RP EQ expr", + /* 150 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 151 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES", + /* 152 */ "upsert ::=", + /* 153 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt", + /* 154 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING", + /* 155 */ "upsert ::= ON CONFLICT DO NOTHING", + /* 156 */ "insert_cmd ::= INSERT orconf", + /* 157 */ "insert_cmd ::= REPLACE", + /* 158 */ "idlist_opt ::=", + /* 159 */ "idlist_opt ::= LP idlist RP", + /* 160 */ "idlist ::= idlist COMMA nm", + /* 161 */ "idlist ::= nm", + /* 162 */ "expr ::= LP expr RP", + /* 163 */ "expr ::= ID|INDEXED", + /* 164 */ "expr ::= JOIN_KW", + /* 165 */ "expr ::= nm DOT nm", + /* 166 */ "expr ::= nm DOT nm DOT nm", + /* 167 */ "term ::= NULL|FLOAT|BLOB", + /* 168 */ "term ::= STRING", + /* 169 */ "term ::= INTEGER", + /* 170 */ "expr ::= VARIABLE", + /* 171 */ "expr ::= expr COLLATE ID|STRING", + /* 172 */ "expr ::= CAST LP expr AS typetoken RP", + /* 173 */ "expr ::= ID|INDEXED LP distinct exprlist RP", + /* 174 */ "expr ::= ID|INDEXED LP STAR RP", + /* 175 */ "expr ::= ID|INDEXED LP distinct exprlist RP over_clause", + /* 176 */ "expr ::= ID|INDEXED LP STAR RP over_clause", + /* 177 */ "term ::= CTIME_KW", + /* 178 */ "expr ::= LP nexprlist COMMA expr RP", + /* 179 */ "expr ::= expr AND expr", + /* 180 */ "expr ::= expr OR expr", + /* 181 */ "expr ::= expr LT|GT|GE|LE expr", + /* 182 */ "expr ::= expr EQ|NE expr", + /* 183 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 184 */ "expr ::= expr PLUS|MINUS expr", + /* 185 */ "expr ::= expr STAR|SLASH|REM expr", + /* 186 */ "expr ::= expr CONCAT expr", + /* 187 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 188 */ "expr ::= expr likeop expr", + /* 189 */ "expr ::= expr likeop expr ESCAPE expr", + /* 190 */ "expr ::= expr ISNULL|NOTNULL", + /* 191 */ "expr ::= expr NOT NULL", + /* 192 */ "expr ::= expr IS expr", + /* 193 */ "expr ::= expr IS NOT expr", + /* 194 */ "expr ::= NOT expr", + /* 195 */ "expr ::= BITNOT expr", + /* 196 */ "expr ::= PLUS|MINUS expr", + /* 197 */ "between_op ::= BETWEEN", + /* 198 */ "between_op ::= NOT BETWEEN", + /* 199 */ "expr ::= expr between_op expr AND expr", + /* 200 */ "in_op ::= IN", + /* 201 */ "in_op ::= NOT IN", + /* 202 */ "expr ::= expr in_op LP exprlist RP", + /* 203 */ "expr ::= LP select RP", + /* 204 */ "expr ::= expr in_op LP select RP", + /* 205 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 206 */ "expr ::= EXISTS LP select RP", + /* 207 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 208 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 209 */ "case_exprlist ::= WHEN expr THEN expr", + /* 210 */ "case_else ::= ELSE expr", + /* 211 */ "case_else ::=", + /* 212 */ "case_operand ::= expr", + /* 213 */ "case_operand ::=", + /* 214 */ "exprlist ::=", + /* 215 */ "nexprlist ::= nexprlist COMMA expr", + /* 216 */ "nexprlist ::= expr", + /* 217 */ "paren_exprlist ::=", + /* 218 */ "paren_exprlist ::= LP exprlist RP", + /* 219 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 220 */ "uniqueflag ::= UNIQUE", + /* 221 */ "uniqueflag ::=", + /* 222 */ "eidlist_opt ::=", + /* 223 */ "eidlist_opt ::= LP eidlist RP", + /* 224 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 225 */ "eidlist ::= nm collate sortorder", + /* 226 */ "collate ::=", + /* 227 */ "collate ::= COLLATE ID|STRING", + /* 228 */ "cmd ::= DROP INDEX ifexists fullname", + /* 229 */ "cmd ::= VACUUM", + /* 230 */ "cmd ::= VACUUM nm", + /* 231 */ "cmd ::= PRAGMA nm dbnm", + /* 232 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 233 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 234 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 235 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 236 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 237 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 238 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 239 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 240 */ "trigger_time ::= BEFORE|AFTER", + /* 241 */ "trigger_time ::= INSTEAD OF", + /* 242 */ "trigger_time ::=", + /* 243 */ "trigger_event ::= DELETE|INSERT", + /* 244 */ "trigger_event ::= UPDATE", + /* 245 */ "trigger_event ::= UPDATE OF idlist", + /* 246 */ "when_clause ::=", + /* 247 */ "when_clause ::= WHEN expr", + /* 248 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 249 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 250 */ "trnm ::= nm DOT nm", + /* 251 */ "tridxby ::= INDEXED BY nm", + /* 252 */ "tridxby ::= NOT INDEXED", + /* 253 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt", + /* 254 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 255 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 256 */ "trigger_cmd ::= scanpt select scanpt", + /* 257 */ "expr ::= RAISE LP IGNORE RP", + /* 258 */ "expr ::= RAISE LP raisetype COMMA nm RP", + /* 259 */ "raisetype ::= ROLLBACK", + /* 260 */ "raisetype ::= ABORT", + /* 261 */ "raisetype ::= FAIL", + /* 262 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 263 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 264 */ "cmd ::= DETACH database_kw_opt expr", + /* 265 */ "key_opt ::=", + /* 266 */ "key_opt ::= KEY expr", + /* 267 */ "cmd ::= REINDEX", + /* 268 */ "cmd ::= REINDEX nm dbnm", + /* 269 */ "cmd ::= ANALYZE", + /* 270 */ "cmd ::= ANALYZE nm dbnm", + /* 271 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 272 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 273 */ "add_column_fullname ::= fullname", + /* 274 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 275 */ "cmd ::= create_vtab", + /* 276 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 277 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 278 */ "vtabarg ::=", + /* 279 */ "vtabargtoken ::= ANY", + /* 280 */ "vtabargtoken ::= lp anylist RP", + /* 281 */ "lp ::= LP", + /* 282 */ "with ::= WITH wqlist", + /* 283 */ "with ::= WITH RECURSIVE wqlist", + /* 284 */ "wqlist ::= nm eidlist_opt AS LP select RP", + /* 285 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", + /* 286 */ "windowdefn_list ::= windowdefn", + /* 287 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 288 */ "windowdefn ::= nm AS window", + /* 289 */ "window ::= LP part_opt orderby_opt frame_opt RP", + /* 290 */ "part_opt ::= PARTITION BY nexprlist", + /* 291 */ "part_opt ::=", + /* 292 */ "frame_opt ::=", + /* 293 */ "frame_opt ::= range_or_rows frame_bound_s", + /* 294 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e", + /* 295 */ "range_or_rows ::= RANGE", + /* 296 */ "range_or_rows ::= ROWS", + /* 297 */ "frame_bound_s ::= frame_bound", + /* 298 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 299 */ "frame_bound_e ::= frame_bound", + /* 300 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 301 */ "frame_bound ::= expr PRECEDING", + /* 302 */ "frame_bound ::= CURRENT ROW", + /* 303 */ "frame_bound ::= expr FOLLOWING", + /* 304 */ "window_clause ::= WINDOW windowdefn_list", + /* 305 */ "over_clause ::= filter_opt OVER window", + /* 306 */ "over_clause ::= filter_opt OVER nm", + /* 307 */ "filter_opt ::=", + /* 308 */ "filter_opt ::= FILTER LP WHERE expr RP", + /* 309 */ "input ::= cmdlist", + /* 310 */ "cmdlist ::= cmdlist ecmd", + /* 311 */ "cmdlist ::= ecmd", + /* 312 */ "ecmd ::= SEMI", + /* 313 */ "ecmd ::= cmdx SEMI", + /* 314 */ "ecmd ::= explain cmdx", + /* 315 */ "trans_opt ::=", + /* 316 */ "trans_opt ::= TRANSACTION", + /* 317 */ "trans_opt ::= TRANSACTION nm", + /* 318 */ "savepoint_opt ::= SAVEPOINT", + /* 319 */ "savepoint_opt ::=", + /* 320 */ "cmd ::= create_table create_table_args", + /* 321 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 322 */ "columnlist ::= columnname carglist", + /* 323 */ "nm ::= ID|INDEXED", + /* 324 */ "nm ::= STRING", + /* 325 */ "nm ::= JOIN_KW", + /* 326 */ "typetoken ::= typename", + /* 327 */ "typename ::= ID|STRING", + /* 328 */ "signed ::= plus_num", + /* 329 */ "signed ::= minus_num", + /* 330 */ "carglist ::= carglist ccons", + /* 331 */ "carglist ::=", + /* 332 */ "ccons ::= NULL onconf", + /* 333 */ "conslist_opt ::= COMMA conslist", + /* 334 */ "conslist ::= conslist tconscomma tcons", + /* 335 */ "conslist ::= tcons", + /* 336 */ "tconscomma ::=", + /* 337 */ "defer_subclause_opt ::= defer_subclause", + /* 338 */ "resolvetype ::= raisetype", + /* 339 */ "selectnowith ::= oneselect", + /* 340 */ "oneselect ::= values", + /* 341 */ "sclp ::= selcollist COMMA", + /* 342 */ "as ::= ID|STRING", + /* 343 */ "expr ::= term", + /* 344 */ "likeop ::= LIKE_KW|MATCH", + /* 345 */ "exprlist ::= nexprlist", + /* 346 */ "nmnum ::= plus_num", + /* 347 */ "nmnum ::= nm", + /* 348 */ "nmnum ::= ON", + /* 349 */ "nmnum ::= DELETE", + /* 350 */ "nmnum ::= DEFAULT", + /* 351 */ "plus_num ::= INTEGER|FLOAT", + /* 352 */ "foreach_clause ::=", + /* 353 */ "foreach_clause ::= FOR EACH ROW", + /* 354 */ "trnm ::= nm", + /* 355 */ "tridxby ::=", + /* 356 */ "database_kw_opt ::= DATABASE", + /* 357 */ "database_kw_opt ::=", + /* 358 */ "kwcolumn_opt ::=", + /* 359 */ "kwcolumn_opt ::= COLUMNKW", + /* 360 */ "vtabarglist ::= vtabarg", + /* 361 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 362 */ "vtabarg ::= vtabarg vtabargtoken", + /* 363 */ "anylist ::=", + /* 364 */ "anylist ::= anylist LP anylist RP", + /* 365 */ "anylist ::= anylist ANY", + /* 366 */ "with ::=", }; #endif /* NDEBUG */ @@ -143136,73 +147801,96 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 164: /* select */ - case 196: /* selectnowith */ - case 197: /* oneselect */ - case 208: /* values */ + case 174: /* select */ + case 206: /* selectnowith */ + case 207: /* oneselect */ + case 219: /* values */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy399)); +sqlite3SelectDelete(pParse->db, (yypminor->yy489)); } break; - case 174: /* term */ - case 175: /* expr */ - case 203: /* where_opt */ - case 205: /* having_opt */ - case 217: /* on_opt */ - case 230: /* case_operand */ - case 232: /* case_else */ - case 241: /* when_clause */ - case 246: /* key_opt */ + case 184: /* term */ + case 185: /* expr */ + case 213: /* where_opt */ + case 215: /* having_opt */ + case 227: /* on_opt */ + case 242: /* case_operand */ + case 244: /* case_else */ + case 253: /* when_clause */ + case 258: /* key_opt */ + case 272: /* filter_opt */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy182)); +sqlite3ExprDelete(pParse->db, (yypminor->yy18)); } break; - case 179: /* eidlist_opt */ - case 188: /* sortlist */ - case 189: /* eidlist */ - case 201: /* selcollist */ - case 204: /* groupby_opt */ - case 206: /* orderby_opt */ - case 209: /* nexprlist */ - case 210: /* exprlist */ - case 211: /* sclp */ - case 222: /* setlist */ - case 229: /* paren_exprlist */ - case 231: /* case_exprlist */ + case 189: /* eidlist_opt */ + case 198: /* sortlist */ + case 199: /* eidlist */ + case 211: /* selcollist */ + case 214: /* groupby_opt */ + case 216: /* orderby_opt */ + case 220: /* nexprlist */ + case 221: /* sclp */ + case 229: /* exprlist */ + case 233: /* setlist */ + case 241: /* paren_exprlist */ + case 243: /* case_exprlist */ + case 271: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy232)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy420)); } break; - case 195: /* fullname */ - case 202: /* from */ - case 213: /* seltablist */ - case 214: /* stl_prefix */ - case 219: /* xfullname */ + case 205: /* fullname */ + case 212: /* from */ + case 223: /* seltablist */ + case 224: /* stl_prefix */ + case 230: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy427)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy135)); } break; - case 198: /* wqlist */ + case 208: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy91)); +sqlite3WithDelete(pParse->db, (yypminor->yy449)); } break; - case 218: /* using_opt */ - case 220: /* idlist */ - case 224: /* idlist_opt */ + case 218: /* window_clause */ + case 267: /* windowdefn_list */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy510)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy327)); } break; - case 237: /* trigger_cmd_list */ - case 242: /* trigger_cmd */ + case 228: /* using_opt */ + case 231: /* idlist */ + case 235: /* idlist_opt */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy47)); +sqlite3IdListDelete(pParse->db, (yypminor->yy48)); } break; - case 239: /* trigger_event */ + case 237: /* over_clause */ + case 268: /* windowdefn */ + case 269: /* window */ + case 270: /* frame_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy300).b); +sqlite3WindowDelete(pParse->db, (yypminor->yy327)); +} + break; + case 249: /* trigger_cmd_list */ + case 254: /* trigger_cmd */ +{ +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy207)); +} + break; + case 251: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy34).b); +} + break; + case 274: /* frame_bound */ + case 275: /* frame_bound_s */ + case 276: /* frame_bound_e */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy119).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -143328,11 +148016,11 @@ static YYACTIONTYPE yy_find_shift_action( do{ i = yy_shift_ofst[stateno]; assert( i>=0 ); - assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) ); + /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */ assert( iLookAhead!=YYNOCODE ); assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( yy_lookahead[i]!=iLookAhead ){ + if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ if( iLookAhead=YY_ACTTAB_COUNT j0 ){ #ifndef NDEBUG @@ -143382,7 +148071,7 @@ static YYACTIONTYPE yy_find_shift_action( ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. */ -static int yy_find_reduce_action( +static YYACTIONTYPE yy_find_reduce_action( YYACTIONTYPE stateno, /* Current state number */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ @@ -143500,347 +148189,373 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 149, -1 }, /* (0) explain ::= EXPLAIN */ - { 149, -3 }, /* (1) explain ::= EXPLAIN QUERY PLAN */ - { 148, -1 }, /* (2) cmdx ::= cmd */ - { 150, -3 }, /* (3) cmd ::= BEGIN transtype trans_opt */ - { 151, 0 }, /* (4) transtype ::= */ - { 151, -1 }, /* (5) transtype ::= DEFERRED */ - { 151, -1 }, /* (6) transtype ::= IMMEDIATE */ - { 151, -1 }, /* (7) transtype ::= EXCLUSIVE */ - { 150, -2 }, /* (8) cmd ::= COMMIT|END trans_opt */ - { 150, -2 }, /* (9) cmd ::= ROLLBACK trans_opt */ - { 150, -2 }, /* (10) cmd ::= SAVEPOINT nm */ - { 150, -3 }, /* (11) cmd ::= RELEASE savepoint_opt nm */ - { 150, -5 }, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ - { 155, -6 }, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ - { 157, -1 }, /* (14) createkw ::= CREATE */ - { 159, 0 }, /* (15) ifnotexists ::= */ - { 159, -3 }, /* (16) ifnotexists ::= IF NOT EXISTS */ - { 158, -1 }, /* (17) temp ::= TEMP */ - { 158, 0 }, /* (18) temp ::= */ - { 156, -5 }, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ - { 156, -2 }, /* (20) create_table_args ::= AS select */ - { 163, 0 }, /* (21) table_options ::= */ - { 163, -2 }, /* (22) table_options ::= WITHOUT nm */ - { 165, -2 }, /* (23) columnname ::= nm typetoken */ - { 167, 0 }, /* (24) typetoken ::= */ - { 167, -4 }, /* (25) typetoken ::= typename LP signed RP */ - { 167, -6 }, /* (26) typetoken ::= typename LP signed COMMA signed RP */ - { 168, -2 }, /* (27) typename ::= typename ID|STRING */ - { 172, 0 }, /* (28) scanpt ::= */ - { 173, -2 }, /* (29) ccons ::= CONSTRAINT nm */ - { 173, -4 }, /* (30) ccons ::= DEFAULT scanpt term scanpt */ - { 173, -4 }, /* (31) ccons ::= DEFAULT LP expr RP */ - { 173, -4 }, /* (32) ccons ::= DEFAULT PLUS term scanpt */ - { 173, -4 }, /* (33) ccons ::= DEFAULT MINUS term scanpt */ - { 173, -3 }, /* (34) ccons ::= DEFAULT scanpt ID|INDEXED */ - { 173, -3 }, /* (35) ccons ::= NOT NULL onconf */ - { 173, -5 }, /* (36) ccons ::= PRIMARY KEY sortorder onconf autoinc */ - { 173, -2 }, /* (37) ccons ::= UNIQUE onconf */ - { 173, -4 }, /* (38) ccons ::= CHECK LP expr RP */ - { 173, -4 }, /* (39) ccons ::= REFERENCES nm eidlist_opt refargs */ - { 173, -1 }, /* (40) ccons ::= defer_subclause */ - { 173, -2 }, /* (41) ccons ::= COLLATE ID|STRING */ - { 178, 0 }, /* (42) autoinc ::= */ - { 178, -1 }, /* (43) autoinc ::= AUTOINCR */ - { 180, 0 }, /* (44) refargs ::= */ - { 180, -2 }, /* (45) refargs ::= refargs refarg */ - { 182, -2 }, /* (46) refarg ::= MATCH nm */ - { 182, -3 }, /* (47) refarg ::= ON INSERT refact */ - { 182, -3 }, /* (48) refarg ::= ON DELETE refact */ - { 182, -3 }, /* (49) refarg ::= ON UPDATE refact */ - { 183, -2 }, /* (50) refact ::= SET NULL */ - { 183, -2 }, /* (51) refact ::= SET DEFAULT */ - { 183, -1 }, /* (52) refact ::= CASCADE */ - { 183, -1 }, /* (53) refact ::= RESTRICT */ - { 183, -2 }, /* (54) refact ::= NO ACTION */ - { 181, -3 }, /* (55) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ - { 181, -2 }, /* (56) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - { 184, 0 }, /* (57) init_deferred_pred_opt ::= */ - { 184, -2 }, /* (58) init_deferred_pred_opt ::= INITIALLY DEFERRED */ - { 184, -2 }, /* (59) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ - { 162, 0 }, /* (60) conslist_opt ::= */ - { 186, -1 }, /* (61) tconscomma ::= COMMA */ - { 187, -2 }, /* (62) tcons ::= CONSTRAINT nm */ - { 187, -7 }, /* (63) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ - { 187, -5 }, /* (64) tcons ::= UNIQUE LP sortlist RP onconf */ - { 187, -5 }, /* (65) tcons ::= CHECK LP expr RP onconf */ - { 187, -10 }, /* (66) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ - { 190, 0 }, /* (67) defer_subclause_opt ::= */ - { 176, 0 }, /* (68) onconf ::= */ - { 176, -3 }, /* (69) onconf ::= ON CONFLICT resolvetype */ - { 191, 0 }, /* (70) orconf ::= */ - { 191, -2 }, /* (71) orconf ::= OR resolvetype */ - { 192, -1 }, /* (72) resolvetype ::= IGNORE */ - { 192, -1 }, /* (73) resolvetype ::= REPLACE */ - { 150, -4 }, /* (74) cmd ::= DROP TABLE ifexists fullname */ - { 194, -2 }, /* (75) ifexists ::= IF EXISTS */ - { 194, 0 }, /* (76) ifexists ::= */ - { 150, -9 }, /* (77) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ - { 150, -4 }, /* (78) cmd ::= DROP VIEW ifexists fullname */ - { 150, -1 }, /* (79) cmd ::= select */ - { 164, -3 }, /* (80) select ::= WITH wqlist selectnowith */ - { 164, -4 }, /* (81) select ::= WITH RECURSIVE wqlist selectnowith */ - { 164, -1 }, /* (82) select ::= selectnowith */ - { 196, -3 }, /* (83) selectnowith ::= selectnowith multiselect_op oneselect */ - { 199, -1 }, /* (84) multiselect_op ::= UNION */ - { 199, -2 }, /* (85) multiselect_op ::= UNION ALL */ - { 199, -1 }, /* (86) multiselect_op ::= EXCEPT|INTERSECT */ - { 197, -9 }, /* (87) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ - { 208, -4 }, /* (88) values ::= VALUES LP nexprlist RP */ - { 208, -5 }, /* (89) values ::= values COMMA LP exprlist RP */ - { 200, -1 }, /* (90) distinct ::= DISTINCT */ - { 200, -1 }, /* (91) distinct ::= ALL */ - { 200, 0 }, /* (92) distinct ::= */ - { 211, 0 }, /* (93) sclp ::= */ - { 201, -5 }, /* (94) selcollist ::= sclp scanpt expr scanpt as */ - { 201, -3 }, /* (95) selcollist ::= sclp scanpt STAR */ - { 201, -5 }, /* (96) selcollist ::= sclp scanpt nm DOT STAR */ - { 212, -2 }, /* (97) as ::= AS nm */ - { 212, 0 }, /* (98) as ::= */ - { 202, 0 }, /* (99) from ::= */ - { 202, -2 }, /* (100) from ::= FROM seltablist */ - { 214, -2 }, /* (101) stl_prefix ::= seltablist joinop */ - { 214, 0 }, /* (102) stl_prefix ::= */ - { 213, -7 }, /* (103) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ - { 213, -9 }, /* (104) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ - { 213, -7 }, /* (105) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ - { 213, -7 }, /* (106) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ - { 160, 0 }, /* (107) dbnm ::= */ - { 160, -2 }, /* (108) dbnm ::= DOT nm */ - { 195, -1 }, /* (109) fullname ::= nm */ - { 195, -3 }, /* (110) fullname ::= nm DOT nm */ - { 219, -1 }, /* (111) xfullname ::= nm */ - { 219, -3 }, /* (112) xfullname ::= nm DOT nm */ - { 219, -5 }, /* (113) xfullname ::= nm DOT nm AS nm */ - { 219, -3 }, /* (114) xfullname ::= nm AS nm */ - { 215, -1 }, /* (115) joinop ::= COMMA|JOIN */ - { 215, -2 }, /* (116) joinop ::= JOIN_KW JOIN */ - { 215, -3 }, /* (117) joinop ::= JOIN_KW nm JOIN */ - { 215, -4 }, /* (118) joinop ::= JOIN_KW nm nm JOIN */ - { 217, -2 }, /* (119) on_opt ::= ON expr */ - { 217, 0 }, /* (120) on_opt ::= */ - { 216, 0 }, /* (121) indexed_opt ::= */ - { 216, -3 }, /* (122) indexed_opt ::= INDEXED BY nm */ - { 216, -2 }, /* (123) indexed_opt ::= NOT INDEXED */ - { 218, -4 }, /* (124) using_opt ::= USING LP idlist RP */ - { 218, 0 }, /* (125) using_opt ::= */ - { 206, 0 }, /* (126) orderby_opt ::= */ - { 206, -3 }, /* (127) orderby_opt ::= ORDER BY sortlist */ - { 188, -4 }, /* (128) sortlist ::= sortlist COMMA expr sortorder */ - { 188, -2 }, /* (129) sortlist ::= expr sortorder */ - { 177, -1 }, /* (130) sortorder ::= ASC */ - { 177, -1 }, /* (131) sortorder ::= DESC */ - { 177, 0 }, /* (132) sortorder ::= */ - { 204, 0 }, /* (133) groupby_opt ::= */ - { 204, -3 }, /* (134) groupby_opt ::= GROUP BY nexprlist */ - { 205, 0 }, /* (135) having_opt ::= */ - { 205, -2 }, /* (136) having_opt ::= HAVING expr */ - { 207, 0 }, /* (137) limit_opt ::= */ - { 207, -2 }, /* (138) limit_opt ::= LIMIT expr */ - { 207, -4 }, /* (139) limit_opt ::= LIMIT expr OFFSET expr */ - { 207, -4 }, /* (140) limit_opt ::= LIMIT expr COMMA expr */ - { 150, -6 }, /* (141) cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ - { 203, 0 }, /* (142) where_opt ::= */ - { 203, -2 }, /* (143) where_opt ::= WHERE expr */ - { 150, -8 }, /* (144) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ - { 222, -5 }, /* (145) setlist ::= setlist COMMA nm EQ expr */ - { 222, -7 }, /* (146) setlist ::= setlist COMMA LP idlist RP EQ expr */ - { 222, -3 }, /* (147) setlist ::= nm EQ expr */ - { 222, -5 }, /* (148) setlist ::= LP idlist RP EQ expr */ - { 150, -7 }, /* (149) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ - { 150, -7 }, /* (150) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ - { 225, 0 }, /* (151) upsert ::= */ - { 225, -11 }, /* (152) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ - { 225, -8 }, /* (153) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ - { 225, -4 }, /* (154) upsert ::= ON CONFLICT DO NOTHING */ - { 223, -2 }, /* (155) insert_cmd ::= INSERT orconf */ - { 223, -1 }, /* (156) insert_cmd ::= REPLACE */ - { 224, 0 }, /* (157) idlist_opt ::= */ - { 224, -3 }, /* (158) idlist_opt ::= LP idlist RP */ - { 220, -3 }, /* (159) idlist ::= idlist COMMA nm */ - { 220, -1 }, /* (160) idlist ::= nm */ - { 175, -3 }, /* (161) expr ::= LP expr RP */ - { 175, -1 }, /* (162) expr ::= ID|INDEXED */ - { 175, -1 }, /* (163) expr ::= JOIN_KW */ - { 175, -3 }, /* (164) expr ::= nm DOT nm */ - { 175, -5 }, /* (165) expr ::= nm DOT nm DOT nm */ - { 174, -1 }, /* (166) term ::= NULL|FLOAT|BLOB */ - { 174, -1 }, /* (167) term ::= STRING */ - { 174, -1 }, /* (168) term ::= INTEGER */ - { 175, -1 }, /* (169) expr ::= VARIABLE */ - { 175, -3 }, /* (170) expr ::= expr COLLATE ID|STRING */ - { 175, -6 }, /* (171) expr ::= CAST LP expr AS typetoken RP */ - { 175, -5 }, /* (172) expr ::= ID|INDEXED LP distinct exprlist RP */ - { 175, -4 }, /* (173) expr ::= ID|INDEXED LP STAR RP */ - { 174, -1 }, /* (174) term ::= CTIME_KW */ - { 175, -5 }, /* (175) expr ::= LP nexprlist COMMA expr RP */ - { 175, -3 }, /* (176) expr ::= expr AND expr */ - { 175, -3 }, /* (177) expr ::= expr OR expr */ - { 175, -3 }, /* (178) expr ::= expr LT|GT|GE|LE expr */ - { 175, -3 }, /* (179) expr ::= expr EQ|NE expr */ - { 175, -3 }, /* (180) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ - { 175, -3 }, /* (181) expr ::= expr PLUS|MINUS expr */ - { 175, -3 }, /* (182) expr ::= expr STAR|SLASH|REM expr */ - { 175, -3 }, /* (183) expr ::= expr CONCAT expr */ - { 226, -2 }, /* (184) likeop ::= NOT LIKE_KW|MATCH */ - { 175, -3 }, /* (185) expr ::= expr likeop expr */ - { 175, -5 }, /* (186) expr ::= expr likeop expr ESCAPE expr */ - { 175, -2 }, /* (187) expr ::= expr ISNULL|NOTNULL */ - { 175, -3 }, /* (188) expr ::= expr NOT NULL */ - { 175, -3 }, /* (189) expr ::= expr IS expr */ - { 175, -4 }, /* (190) expr ::= expr IS NOT expr */ - { 175, -2 }, /* (191) expr ::= NOT expr */ - { 175, -2 }, /* (192) expr ::= BITNOT expr */ - { 175, -2 }, /* (193) expr ::= MINUS expr */ - { 175, -2 }, /* (194) expr ::= PLUS expr */ - { 227, -1 }, /* (195) between_op ::= BETWEEN */ - { 227, -2 }, /* (196) between_op ::= NOT BETWEEN */ - { 175, -5 }, /* (197) expr ::= expr between_op expr AND expr */ - { 228, -1 }, /* (198) in_op ::= IN */ - { 228, -2 }, /* (199) in_op ::= NOT IN */ - { 175, -5 }, /* (200) expr ::= expr in_op LP exprlist RP */ - { 175, -3 }, /* (201) expr ::= LP select RP */ - { 175, -5 }, /* (202) expr ::= expr in_op LP select RP */ - { 175, -5 }, /* (203) expr ::= expr in_op nm dbnm paren_exprlist */ - { 175, -4 }, /* (204) expr ::= EXISTS LP select RP */ - { 175, -5 }, /* (205) expr ::= CASE case_operand case_exprlist case_else END */ - { 231, -5 }, /* (206) case_exprlist ::= case_exprlist WHEN expr THEN expr */ - { 231, -4 }, /* (207) case_exprlist ::= WHEN expr THEN expr */ - { 232, -2 }, /* (208) case_else ::= ELSE expr */ - { 232, 0 }, /* (209) case_else ::= */ - { 230, -1 }, /* (210) case_operand ::= expr */ - { 230, 0 }, /* (211) case_operand ::= */ - { 210, 0 }, /* (212) exprlist ::= */ - { 209, -3 }, /* (213) nexprlist ::= nexprlist COMMA expr */ - { 209, -1 }, /* (214) nexprlist ::= expr */ - { 229, 0 }, /* (215) paren_exprlist ::= */ - { 229, -3 }, /* (216) paren_exprlist ::= LP exprlist RP */ - { 150, -12 }, /* (217) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ - { 233, -1 }, /* (218) uniqueflag ::= UNIQUE */ - { 233, 0 }, /* (219) uniqueflag ::= */ - { 179, 0 }, /* (220) eidlist_opt ::= */ - { 179, -3 }, /* (221) eidlist_opt ::= LP eidlist RP */ - { 189, -5 }, /* (222) eidlist ::= eidlist COMMA nm collate sortorder */ - { 189, -3 }, /* (223) eidlist ::= nm collate sortorder */ - { 234, 0 }, /* (224) collate ::= */ - { 234, -2 }, /* (225) collate ::= COLLATE ID|STRING */ - { 150, -4 }, /* (226) cmd ::= DROP INDEX ifexists fullname */ - { 150, -1 }, /* (227) cmd ::= VACUUM */ - { 150, -2 }, /* (228) cmd ::= VACUUM nm */ - { 150, -3 }, /* (229) cmd ::= PRAGMA nm dbnm */ - { 150, -5 }, /* (230) cmd ::= PRAGMA nm dbnm EQ nmnum */ - { 150, -6 }, /* (231) cmd ::= PRAGMA nm dbnm LP nmnum RP */ - { 150, -5 }, /* (232) cmd ::= PRAGMA nm dbnm EQ minus_num */ - { 150, -6 }, /* (233) cmd ::= PRAGMA nm dbnm LP minus_num RP */ - { 170, -2 }, /* (234) plus_num ::= PLUS INTEGER|FLOAT */ - { 171, -2 }, /* (235) minus_num ::= MINUS INTEGER|FLOAT */ - { 150, -5 }, /* (236) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ - { 236, -11 }, /* (237) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ - { 238, -1 }, /* (238) trigger_time ::= BEFORE|AFTER */ - { 238, -2 }, /* (239) trigger_time ::= INSTEAD OF */ - { 238, 0 }, /* (240) trigger_time ::= */ - { 239, -1 }, /* (241) trigger_event ::= DELETE|INSERT */ - { 239, -1 }, /* (242) trigger_event ::= UPDATE */ - { 239, -3 }, /* (243) trigger_event ::= UPDATE OF idlist */ - { 241, 0 }, /* (244) when_clause ::= */ - { 241, -2 }, /* (245) when_clause ::= WHEN expr */ - { 237, -3 }, /* (246) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ - { 237, -2 }, /* (247) trigger_cmd_list ::= trigger_cmd SEMI */ - { 243, -3 }, /* (248) trnm ::= nm DOT nm */ - { 244, -3 }, /* (249) tridxby ::= INDEXED BY nm */ - { 244, -2 }, /* (250) tridxby ::= NOT INDEXED */ - { 242, -8 }, /* (251) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ - { 242, -8 }, /* (252) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ - { 242, -6 }, /* (253) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ - { 242, -3 }, /* (254) trigger_cmd ::= scanpt select scanpt */ - { 175, -4 }, /* (255) expr ::= RAISE LP IGNORE RP */ - { 175, -6 }, /* (256) expr ::= RAISE LP raisetype COMMA nm RP */ - { 193, -1 }, /* (257) raisetype ::= ROLLBACK */ - { 193, -1 }, /* (258) raisetype ::= ABORT */ - { 193, -1 }, /* (259) raisetype ::= FAIL */ - { 150, -4 }, /* (260) cmd ::= DROP TRIGGER ifexists fullname */ - { 150, -6 }, /* (261) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ - { 150, -3 }, /* (262) cmd ::= DETACH database_kw_opt expr */ - { 246, 0 }, /* (263) key_opt ::= */ - { 246, -2 }, /* (264) key_opt ::= KEY expr */ - { 150, -1 }, /* (265) cmd ::= REINDEX */ - { 150, -3 }, /* (266) cmd ::= REINDEX nm dbnm */ - { 150, -1 }, /* (267) cmd ::= ANALYZE */ - { 150, -3 }, /* (268) cmd ::= ANALYZE nm dbnm */ - { 150, -6 }, /* (269) cmd ::= ALTER TABLE fullname RENAME TO nm */ - { 150, -7 }, /* (270) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ - { 247, -1 }, /* (271) add_column_fullname ::= fullname */ - { 150, -1 }, /* (272) cmd ::= create_vtab */ - { 150, -4 }, /* (273) cmd ::= create_vtab LP vtabarglist RP */ - { 249, -8 }, /* (274) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ - { 251, 0 }, /* (275) vtabarg ::= */ - { 252, -1 }, /* (276) vtabargtoken ::= ANY */ - { 252, -3 }, /* (277) vtabargtoken ::= lp anylist RP */ - { 253, -1 }, /* (278) lp ::= LP */ - { 221, -2 }, /* (279) with ::= WITH wqlist */ - { 221, -3 }, /* (280) with ::= WITH RECURSIVE wqlist */ - { 198, -6 }, /* (281) wqlist ::= nm eidlist_opt AS LP select RP */ - { 198, -8 }, /* (282) wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ - { 145, -1 }, /* (283) input ::= cmdlist */ - { 146, -2 }, /* (284) cmdlist ::= cmdlist ecmd */ - { 146, -1 }, /* (285) cmdlist ::= ecmd */ - { 147, -1 }, /* (286) ecmd ::= SEMI */ - { 147, -2 }, /* (287) ecmd ::= cmdx SEMI */ - { 147, -2 }, /* (288) ecmd ::= explain cmdx */ - { 152, 0 }, /* (289) trans_opt ::= */ - { 152, -1 }, /* (290) trans_opt ::= TRANSACTION */ - { 152, -2 }, /* (291) trans_opt ::= TRANSACTION nm */ - { 154, -1 }, /* (292) savepoint_opt ::= SAVEPOINT */ - { 154, 0 }, /* (293) savepoint_opt ::= */ - { 150, -2 }, /* (294) cmd ::= create_table create_table_args */ - { 161, -4 }, /* (295) columnlist ::= columnlist COMMA columnname carglist */ - { 161, -2 }, /* (296) columnlist ::= columnname carglist */ - { 153, -1 }, /* (297) nm ::= ID|INDEXED */ - { 153, -1 }, /* (298) nm ::= STRING */ - { 153, -1 }, /* (299) nm ::= JOIN_KW */ - { 167, -1 }, /* (300) typetoken ::= typename */ - { 168, -1 }, /* (301) typename ::= ID|STRING */ - { 169, -1 }, /* (302) signed ::= plus_num */ - { 169, -1 }, /* (303) signed ::= minus_num */ - { 166, -2 }, /* (304) carglist ::= carglist ccons */ - { 166, 0 }, /* (305) carglist ::= */ - { 173, -2 }, /* (306) ccons ::= NULL onconf */ - { 162, -2 }, /* (307) conslist_opt ::= COMMA conslist */ - { 185, -3 }, /* (308) conslist ::= conslist tconscomma tcons */ - { 185, -1 }, /* (309) conslist ::= tcons */ - { 186, 0 }, /* (310) tconscomma ::= */ - { 190, -1 }, /* (311) defer_subclause_opt ::= defer_subclause */ - { 192, -1 }, /* (312) resolvetype ::= raisetype */ - { 196, -1 }, /* (313) selectnowith ::= oneselect */ - { 197, -1 }, /* (314) oneselect ::= values */ - { 211, -2 }, /* (315) sclp ::= selcollist COMMA */ - { 212, -1 }, /* (316) as ::= ID|STRING */ - { 175, -1 }, /* (317) expr ::= term */ - { 226, -1 }, /* (318) likeop ::= LIKE_KW|MATCH */ - { 210, -1 }, /* (319) exprlist ::= nexprlist */ - { 235, -1 }, /* (320) nmnum ::= plus_num */ - { 235, -1 }, /* (321) nmnum ::= nm */ - { 235, -1 }, /* (322) nmnum ::= ON */ - { 235, -1 }, /* (323) nmnum ::= DELETE */ - { 235, -1 }, /* (324) nmnum ::= DEFAULT */ - { 170, -1 }, /* (325) plus_num ::= INTEGER|FLOAT */ - { 240, 0 }, /* (326) foreach_clause ::= */ - { 240, -3 }, /* (327) foreach_clause ::= FOR EACH ROW */ - { 243, -1 }, /* (328) trnm ::= nm */ - { 244, 0 }, /* (329) tridxby ::= */ - { 245, -1 }, /* (330) database_kw_opt ::= DATABASE */ - { 245, 0 }, /* (331) database_kw_opt ::= */ - { 248, 0 }, /* (332) kwcolumn_opt ::= */ - { 248, -1 }, /* (333) kwcolumn_opt ::= COLUMNKW */ - { 250, -1 }, /* (334) vtabarglist ::= vtabarg */ - { 250, -3 }, /* (335) vtabarglist ::= vtabarglist COMMA vtabarg */ - { 251, -2 }, /* (336) vtabarg ::= vtabarg vtabargtoken */ - { 254, 0 }, /* (337) anylist ::= */ - { 254, -4 }, /* (338) anylist ::= anylist LP anylist RP */ - { 254, -2 }, /* (339) anylist ::= anylist ANY */ - { 221, 0 }, /* (340) with ::= */ + { 159, -1 }, /* (0) explain ::= EXPLAIN */ + { 159, -3 }, /* (1) explain ::= EXPLAIN QUERY PLAN */ + { 158, -1 }, /* (2) cmdx ::= cmd */ + { 160, -3 }, /* (3) cmd ::= BEGIN transtype trans_opt */ + { 161, 0 }, /* (4) transtype ::= */ + { 161, -1 }, /* (5) transtype ::= DEFERRED */ + { 161, -1 }, /* (6) transtype ::= IMMEDIATE */ + { 161, -1 }, /* (7) transtype ::= EXCLUSIVE */ + { 160, -2 }, /* (8) cmd ::= COMMIT|END trans_opt */ + { 160, -2 }, /* (9) cmd ::= ROLLBACK trans_opt */ + { 160, -2 }, /* (10) cmd ::= SAVEPOINT nm */ + { 160, -3 }, /* (11) cmd ::= RELEASE savepoint_opt nm */ + { 160, -5 }, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + { 165, -6 }, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + { 167, -1 }, /* (14) createkw ::= CREATE */ + { 169, 0 }, /* (15) ifnotexists ::= */ + { 169, -3 }, /* (16) ifnotexists ::= IF NOT EXISTS */ + { 168, -1 }, /* (17) temp ::= TEMP */ + { 168, 0 }, /* (18) temp ::= */ + { 166, -5 }, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_options */ + { 166, -2 }, /* (20) create_table_args ::= AS select */ + { 173, 0 }, /* (21) table_options ::= */ + { 173, -2 }, /* (22) table_options ::= WITHOUT nm */ + { 175, -2 }, /* (23) columnname ::= nm typetoken */ + { 177, 0 }, /* (24) typetoken ::= */ + { 177, -4 }, /* (25) typetoken ::= typename LP signed RP */ + { 177, -6 }, /* (26) typetoken ::= typename LP signed COMMA signed RP */ + { 178, -2 }, /* (27) typename ::= typename ID|STRING */ + { 182, 0 }, /* (28) scanpt ::= */ + { 183, -2 }, /* (29) ccons ::= CONSTRAINT nm */ + { 183, -4 }, /* (30) ccons ::= DEFAULT scanpt term scanpt */ + { 183, -4 }, /* (31) ccons ::= DEFAULT LP expr RP */ + { 183, -4 }, /* (32) ccons ::= DEFAULT PLUS term scanpt */ + { 183, -4 }, /* (33) ccons ::= DEFAULT MINUS term scanpt */ + { 183, -3 }, /* (34) ccons ::= DEFAULT scanpt ID|INDEXED */ + { 183, -3 }, /* (35) ccons ::= NOT NULL onconf */ + { 183, -5 }, /* (36) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + { 183, -2 }, /* (37) ccons ::= UNIQUE onconf */ + { 183, -4 }, /* (38) ccons ::= CHECK LP expr RP */ + { 183, -4 }, /* (39) ccons ::= REFERENCES nm eidlist_opt refargs */ + { 183, -1 }, /* (40) ccons ::= defer_subclause */ + { 183, -2 }, /* (41) ccons ::= COLLATE ID|STRING */ + { 188, 0 }, /* (42) autoinc ::= */ + { 188, -1 }, /* (43) autoinc ::= AUTOINCR */ + { 190, 0 }, /* (44) refargs ::= */ + { 190, -2 }, /* (45) refargs ::= refargs refarg */ + { 192, -2 }, /* (46) refarg ::= MATCH nm */ + { 192, -3 }, /* (47) refarg ::= ON INSERT refact */ + { 192, -3 }, /* (48) refarg ::= ON DELETE refact */ + { 192, -3 }, /* (49) refarg ::= ON UPDATE refact */ + { 193, -2 }, /* (50) refact ::= SET NULL */ + { 193, -2 }, /* (51) refact ::= SET DEFAULT */ + { 193, -1 }, /* (52) refact ::= CASCADE */ + { 193, -1 }, /* (53) refact ::= RESTRICT */ + { 193, -2 }, /* (54) refact ::= NO ACTION */ + { 191, -3 }, /* (55) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + { 191, -2 }, /* (56) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + { 194, 0 }, /* (57) init_deferred_pred_opt ::= */ + { 194, -2 }, /* (58) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + { 194, -2 }, /* (59) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + { 172, 0 }, /* (60) conslist_opt ::= */ + { 196, -1 }, /* (61) tconscomma ::= COMMA */ + { 197, -2 }, /* (62) tcons ::= CONSTRAINT nm */ + { 197, -7 }, /* (63) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + { 197, -5 }, /* (64) tcons ::= UNIQUE LP sortlist RP onconf */ + { 197, -5 }, /* (65) tcons ::= CHECK LP expr RP onconf */ + { 197, -10 }, /* (66) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + { 200, 0 }, /* (67) defer_subclause_opt ::= */ + { 186, 0 }, /* (68) onconf ::= */ + { 186, -3 }, /* (69) onconf ::= ON CONFLICT resolvetype */ + { 201, 0 }, /* (70) orconf ::= */ + { 201, -2 }, /* (71) orconf ::= OR resolvetype */ + { 202, -1 }, /* (72) resolvetype ::= IGNORE */ + { 202, -1 }, /* (73) resolvetype ::= REPLACE */ + { 160, -4 }, /* (74) cmd ::= DROP TABLE ifexists fullname */ + { 204, -2 }, /* (75) ifexists ::= IF EXISTS */ + { 204, 0 }, /* (76) ifexists ::= */ + { 160, -9 }, /* (77) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + { 160, -4 }, /* (78) cmd ::= DROP VIEW ifexists fullname */ + { 160, -1 }, /* (79) cmd ::= select */ + { 174, -3 }, /* (80) select ::= WITH wqlist selectnowith */ + { 174, -4 }, /* (81) select ::= WITH RECURSIVE wqlist selectnowith */ + { 174, -1 }, /* (82) select ::= selectnowith */ + { 206, -3 }, /* (83) selectnowith ::= selectnowith multiselect_op oneselect */ + { 209, -1 }, /* (84) multiselect_op ::= UNION */ + { 209, -2 }, /* (85) multiselect_op ::= UNION ALL */ + { 209, -1 }, /* (86) multiselect_op ::= EXCEPT|INTERSECT */ + { 207, -9 }, /* (87) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + { 207, -10 }, /* (88) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + { 219, -4 }, /* (89) values ::= VALUES LP nexprlist RP */ + { 219, -5 }, /* (90) values ::= values COMMA LP nexprlist RP */ + { 210, -1 }, /* (91) distinct ::= DISTINCT */ + { 210, -1 }, /* (92) distinct ::= ALL */ + { 210, 0 }, /* (93) distinct ::= */ + { 221, 0 }, /* (94) sclp ::= */ + { 211, -5 }, /* (95) selcollist ::= sclp scanpt expr scanpt as */ + { 211, -3 }, /* (96) selcollist ::= sclp scanpt STAR */ + { 211, -5 }, /* (97) selcollist ::= sclp scanpt nm DOT STAR */ + { 222, -2 }, /* (98) as ::= AS nm */ + { 222, 0 }, /* (99) as ::= */ + { 212, 0 }, /* (100) from ::= */ + { 212, -2 }, /* (101) from ::= FROM seltablist */ + { 224, -2 }, /* (102) stl_prefix ::= seltablist joinop */ + { 224, 0 }, /* (103) stl_prefix ::= */ + { 223, -7 }, /* (104) seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + { 223, -9 }, /* (105) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + { 223, -7 }, /* (106) seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + { 223, -7 }, /* (107) seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + { 170, 0 }, /* (108) dbnm ::= */ + { 170, -2 }, /* (109) dbnm ::= DOT nm */ + { 205, -1 }, /* (110) fullname ::= nm */ + { 205, -3 }, /* (111) fullname ::= nm DOT nm */ + { 230, -1 }, /* (112) xfullname ::= nm */ + { 230, -3 }, /* (113) xfullname ::= nm DOT nm */ + { 230, -5 }, /* (114) xfullname ::= nm DOT nm AS nm */ + { 230, -3 }, /* (115) xfullname ::= nm AS nm */ + { 225, -1 }, /* (116) joinop ::= COMMA|JOIN */ + { 225, -2 }, /* (117) joinop ::= JOIN_KW JOIN */ + { 225, -3 }, /* (118) joinop ::= JOIN_KW nm JOIN */ + { 225, -4 }, /* (119) joinop ::= JOIN_KW nm nm JOIN */ + { 227, -2 }, /* (120) on_opt ::= ON expr */ + { 227, 0 }, /* (121) on_opt ::= */ + { 226, 0 }, /* (122) indexed_opt ::= */ + { 226, -3 }, /* (123) indexed_opt ::= INDEXED BY nm */ + { 226, -2 }, /* (124) indexed_opt ::= NOT INDEXED */ + { 228, -4 }, /* (125) using_opt ::= USING LP idlist RP */ + { 228, 0 }, /* (126) using_opt ::= */ + { 216, 0 }, /* (127) orderby_opt ::= */ + { 216, -3 }, /* (128) orderby_opt ::= ORDER BY sortlist */ + { 198, -4 }, /* (129) sortlist ::= sortlist COMMA expr sortorder */ + { 198, -2 }, /* (130) sortlist ::= expr sortorder */ + { 187, -1 }, /* (131) sortorder ::= ASC */ + { 187, -1 }, /* (132) sortorder ::= DESC */ + { 187, 0 }, /* (133) sortorder ::= */ + { 214, 0 }, /* (134) groupby_opt ::= */ + { 214, -3 }, /* (135) groupby_opt ::= GROUP BY nexprlist */ + { 215, 0 }, /* (136) having_opt ::= */ + { 215, -2 }, /* (137) having_opt ::= HAVING expr */ + { 217, 0 }, /* (138) limit_opt ::= */ + { 217, -2 }, /* (139) limit_opt ::= LIMIT expr */ + { 217, -4 }, /* (140) limit_opt ::= LIMIT expr OFFSET expr */ + { 217, -4 }, /* (141) limit_opt ::= LIMIT expr COMMA expr */ + { 160, -6 }, /* (142) cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ + { 213, 0 }, /* (143) where_opt ::= */ + { 213, -2 }, /* (144) where_opt ::= WHERE expr */ + { 160, -8 }, /* (145) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ + { 233, -5 }, /* (146) setlist ::= setlist COMMA nm EQ expr */ + { 233, -7 }, /* (147) setlist ::= setlist COMMA LP idlist RP EQ expr */ + { 233, -3 }, /* (148) setlist ::= nm EQ expr */ + { 233, -5 }, /* (149) setlist ::= LP idlist RP EQ expr */ + { 160, -7 }, /* (150) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + { 160, -7 }, /* (151) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ + { 236, 0 }, /* (152) upsert ::= */ + { 236, -11 }, /* (153) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ + { 236, -8 }, /* (154) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ + { 236, -4 }, /* (155) upsert ::= ON CONFLICT DO NOTHING */ + { 234, -2 }, /* (156) insert_cmd ::= INSERT orconf */ + { 234, -1 }, /* (157) insert_cmd ::= REPLACE */ + { 235, 0 }, /* (158) idlist_opt ::= */ + { 235, -3 }, /* (159) idlist_opt ::= LP idlist RP */ + { 231, -3 }, /* (160) idlist ::= idlist COMMA nm */ + { 231, -1 }, /* (161) idlist ::= nm */ + { 185, -3 }, /* (162) expr ::= LP expr RP */ + { 185, -1 }, /* (163) expr ::= ID|INDEXED */ + { 185, -1 }, /* (164) expr ::= JOIN_KW */ + { 185, -3 }, /* (165) expr ::= nm DOT nm */ + { 185, -5 }, /* (166) expr ::= nm DOT nm DOT nm */ + { 184, -1 }, /* (167) term ::= NULL|FLOAT|BLOB */ + { 184, -1 }, /* (168) term ::= STRING */ + { 184, -1 }, /* (169) term ::= INTEGER */ + { 185, -1 }, /* (170) expr ::= VARIABLE */ + { 185, -3 }, /* (171) expr ::= expr COLLATE ID|STRING */ + { 185, -6 }, /* (172) expr ::= CAST LP expr AS typetoken RP */ + { 185, -5 }, /* (173) expr ::= ID|INDEXED LP distinct exprlist RP */ + { 185, -4 }, /* (174) expr ::= ID|INDEXED LP STAR RP */ + { 185, -6 }, /* (175) expr ::= ID|INDEXED LP distinct exprlist RP over_clause */ + { 185, -5 }, /* (176) expr ::= ID|INDEXED LP STAR RP over_clause */ + { 184, -1 }, /* (177) term ::= CTIME_KW */ + { 185, -5 }, /* (178) expr ::= LP nexprlist COMMA expr RP */ + { 185, -3 }, /* (179) expr ::= expr AND expr */ + { 185, -3 }, /* (180) expr ::= expr OR expr */ + { 185, -3 }, /* (181) expr ::= expr LT|GT|GE|LE expr */ + { 185, -3 }, /* (182) expr ::= expr EQ|NE expr */ + { 185, -3 }, /* (183) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + { 185, -3 }, /* (184) expr ::= expr PLUS|MINUS expr */ + { 185, -3 }, /* (185) expr ::= expr STAR|SLASH|REM expr */ + { 185, -3 }, /* (186) expr ::= expr CONCAT expr */ + { 238, -2 }, /* (187) likeop ::= NOT LIKE_KW|MATCH */ + { 185, -3 }, /* (188) expr ::= expr likeop expr */ + { 185, -5 }, /* (189) expr ::= expr likeop expr ESCAPE expr */ + { 185, -2 }, /* (190) expr ::= expr ISNULL|NOTNULL */ + { 185, -3 }, /* (191) expr ::= expr NOT NULL */ + { 185, -3 }, /* (192) expr ::= expr IS expr */ + { 185, -4 }, /* (193) expr ::= expr IS NOT expr */ + { 185, -2 }, /* (194) expr ::= NOT expr */ + { 185, -2 }, /* (195) expr ::= BITNOT expr */ + { 185, -2 }, /* (196) expr ::= PLUS|MINUS expr */ + { 239, -1 }, /* (197) between_op ::= BETWEEN */ + { 239, -2 }, /* (198) between_op ::= NOT BETWEEN */ + { 185, -5 }, /* (199) expr ::= expr between_op expr AND expr */ + { 240, -1 }, /* (200) in_op ::= IN */ + { 240, -2 }, /* (201) in_op ::= NOT IN */ + { 185, -5 }, /* (202) expr ::= expr in_op LP exprlist RP */ + { 185, -3 }, /* (203) expr ::= LP select RP */ + { 185, -5 }, /* (204) expr ::= expr in_op LP select RP */ + { 185, -5 }, /* (205) expr ::= expr in_op nm dbnm paren_exprlist */ + { 185, -4 }, /* (206) expr ::= EXISTS LP select RP */ + { 185, -5 }, /* (207) expr ::= CASE case_operand case_exprlist case_else END */ + { 243, -5 }, /* (208) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + { 243, -4 }, /* (209) case_exprlist ::= WHEN expr THEN expr */ + { 244, -2 }, /* (210) case_else ::= ELSE expr */ + { 244, 0 }, /* (211) case_else ::= */ + { 242, -1 }, /* (212) case_operand ::= expr */ + { 242, 0 }, /* (213) case_operand ::= */ + { 229, 0 }, /* (214) exprlist ::= */ + { 220, -3 }, /* (215) nexprlist ::= nexprlist COMMA expr */ + { 220, -1 }, /* (216) nexprlist ::= expr */ + { 241, 0 }, /* (217) paren_exprlist ::= */ + { 241, -3 }, /* (218) paren_exprlist ::= LP exprlist RP */ + { 160, -12 }, /* (219) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + { 245, -1 }, /* (220) uniqueflag ::= UNIQUE */ + { 245, 0 }, /* (221) uniqueflag ::= */ + { 189, 0 }, /* (222) eidlist_opt ::= */ + { 189, -3 }, /* (223) eidlist_opt ::= LP eidlist RP */ + { 199, -5 }, /* (224) eidlist ::= eidlist COMMA nm collate sortorder */ + { 199, -3 }, /* (225) eidlist ::= nm collate sortorder */ + { 246, 0 }, /* (226) collate ::= */ + { 246, -2 }, /* (227) collate ::= COLLATE ID|STRING */ + { 160, -4 }, /* (228) cmd ::= DROP INDEX ifexists fullname */ + { 160, -1 }, /* (229) cmd ::= VACUUM */ + { 160, -2 }, /* (230) cmd ::= VACUUM nm */ + { 160, -3 }, /* (231) cmd ::= PRAGMA nm dbnm */ + { 160, -5 }, /* (232) cmd ::= PRAGMA nm dbnm EQ nmnum */ + { 160, -6 }, /* (233) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + { 160, -5 }, /* (234) cmd ::= PRAGMA nm dbnm EQ minus_num */ + { 160, -6 }, /* (235) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + { 180, -2 }, /* (236) plus_num ::= PLUS INTEGER|FLOAT */ + { 181, -2 }, /* (237) minus_num ::= MINUS INTEGER|FLOAT */ + { 160, -5 }, /* (238) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + { 248, -11 }, /* (239) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + { 250, -1 }, /* (240) trigger_time ::= BEFORE|AFTER */ + { 250, -2 }, /* (241) trigger_time ::= INSTEAD OF */ + { 250, 0 }, /* (242) trigger_time ::= */ + { 251, -1 }, /* (243) trigger_event ::= DELETE|INSERT */ + { 251, -1 }, /* (244) trigger_event ::= UPDATE */ + { 251, -3 }, /* (245) trigger_event ::= UPDATE OF idlist */ + { 253, 0 }, /* (246) when_clause ::= */ + { 253, -2 }, /* (247) when_clause ::= WHEN expr */ + { 249, -3 }, /* (248) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + { 249, -2 }, /* (249) trigger_cmd_list ::= trigger_cmd SEMI */ + { 255, -3 }, /* (250) trnm ::= nm DOT nm */ + { 256, -3 }, /* (251) tridxby ::= INDEXED BY nm */ + { 256, -2 }, /* (252) tridxby ::= NOT INDEXED */ + { 254, -8 }, /* (253) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ + { 254, -8 }, /* (254) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + { 254, -6 }, /* (255) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + { 254, -3 }, /* (256) trigger_cmd ::= scanpt select scanpt */ + { 185, -4 }, /* (257) expr ::= RAISE LP IGNORE RP */ + { 185, -6 }, /* (258) expr ::= RAISE LP raisetype COMMA nm RP */ + { 203, -1 }, /* (259) raisetype ::= ROLLBACK */ + { 203, -1 }, /* (260) raisetype ::= ABORT */ + { 203, -1 }, /* (261) raisetype ::= FAIL */ + { 160, -4 }, /* (262) cmd ::= DROP TRIGGER ifexists fullname */ + { 160, -6 }, /* (263) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + { 160, -3 }, /* (264) cmd ::= DETACH database_kw_opt expr */ + { 258, 0 }, /* (265) key_opt ::= */ + { 258, -2 }, /* (266) key_opt ::= KEY expr */ + { 160, -1 }, /* (267) cmd ::= REINDEX */ + { 160, -3 }, /* (268) cmd ::= REINDEX nm dbnm */ + { 160, -1 }, /* (269) cmd ::= ANALYZE */ + { 160, -3 }, /* (270) cmd ::= ANALYZE nm dbnm */ + { 160, -6 }, /* (271) cmd ::= ALTER TABLE fullname RENAME TO nm */ + { 160, -7 }, /* (272) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + { 259, -1 }, /* (273) add_column_fullname ::= fullname */ + { 160, -8 }, /* (274) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + { 160, -1 }, /* (275) cmd ::= create_vtab */ + { 160, -4 }, /* (276) cmd ::= create_vtab LP vtabarglist RP */ + { 261, -8 }, /* (277) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + { 263, 0 }, /* (278) vtabarg ::= */ + { 264, -1 }, /* (279) vtabargtoken ::= ANY */ + { 264, -3 }, /* (280) vtabargtoken ::= lp anylist RP */ + { 265, -1 }, /* (281) lp ::= LP */ + { 232, -2 }, /* (282) with ::= WITH wqlist */ + { 232, -3 }, /* (283) with ::= WITH RECURSIVE wqlist */ + { 208, -6 }, /* (284) wqlist ::= nm eidlist_opt AS LP select RP */ + { 208, -8 }, /* (285) wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ + { 267, -1 }, /* (286) windowdefn_list ::= windowdefn */ + { 267, -3 }, /* (287) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + { 268, -3 }, /* (288) windowdefn ::= nm AS window */ + { 269, -5 }, /* (289) window ::= LP part_opt orderby_opt frame_opt RP */ + { 271, -3 }, /* (290) part_opt ::= PARTITION BY nexprlist */ + { 271, 0 }, /* (291) part_opt ::= */ + { 270, 0 }, /* (292) frame_opt ::= */ + { 270, -2 }, /* (293) frame_opt ::= range_or_rows frame_bound_s */ + { 270, -5 }, /* (294) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e */ + { 273, -1 }, /* (295) range_or_rows ::= RANGE */ + { 273, -1 }, /* (296) range_or_rows ::= ROWS */ + { 275, -1 }, /* (297) frame_bound_s ::= frame_bound */ + { 275, -2 }, /* (298) frame_bound_s ::= UNBOUNDED PRECEDING */ + { 276, -1 }, /* (299) frame_bound_e ::= frame_bound */ + { 276, -2 }, /* (300) frame_bound_e ::= UNBOUNDED FOLLOWING */ + { 274, -2 }, /* (301) frame_bound ::= expr PRECEDING */ + { 274, -2 }, /* (302) frame_bound ::= CURRENT ROW */ + { 274, -2 }, /* (303) frame_bound ::= expr FOLLOWING */ + { 218, -2 }, /* (304) window_clause ::= WINDOW windowdefn_list */ + { 237, -3 }, /* (305) over_clause ::= filter_opt OVER window */ + { 237, -3 }, /* (306) over_clause ::= filter_opt OVER nm */ + { 272, 0 }, /* (307) filter_opt ::= */ + { 272, -5 }, /* (308) filter_opt ::= FILTER LP WHERE expr RP */ + { 155, -1 }, /* (309) input ::= cmdlist */ + { 156, -2 }, /* (310) cmdlist ::= cmdlist ecmd */ + { 156, -1 }, /* (311) cmdlist ::= ecmd */ + { 157, -1 }, /* (312) ecmd ::= SEMI */ + { 157, -2 }, /* (313) ecmd ::= cmdx SEMI */ + { 157, -2 }, /* (314) ecmd ::= explain cmdx */ + { 162, 0 }, /* (315) trans_opt ::= */ + { 162, -1 }, /* (316) trans_opt ::= TRANSACTION */ + { 162, -2 }, /* (317) trans_opt ::= TRANSACTION nm */ + { 164, -1 }, /* (318) savepoint_opt ::= SAVEPOINT */ + { 164, 0 }, /* (319) savepoint_opt ::= */ + { 160, -2 }, /* (320) cmd ::= create_table create_table_args */ + { 171, -4 }, /* (321) columnlist ::= columnlist COMMA columnname carglist */ + { 171, -2 }, /* (322) columnlist ::= columnname carglist */ + { 163, -1 }, /* (323) nm ::= ID|INDEXED */ + { 163, -1 }, /* (324) nm ::= STRING */ + { 163, -1 }, /* (325) nm ::= JOIN_KW */ + { 177, -1 }, /* (326) typetoken ::= typename */ + { 178, -1 }, /* (327) typename ::= ID|STRING */ + { 179, -1 }, /* (328) signed ::= plus_num */ + { 179, -1 }, /* (329) signed ::= minus_num */ + { 176, -2 }, /* (330) carglist ::= carglist ccons */ + { 176, 0 }, /* (331) carglist ::= */ + { 183, -2 }, /* (332) ccons ::= NULL onconf */ + { 172, -2 }, /* (333) conslist_opt ::= COMMA conslist */ + { 195, -3 }, /* (334) conslist ::= conslist tconscomma tcons */ + { 195, -1 }, /* (335) conslist ::= tcons */ + { 196, 0 }, /* (336) tconscomma ::= */ + { 200, -1 }, /* (337) defer_subclause_opt ::= defer_subclause */ + { 202, -1 }, /* (338) resolvetype ::= raisetype */ + { 206, -1 }, /* (339) selectnowith ::= oneselect */ + { 207, -1 }, /* (340) oneselect ::= values */ + { 221, -2 }, /* (341) sclp ::= selcollist COMMA */ + { 222, -1 }, /* (342) as ::= ID|STRING */ + { 185, -1 }, /* (343) expr ::= term */ + { 238, -1 }, /* (344) likeop ::= LIKE_KW|MATCH */ + { 229, -1 }, /* (345) exprlist ::= nexprlist */ + { 247, -1 }, /* (346) nmnum ::= plus_num */ + { 247, -1 }, /* (347) nmnum ::= nm */ + { 247, -1 }, /* (348) nmnum ::= ON */ + { 247, -1 }, /* (349) nmnum ::= DELETE */ + { 247, -1 }, /* (350) nmnum ::= DEFAULT */ + { 180, -1 }, /* (351) plus_num ::= INTEGER|FLOAT */ + { 252, 0 }, /* (352) foreach_clause ::= */ + { 252, -3 }, /* (353) foreach_clause ::= FOR EACH ROW */ + { 255, -1 }, /* (354) trnm ::= nm */ + { 256, 0 }, /* (355) tridxby ::= */ + { 257, -1 }, /* (356) database_kw_opt ::= DATABASE */ + { 257, 0 }, /* (357) database_kw_opt ::= */ + { 260, 0 }, /* (358) kwcolumn_opt ::= */ + { 260, -1 }, /* (359) kwcolumn_opt ::= COLUMNKW */ + { 262, -1 }, /* (360) vtabarglist ::= vtabarg */ + { 262, -3 }, /* (361) vtabarglist ::= vtabarglist COMMA vtabarg */ + { 263, -2 }, /* (362) vtabarg ::= vtabarg vtabargtoken */ + { 266, 0 }, /* (363) anylist ::= */ + { 266, -4 }, /* (364) anylist ::= anylist LP anylist RP */ + { 266, -2 }, /* (365) anylist ::= anylist ANY */ + { 232, 0 }, /* (366) with ::= */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -143863,7 +148578,7 @@ static YYACTIONTYPE yy_reduce( sqlite3ParserCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ - int yyact; /* The next action */ + YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ sqlite3ParserARG_FETCH @@ -143937,15 +148652,15 @@ static YYACTIONTYPE yy_reduce( { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy502);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy70);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy502 = TK_DEFERRED;} +{yymsp[1].minor.yy70 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); -{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/} +{yymsp[0].minor.yy70 = yymsp[0].major; /*A-overwrites-X*/} break; case 8: /* cmd ::= COMMIT|END trans_opt */ case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); @@ -143968,7 +148683,7 @@ static YYACTIONTYPE yy_reduce( break; case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy502,0,0,yymsp[-2].minor.yy502); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy70,0,0,yymsp[-2].minor.yy70); } break; case 14: /* createkw ::= CREATE */ @@ -143981,34 +148696,34 @@ static YYACTIONTYPE yy_reduce( case 57: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==57); case 67: /* defer_subclause_opt ::= */ yytestcase(yyruleno==67); case 76: /* ifexists ::= */ yytestcase(yyruleno==76); - case 92: /* distinct ::= */ yytestcase(yyruleno==92); - case 224: /* collate ::= */ yytestcase(yyruleno==224); -{yymsp[1].minor.yy502 = 0;} + case 93: /* distinct ::= */ yytestcase(yyruleno==93); + case 226: /* collate ::= */ yytestcase(yyruleno==226); +{yymsp[1].minor.yy70 = 0;} break; case 16: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy502 = 1;} +{yymsp[-2].minor.yy70 = 1;} break; case 17: /* temp ::= TEMP */ case 43: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==43); -{yymsp[0].minor.yy502 = 1;} +{yymsp[0].minor.yy70 = 1;} break; case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy502,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy70,0); } break; case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy399); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy399); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy489); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy489); } break; case 22: /* table_options ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy502 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy70 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy502 = 0; + yymsp[-1].minor.yy70 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } @@ -144018,7 +148733,7 @@ static YYACTIONTYPE yy_reduce( break; case 24: /* typetoken ::= */ case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); - case 98: /* as ::= */ yytestcase(yyruleno==98); + case 99: /* as ::= */ yytestcase(yyruleno==99); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; case 25: /* typetoken ::= typename LP signed RP */ @@ -144037,7 +148752,7 @@ static YYACTIONTYPE yy_reduce( case 28: /* scanpt ::= */ { assert( yyLookahead!=YYNOCODE ); - yymsp[1].minor.yy36 = yyLookaheadToken.z; + yymsp[1].minor.yy392 = yyLookaheadToken.z; } break; case 29: /* ccons ::= CONSTRAINT nm */ @@ -144045,18 +148760,18 @@ static YYACTIONTYPE yy_reduce( {pParse->constraintName = yymsp[0].minor.yy0;} break; case 30: /* ccons ::= DEFAULT scanpt term scanpt */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy182,yymsp[-2].minor.yy36,yymsp[0].minor.yy36);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy18,yymsp[-2].minor.yy392,yymsp[0].minor.yy392);} break; case 31: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy182,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy18,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; case 32: /* ccons ::= DEFAULT PLUS term scanpt */ -{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy182,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy36);} +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy18,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy392);} break; case 33: /* ccons ::= DEFAULT MINUS term scanpt */ { - Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[-1].minor.yy182, 0); - sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy36); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[-1].minor.yy18, 0); + sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy392); } break; case 34: /* ccons ::= DEFAULT scanpt ID|INDEXED */ @@ -144066,174 +148781,174 @@ static YYACTIONTYPE yy_reduce( sqlite3ExprIdToTrueFalse(p); testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); } - sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); + sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); } break; case 35: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy502);} +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy70);} break; case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy502,yymsp[0].minor.yy502,yymsp[-2].minor.yy502);} +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy70,yymsp[0].minor.yy70,yymsp[-2].minor.yy70);} break; case 37: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy502,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy70,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 38: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy182);} +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy18);} break; case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy232,yymsp[0].minor.yy502);} +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy420,yymsp[0].minor.yy70);} break; case 40: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy502);} +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy70);} break; case 41: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; case 44: /* refargs ::= */ -{ yymsp[1].minor.yy502 = OE_None*0x0101; /* EV: R-19803-45884 */} +{ yymsp[1].minor.yy70 = OE_None*0x0101; /* EV: R-19803-45884 */} break; case 45: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy502 = (yymsp[-1].minor.yy502 & ~yymsp[0].minor.yy107.mask) | yymsp[0].minor.yy107.value; } +{ yymsp[-1].minor.yy70 = (yymsp[-1].minor.yy70 & ~yymsp[0].minor.yy111.mask) | yymsp[0].minor.yy111.value; } break; case 46: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy107.value = 0; yymsp[-1].minor.yy107.mask = 0x000000; } +{ yymsp[-1].minor.yy111.value = 0; yymsp[-1].minor.yy111.mask = 0x000000; } break; case 47: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy107.value = 0; yymsp[-2].minor.yy107.mask = 0x000000; } +{ yymsp[-2].minor.yy111.value = 0; yymsp[-2].minor.yy111.mask = 0x000000; } break; case 48: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy107.value = yymsp[0].minor.yy502; yymsp[-2].minor.yy107.mask = 0x0000ff; } +{ yymsp[-2].minor.yy111.value = yymsp[0].minor.yy70; yymsp[-2].minor.yy111.mask = 0x0000ff; } break; case 49: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy107.value = yymsp[0].minor.yy502<<8; yymsp[-2].minor.yy107.mask = 0x00ff00; } +{ yymsp[-2].minor.yy111.value = yymsp[0].minor.yy70<<8; yymsp[-2].minor.yy111.mask = 0x00ff00; } break; case 50: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy502 = OE_SetNull; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy70 = OE_SetNull; /* EV: R-33326-45252 */} break; case 51: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy502 = OE_SetDflt; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy70 = OE_SetDflt; /* EV: R-33326-45252 */} break; case 52: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy502 = OE_Cascade; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy70 = OE_Cascade; /* EV: R-33326-45252 */} break; case 53: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy502 = OE_Restrict; /* EV: R-33326-45252 */} +{ yymsp[0].minor.yy70 = OE_Restrict; /* EV: R-33326-45252 */} break; case 54: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy502 = OE_None; /* EV: R-33326-45252 */} +{ yymsp[-1].minor.yy70 = OE_None; /* EV: R-33326-45252 */} break; case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy502 = 0;} +{yymsp[-2].minor.yy70 = 0;} break; case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); - case 155: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==155); -{yymsp[-1].minor.yy502 = yymsp[0].minor.yy502;} + case 156: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==156); +{yymsp[-1].minor.yy70 = yymsp[0].minor.yy70;} break; case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); - case 196: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==196); - case 199: /* in_op ::= NOT IN */ yytestcase(yyruleno==199); - case 225: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==225); -{yymsp[-1].minor.yy502 = 1;} + case 198: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==198); + case 201: /* in_op ::= NOT IN */ yytestcase(yyruleno==201); + case 227: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==227); +{yymsp[-1].minor.yy70 = 1;} break; case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy502 = 0;} +{yymsp[-1].minor.yy70 = 0;} break; case 61: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy232,yymsp[0].minor.yy502,yymsp[-2].minor.yy502,0);} +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy420,yymsp[0].minor.yy70,yymsp[-2].minor.yy70,0);} break; case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy232,yymsp[0].minor.yy502,0,0,0,0, +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy420,yymsp[0].minor.yy70,0,0,0,0, SQLITE_IDXTYPE_UNIQUE);} break; case 65: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy182);} +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy18);} break; case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy232, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy232, yymsp[-1].minor.yy502); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy502); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy420, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy420, yymsp[-1].minor.yy70); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy70); } break; case 68: /* onconf ::= */ case 70: /* orconf ::= */ yytestcase(yyruleno==70); -{yymsp[1].minor.yy502 = OE_Default;} +{yymsp[1].minor.yy70 = OE_Default;} break; case 69: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy502 = yymsp[0].minor.yy502;} +{yymsp[-2].minor.yy70 = yymsp[0].minor.yy70;} break; case 72: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy502 = OE_Ignore;} +{yymsp[0].minor.yy70 = OE_Ignore;} break; case 73: /* resolvetype ::= REPLACE */ - case 156: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==156); -{yymsp[0].minor.yy502 = OE_Replace;} + case 157: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==157); +{yymsp[0].minor.yy70 = OE_Replace;} break; case 74: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy427, 0, yymsp[-1].minor.yy502); + sqlite3DropTable(pParse, yymsp[0].minor.yy135, 0, yymsp[-1].minor.yy70); } break; case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy232, yymsp[0].minor.yy399, yymsp[-7].minor.yy502, yymsp[-5].minor.yy502); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy420, yymsp[0].minor.yy489, yymsp[-7].minor.yy70, yymsp[-5].minor.yy70); } break; case 78: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy427, 1, yymsp[-1].minor.yy502); + sqlite3DropTable(pParse, yymsp[0].minor.yy135, 1, yymsp[-1].minor.yy70); } break; case 79: /* cmd ::= select */ { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy399, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy399); + sqlite3Select(pParse, yymsp[0].minor.yy489, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy489); } break; case 80: /* select ::= WITH wqlist selectnowith */ { - Select *p = yymsp[0].minor.yy399; + Select *p = yymsp[0].minor.yy489; if( p ){ - p->pWith = yymsp[-1].minor.yy91; + p->pWith = yymsp[-1].minor.yy449; parserDoubleLinkSelect(pParse, p); }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy91); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy449); } - yymsp[-2].minor.yy399 = p; + yymsp[-2].minor.yy489 = p; } break; case 81: /* select ::= WITH RECURSIVE wqlist selectnowith */ { - Select *p = yymsp[0].minor.yy399; + Select *p = yymsp[0].minor.yy489; if( p ){ - p->pWith = yymsp[-1].minor.yy91; + p->pWith = yymsp[-1].minor.yy449; parserDoubleLinkSelect(pParse, p); }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy91); + sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy449); } - yymsp[-3].minor.yy399 = p; + yymsp[-3].minor.yy489 = p; } break; case 82: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy399; + Select *p = yymsp[0].minor.yy489; if( p ){ parserDoubleLinkSelect(pParse, p); } - yymsp[0].minor.yy399 = p; /*A-overwrites-X*/ + yymsp[0].minor.yy489 = p; /*A-overwrites-X*/ } break; case 83: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy399; - Select *pLhs = yymsp[-2].minor.yy399; + Select *pRhs = yymsp[0].minor.yy489; + Select *pLhs = yymsp[-2].minor.yy489; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; @@ -144243,378 +148958,382 @@ static YYACTIONTYPE yy_reduce( pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy502; + pRhs->op = (u8)yymsp[-1].minor.yy70; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy502!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy70!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy399 = pRhs; + yymsp[-2].minor.yy489 = pRhs; } break; case 84: /* multiselect_op ::= UNION */ case 86: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==86); -{yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-OP*/} +{yymsp[0].minor.yy70 = yymsp[0].major; /*A-overwrites-OP*/} break; case 85: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy502 = TK_ALL;} +{yymsp[-1].minor.yy70 = TK_ALL;} break; case 87: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { -#if SELECTTRACE_ENABLED - Token s = yymsp[-8].minor.yy0; /*A-overwrites-S*/ -#endif - yymsp[-8].minor.yy399 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy232,yymsp[-5].minor.yy427,yymsp[-4].minor.yy182,yymsp[-3].minor.yy232,yymsp[-2].minor.yy182,yymsp[-1].minor.yy232,yymsp[-7].minor.yy502,yymsp[0].minor.yy182); -#if SELECTTRACE_ENABLED - /* Populate the Select.zSelName[] string that is used to help with - ** query planner debugging, to differentiate between multiple Select - ** objects in a complex query. - ** - ** If the SELECT keyword is immediately followed by a C-style comment - ** then extract the first few alphanumeric characters from within that - ** comment to be the zSelName value. Otherwise, the label is #N where - ** is an integer that is incremented with each SELECT statement seen. - */ - if( yymsp[-8].minor.yy399!=0 ){ - const char *z = s.z+6; - int i; - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy399->zSelName), yymsp[-8].minor.yy399->zSelName,"#%d",++pParse->nSelect); - while( z[0]==' ' ) z++; - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]==' ' ) z++; - for(i=0; sqlite3Isalnum(z[i]); i++){} - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy399->zSelName), yymsp[-8].minor.yy399->zSelName, "%.*s", i, z); - } + yymsp[-8].minor.yy489 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy420,yymsp[-5].minor.yy135,yymsp[-4].minor.yy18,yymsp[-3].minor.yy420,yymsp[-2].minor.yy18,yymsp[-1].minor.yy420,yymsp[-7].minor.yy70,yymsp[0].minor.yy18); +} + break; + case 88: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ +{ + yymsp[-9].minor.yy489 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy420,yymsp[-6].minor.yy135,yymsp[-5].minor.yy18,yymsp[-4].minor.yy420,yymsp[-3].minor.yy18,yymsp[-1].minor.yy420,yymsp[-8].minor.yy70,yymsp[0].minor.yy18); + if( yymsp[-9].minor.yy489 ){ + yymsp[-9].minor.yy489->pWinDefn = yymsp[-2].minor.yy327; + }else{ + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy327); } -#endif /* SELECTRACE_ENABLED */ } break; - case 88: /* values ::= VALUES LP nexprlist RP */ + case 89: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-3].minor.yy399 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy232,0,0,0,0,0,SF_Values,0); + yymsp[-3].minor.yy489 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy420,0,0,0,0,0,SF_Values,0); } break; - case 89: /* values ::= values COMMA LP exprlist RP */ + case 90: /* values ::= values COMMA LP nexprlist RP */ { - Select *pRight, *pLeft = yymsp[-4].minor.yy399; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy232,0,0,0,0,0,SF_Values|SF_MultiValue,0); + Select *pRight, *pLeft = yymsp[-4].minor.yy489; + pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy420,0,0,0,0,0,SF_Values|SF_MultiValue,0); if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; if( pRight ){ pRight->op = TK_ALL; pRight->pPrior = pLeft; - yymsp[-4].minor.yy399 = pRight; + yymsp[-4].minor.yy489 = pRight; }else{ - yymsp[-4].minor.yy399 = pLeft; + yymsp[-4].minor.yy489 = pLeft; } } break; - case 90: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy502 = SF_Distinct;} + case 91: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy70 = SF_Distinct;} break; - case 91: /* distinct ::= ALL */ -{yymsp[0].minor.yy502 = SF_All;} + case 92: /* distinct ::= ALL */ +{yymsp[0].minor.yy70 = SF_All;} break; - case 93: /* sclp ::= */ - case 126: /* orderby_opt ::= */ yytestcase(yyruleno==126); - case 133: /* groupby_opt ::= */ yytestcase(yyruleno==133); - case 212: /* exprlist ::= */ yytestcase(yyruleno==212); - case 215: /* paren_exprlist ::= */ yytestcase(yyruleno==215); - case 220: /* eidlist_opt ::= */ yytestcase(yyruleno==220); -{yymsp[1].minor.yy232 = 0;} + case 94: /* sclp ::= */ + case 127: /* orderby_opt ::= */ yytestcase(yyruleno==127); + case 134: /* groupby_opt ::= */ yytestcase(yyruleno==134); + case 214: /* exprlist ::= */ yytestcase(yyruleno==214); + case 217: /* paren_exprlist ::= */ yytestcase(yyruleno==217); + case 222: /* eidlist_opt ::= */ yytestcase(yyruleno==222); +{yymsp[1].minor.yy420 = 0;} break; - case 94: /* selcollist ::= sclp scanpt expr scanpt as */ + case 95: /* selcollist ::= sclp scanpt expr scanpt as */ { - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy232, yymsp[-2].minor.yy182); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy232, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy232,yymsp[-3].minor.yy36,yymsp[-1].minor.yy36); + yymsp[-4].minor.yy420 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy420, yymsp[-2].minor.yy18); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy420, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy420,yymsp[-3].minor.yy392,yymsp[-1].minor.yy392); } break; - case 95: /* selcollist ::= sclp scanpt STAR */ + case 96: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - yymsp[-2].minor.yy232 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy232, p); + yymsp[-2].minor.yy420 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy420, p); } break; - case 96: /* selcollist ::= sclp scanpt nm DOT STAR */ + case 97: /* selcollist ::= sclp scanpt nm DOT STAR */ { Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); Expr *pLeft = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy232, pDot); + yymsp[-4].minor.yy420 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy420, pDot); } break; - case 97: /* as ::= AS nm */ - case 108: /* dbnm ::= DOT nm */ yytestcase(yyruleno==108); - case 234: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==234); - case 235: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==235); + case 98: /* as ::= AS nm */ + case 109: /* dbnm ::= DOT nm */ yytestcase(yyruleno==109); + case 236: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==236); + case 237: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==237); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 99: /* from ::= */ -{yymsp[1].minor.yy427 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy427));} + case 100: /* from ::= */ +{yymsp[1].minor.yy135 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy135));} break; - case 100: /* from ::= FROM seltablist */ + case 101: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy427 = yymsp[0].minor.yy427; - sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy427); + yymsp[-1].minor.yy135 = yymsp[0].minor.yy135; + sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy135); } break; - case 101: /* stl_prefix ::= seltablist joinop */ + case 102: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy427 && yymsp[-1].minor.yy427->nSrc>0) ) yymsp[-1].minor.yy427->a[yymsp[-1].minor.yy427->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy502; + if( ALWAYS(yymsp[-1].minor.yy135 && yymsp[-1].minor.yy135->nSrc>0) ) yymsp[-1].minor.yy135->a[yymsp[-1].minor.yy135->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy70; } break; - case 102: /* stl_prefix ::= */ -{yymsp[1].minor.yy427 = 0;} + case 103: /* stl_prefix ::= */ +{yymsp[1].minor.yy135 = 0;} break; - case 103: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 104: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ { - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); - sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy427, &yymsp[-2].minor.yy0); + yymsp[-6].minor.yy135 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy135,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy18,yymsp[0].minor.yy48); + sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy135, &yymsp[-2].minor.yy0); } break; - case 104: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + case 105: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ { - yymsp[-8].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy427,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); - sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy427, yymsp[-4].minor.yy232); + yymsp[-8].minor.yy135 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy135,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy18,yymsp[0].minor.yy48); + sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy135, yymsp[-4].minor.yy420); } break; - case 105: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 106: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ { - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy399,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); + yymsp[-6].minor.yy135 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy135,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy489,yymsp[-1].minor.yy18,yymsp[0].minor.yy48); } break; - case 106: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 107: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ { - if( yymsp[-6].minor.yy427==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy182==0 && yymsp[0].minor.yy510==0 ){ - yymsp[-6].minor.yy427 = yymsp[-4].minor.yy427; - }else if( yymsp[-4].minor.yy427->nSrc==1 ){ - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); - if( yymsp[-6].minor.yy427 ){ - struct SrcList_item *pNew = &yymsp[-6].minor.yy427->a[yymsp[-6].minor.yy427->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy427->a; + if( yymsp[-6].minor.yy135==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy18==0 && yymsp[0].minor.yy48==0 ){ + yymsp[-6].minor.yy135 = yymsp[-4].minor.yy135; + }else if( yymsp[-4].minor.yy135->nSrc==1 ){ + yymsp[-6].minor.yy135 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy135,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy18,yymsp[0].minor.yy48); + if( yymsp[-6].minor.yy135 ){ + struct SrcList_item *pNew = &yymsp[-6].minor.yy135->a[yymsp[-6].minor.yy135->nSrc-1]; + struct SrcList_item *pOld = yymsp[-4].minor.yy135->a; pNew->zName = pOld->zName; pNew->zDatabase = pOld->zDatabase; pNew->pSelect = pOld->pSelect; pOld->zName = pOld->zDatabase = 0; pOld->pSelect = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy427); + sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy135); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy427); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy427,0,0,0,0,SF_NestedFrom,0); - yymsp[-6].minor.yy427 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy427,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy182,yymsp[0].minor.yy510); + sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy135); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy135,0,0,0,0,SF_NestedFrom,0); + yymsp[-6].minor.yy135 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy135,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy18,yymsp[0].minor.yy48); } } break; - case 107: /* dbnm ::= */ - case 121: /* indexed_opt ::= */ yytestcase(yyruleno==121); + case 108: /* dbnm ::= */ + case 122: /* indexed_opt ::= */ yytestcase(yyruleno==122); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 109: /* fullname ::= nm */ - case 111: /* xfullname ::= nm */ yytestcase(yyruleno==111); -{yymsp[0].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} - break; - case 110: /* fullname ::= nm DOT nm */ - case 112: /* xfullname ::= nm DOT nm */ yytestcase(yyruleno==112); -{yymsp[-2].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 113: /* xfullname ::= nm DOT nm AS nm */ + case 110: /* fullname ::= nm */ { - yymsp[-4].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ - if( yymsp[-4].minor.yy427 ) yymsp[-4].minor.yy427->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yylhsminor.yy135 = sqlite3SrcListAppend(pParse->db,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy135 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy135->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[0].minor.yy135 = yylhsminor.yy135; + break; + case 111: /* fullname ::= nm DOT nm */ +{ + yylhsminor.yy135 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy135 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy135->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[-2].minor.yy135 = yylhsminor.yy135; + break; + case 112: /* xfullname ::= nm */ +{yymsp[0].minor.yy135 = sqlite3SrcListAppend(pParse->db,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} + break; + case 113: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy135 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 114: /* xfullname ::= nm DOT nm AS nm */ +{ + yymsp[-4].minor.yy135 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy135 ) yymsp[-4].minor.yy135->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 114: /* xfullname ::= nm AS nm */ + case 115: /* xfullname ::= nm AS nm */ { - yymsp[-2].minor.yy427 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ - if( yymsp[-2].minor.yy427 ) yymsp[-2].minor.yy427->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); + yymsp[-2].minor.yy135 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy135 ) yymsp[-2].minor.yy135->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); } break; - case 115: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy502 = JT_INNER; } + case 116: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy70 = JT_INNER; } break; - case 116: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 117: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy70 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} break; - case 117: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 118: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy70 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} break; - case 118: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy502 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 119: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy70 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} break; - case 119: /* on_opt ::= ON expr */ - case 136: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==136); - case 143: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==143); - case 208: /* case_else ::= ELSE expr */ yytestcase(yyruleno==208); -{yymsp[-1].minor.yy182 = yymsp[0].minor.yy182;} + case 120: /* on_opt ::= ON expr */ + case 137: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==137); + case 144: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==144); + case 210: /* case_else ::= ELSE expr */ yytestcase(yyruleno==210); +{yymsp[-1].minor.yy18 = yymsp[0].minor.yy18;} break; - case 120: /* on_opt ::= */ - case 135: /* having_opt ::= */ yytestcase(yyruleno==135); - case 137: /* limit_opt ::= */ yytestcase(yyruleno==137); - case 142: /* where_opt ::= */ yytestcase(yyruleno==142); - case 209: /* case_else ::= */ yytestcase(yyruleno==209); - case 211: /* case_operand ::= */ yytestcase(yyruleno==211); -{yymsp[1].minor.yy182 = 0;} + case 121: /* on_opt ::= */ + case 136: /* having_opt ::= */ yytestcase(yyruleno==136); + case 138: /* limit_opt ::= */ yytestcase(yyruleno==138); + case 143: /* where_opt ::= */ yytestcase(yyruleno==143); + case 211: /* case_else ::= */ yytestcase(yyruleno==211); + case 213: /* case_operand ::= */ yytestcase(yyruleno==213); +{yymsp[1].minor.yy18 = 0;} break; - case 122: /* indexed_opt ::= INDEXED BY nm */ + case 123: /* indexed_opt ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 123: /* indexed_opt ::= NOT INDEXED */ + case 124: /* indexed_opt ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 124: /* using_opt ::= USING LP idlist RP */ -{yymsp[-3].minor.yy510 = yymsp[-1].minor.yy510;} + case 125: /* using_opt ::= USING LP idlist RP */ +{yymsp[-3].minor.yy48 = yymsp[-1].minor.yy48;} break; - case 125: /* using_opt ::= */ - case 157: /* idlist_opt ::= */ yytestcase(yyruleno==157); -{yymsp[1].minor.yy510 = 0;} + case 126: /* using_opt ::= */ + case 158: /* idlist_opt ::= */ yytestcase(yyruleno==158); +{yymsp[1].minor.yy48 = 0;} break; - case 127: /* orderby_opt ::= ORDER BY sortlist */ - case 134: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==134); -{yymsp[-2].minor.yy232 = yymsp[0].minor.yy232;} + case 128: /* orderby_opt ::= ORDER BY sortlist */ + case 135: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==135); +{yymsp[-2].minor.yy420 = yymsp[0].minor.yy420;} break; - case 128: /* sortlist ::= sortlist COMMA expr sortorder */ + case 129: /* sortlist ::= sortlist COMMA expr sortorder */ { - yymsp[-3].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy232,yymsp[-1].minor.yy182); - sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy232,yymsp[0].minor.yy502); + yymsp[-3].minor.yy420 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy420,yymsp[-1].minor.yy18); + sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy420,yymsp[0].minor.yy70); } break; - case 129: /* sortlist ::= expr sortorder */ + case 130: /* sortlist ::= expr sortorder */ { - yymsp[-1].minor.yy232 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy182); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy232,yymsp[0].minor.yy502); + yymsp[-1].minor.yy420 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy18); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy420,yymsp[0].minor.yy70); } break; - case 130: /* sortorder ::= ASC */ -{yymsp[0].minor.yy502 = SQLITE_SO_ASC;} + case 131: /* sortorder ::= ASC */ +{yymsp[0].minor.yy70 = SQLITE_SO_ASC;} break; - case 131: /* sortorder ::= DESC */ -{yymsp[0].minor.yy502 = SQLITE_SO_DESC;} + case 132: /* sortorder ::= DESC */ +{yymsp[0].minor.yy70 = SQLITE_SO_DESC;} break; - case 132: /* sortorder ::= */ -{yymsp[1].minor.yy502 = SQLITE_SO_UNDEFINED;} + case 133: /* sortorder ::= */ +{yymsp[1].minor.yy70 = SQLITE_SO_UNDEFINED;} break; - case 138: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy182,0);} + case 139: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy18 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy18,0);} break; - case 139: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy182 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy182,yymsp[0].minor.yy182);} + case 140: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy18 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy18,yymsp[0].minor.yy18);} break; - case 140: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy182 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy182,yymsp[-2].minor.yy182);} + case 141: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy18 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy18,yymsp[-2].minor.yy18);} break; - case 141: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ + case 142: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy427, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy427,yymsp[0].minor.yy182,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy135, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy135,yymsp[0].minor.yy18,0,0); } break; - case 144: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ + case 145: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist where_opt */ { - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy427, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy232,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy427,yymsp[-1].minor.yy232,yymsp[0].minor.yy182,yymsp[-5].minor.yy502,0,0,0); + sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy135, &yymsp[-3].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy420,"set list"); + sqlite3Update(pParse,yymsp[-4].minor.yy135,yymsp[-1].minor.yy420,yymsp[0].minor.yy18,yymsp[-5].minor.yy70,0,0,0); } break; - case 145: /* setlist ::= setlist COMMA nm EQ expr */ + case 146: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy232, yymsp[0].minor.yy182); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy232, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy420 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy420, yymsp[0].minor.yy18); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy420, &yymsp[-2].minor.yy0, 1); } break; - case 146: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ + case 147: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yymsp[-6].minor.yy232 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy232, yymsp[-3].minor.yy510, yymsp[0].minor.yy182); + yymsp[-6].minor.yy420 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy420, yymsp[-3].minor.yy48, yymsp[0].minor.yy18); } break; - case 147: /* setlist ::= nm EQ expr */ + case 148: /* setlist ::= nm EQ expr */ { - yylhsminor.yy232 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy182); - sqlite3ExprListSetName(pParse, yylhsminor.yy232, &yymsp[-2].minor.yy0, 1); + yylhsminor.yy420 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy18); + sqlite3ExprListSetName(pParse, yylhsminor.yy420, &yymsp[-2].minor.yy0, 1); } - yymsp[-2].minor.yy232 = yylhsminor.yy232; + yymsp[-2].minor.yy420 = yylhsminor.yy420; break; - case 148: /* setlist ::= LP idlist RP EQ expr */ + case 149: /* setlist ::= LP idlist RP EQ expr */ { - yymsp[-4].minor.yy232 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy510, yymsp[0].minor.yy182); + yymsp[-4].minor.yy420 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy48, yymsp[0].minor.yy18); } break; - case 149: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + case 150: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy427, yymsp[-1].minor.yy399, yymsp[-2].minor.yy510, yymsp[-5].minor.yy502, yymsp[0].minor.yy198); + sqlite3Insert(pParse, yymsp[-3].minor.yy135, yymsp[-1].minor.yy489, yymsp[-2].minor.yy48, yymsp[-5].minor.yy70, yymsp[0].minor.yy340); } break; - case 150: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ + case 151: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES */ { - sqlite3Insert(pParse, yymsp[-3].minor.yy427, 0, yymsp[-2].minor.yy510, yymsp[-5].minor.yy502, 0); + sqlite3Insert(pParse, yymsp[-3].minor.yy135, 0, yymsp[-2].minor.yy48, yymsp[-5].minor.yy70, 0); } break; - case 151: /* upsert ::= */ -{ yymsp[1].minor.yy198 = 0; } + case 152: /* upsert ::= */ +{ yymsp[1].minor.yy340 = 0; } break; - case 152: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ -{ yymsp[-10].minor.yy198 = sqlite3UpsertNew(pParse->db,yymsp[-7].minor.yy232,yymsp[-5].minor.yy182,yymsp[-1].minor.yy232,yymsp[0].minor.yy182);} + case 153: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt */ +{ yymsp[-10].minor.yy340 = sqlite3UpsertNew(pParse->db,yymsp[-7].minor.yy420,yymsp[-5].minor.yy18,yymsp[-1].minor.yy420,yymsp[0].minor.yy18);} break; - case 153: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ -{ yymsp[-7].minor.yy198 = sqlite3UpsertNew(pParse->db,yymsp[-4].minor.yy232,yymsp[-2].minor.yy182,0,0); } + case 154: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING */ +{ yymsp[-7].minor.yy340 = sqlite3UpsertNew(pParse->db,yymsp[-4].minor.yy420,yymsp[-2].minor.yy18,0,0); } break; - case 154: /* upsert ::= ON CONFLICT DO NOTHING */ -{ yymsp[-3].minor.yy198 = sqlite3UpsertNew(pParse->db,0,0,0,0); } + case 155: /* upsert ::= ON CONFLICT DO NOTHING */ +{ yymsp[-3].minor.yy340 = sqlite3UpsertNew(pParse->db,0,0,0,0); } break; - case 158: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy510 = yymsp[-1].minor.yy510;} + case 159: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy48 = yymsp[-1].minor.yy48;} break; - case 159: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy510 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy510,&yymsp[0].minor.yy0);} + case 160: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy48 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy48,&yymsp[0].minor.yy0);} break; - case 160: /* idlist ::= nm */ -{yymsp[0].minor.yy510 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + case 161: /* idlist ::= nm */ +{yymsp[0].minor.yy48 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} break; - case 161: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy182 = yymsp[-1].minor.yy182;} + case 162: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy18 = yymsp[-1].minor.yy18;} break; - case 162: /* expr ::= ID|INDEXED */ - case 163: /* expr ::= JOIN_KW */ yytestcase(yyruleno==163); -{yymsp[0].minor.yy182=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 163: /* expr ::= ID|INDEXED */ + case 164: /* expr ::= JOIN_KW */ yytestcase(yyruleno==164); +{yymsp[0].minor.yy18=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 164: /* expr ::= nm DOT nm */ + case 165: /* expr ::= nm DOT nm */ { Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); - yylhsminor.yy182 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp1, &yymsp[-2].minor.yy0); + } + yylhsminor.yy18 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); } - yymsp[-2].minor.yy182 = yylhsminor.yy182; + yymsp[-2].minor.yy18 = yylhsminor.yy18; break; - case 165: /* expr ::= nm DOT nm DOT nm */ + case 166: /* expr ::= nm DOT nm DOT nm */ { Expr *temp1 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-4].minor.yy0, 1); Expr *temp2 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[-2].minor.yy0, 1); Expr *temp3 = sqlite3ExprAlloc(pParse->db, TK_ID, &yymsp[0].minor.yy0, 1); Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); - yylhsminor.yy182 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)temp3, &yymsp[0].minor.yy0); + sqlite3RenameTokenMap(pParse, (void*)temp2, &yymsp[-2].minor.yy0); + } + yylhsminor.yy18 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); } - yymsp[-4].minor.yy182 = yylhsminor.yy182; + yymsp[-4].minor.yy18 = yylhsminor.yy18; break; - case 166: /* term ::= NULL|FLOAT|BLOB */ - case 167: /* term ::= STRING */ yytestcase(yyruleno==167); -{yymsp[0].minor.yy182=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 167: /* term ::= NULL|FLOAT|BLOB */ + case 168: /* term ::= STRING */ yytestcase(yyruleno==168); +{yymsp[0].minor.yy18=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 168: /* term ::= INTEGER */ + case 169: /* term ::= INTEGER */ { - yylhsminor.yy182 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + yylhsminor.yy18 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); } - yymsp[0].minor.yy182 = yylhsminor.yy182; + yymsp[0].minor.yy18 = yylhsminor.yy18; break; - case 169: /* expr ::= VARIABLE */ + case 170: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ u32 n = yymsp[0].minor.yy0.n; - yymsp[0].minor.yy182 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy182, n); + yymsp[0].minor.yy18 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy18, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers @@ -144623,146 +149342,154 @@ static YYACTIONTYPE yy_reduce( assert( t.n>=2 ); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy182 = 0; + yymsp[0].minor.yy18 = 0; }else{ - yymsp[0].minor.yy182 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); - if( yymsp[0].minor.yy182 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy182->iTable); + yymsp[0].minor.yy18 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy18 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy18->iTable); } } } break; - case 170: /* expr ::= expr COLLATE ID|STRING */ + case 171: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy182 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy182, &yymsp[0].minor.yy0, 1); + yymsp[-2].minor.yy18 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy18, &yymsp[0].minor.yy0, 1); } break; - case 171: /* expr ::= CAST LP expr AS typetoken RP */ + case 172: /* expr ::= CAST LP expr AS typetoken RP */ { - yymsp[-5].minor.yy182 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); - sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy182, yymsp[-3].minor.yy182, 0); + yymsp[-5].minor.yy18 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy18, yymsp[-3].minor.yy18, 0); } break; - case 172: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 173: /* expr ::= ID|INDEXED LP distinct exprlist RP */ { - if( yymsp[-1].minor.yy232 && yymsp[-1].minor.yy232->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); - } - yylhsminor.yy182 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy232, &yymsp[-4].minor.yy0); - if( yymsp[-2].minor.yy502==SF_Distinct && yylhsminor.yy182 ){ - yylhsminor.yy182->flags |= EP_Distinct; - } + yylhsminor.yy18 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy420, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy70); } - yymsp[-4].minor.yy182 = yylhsminor.yy182; + yymsp[-4].minor.yy18 = yylhsminor.yy18; break; - case 173: /* expr ::= ID|INDEXED LP STAR RP */ + case 174: /* expr ::= ID|INDEXED LP STAR RP */ { - yylhsminor.yy182 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); + yylhsminor.yy18 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[-3].minor.yy182 = yylhsminor.yy182; + yymsp[-3].minor.yy18 = yylhsminor.yy18; break; - case 174: /* term ::= CTIME_KW */ + case 175: /* expr ::= ID|INDEXED LP distinct exprlist RP over_clause */ { - yylhsminor.yy182 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); + yylhsminor.yy18 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy420, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy70); + sqlite3WindowAttach(pParse, yylhsminor.yy18, yymsp[0].minor.yy327); } - yymsp[0].minor.yy182 = yylhsminor.yy182; + yymsp[-5].minor.yy18 = yylhsminor.yy18; break; - case 175: /* expr ::= LP nexprlist COMMA expr RP */ + case 176: /* expr ::= ID|INDEXED LP STAR RP over_clause */ { - ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy232, yymsp[-1].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = pList; + yylhsminor.yy18 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy18, yymsp[0].minor.yy327); +} + yymsp[-4].minor.yy18 = yylhsminor.yy18; + break; + case 177: /* term ::= CTIME_KW */ +{ + yylhsminor.yy18 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); +} + yymsp[0].minor.yy18 = yylhsminor.yy18; + break; + case 178: /* expr ::= LP nexprlist COMMA expr RP */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy420, yymsp[-1].minor.yy18); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy18 ){ + yymsp[-4].minor.yy18->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } } break; - case 176: /* expr ::= expr AND expr */ - case 177: /* expr ::= expr OR expr */ yytestcase(yyruleno==177); - case 178: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==178); - case 179: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==179); - case 180: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==180); - case 181: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==181); - case 182: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==182); - case 183: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==183); -{yymsp[-2].minor.yy182=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy182,yymsp[0].minor.yy182);} + case 179: /* expr ::= expr AND expr */ + case 180: /* expr ::= expr OR expr */ yytestcase(yyruleno==180); + case 181: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==181); + case 182: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==182); + case 183: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==183); + case 184: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==184); + case 185: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==185); + case 186: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==186); +{yymsp[-2].minor.yy18=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy18,yymsp[0].minor.yy18);} break; - case 184: /* likeop ::= NOT LIKE_KW|MATCH */ + case 187: /* likeop ::= NOT LIKE_KW|MATCH */ {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} break; - case 185: /* expr ::= expr likeop expr */ + case 188: /* expr ::= expr likeop expr */ { ExprList *pList; int bNot = yymsp[-1].minor.yy0.n & 0x80000000; yymsp[-1].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy182); - yymsp[-2].minor.yy182 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0); - if( bNot ) yymsp[-2].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy182, 0); - if( yymsp[-2].minor.yy182 ) yymsp[-2].minor.yy182->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy18); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy18); + yymsp[-2].minor.yy18 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy18 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy18, 0); + if( yymsp[-2].minor.yy18 ) yymsp[-2].minor.yy18->flags |= EP_InfixFunc; } break; - case 186: /* expr ::= expr likeop expr ESCAPE expr */ + case 189: /* expr ::= expr likeop expr ESCAPE expr */ { ExprList *pList; int bNot = yymsp[-3].minor.yy0.n & 0x80000000; yymsp[-3].minor.yy0.n &= 0x7fffffff; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0); - if( bNot ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ) yymsp[-4].minor.yy182->flags |= EP_InfixFunc; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy18); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy18); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy18); + yymsp[-4].minor.yy18 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy18, 0); + if( yymsp[-4].minor.yy18 ) yymsp[-4].minor.yy18->flags |= EP_InfixFunc; } break; - case 187: /* expr ::= expr ISNULL|NOTNULL */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy182,0);} + case 190: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy18 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy18,0);} break; - case 188: /* expr ::= expr NOT NULL */ -{yymsp[-2].minor.yy182 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy182,0);} + case 191: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy18 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy18,0);} break; - case 189: /* expr ::= expr IS expr */ + case 192: /* expr ::= expr IS expr */ { - yymsp[-2].minor.yy182 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy182,yymsp[0].minor.yy182); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy182, yymsp[-2].minor.yy182, TK_ISNULL); + yymsp[-2].minor.yy18 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy18,yymsp[0].minor.yy18); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy18, yymsp[-2].minor.yy18, TK_ISNULL); } break; - case 190: /* expr ::= expr IS NOT expr */ + case 193: /* expr ::= expr IS NOT expr */ { - yymsp[-3].minor.yy182 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy182,yymsp[0].minor.yy182); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy182, yymsp[-3].minor.yy182, TK_NOTNULL); + yymsp[-3].minor.yy18 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy18,yymsp[0].minor.yy18); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy18, yymsp[-3].minor.yy18, TK_NOTNULL); } break; - case 191: /* expr ::= NOT expr */ - case 192: /* expr ::= BITNOT expr */ yytestcase(yyruleno==192); -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy182, 0);/*A-overwrites-B*/} + case 194: /* expr ::= NOT expr */ + case 195: /* expr ::= BITNOT expr */ yytestcase(yyruleno==195); +{yymsp[-1].minor.yy18 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy18, 0);/*A-overwrites-B*/} break; - case 193: /* expr ::= MINUS expr */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy182, 0);} - break; - case 194: /* expr ::= PLUS expr */ -{yymsp[-1].minor.yy182 = sqlite3PExpr(pParse, TK_UPLUS, yymsp[0].minor.yy182, 0);} - break; - case 195: /* between_op ::= BETWEEN */ - case 198: /* in_op ::= IN */ yytestcase(yyruleno==198); -{yymsp[0].minor.yy502 = 0;} - break; - case 197: /* expr ::= expr between_op expr AND expr */ + case 196: /* expr ::= PLUS|MINUS expr */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy182); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = pList; + yymsp[-1].minor.yy18 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy18, 0); + /*A-overwrites-B*/ +} + break; + case 197: /* between_op ::= BETWEEN */ + case 200: /* in_op ::= IN */ yytestcase(yyruleno==200); +{yymsp[0].minor.yy70 = 0;} + break; + case 199: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy18); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy18); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy18, 0); + if( yymsp[-4].minor.yy18 ){ + yymsp[-4].minor.yy18->x.pList = pList; }else{ sqlite3ExprListDelete(pParse->db, pList); } - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + if( yymsp[-3].minor.yy70 ) yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy18, 0); } break; - case 200: /* expr ::= expr in_op LP exprlist RP */ + case 202: /* expr ::= expr in_op LP exprlist RP */ { - if( yymsp[-1].minor.yy232==0 ){ + if( yymsp[-1].minor.yy420==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -144771,9 +149498,9 @@ static YYACTIONTYPE yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy182); - yymsp[-4].minor.yy182 = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[yymsp[-3].minor.yy502],1); - }else if( yymsp[-1].minor.yy232->nExpr==1 ){ + sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy18); + yymsp[-4].minor.yy18 = sqlite3ExprAlloc(pParse->db, TK_INTEGER,&sqlite3IntTokens[yymsp[-3].minor.yy70],1); + }else if( yymsp[-1].minor.yy420->nExpr==1 ){ /* Expressions of the form: ** ** expr1 IN (?1) @@ -144790,195 +149517,199 @@ static YYACTIONTYPE yy_reduce( ** affinity or the collating sequence to use for comparison. Otherwise, ** the semantics would be subtly different from IN or NOT IN. */ - Expr *pRHS = yymsp[-1].minor.yy232->a[0].pExpr; - yymsp[-1].minor.yy232->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy232); + Expr *pRHS = yymsp[-1].minor.yy420->a[0].pExpr; + yymsp[-1].minor.yy420->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy420); /* pRHS cannot be NULL because a malloc error would have been detected ** before now and control would have never reached this point */ if( ALWAYS(pRHS) ){ pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, yymsp[-3].minor.yy502 ? TK_NE : TK_EQ, yymsp[-4].minor.yy182, pRHS); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, yymsp[-3].minor.yy70 ? TK_NE : TK_EQ, yymsp[-4].minor.yy18, pRHS); }else{ - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = yymsp[-1].minor.yy232; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy182); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy18, 0); + if( yymsp[-4].minor.yy18 ){ + yymsp[-4].minor.yy18->x.pList = yymsp[-1].minor.yy420; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy18); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy232); + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy420); } - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + if( yymsp[-3].minor.yy70 ) yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy18, 0); } } break; - case 201: /* expr ::= LP select RP */ + case 203: /* expr ::= LP select RP */ { - yymsp[-2].minor.yy182 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy182, yymsp[-1].minor.yy399); + yymsp[-2].minor.yy18 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy18, yymsp[-1].minor.yy489); } break; - case 202: /* expr ::= expr in_op LP select RP */ + case 204: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy182, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy182, yymsp[-1].minor.yy399); - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy18, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy18, yymsp[-1].minor.yy489); + if( yymsp[-3].minor.yy70 ) yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy18, 0); } break; - case 203: /* expr ::= expr in_op nm dbnm paren_exprlist */ + case 205: /* expr ::= expr in_op nm dbnm paren_exprlist */ { SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); - if( yymsp[0].minor.yy232 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy232); - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy182, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy182, pSelect); - if( yymsp[-3].minor.yy502 ) yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy182, 0); + if( yymsp[0].minor.yy420 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy420); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy18, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy18, pSelect); + if( yymsp[-3].minor.yy70 ) yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy18, 0); } break; - case 204: /* expr ::= EXISTS LP select RP */ + case 206: /* expr ::= EXISTS LP select RP */ { Expr *p; - p = yymsp[-3].minor.yy182 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy399); + p = yymsp[-3].minor.yy18 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy489); } break; - case 205: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 207: /* expr ::= CASE case_operand case_exprlist case_else END */ { - yymsp[-4].minor.yy182 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy182, 0); - if( yymsp[-4].minor.yy182 ){ - yymsp[-4].minor.yy182->x.pList = yymsp[-1].minor.yy182 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy232,yymsp[-1].minor.yy182) : yymsp[-2].minor.yy232; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy182); + yymsp[-4].minor.yy18 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy18, 0); + if( yymsp[-4].minor.yy18 ){ + yymsp[-4].minor.yy18->x.pList = yymsp[-1].minor.yy18 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy420,yymsp[-1].minor.yy18) : yymsp[-2].minor.yy420; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy18); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy232); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy182); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy420); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy18); } } break; - case 206: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 208: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy232, yymsp[-2].minor.yy182); - yymsp[-4].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy232, yymsp[0].minor.yy182); + yymsp[-4].minor.yy420 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy420, yymsp[-2].minor.yy18); + yymsp[-4].minor.yy420 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy420, yymsp[0].minor.yy18); } break; - case 207: /* case_exprlist ::= WHEN expr THEN expr */ + case 209: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy232 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy182); - yymsp[-3].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy232, yymsp[0].minor.yy182); + yymsp[-3].minor.yy420 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy18); + yymsp[-3].minor.yy420 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy420, yymsp[0].minor.yy18); } break; - case 210: /* case_operand ::= expr */ -{yymsp[0].minor.yy182 = yymsp[0].minor.yy182; /*A-overwrites-X*/} + case 212: /* case_operand ::= expr */ +{yymsp[0].minor.yy18 = yymsp[0].minor.yy18; /*A-overwrites-X*/} break; - case 213: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy232 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy232,yymsp[0].minor.yy182);} + case 215: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy420 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy420,yymsp[0].minor.yy18);} break; - case 214: /* nexprlist ::= expr */ -{yymsp[0].minor.yy232 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy182); /*A-overwrites-Y*/} + case 216: /* nexprlist ::= expr */ +{yymsp[0].minor.yy420 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy18); /*A-overwrites-Y*/} break; - case 216: /* paren_exprlist ::= LP exprlist RP */ - case 221: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==221); -{yymsp[-2].minor.yy232 = yymsp[-1].minor.yy232;} + case 218: /* paren_exprlist ::= LP exprlist RP */ + case 223: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==223); +{yymsp[-2].minor.yy420 = yymsp[-1].minor.yy420;} break; - case 217: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 219: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy232, yymsp[-10].minor.yy502, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy182, SQLITE_SO_ASC, yymsp[-8].minor.yy502, SQLITE_IDXTYPE_APPDEF); + sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy420, yymsp[-10].minor.yy70, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy18, SQLITE_SO_ASC, yymsp[-8].minor.yy70, SQLITE_IDXTYPE_APPDEF); + if( IN_RENAME_OBJECT && pParse->pNewIndex ){ + sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); + } } break; - case 218: /* uniqueflag ::= UNIQUE */ - case 258: /* raisetype ::= ABORT */ yytestcase(yyruleno==258); -{yymsp[0].minor.yy502 = OE_Abort;} + case 220: /* uniqueflag ::= UNIQUE */ + case 260: /* raisetype ::= ABORT */ yytestcase(yyruleno==260); +{yymsp[0].minor.yy70 = OE_Abort;} break; - case 219: /* uniqueflag ::= */ -{yymsp[1].minor.yy502 = OE_None;} + case 221: /* uniqueflag ::= */ +{yymsp[1].minor.yy70 = OE_None;} break; - case 222: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 224: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy232 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy232, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); + yymsp[-4].minor.yy420 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy420, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy70, yymsp[0].minor.yy70); } break; - case 223: /* eidlist ::= nm collate sortorder */ + case 225: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy232 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy502, yymsp[0].minor.yy502); /*A-overwrites-Y*/ + yymsp[-2].minor.yy420 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy70, yymsp[0].minor.yy70); /*A-overwrites-Y*/ } break; - case 226: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy427, yymsp[-1].minor.yy502);} + case 228: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy135, yymsp[-1].minor.yy70);} break; - case 227: /* cmd ::= VACUUM */ + case 229: /* cmd ::= VACUUM */ {sqlite3Vacuum(pParse,0);} break; - case 228: /* cmd ::= VACUUM nm */ + case 230: /* cmd ::= VACUUM nm */ {sqlite3Vacuum(pParse,&yymsp[0].minor.yy0);} break; - case 229: /* cmd ::= PRAGMA nm dbnm */ + case 231: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 230: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 232: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 231: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 233: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 232: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 234: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 233: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 235: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 236: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 238: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy47, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy207, &all); } break; - case 237: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 239: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy502, yymsp[-4].minor.yy300.a, yymsp[-4].minor.yy300.b, yymsp[-2].minor.yy427, yymsp[0].minor.yy182, yymsp[-10].minor.yy502, yymsp[-8].minor.yy502); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy70, yymsp[-4].minor.yy34.a, yymsp[-4].minor.yy34.b, yymsp[-2].minor.yy135, yymsp[0].minor.yy18, yymsp[-10].minor.yy70, yymsp[-8].minor.yy70); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 238: /* trigger_time ::= BEFORE|AFTER */ -{ yymsp[0].minor.yy502 = yymsp[0].major; /*A-overwrites-X*/ } + case 240: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy70 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 239: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy502 = TK_INSTEAD;} + case 241: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy70 = TK_INSTEAD;} break; - case 240: /* trigger_time ::= */ -{ yymsp[1].minor.yy502 = TK_BEFORE; } + case 242: /* trigger_time ::= */ +{ yymsp[1].minor.yy70 = TK_BEFORE; } break; - case 241: /* trigger_event ::= DELETE|INSERT */ - case 242: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==242); -{yymsp[0].minor.yy300.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy300.b = 0;} + case 243: /* trigger_event ::= DELETE|INSERT */ + case 244: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==244); +{yymsp[0].minor.yy34.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy34.b = 0;} break; - case 243: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy300.a = TK_UPDATE; yymsp[-2].minor.yy300.b = yymsp[0].minor.yy510;} + case 245: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy34.a = TK_UPDATE; yymsp[-2].minor.yy34.b = yymsp[0].minor.yy48;} break; - case 244: /* when_clause ::= */ - case 263: /* key_opt ::= */ yytestcase(yyruleno==263); -{ yymsp[1].minor.yy182 = 0; } + case 246: /* when_clause ::= */ + case 265: /* key_opt ::= */ yytestcase(yyruleno==265); + case 307: /* filter_opt ::= */ yytestcase(yyruleno==307); +{ yymsp[1].minor.yy18 = 0; } break; - case 245: /* when_clause ::= WHEN expr */ - case 264: /* key_opt ::= KEY expr */ yytestcase(yyruleno==264); -{ yymsp[-1].minor.yy182 = yymsp[0].minor.yy182; } + case 247: /* when_clause ::= WHEN expr */ + case 266: /* key_opt ::= KEY expr */ yytestcase(yyruleno==266); +{ yymsp[-1].minor.yy18 = yymsp[0].minor.yy18; } break; - case 246: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 248: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy47!=0 ); - yymsp[-2].minor.yy47->pLast->pNext = yymsp[-1].minor.yy47; - yymsp[-2].minor.yy47->pLast = yymsp[-1].minor.yy47; + assert( yymsp[-2].minor.yy207!=0 ); + yymsp[-2].minor.yy207->pLast->pNext = yymsp[-1].minor.yy207; + yymsp[-2].minor.yy207->pLast = yymsp[-1].minor.yy207; } break; - case 247: /* trigger_cmd_list ::= trigger_cmd SEMI */ + case 249: /* trigger_cmd_list ::= trigger_cmd SEMI */ { - assert( yymsp[-1].minor.yy47!=0 ); - yymsp[-1].minor.yy47->pLast = yymsp[-1].minor.yy47; + assert( yymsp[-1].minor.yy207!=0 ); + yymsp[-1].minor.yy207->pLast = yymsp[-1].minor.yy207; } break; - case 248: /* trnm ::= nm DOT nm */ + case 250: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; sqlite3ErrorMsg(pParse, @@ -144986,196 +149717,306 @@ static YYACTIONTYPE yy_reduce( "statements within triggers"); } break; - case 249: /* tridxby ::= INDEXED BY nm */ + case 251: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 250: /* tridxby ::= NOT INDEXED */ + case 252: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 251: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ -{yylhsminor.yy47 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-5].minor.yy0, yymsp[-2].minor.yy232, yymsp[-1].minor.yy182, yymsp[-6].minor.yy502, yymsp[-7].minor.yy0.z, yymsp[0].minor.yy36);} - yymsp[-7].minor.yy47 = yylhsminor.yy47; + case 253: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt scanpt */ +{yylhsminor.yy207 = sqlite3TriggerUpdateStep(pParse, &yymsp[-5].minor.yy0, yymsp[-2].minor.yy420, yymsp[-1].minor.yy18, yymsp[-6].minor.yy70, yymsp[-7].minor.yy0.z, yymsp[0].minor.yy392);} + yymsp[-7].minor.yy207 = yylhsminor.yy207; break; - case 252: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + case 254: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - yylhsminor.yy47 = sqlite3TriggerInsertStep(pParse->db,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy510,yymsp[-2].minor.yy399,yymsp[-6].minor.yy502,yymsp[-1].minor.yy198,yymsp[-7].minor.yy36,yymsp[0].minor.yy36);/*yylhsminor.yy47-overwrites-yymsp[-6].minor.yy502*/ + yylhsminor.yy207 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy48,yymsp[-2].minor.yy489,yymsp[-6].minor.yy70,yymsp[-1].minor.yy340,yymsp[-7].minor.yy392,yymsp[0].minor.yy392);/*yylhsminor.yy207-overwrites-yymsp[-6].minor.yy70*/ } - yymsp[-7].minor.yy47 = yylhsminor.yy47; + yymsp[-7].minor.yy207 = yylhsminor.yy207; break; - case 253: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ -{yylhsminor.yy47 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy182, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy36);} - yymsp[-5].minor.yy47 = yylhsminor.yy47; + case 255: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy207 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy18, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy392);} + yymsp[-5].minor.yy207 = yylhsminor.yy207; break; - case 254: /* trigger_cmd ::= scanpt select scanpt */ -{yylhsminor.yy47 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy399, yymsp[-2].minor.yy36, yymsp[0].minor.yy36); /*yylhsminor.yy47-overwrites-yymsp[-1].minor.yy399*/} - yymsp[-2].minor.yy47 = yylhsminor.yy47; + case 256: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy207 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy489, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); /*yylhsminor.yy207-overwrites-yymsp[-1].minor.yy489*/} + yymsp[-2].minor.yy207 = yylhsminor.yy207; break; - case 255: /* expr ::= RAISE LP IGNORE RP */ + case 257: /* expr ::= RAISE LP IGNORE RP */ { - yymsp[-3].minor.yy182 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); - if( yymsp[-3].minor.yy182 ){ - yymsp[-3].minor.yy182->affinity = OE_Ignore; + yymsp[-3].minor.yy18 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy18 ){ + yymsp[-3].minor.yy18->affinity = OE_Ignore; } } break; - case 256: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 258: /* expr ::= RAISE LP raisetype COMMA nm RP */ { - yymsp[-5].minor.yy182 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); - if( yymsp[-5].minor.yy182 ) { - yymsp[-5].minor.yy182->affinity = (char)yymsp[-3].minor.yy502; + yymsp[-5].minor.yy18 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1); + if( yymsp[-5].minor.yy18 ) { + yymsp[-5].minor.yy18->affinity = (char)yymsp[-3].minor.yy70; } } break; - case 257: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy502 = OE_Rollback;} + case 259: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy70 = OE_Rollback;} break; - case 259: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy502 = OE_Fail;} + case 261: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy70 = OE_Fail;} break; - case 260: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 262: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy427,yymsp[-1].minor.yy502); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy135,yymsp[-1].minor.yy70); } break; - case 261: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 263: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy182, yymsp[-1].minor.yy182, yymsp[0].minor.yy182); + sqlite3Attach(pParse, yymsp[-3].minor.yy18, yymsp[-1].minor.yy18, yymsp[0].minor.yy18); } break; - case 262: /* cmd ::= DETACH database_kw_opt expr */ + case 264: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy182); + sqlite3Detach(pParse, yymsp[0].minor.yy18); } break; - case 265: /* cmd ::= REINDEX */ + case 267: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 266: /* cmd ::= REINDEX nm dbnm */ + case 268: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 267: /* cmd ::= ANALYZE */ + case 269: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 268: /* cmd ::= ANALYZE nm dbnm */ + case 270: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 269: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 271: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy427,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy135,&yymsp[0].minor.yy0); } break; - case 270: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 272: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 271: /* add_column_fullname ::= fullname */ + case 273: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy427); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy135); } break; - case 272: /* cmd ::= create_vtab */ + case 274: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ +{ + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy135, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 275: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 273: /* cmd ::= create_vtab LP vtabarglist RP */ + case 276: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 274: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 277: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy502); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy70); } break; - case 275: /* vtabarg ::= */ + case 278: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 276: /* vtabargtoken ::= ANY */ - case 277: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==277); - case 278: /* lp ::= LP */ yytestcase(yyruleno==278); + case 279: /* vtabargtoken ::= ANY */ + case 280: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==280); + case 281: /* lp ::= LP */ yytestcase(yyruleno==281); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 279: /* with ::= WITH wqlist */ - case 280: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==280); -{ sqlite3WithPush(pParse, yymsp[0].minor.yy91, 1); } + case 282: /* with ::= WITH wqlist */ + case 283: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==283); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy449, 1); } break; - case 281: /* wqlist ::= nm eidlist_opt AS LP select RP */ + case 284: /* wqlist ::= nm eidlist_opt AS LP select RP */ { - yymsp[-5].minor.yy91 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy232, yymsp[-1].minor.yy399); /*A-overwrites-X*/ + yymsp[-5].minor.yy449 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy420, yymsp[-1].minor.yy489); /*A-overwrites-X*/ } break; - case 282: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ + case 285: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ { - yymsp[-7].minor.yy91 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy91, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy232, yymsp[-1].minor.yy399); + yymsp[-7].minor.yy449 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy449, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy420, yymsp[-1].minor.yy489); } break; + case 286: /* windowdefn_list ::= windowdefn */ +{ yylhsminor.yy327 = yymsp[0].minor.yy327; } + yymsp[0].minor.yy327 = yylhsminor.yy327; + break; + case 287: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ +{ + assert( yymsp[0].minor.yy327!=0 ); + yymsp[0].minor.yy327->pNextWin = yymsp[-2].minor.yy327; + yylhsminor.yy327 = yymsp[0].minor.yy327; +} + yymsp[-2].minor.yy327 = yylhsminor.yy327; + break; + case 288: /* windowdefn ::= nm AS window */ +{ + if( ALWAYS(yymsp[0].minor.yy327) ){ + yymsp[0].minor.yy327->zName = sqlite3DbStrNDup(pParse->db, yymsp[-2].minor.yy0.z, yymsp[-2].minor.yy0.n); + } + yylhsminor.yy327 = yymsp[0].minor.yy327; +} + yymsp[-2].minor.yy327 = yylhsminor.yy327; + break; + case 289: /* window ::= LP part_opt orderby_opt frame_opt RP */ +{ + yymsp[-4].minor.yy327 = yymsp[-1].minor.yy327; + if( ALWAYS(yymsp[-4].minor.yy327) ){ + yymsp[-4].minor.yy327->pPartition = yymsp[-3].minor.yy420; + yymsp[-4].minor.yy327->pOrderBy = yymsp[-2].minor.yy420; + } +} + break; + case 290: /* part_opt ::= PARTITION BY nexprlist */ +{ yymsp[-2].minor.yy420 = yymsp[0].minor.yy420; } + break; + case 291: /* part_opt ::= */ +{ yymsp[1].minor.yy420 = 0; } + break; + case 292: /* frame_opt ::= */ +{ + yymsp[1].minor.yy327 = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0); +} + break; + case 293: /* frame_opt ::= range_or_rows frame_bound_s */ +{ + yylhsminor.yy327 = sqlite3WindowAlloc(pParse, yymsp[-1].minor.yy70, yymsp[0].minor.yy119.eType, yymsp[0].minor.yy119.pExpr, TK_CURRENT, 0); +} + yymsp[-1].minor.yy327 = yylhsminor.yy327; + break; + case 294: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e */ +{ + yylhsminor.yy327 = sqlite3WindowAlloc(pParse, yymsp[-4].minor.yy70, yymsp[-2].minor.yy119.eType, yymsp[-2].minor.yy119.pExpr, yymsp[0].minor.yy119.eType, yymsp[0].minor.yy119.pExpr); +} + yymsp[-4].minor.yy327 = yylhsminor.yy327; + break; + case 295: /* range_or_rows ::= RANGE */ +{ yymsp[0].minor.yy70 = TK_RANGE; } + break; + case 296: /* range_or_rows ::= ROWS */ +{ yymsp[0].minor.yy70 = TK_ROWS; } + break; + case 297: /* frame_bound_s ::= frame_bound */ + case 299: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==299); +{ yylhsminor.yy119 = yymsp[0].minor.yy119; } + yymsp[0].minor.yy119 = yylhsminor.yy119; + break; + case 298: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 300: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==300); +{yymsp[-1].minor.yy119.eType = TK_UNBOUNDED; yymsp[-1].minor.yy119.pExpr = 0;} + break; + case 301: /* frame_bound ::= expr PRECEDING */ +{ yylhsminor.yy119.eType = TK_PRECEDING; yylhsminor.yy119.pExpr = yymsp[-1].minor.yy18; } + yymsp[-1].minor.yy119 = yylhsminor.yy119; + break; + case 302: /* frame_bound ::= CURRENT ROW */ +{ yymsp[-1].minor.yy119.eType = TK_CURRENT ; yymsp[-1].minor.yy119.pExpr = 0; } + break; + case 303: /* frame_bound ::= expr FOLLOWING */ +{ yylhsminor.yy119.eType = TK_FOLLOWING; yylhsminor.yy119.pExpr = yymsp[-1].minor.yy18; } + yymsp[-1].minor.yy119 = yylhsminor.yy119; + break; + case 304: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy327 = yymsp[0].minor.yy327; } + break; + case 305: /* over_clause ::= filter_opt OVER window */ +{ + yylhsminor.yy327 = yymsp[0].minor.yy327; + assert( yylhsminor.yy327!=0 ); + yylhsminor.yy327->pFilter = yymsp[-2].minor.yy18; +} + yymsp[-2].minor.yy327 = yylhsminor.yy327; + break; + case 306: /* over_clause ::= filter_opt OVER nm */ +{ + yylhsminor.yy327 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy327 ){ + yylhsminor.yy327->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + yylhsminor.yy327->pFilter = yymsp[-2].minor.yy18; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[-2].minor.yy18); + } +} + yymsp[-2].minor.yy327 = yylhsminor.yy327; + break; + case 308: /* filter_opt ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy18 = yymsp[-1].minor.yy18; } + break; default: - /* (283) input ::= cmdlist */ yytestcase(yyruleno==283); - /* (284) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==284); - /* (285) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=285); - /* (286) ecmd ::= SEMI */ yytestcase(yyruleno==286); - /* (287) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==287); - /* (288) ecmd ::= explain cmdx */ yytestcase(yyruleno==288); - /* (289) trans_opt ::= */ yytestcase(yyruleno==289); - /* (290) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==290); - /* (291) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==291); - /* (292) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==292); - /* (293) savepoint_opt ::= */ yytestcase(yyruleno==293); - /* (294) cmd ::= create_table create_table_args */ yytestcase(yyruleno==294); - /* (295) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==295); - /* (296) columnlist ::= columnname carglist */ yytestcase(yyruleno==296); - /* (297) nm ::= ID|INDEXED */ yytestcase(yyruleno==297); - /* (298) nm ::= STRING */ yytestcase(yyruleno==298); - /* (299) nm ::= JOIN_KW */ yytestcase(yyruleno==299); - /* (300) typetoken ::= typename */ yytestcase(yyruleno==300); - /* (301) typename ::= ID|STRING */ yytestcase(yyruleno==301); - /* (302) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=302); - /* (303) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=303); - /* (304) carglist ::= carglist ccons */ yytestcase(yyruleno==304); - /* (305) carglist ::= */ yytestcase(yyruleno==305); - /* (306) ccons ::= NULL onconf */ yytestcase(yyruleno==306); - /* (307) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==307); - /* (308) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==308); - /* (309) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=309); - /* (310) tconscomma ::= */ yytestcase(yyruleno==310); - /* (311) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=311); - /* (312) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=312); - /* (313) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=313); - /* (314) oneselect ::= values */ yytestcase(yyruleno==314); - /* (315) sclp ::= selcollist COMMA */ yytestcase(yyruleno==315); - /* (316) as ::= ID|STRING */ yytestcase(yyruleno==316); - /* (317) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=317); - /* (318) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==318); - /* (319) exprlist ::= nexprlist */ yytestcase(yyruleno==319); - /* (320) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=320); - /* (321) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=321); - /* (322) nmnum ::= ON */ yytestcase(yyruleno==322); - /* (323) nmnum ::= DELETE */ yytestcase(yyruleno==323); - /* (324) nmnum ::= DEFAULT */ yytestcase(yyruleno==324); - /* (325) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==325); - /* (326) foreach_clause ::= */ yytestcase(yyruleno==326); - /* (327) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==327); - /* (328) trnm ::= nm */ yytestcase(yyruleno==328); - /* (329) tridxby ::= */ yytestcase(yyruleno==329); - /* (330) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==330); - /* (331) database_kw_opt ::= */ yytestcase(yyruleno==331); - /* (332) kwcolumn_opt ::= */ yytestcase(yyruleno==332); - /* (333) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==333); - /* (334) vtabarglist ::= vtabarg */ yytestcase(yyruleno==334); - /* (335) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==335); - /* (336) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==336); - /* (337) anylist ::= */ yytestcase(yyruleno==337); - /* (338) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==338); - /* (339) anylist ::= anylist ANY */ yytestcase(yyruleno==339); - /* (340) with ::= */ yytestcase(yyruleno==340); + /* (309) input ::= cmdlist */ yytestcase(yyruleno==309); + /* (310) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==310); + /* (311) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=311); + /* (312) ecmd ::= SEMI */ yytestcase(yyruleno==312); + /* (313) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==313); + /* (314) ecmd ::= explain cmdx */ yytestcase(yyruleno==314); + /* (315) trans_opt ::= */ yytestcase(yyruleno==315); + /* (316) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==316); + /* (317) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==317); + /* (318) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==318); + /* (319) savepoint_opt ::= */ yytestcase(yyruleno==319); + /* (320) cmd ::= create_table create_table_args */ yytestcase(yyruleno==320); + /* (321) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==321); + /* (322) columnlist ::= columnname carglist */ yytestcase(yyruleno==322); + /* (323) nm ::= ID|INDEXED */ yytestcase(yyruleno==323); + /* (324) nm ::= STRING */ yytestcase(yyruleno==324); + /* (325) nm ::= JOIN_KW */ yytestcase(yyruleno==325); + /* (326) typetoken ::= typename */ yytestcase(yyruleno==326); + /* (327) typename ::= ID|STRING */ yytestcase(yyruleno==327); + /* (328) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=328); + /* (329) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=329); + /* (330) carglist ::= carglist ccons */ yytestcase(yyruleno==330); + /* (331) carglist ::= */ yytestcase(yyruleno==331); + /* (332) ccons ::= NULL onconf */ yytestcase(yyruleno==332); + /* (333) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==333); + /* (334) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==334); + /* (335) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=335); + /* (336) tconscomma ::= */ yytestcase(yyruleno==336); + /* (337) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=337); + /* (338) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=338); + /* (339) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=339); + /* (340) oneselect ::= values */ yytestcase(yyruleno==340); + /* (341) sclp ::= selcollist COMMA */ yytestcase(yyruleno==341); + /* (342) as ::= ID|STRING */ yytestcase(yyruleno==342); + /* (343) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=343); + /* (344) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==344); + /* (345) exprlist ::= nexprlist */ yytestcase(yyruleno==345); + /* (346) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=346); + /* (347) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=347); + /* (348) nmnum ::= ON */ yytestcase(yyruleno==348); + /* (349) nmnum ::= DELETE */ yytestcase(yyruleno==349); + /* (350) nmnum ::= DEFAULT */ yytestcase(yyruleno==350); + /* (351) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==351); + /* (352) foreach_clause ::= */ yytestcase(yyruleno==352); + /* (353) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==353); + /* (354) trnm ::= nm */ yytestcase(yyruleno==354); + /* (355) tridxby ::= */ yytestcase(yyruleno==355); + /* (356) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==356); + /* (357) database_kw_opt ::= */ yytestcase(yyruleno==357); + /* (358) kwcolumn_opt ::= */ yytestcase(yyruleno==358); + /* (359) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==359); + /* (360) vtabarglist ::= vtabarg */ yytestcase(yyruleno==360); + /* (361) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==361); + /* (362) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==362); + /* (363) anylist ::= */ yytestcase(yyruleno==363); + /* (364) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==364); + /* (365) anylist ::= anylist ANY */ yytestcase(yyruleno==365); + /* (366) with ::= */ yytestcase(yyruleno==366); break; /********** End reduce actions ************************************************/ }; @@ -145329,12 +150170,12 @@ SQLITE_PRIVATE void sqlite3Parser( do{ assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action(yymajor,yyact); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); if( yyact >= YY_MIN_REDUCE ){ yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, yyminor sqlite3ParserCTX_PARAM); }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,yymajor,yyminor); + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif @@ -145462,6 +150303,21 @@ SQLITE_PRIVATE void sqlite3Parser( return; } +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +SQLITE_PRIVATE int sqlite3ParserFallback(int iToken){ +#ifdef YYFALLBACK + if( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ){ + return yyFallback[iToken]; + } +#else + (void)iToken; +#endif + return 0; +} + /************** End of parse.c ***********************************************/ /************** Begin file tokenize.c ****************************************/ /* @@ -145520,11 +150376,12 @@ SQLITE_PRIVATE void sqlite3Parser( #define CC_TILDA 25 /* '~' */ #define CC_DOT 26 /* '.' */ #define CC_ILLEGAL 27 /* Illegal character */ +#define CC_NUL 28 /* 0x00 */ static const unsigned char aiClass[] = { #ifdef SQLITE_ASCII /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 0x */ 28, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, /* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, /* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, @@ -145623,19 +150480,20 @@ const unsigned char ebcdicToAscii[] = { ** is substantially reduced. This is important for embedded applications ** on platforms with limited memory. */ -/* Hash score: 185 */ -/* zKWText[] encodes 845 bytes of keyword text in 561 bytes */ +/* Hash score: 208 */ +/* zKWText[] encodes 923 bytes of keyword text in 614 bytes */ /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ -/* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ -/* BETWEENOTHINGLOBYCASCADELETECASECOLLATECREATECURRENT_DATE */ -/* DETACHIMMEDIATEJOINSERTLIKEMATCHPLANALYZEPRAGMABORTVALUES */ -/* VIRTUALIMITWHENOTNULLWHERENAMEAFTEREPLACEANDEFAULT */ -/* AUTOINCREMENTCASTCOLUMNCOMMITCONFLICTCROSSCURRENT_TIMESTAMP */ -/* RIMARYDEFERREDISTINCTDORDERESTRICTDROPFAILFROMFULLIFISNULL */ -/* RIGHTROLLBACKROWUNIONUSINGVACUUMVIEWINITIALLY */ -static const char zKWText[560] = { +/* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERANGEBETWEEN */ +/* OTHINGLOBYCASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ +/* IMMEDIATEJOINSERTLIKEMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMIT */ +/* WHENOTNULLWHERECURSIVEAFTERENAMEANDEFAULTAUTOINCREMENTCAST */ +/* COLUMNCOMMITCONFLICTCROSSCURRENT_TIMESTAMPARTITIONDEFERRED */ +/* ISTINCTDROPRECEDINGFAILFILTEREPLACEFOLLOWINGFROMFULLIFISNULL */ +/* ORDERESTRICTOVERIGHTROLLBACKROWSUNBOUNDEDUNIONUSINGVACUUMVIEW */ +/* INDOWINITIALLYPRIMARY */ +static const char zKWText[613] = { 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', @@ -145648,84 +150506,90 @@ static const char zKWText[560] = { 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', - 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', - 'B','E','T','W','E','E','N','O','T','H','I','N','G','L','O','B','Y','C', - 'A','S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L', - 'A','T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D', - 'A','T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E', - 'J','O','I','N','S','E','R','T','L','I','K','E','M','A','T','C','H','P', - 'L','A','N','A','L','Y','Z','E','P','R','A','G','M','A','B','O','R','T', - 'V','A','L','U','E','S','V','I','R','T','U','A','L','I','M','I','T','W', - 'H','E','N','O','T','N','U','L','L','W','H','E','R','E','N','A','M','E', - 'A','F','T','E','R','E','P','L','A','C','E','A','N','D','E','F','A','U', - 'L','T','A','U','T','O','I','N','C','R','E','M','E','N','T','C','A','S', - 'T','C','O','L','U','M','N','C','O','M','M','I','T','C','O','N','F','L', - 'I','C','T','C','R','O','S','S','C','U','R','R','E','N','T','_','T','I', - 'M','E','S','T','A','M','P','R','I','M','A','R','Y','D','E','F','E','R', - 'R','E','D','I','S','T','I','N','C','T','D','O','R','D','E','R','E','S', - 'T','R','I','C','T','D','R','O','P','F','A','I','L','F','R','O','M','F', - 'U','L','L','I','F','I','S','N','U','L','L','R','I','G','H','T','R','O', - 'L','L','B','A','C','K','R','O','W','U','N','I','O','N','U','S','I','N', - 'G','V','A','C','U','U','M','V','I','E','W','I','N','I','T','I','A','L', - 'L','Y', + 'T','E','B','E','G','I','N','N','E','R','A','N','G','E','B','E','T','W', + 'E','E','N','O','T','H','I','N','G','L','O','B','Y','C','A','S','C','A', + 'D','E','L','E','T','E','C','A','S','E','C','O','L','L','A','T','E','C', + 'R','E','A','T','E','C','U','R','R','E','N','T','_','D','A','T','E','D', + 'E','T','A','C','H','I','M','M','E','D','I','A','T','E','J','O','I','N', + 'S','E','R','T','L','I','K','E','M','A','T','C','H','P','L','A','N','A', + 'L','Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U', + 'E','S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','O', + 'T','N','U','L','L','W','H','E','R','E','C','U','R','S','I','V','E','A', + 'F','T','E','R','E','N','A','M','E','A','N','D','E','F','A','U','L','T', + 'A','U','T','O','I','N','C','R','E','M','E','N','T','C','A','S','T','C', + 'O','L','U','M','N','C','O','M','M','I','T','C','O','N','F','L','I','C', + 'T','C','R','O','S','S','C','U','R','R','E','N','T','_','T','I','M','E', + 'S','T','A','M','P','A','R','T','I','T','I','O','N','D','E','F','E','R', + 'R','E','D','I','S','T','I','N','C','T','D','R','O','P','R','E','C','E', + 'D','I','N','G','F','A','I','L','F','I','L','T','E','R','E','P','L','A', + 'C','E','F','O','L','L','O','W','I','N','G','F','R','O','M','F','U','L', + 'L','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T','R', + 'I','C','T','O','V','E','R','I','G','H','T','R','O','L','L','B','A','C', + 'K','R','O','W','S','U','N','B','O','U','N','D','E','D','U','N','I','O', + 'N','U','S','I','N','G','V','A','C','U','U','M','V','I','E','W','I','N', + 'D','O','W','I','N','I','T','I','A','L','L','Y','P','R','I','M','A','R', + 'Y', }; /* aKWHash[i] is the hash value for the i-th keyword */ static const unsigned char aKWHash[127] = { - 74, 108, 119, 72, 0, 45, 0, 0, 81, 0, 76, 61, 0, - 42, 12, 77, 15, 0, 118, 84, 54, 116, 0, 19, 0, 0, - 123, 0, 121, 111, 0, 22, 96, 0, 9, 0, 0, 68, 69, - 0, 67, 6, 0, 48, 93, 105, 0, 120, 104, 0, 0, 44, - 0, 106, 24, 0, 17, 0, 124, 53, 23, 0, 5, 62, 25, - 99, 0, 0, 126, 112, 60, 125, 57, 28, 55, 0, 94, 0, - 103, 26, 0, 102, 0, 0, 0, 98, 95, 100, 91, 115, 14, - 39, 114, 0, 80, 0, 109, 92, 90, 32, 0, 122, 79, 117, - 86, 46, 83, 0, 0, 97, 40, 59, 110, 0, 36, 0, 0, - 29, 0, 89, 87, 88, 0, 20, 85, 0, 56, + 74, 109, 124, 72, 106, 45, 0, 0, 81, 0, 76, 61, 0, + 42, 12, 77, 15, 0, 123, 84, 54, 118, 125, 19, 0, 0, + 130, 0, 128, 121, 0, 22, 96, 0, 9, 0, 0, 115, 69, + 0, 67, 6, 0, 48, 93, 136, 0, 126, 104, 0, 0, 44, + 0, 107, 24, 0, 17, 0, 131, 53, 23, 0, 5, 62, 132, + 99, 0, 0, 135, 110, 60, 134, 57, 113, 55, 0, 94, 0, + 103, 26, 0, 102, 0, 0, 0, 98, 95, 100, 105, 117, 14, + 39, 116, 0, 80, 0, 133, 114, 92, 59, 0, 129, 79, 119, + 86, 46, 83, 0, 0, 97, 40, 122, 120, 0, 127, 0, 0, + 29, 0, 89, 87, 88, 0, 20, 85, 111, 56, }; /* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 ** then the i-th keyword has no more hash collisions. Otherwise, ** the next keyword with the same hash is aKWHash[i]-1. */ -static const unsigned char aKWNext[126] = { +static const unsigned char aKWNext[136] = { 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, - 0, 43, 3, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 43, 3, 47, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 1, 64, 0, 0, 65, 0, 41, 0, 38, 0, 0, 0, - 0, 0, 49, 75, 0, 0, 30, 0, 58, 0, 0, 63, 31, - 52, 16, 34, 10, 0, 0, 0, 0, 0, 0, 0, 11, 70, - 78, 0, 8, 0, 18, 51, 0, 107, 101, 0, 113, 0, 73, - 27, 37, 71, 82, 0, 35, 66, 0, 0, + 0, 0, 49, 75, 0, 0, 30, 0, 58, 0, 0, 0, 31, + 63, 16, 34, 10, 0, 0, 0, 0, 0, 0, 0, 11, 70, + 91, 0, 0, 8, 0, 108, 0, 101, 28, 52, 68, 0, 112, + 0, 73, 51, 0, 90, 27, 37, 0, 71, 36, 82, 0, 35, + 66, 25, 18, 0, 0, 78, }; /* aKWLen[i] is the length (in bytes) of the i-th keyword */ -static const unsigned char aKWLen[126] = { +static const unsigned char aKWLen[136] = { 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, - 6, 6, 5, 6, 5, 5, 9, 7, 7, 4, 2, 7, 3, + 6, 6, 5, 6, 5, 5, 5, 7, 7, 4, 2, 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 4, 5, 4, 7, - 6, 5, 6, 7, 5, 4, 7, 3, 2, 4, 5, 6, 5, - 7, 3, 7, 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, - 7, 8, 8, 2, 2, 5, 8, 4, 4, 4, 4, 2, 6, - 5, 8, 3, 5, 5, 6, 4, 9, 3, + 6, 5, 6, 7, 5, 4, 7, 3, 2, 4, 5, 9, 5, + 6, 3, 7, 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, + 7, 9, 8, 8, 2, 4, 9, 4, 6, 7, 9, 4, 4, + 2, 6, 5, 8, 4, 5, 8, 4, 3, 9, 5, 5, 6, + 4, 6, 2, 9, 3, 7, }; /* aKWOffset[i] is the index into zKWText[] of the start of ** the text for the i-th keyword. */ -static const unsigned short int aKWOffset[126] = { +static const unsigned short int aKWOffset[136] = { 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, - 199, 204, 209, 212, 218, 221, 225, 234, 240, 246, 249, 251, 252, - 256, 262, 266, 273, 279, 291, 297, 306, 308, 314, 318, 323, 325, - 332, 337, 342, 348, 354, 359, 362, 362, 362, 365, 369, 372, 378, - 382, 389, 391, 398, 400, 402, 411, 415, 421, 427, 435, 440, 440, - 456, 463, 470, 471, 478, 479, 483, 491, 495, 499, 503, 507, 509, - 515, 520, 528, 531, 536, 541, 547, 551, 556, + 199, 204, 209, 212, 218, 221, 225, 230, 236, 242, 245, 247, 248, + 252, 258, 262, 269, 275, 287, 293, 302, 304, 310, 314, 319, 321, + 328, 333, 338, 344, 350, 355, 358, 358, 358, 361, 365, 368, 377, + 381, 387, 389, 396, 398, 400, 409, 413, 419, 425, 433, 438, 438, + 438, 454, 463, 470, 471, 478, 481, 490, 494, 499, 506, 515, 519, + 523, 525, 531, 535, 543, 546, 551, 559, 559, 563, 572, 577, 582, + 588, 591, 594, 597, 602, 606, }; /* aKWCode[i] is the parser symbol code for the i-th keyword */ -static const unsigned char aKWCode[126] = { +static const unsigned char aKWCode[136] = { TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, @@ -145737,21 +150601,23 @@ static const unsigned char aKWCode[126] = { TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, - TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, + TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RANGE, TK_BETWEEN, TK_NOTHING, TK_LIKE_KW, TK_BY, TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_LIKE_KW, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, TK_NOTNULL, - TK_NOT, TK_NO, TK_NULL, TK_WHERE, TK_RENAME, - TK_AFTER, TK_REPLACE, TK_AND, TK_DEFAULT, TK_AUTOINCR, + TK_NOT, TK_NO, TK_NULL, TK_WHERE, TK_RECURSIVE, + TK_AFTER, TK_RENAME, TK_AND, TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, TK_COLUMNKW, TK_COMMIT, - TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, TK_PRIMARY, - TK_DEFERRED, TK_DISTINCT, TK_IS, TK_DO, TK_ORDER, - TK_RESTRICT, TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, - TK_IF, TK_ISNULL, TK_JOIN_KW, TK_ROLLBACK, TK_ROW, - TK_UNION, TK_USING, TK_VACUUM, TK_VIEW, TK_INITIALLY, - TK_ALL, + TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, TK_CURRENT, + TK_PARTITION, TK_DEFERRED, TK_DISTINCT, TK_IS, TK_DROP, + TK_PRECEDING, TK_FAIL, TK_FILTER, TK_REPLACE, TK_FOLLOWING, + TK_FROM, TK_JOIN_KW, TK_IF, TK_ISNULL, TK_ORDER, + TK_RESTRICT, TK_OVER, TK_JOIN_KW, TK_ROLLBACK, TK_ROWS, + TK_ROW, TK_UNBOUNDED, TK_UNION, TK_USING, TK_VACUUM, + TK_VIEW, TK_WINDOW, TK_DO, TK_INITIALLY, TK_ALL, + TK_PRIMARY, }; /* Check to see if z[0..n-1] is a keyword. If it is, write the ** parser symbol code for that keyword into *pType. Always @@ -145830,7 +150696,7 @@ static int keywordCode(const char *z, int n, int *pType){ testcase( i==55 ); /* UPDATE */ testcase( i==56 ); /* BEGIN */ testcase( i==57 ); /* INNER */ - testcase( i==58 ); /* RECURSIVE */ + testcase( i==58 ); /* RANGE */ testcase( i==59 ); /* BETWEEN */ testcase( i==60 ); /* NOTHING */ testcase( i==61 ); /* GLOB */ @@ -145861,9 +150727,9 @@ static int keywordCode(const char *z, int n, int *pType){ testcase( i==86 ); /* NO */ testcase( i==87 ); /* NULL */ testcase( i==88 ); /* WHERE */ - testcase( i==89 ); /* RENAME */ + testcase( i==89 ); /* RECURSIVE */ testcase( i==90 ); /* AFTER */ - testcase( i==91 ); /* REPLACE */ + testcase( i==91 ); /* RENAME */ testcase( i==92 ); /* AND */ testcase( i==93 ); /* DEFAULT */ testcase( i==94 ); /* AUTOINCREMENT */ @@ -145876,28 +150742,38 @@ static int keywordCode(const char *z, int n, int *pType){ testcase( i==101 ); /* CROSS */ testcase( i==102 ); /* CURRENT_TIMESTAMP */ testcase( i==103 ); /* CURRENT_TIME */ - testcase( i==104 ); /* PRIMARY */ - testcase( i==105 ); /* DEFERRED */ - testcase( i==106 ); /* DISTINCT */ - testcase( i==107 ); /* IS */ - testcase( i==108 ); /* DO */ - testcase( i==109 ); /* ORDER */ - testcase( i==110 ); /* RESTRICT */ - testcase( i==111 ); /* DROP */ - testcase( i==112 ); /* FAIL */ - testcase( i==113 ); /* FROM */ - testcase( i==114 ); /* FULL */ - testcase( i==115 ); /* IF */ - testcase( i==116 ); /* ISNULL */ - testcase( i==117 ); /* RIGHT */ - testcase( i==118 ); /* ROLLBACK */ - testcase( i==119 ); /* ROW */ - testcase( i==120 ); /* UNION */ - testcase( i==121 ); /* USING */ - testcase( i==122 ); /* VACUUM */ - testcase( i==123 ); /* VIEW */ - testcase( i==124 ); /* INITIALLY */ - testcase( i==125 ); /* ALL */ + testcase( i==104 ); /* CURRENT */ + testcase( i==105 ); /* PARTITION */ + testcase( i==106 ); /* DEFERRED */ + testcase( i==107 ); /* DISTINCT */ + testcase( i==108 ); /* IS */ + testcase( i==109 ); /* DROP */ + testcase( i==110 ); /* PRECEDING */ + testcase( i==111 ); /* FAIL */ + testcase( i==112 ); /* FILTER */ + testcase( i==113 ); /* REPLACE */ + testcase( i==114 ); /* FOLLOWING */ + testcase( i==115 ); /* FROM */ + testcase( i==116 ); /* FULL */ + testcase( i==117 ); /* IF */ + testcase( i==118 ); /* ISNULL */ + testcase( i==119 ); /* ORDER */ + testcase( i==120 ); /* RESTRICT */ + testcase( i==121 ); /* OVER */ + testcase( i==122 ); /* RIGHT */ + testcase( i==123 ); /* ROLLBACK */ + testcase( i==124 ); /* ROWS */ + testcase( i==125 ); /* ROW */ + testcase( i==126 ); /* UNBOUNDED */ + testcase( i==127 ); /* UNION */ + testcase( i==128 ); /* USING */ + testcase( i==129 ); /* VACUUM */ + testcase( i==130 ); /* VIEW */ + testcase( i==131 ); /* WINDOW */ + testcase( i==132 ); /* DO */ + testcase( i==133 ); /* INITIALLY */ + testcase( i==134 ); /* ALL */ + testcase( i==135 ); /* PRIMARY */ *pType = aKWCode[i]; break; } @@ -145909,7 +150785,7 @@ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ keywordCode((char*)z, n, &id); return id; } -#define SQLITE_N_KEYWORD 126 +#define SQLITE_N_KEYWORD 136 SQLITE_API int sqlite3_keyword_name(int i,const char **pzName,int *pnName){ if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR; *pzName = zKWText + aKWOffset[i]; @@ -145963,11 +150839,85 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif -/* Make the IdChar function accessible from ctime.c */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* Make the IdChar function accessible from ctime.c and alter.c */ SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } -#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Return the id of the next token in string (*pz). Before returning, set +** (*pz) to point to the byte following the parsed token. +*/ +static int getToken(const unsigned char **pz){ + const unsigned char *z = *pz; + int t; /* Token type to return */ + do { + z += sqlite3GetToken(z, &t); + }while( t==TK_SPACE ); + if( t==TK_ID + || t==TK_STRING + || t==TK_JOIN_KW + || t==TK_WINDOW + || t==TK_OVER + || sqlite3ParserFallback(t)==TK_ID + ){ + t = TK_ID; + } + *pz = z; + return t; +} + +/* +** The following three functions are called immediately after the tokenizer +** reads the keywords WINDOW, OVER and FILTER, respectively, to determine +** whether the token should be treated as a keyword or an SQL identifier. +** This cannot be handled by the usual lemon %fallback method, due to +** the ambiguity in some constructions. e.g. +** +** SELECT sum(x) OVER ... +** +** In the above, "OVER" might be a keyword, or it might be an alias for the +** sum(x) expression. If a "%fallback ID OVER" directive were added to +** grammar, then SQLite would always treat "OVER" as an alias, making it +** impossible to call a window-function without a FILTER clause. +** +** WINDOW is treated as a keyword if: +** +** * the following token is an identifier, or a keyword that can fallback +** to being an identifier, and +** * the token after than one is TK_AS. +** +** OVER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is either TK_LP or an identifier. +** +** FILTER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is TK_LP. +*/ +static int analyzeWindowKeyword(const unsigned char *z){ + int t; + t = getToken(&z); + if( t!=TK_ID ) return TK_ID; + t = getToken(&z); + if( t!=TK_AS ) return TK_ID; + return TK_WINDOW; +} +static int analyzeOverKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP ){ + int t = getToken(&z); + if( t==TK_LP || t==TK_ID ) return TK_OVER; + } + return TK_ID; +} +static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP && getToken(&z)==TK_LP ){ + return TK_FILTER; + } + return TK_ID; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** Return the length (in bytes) of the token that begins at z[0]. @@ -146236,6 +151186,10 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ i = 1; break; } + case CC_NUL: { + *tokenType = TK_ILLEGAL; + return 0; + } default: { *tokenType = TK_ILLEGAL; return 1; @@ -146289,47 +151243,64 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr assert( pParse->nVar==0 ); assert( pParse->pVList==0 ); while( 1 ){ - if( zSql[0]!=0 ){ - n = sqlite3GetToken((u8*)zSql, &tokenType); - mxSqlLen -= n; - if( mxSqlLen<0 ){ - pParse->rc = SQLITE_TOOBIG; - break; - } - }else{ - /* Upon reaching the end of input, call the parser two more times - ** with tokens TK_SEMI and 0, in that order. */ - if( lastTokenParsed==TK_SEMI ){ - tokenType = 0; - }else if( lastTokenParsed==0 ){ - break; - }else{ - tokenType = TK_SEMI; - } - n = 0; + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ + pParse->rc = SQLITE_TOOBIG; + break; } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( tokenType>=TK_WINDOW ){ + assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER + || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + ); +#else if( tokenType>=TK_SPACE ){ assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); +#endif /* SQLITE_OMIT_WINDOWFUNC */ if( db->u1.isInterrupted ){ pParse->rc = SQLITE_INTERRUPT; break; } - if( tokenType==TK_ILLEGAL ){ + if( tokenType==TK_SPACE ){ + zSql += n; + continue; + } + if( zSql[0]==0 ){ + /* Upon reaching the end of input, call the parser two more times + ** with tokens TK_SEMI and 0, in that order. */ + if( lastTokenParsed==TK_SEMI ){ + tokenType = 0; + }else if( lastTokenParsed==0 ){ + break; + }else{ + tokenType = TK_SEMI; + } + n = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + }else if( tokenType==TK_WINDOW ){ + assert( n==6 ); + tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); + }else if( tokenType==TK_OVER ){ + assert( n==4 ); + tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); + }else if( tokenType==TK_FILTER ){ + assert( n==6 ); + tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + }else{ sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql); break; } - zSql += n; - }else{ - pParse->sLastToken.z = zSql; - pParse->sLastToken.n = n; - sqlite3Parser(pEngine, tokenType, pParse->sLastToken); - lastTokenParsed = tokenType; - zSql += n; - if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; + sqlite3Parser(pEngine, tokenType, pParse->sLastToken); + lastTokenParsed = tokenType; + zSql += n; + if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } assert( nErr==0 ); - pParse->zTail = zSql; #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, @@ -146351,10 +151322,12 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr assert( pzErrMsg!=0 ); if( pParse->zErrMsg ){ *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); + sqlite3_log(pParse->rc, "%s in \"%s\"", + *pzErrMsg, pParse->zTail); pParse->zErrMsg = 0; nErr++; } + pParse->zTail = zSql; if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ sqlite3VdbeDelete(pParse->pVdbe); pParse->pVdbe = 0; @@ -146370,16 +151343,18 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr sqlite3_free(pParse->apVtabLock); #endif - if( !IN_DECLARE_VTAB ){ + if( !IN_SPECIAL_PARSE ){ /* If the pParse->declareVtab flag is set, do not delete any table ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ** will take responsibility for freeing the Table structure. */ sqlite3DeleteTable(db, pParse->pNewTable); } + if( !IN_RENAME_OBJECT ){ + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + } if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); sqlite3DbFree(db, pParse->pVList); while( pParse->pAinc ){ AutoincInfo *p = pParse->pAinc; @@ -147636,7 +152611,7 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ db->flags &= ~aFlagOp[i].mask; } if( oldFlags!=db->flags ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); } if( pRes ){ *pRes = (db->flags & aFlagOp[i].mask)!=0; @@ -147697,6 +152672,15 @@ static int binCollFunc( return rc; } +/* +** Return true if CollSeq is the default built-in BINARY. +*/ +SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq *p){ + assert( p==0 || p->xCmp!=binCollFunc || p->pUser!=0 + || strcmp(p->zName,"BINARY")==0 ); + return p==0 || (p->xCmp==binCollFunc && p->pUser==0); +} + /* ** Another built-in collating sequence: NOCASE. ** @@ -147818,7 +152802,7 @@ static void disconnectAllVtab(sqlite3 *db){ sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; - if( db->aDb[i].pSchema ){ + if( pSchema ){ for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ Table *pTab = (Table *)sqliteHashData(p); if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); @@ -148078,8 +153062,8 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( (db->mDbFlags&DBFLAG_SchemaChange)!=0 && db->init.busy==0 ){ - sqlite3ExpirePreparedStatements(db); + if( schemaChange ){ + sqlite3ExpirePreparedStatements(db, 0); sqlite3ResetAllSchemasOfConnection(db); } sqlite3BtreeLeaveAll(db); @@ -148107,6 +153091,7 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ switch( rc ){ case SQLITE_OK: zName = "SQLITE_OK"; break; case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; case SQLITE_PERM: zName = "SQLITE_PERM"; break; case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; @@ -148470,6 +153455,8 @@ SQLITE_PRIVATE int sqlite3CreateFunc( void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), FuncDestructor *pDestructor ){ FuncDef *p; @@ -148477,12 +153464,14 @@ SQLITE_PRIVATE int sqlite3CreateFunc( int extraFlags; assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xSFunc && (xFinal || xStep)) || - (!xSFunc && (xFinal && !xStep)) || - (!xSFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + assert( xValue==0 || xSFunc==0 ); + if( zFunctionName==0 /* Must have a valid name */ + || (xSFunc!=0 && xFinal!=0) /* Not both xSFunc and xFinal */ + || ((xFinal==0)!=(xStep==0)) /* Both or neither of xFinal and xStep */ + || ((xValue==0)!=(xInverse==0)) /* Both or neither of xValue, xInverse */ + || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) + || (255<(nName = sqlite3Strlen30( zFunctionName))) + ){ return SQLITE_MISUSE_BKPT; } @@ -148503,10 +153492,10 @@ SQLITE_PRIVATE int sqlite3CreateFunc( }else if( enc==SQLITE_ANY ){ int rc; rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF8|extraFlags, - pUserData, xSFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); if( rc==SQLITE_OK ){ rc = sqlite3CreateFunc(db, zFunctionName, nArg, SQLITE_UTF16LE|extraFlags, - pUserData, xSFunc, xStep, xFinal, pDestructor); + pUserData, xSFunc, xStep, xFinal, xValue, xInverse, pDestructor); } if( rc!=SQLITE_OK ){ return rc; @@ -148523,14 +153512,14 @@ SQLITE_PRIVATE int sqlite3CreateFunc( ** operation to continue but invalidate all precompiled statements. */ p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 0); - if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==(u32)enc && p->nArg==nArg ){ if( db->nVdbeActive ){ sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; }else{ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); } } @@ -148552,38 +153541,32 @@ SQLITE_PRIVATE int sqlite3CreateFunc( testcase( p->funcFlags & SQLITE_DETERMINISTIC ); p->xSFunc = xSFunc ? xSFunc : xStep; p->xFinalize = xFinal; + p->xValue = xValue; + p->xInverse = xInverse; p->pUserData = pUserData; p->nArg = (u16)nArg; return SQLITE_OK; } /* -** Create new user functions. +** Worker function used by utf-8 APIs that create new functions: +** +** sqlite3_create_function() +** sqlite3_create_function_v2() +** sqlite3_create_window_function() */ -SQLITE_API int sqlite3_create_function( +static int createFunctionApi( sqlite3 *db, const char *zFunc, int nArg, int enc, void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) -){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, - xFinal, 0); -} - -SQLITE_API int sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; @@ -148605,7 +153588,9 @@ SQLITE_API int sqlite3_create_function_v2( pArg->xDestroy = xDestroy; pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, + xSFunc, xStep, xFinal, xValue, xInverse, pArg + ); if( pArg && pArg->nRef==0 ){ assert( rc!=SQLITE_OK ); xDestroy(p); @@ -148618,6 +153603,52 @@ SQLITE_API int sqlite3_create_function_v2( return rc; } +/* +** Create new user functions. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, 0); +} +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, xDestroy); +} +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, 0, xStep, + xFinal, xValue, xInverse, xDestroy); +} + #ifndef SQLITE_OMIT_UTF16 SQLITE_API int sqlite3_create_function16( sqlite3 *db, @@ -148638,7 +153669,7 @@ SQLITE_API int sqlite3_create_function16( sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0,0,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -149263,7 +154294,7 @@ static int createCollation( "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); /* If collation sequence pColl was created directly by a call to ** sqlite3_create_collation, and not generated by synthCollSeq(), @@ -149752,6 +154783,7 @@ static int openDatabase( db->nDb = 2; db->magic = SQLITE_MAGIC_BUSY; db->aDb = db->aDbStatic; + db->lookaside.bDisable = 1; assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); @@ -150452,6 +155484,9 @@ SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, vo }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_DATA_VERSION ){ + *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); + rc = SQLITE_OK; }else{ rc = sqlite3OsFileControl(fd, op, pArg); } @@ -150715,7 +155750,8 @@ SQLITE_API int sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_VDBE_COVERAGE: { #ifdef SQLITE_VDBE_COVERAGE - typedef void (*branch_callback)(void*,int,u8,u8); + typedef void (*branch_callback)(void*,unsigned int, + unsigned char,unsigned char); sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); #endif @@ -150902,7 +155938,7 @@ SQLITE_API int sqlite3_snapshot_get( if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; if( 0==sqlite3BtreeIsInTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); } @@ -150937,11 +155973,29 @@ SQLITE_API int sqlite3_snapshot_open( iDb = sqlite3FindDbName(db, zDb); if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), pSnapshot); + if( sqlite3BtreeIsInTrans(pBt)==0 ){ + Pager *pPager = sqlite3BtreePager(pBt); + int bUnlock = 0; + if( sqlite3BtreeIsInReadTrans(pBt) ){ + if( db->nVdbeActive==0 ){ + rc = sqlite3PagerSnapshotCheck(pPager, pSnapshot); + if( rc==SQLITE_OK ){ + bUnlock = 1; + rc = sqlite3BtreeCommit(pBt); + } + } + }else{ + rc = SQLITE_OK; + } if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), 0); + rc = sqlite3PagerSnapshotOpen(pPager, pSnapshot); + } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); + } + if( bUnlock ){ + sqlite3PagerSnapshotUnlock(pPager); } } } @@ -150972,7 +156026,7 @@ SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); sqlite3BtreeCommit(pBt); @@ -156095,7 +161149,7 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ int rc = SQLITE_OK; UNUSED_PARAMETER(iSavepoint); assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); + assert( ((Fts3Table *)pVtab)->mxSavepoint <= iSavepoint ); TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ rc = fts3SyncMethod(pVtab); @@ -170518,6 +175572,2526 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ /************** End of fts3_unicode2.c ***************************************/ +/************** Begin file json1.c *******************************************/ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements JSON functions. The interface is +** modeled after MySQL JSON functions: +** +** https://dev.mysql.com/doc/refman/5.7/en/json.html +** +** For the time being, all JSON is stored as pure text. (We might add +** a JSONB type in the future which stores a binary encoding of JSON in +** a BLOB, but there is no support for JSONB in the current implementation. +** This implementation parses JSON text at 250 MB/s, so it is hard to see +** how JSONB might improve on that.) +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) +#if !defined(SQLITEINT_H) +/* #include "sqlite3ext.h" */ +#endif +SQLITE_EXTENSION_INIT1 +/* #include */ +/* #include */ +/* #include */ +/* #include */ + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAM +# define UNUSED_PARAM(X) (void)(X) +#endif + +#ifndef LARGEST_INT64 +# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) +# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) +#endif + +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) + +#ifndef SQLITE_AMALGAMATION + /* Unsigned integer types. These are already defined in the sqliteInt.h, + ** but the definitions need to be repeated for separate compilation. */ + typedef sqlite3_uint64 u64; + typedef unsigned int u32; + typedef unsigned short int u16; + typedef unsigned char u8; +#endif + +/* Objects */ +typedef struct JsonString JsonString; +typedef struct JsonNode JsonNode; +typedef struct JsonParse JsonParse; + +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. +*/ +struct JsonString { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON content here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 bErr; /* True if an error has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* JSON type values +*/ +#define JSON_NULL 0 +#define JSON_TRUE 1 +#define JSON_FALSE 2 +#define JSON_INT 3 +#define JSON_REAL 4 +#define JSON_STRING 5 +#define JSON_ARRAY 6 +#define JSON_OBJECT 7 + +/* The "subtype" set for JSON values */ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + +/* +** Names of the various JSON types: +*/ +static const char * const jsonType[] = { + "null", "true", "false", "integer", "real", "text", "array", "object" +}; + +/* Bit values for the JsonNode.jnFlag field +*/ +#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ +#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ +#define JNODE_REMOVE 0x04 /* Do not output */ +#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ +#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ +#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ +#define JNODE_LABEL 0x40 /* Is a label of an object */ + + +/* A single node of parsed JSON +*/ +struct JsonNode { + u8 eType; /* One of the JSON_ type values */ + u8 jnFlags; /* JNODE flags */ + u32 n; /* Bytes of content, or number of sub-nodes */ + union { + const char *zJContent; /* Content for INT, REAL, and STRING */ + u32 iAppend; /* More terms for ARRAY and OBJECT */ + u32 iKey; /* Key for ARRAY objects in json_tree() */ + u32 iReplace; /* Replacement content for JNODE_REPLACE */ + JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ + } u; +}; + +/* A completely parsed JSON string +*/ +struct JsonParse { + u32 nNode; /* Number of slots of aNode[] used */ + u32 nAlloc; /* Number of slots of aNode[] allocated */ + JsonNode *aNode; /* Array of nodes containing the parse */ + const char *zJson; /* Original JSON string */ + u32 *aUp; /* Index of parent of each node */ + u8 oom; /* Set to true if out of memory */ + u8 nErr; /* Number of errors seen */ + u16 iDepth; /* Nesting depth */ + int nJson; /* Length of the zJson string in bytes */ + u32 iHold; /* Replace cache line with the lowest iHold value */ +}; + +/* +** Maximum nesting depth of JSON for this implementation. +** +** This limit is needed to avoid a stack overflow in the recursive +** descent parser. A depth of 2000 is far deeper than any sane JSON +** should go. +*/ +#define JSON_MAX_DEPTH 2000 + +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ + +/* Set the JsonString object to an empty string +*/ +static void jsonZero(JsonString *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the JsonString object +*/ +static void jsonInit(JsonString *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->bErr = 0; + jsonZero(p); +} + + +/* Free all allocated memory and reset the JsonString object back to its +** initial state. +*/ +static void jsonReset(JsonString *p){ + if( !p->bStatic ) sqlite3_free(p->zBuf); + jsonZero(p); +} + + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonOom(JsonString *p){ + p->bErr = 1; + sqlite3_result_error_nomem(p->pCtx); + jsonReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonGrow(JsonString *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; + char *zNew; + if( p->bStatic ){ + if( p->bErr ) return 1; + zNew = sqlite3_malloc64(nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, (size_t)p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + zNew = sqlite3_realloc64(p->zBuf, nTotal); + if( zNew==0 ){ + jsonOom(p); + return SQLITE_NOMEM; + } + p->zBuf = zNew; + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the JsonString string. +*/ +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} + +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + +/* Append a single character +*/ +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; + p->zBuf[p->nUsed++] = c; +} + +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(JsonString *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c!='[' && c!='{' ) jsonAppendChar(p, ','); +} + +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in "..." and escape +** any double-quotes or backslash characters contained within the +** string. +*/ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ + u32 i; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + }else if( c<=0x1f ){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + if( aSpecial[c] ){ + c = aSpecial[c]; + goto json_simple_escape; + } + if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = 'u'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0'; + p->zBuf[p->nUsed++] = '0' + (c>>4); + c = "0123456789abcdef"[c&0xf]; + } + p->zBuf[p->nUsed++] = c; + } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsednAlloc ); +} + +/* +** Append a function parameter value to the JSON string under +** construction. +*/ +static void jsonAppendValue( + JsonString *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRaw(p, "null", 4); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } + break; + } + default: { + if( p->bErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->bErr = 2; + jsonReset(p); + } + break; + } + } +} + + +/* Make the JSON in p the result of the SQL function. +*/ +static void jsonResult(JsonString *p){ + if( p->bErr==0 ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, + SQLITE_UTF8); + jsonZero(p); + } + assert( p->bStatic ); +} + +/************************************************************************** +** Utility routines for dealing with JsonNode and JsonParse objects +**************************************************************************/ + +/* +** Return the number of consecutive JsonNode slots need to represent +** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and +** OBJECT types, the number might be larger. +** +** Appended elements are not counted. The value returned is the number +** by which the JsonNode counter should increment in order to go to the +** next peer value. +*/ +static u32 jsonNodeSize(JsonNode *pNode){ + return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; +} + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + sqlite3_free(pParse->aNode); + pParse->aNode = 0; + pParse->nNode = 0; + pParse->nAlloc = 0; + sqlite3_free(pParse->aUp); + pParse->aUp = 0; +} + +/* +** Free a JsonParse object that was obtained from sqlite3_malloc(). +*/ +static void jsonParseFree(JsonParse *pParse){ + jsonParseReset(pParse); + sqlite3_free(pParse); +} + +/* +** Convert the JsonNode pNode into a pure JSON string and +** append to pOut. Subsubstructure is also included. Return +** the number of JsonNode objects that are encoded. +*/ +static void jsonRenderNode( + JsonNode *pNode, /* The node to render */ + JsonString *pOut, /* Write JSON here */ + sqlite3_value **aReplace /* Replacement values */ +){ + if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ + if( pNode->jnFlags & JNODE_REPLACE ){ + jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); + return; + } + pNode = pNode->u.pPatch; + } + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + jsonAppendRaw(pOut, "null", 4); + break; + } + case JSON_TRUE: { + jsonAppendRaw(pOut, "true", 4); + break; + } + case JSON_FALSE: { + jsonAppendRaw(pOut, "false", 5); + break; + } + case JSON_STRING: { + if( pNode->jnFlags & JNODE_RAW ){ + jsonAppendString(pOut, pNode->u.zJContent, pNode->n); + break; + } + /* Fall through into the next case */ + } + case JSON_REAL: + case JSON_INT: { + jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); + break; + } + case JSON_ARRAY: { + u32 j = 1; + jsonAppendChar(pOut, '['); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + } + j += jsonNodeSize(&pNode[j]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, ']'); + break; + } + case JSON_OBJECT: { + u32 j = 1; + jsonAppendChar(pOut, '{'); + for(;;){ + while( j<=pNode->n ){ + if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ + jsonAppendSeparator(pOut); + jsonRenderNode(&pNode[j], pOut, aReplace); + jsonAppendChar(pOut, ':'); + jsonRenderNode(&pNode[j+1], pOut, aReplace); + } + j += 1 + jsonNodeSize(&pNode[j+1]); + } + if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; + pNode = &pNode[pNode->u.iAppend]; + j = 1; + } + jsonAppendChar(pOut, '}'); + break; + } + } +} + +/* +** Return a JsonNode and all its descendents as a JSON string. +*/ +static void jsonReturnJson( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + JsonString s; + jsonInit(&s, pCtx); + jsonRenderNode(pNode, &s, aReplace); + jsonResult(&s); + sqlite3_result_subtype(pCtx, JSON_SUBTYPE); +} + +/* +** Make the JsonNode the return value of the function. +*/ +static void jsonReturn( + JsonNode *pNode, /* Node to return */ + sqlite3_context *pCtx, /* Return value for this function */ + sqlite3_value **aReplace /* Array of replacement values */ +){ + switch( pNode->eType ){ + default: { + assert( pNode->eType==JSON_NULL ); + sqlite3_result_null(pCtx); + break; + } + case JSON_TRUE: { + sqlite3_result_int(pCtx, 1); + break; + } + case JSON_FALSE: { + sqlite3_result_int(pCtx, 0); + break; + } + case JSON_INT: { + sqlite3_int64 i = 0; + const char *z = pNode->u.zJContent; + if( z[0]=='-' ){ z++; } + while( z[0]>='0' && z[0]<='9' ){ + unsigned v = *(z++) - '0'; + if( i>=LARGEST_INT64/10 ){ + if( i>LARGEST_INT64/10 ) goto int_as_real; + if( z[0]>='0' && z[0]<='9' ) goto int_as_real; + if( v==9 ) goto int_as_real; + if( v==8 ){ + if( pNode->u.zJContent[0]=='-' ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + goto int_done; + }else{ + goto int_as_real; + } + } + } + i = i*10 + v; + } + if( pNode->u.zJContent[0]=='-' ){ i = -i; } + sqlite3_result_int64(pCtx, i); + int_done: + break; + int_as_real: /* fall through to real */; + } + case JSON_REAL: { + double r; +#ifdef SQLITE_AMALGAMATION + const char *z = pNode->u.zJContent; + sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); +#else + r = strtod(pNode->u.zJContent, 0); +#endif + sqlite3_result_double(pCtx, r); + break; + } + case JSON_STRING: { +#if 0 /* Never happens because JNODE_RAW is only set by json_set(), + ** json_insert() and json_replace() and those routines do not + ** call jsonReturn() */ + if( pNode->jnFlags & JNODE_RAW ){ + sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, + SQLITE_TRANSIENT); + }else +#endif + assert( (pNode->jnFlags & JNODE_RAW)==0 ); + if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ + /* JSON formatted without any backslash-escapes */ + sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, + SQLITE_TRANSIENT); + }else{ + /* Translate JSON formatted string into raw text */ + u32 i; + u32 n = pNode->n; + const char *z = pNode->u.zJContent; + char *zOut; + u32 j; + zOut = sqlite3_malloc( n+1 ); + if( zOut==0 ){ + sqlite3_result_error_nomem(pCtx); + break; + } + for(i=1, j=0; i>6)); + zOut[j++] = 0x80 | (v&0x3f); + }else{ + zOut[j++] = (char)(0xe0 | (v>>12)); + zOut[j++] = 0x80 | ((v>>6)&0x3f); + zOut[j++] = 0x80 | (v&0x3f); + } + }else{ + if( c=='b' ){ + c = '\b'; + }else if( c=='f' ){ + c = '\f'; + }else if( c=='n' ){ + c = '\n'; + }else if( c=='r' ){ + c = '\r'; + }else if( c=='t' ){ + c = '\t'; + } + zOut[j++] = c; + } + } + } + zOut[j] = 0; + sqlite3_result_text(pCtx, zOut, j, sqlite3_free); + } + break; + } + case JSON_ARRAY: + case JSON_OBJECT: { + jsonReturnJson(pNode, pCtx, aReplace); + break; + } + } +} + +/* Forward reference */ +static int jsonParseAddNode(JsonParse*,u32,u32,const char*); + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define JSON_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define JSON_NOINLINE __declspec(noinline) +#else +# define JSON_NOINLINE +#endif + + +static JSON_NOINLINE int jsonParseAddNodeExpand( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + u32 nNew; + JsonNode *pNew; + assert( pParse->nNode>=pParse->nAlloc ); + if( pParse->oom ) return -1; + nNew = pParse->nAlloc*2 + 10; + pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); + if( pNew==0 ){ + pParse->oom = 1; + return -1; + } + pParse->nAlloc = nNew; + pParse->aNode = pNew; + assert( pParse->nNodenAlloc ); + return jsonParseAddNode(pParse, eType, n, zContent); +} + +/* +** Create a new JsonNode instance based on the arguments and append that +** instance to the JsonParse. Return the index in pParse->aNode[] of the +** new node, or -1 if a memory allocation fails. +*/ +static int jsonParseAddNode( + JsonParse *pParse, /* Append the node to this object */ + u32 eType, /* Node type */ + u32 n, /* Content size or sub-node count */ + const char *zContent /* Content */ +){ + JsonNode *p; + if( pParse->nNode>=pParse->nAlloc ){ + return jsonParseAddNodeExpand(pParse, eType, n, zContent); + } + p = &pParse->aNode[pParse->nNode]; + p->eType = (u8)eType; + p->jnFlags = 0; + p->n = n; + p->u.zJContent = zContent; + return pParse->nNode++; +} + +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + int i; + for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; + return 1; +} + +/* +** Parse a single JSON value which begins at pParse->zJson[i]. Return the +** index of the first character past the end of the value parsed. +** +** Return negative for a syntax error. Special cases: return -2 if the +** first non-whitespace character is '}' and return -3 if the first +** non-whitespace character is ']'. +*/ +static int jsonParseValue(JsonParse *pParse, u32 i){ + char c; + u32 j; + int iThis; + int x; + JsonNode *pNode; + const char *z = pParse->zJson; + while( safe_isspace(z[i]) ){ i++; } + if( (c = z[i])=='{' ){ + /* Parse object */ + iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + if( x<0 ){ + pParse->iDepth--; + if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + if( pParse->oom ) return -1; + pNode = &pParse->aNode[pParse->nNode-1]; + if( pNode->eType!=JSON_STRING ) return -1; + pNode->jnFlags |= JNODE_LABEL; + j = x; + while( safe_isspace(z[j]) ){ j++; } + if( z[j]!=':' ) return -1; + j++; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ) return -1; + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!='}' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='[' ){ + /* Parse array */ + iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + if( iThis<0 ) return -1; + for(j=i+1;;j++){ + while( safe_isspace(z[j]) ){ j++; } + if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; + x = jsonParseValue(pParse, j); + pParse->iDepth--; + if( x<0 ){ + if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; + return -1; + } + j = x; + while( safe_isspace(z[j]) ){ j++; } + c = z[j]; + if( c==',' ) continue; + if( c!=']' ) return -1; + break; + } + pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; + return j+1; + }else if( c=='"' ){ + /* Parse string */ + u8 jnFlags = 0; + j = i+1; + for(;;){ + c = z[j]; + if( (c & ~0x1f)==0 ){ + /* Control characters are not allowed in strings */ + return -1; + } + if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(z+j+1)) ){ + jnFlags = JNODE_ESCAPE; + }else{ + return -1; + } + }else if( c=='"' ){ + break; + } + j++; + } + jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); + if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; + return j+1; + }else if( c=='n' + && strncmp(z+i,"null",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return i+4; + }else if( c=='t' + && strncmp(z+i,"true",4)==0 + && !safe_isalnum(z[i+4]) ){ + jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + return i+4; + }else if( c=='f' + && strncmp(z+i,"false",5)==0 + && !safe_isalnum(z[i+5]) ){ + jsonParseAddNode(pParse, JSON_FALSE, 0, 0); + return i+5; + }else if( c=='-' || (c>='0' && c<='9') ){ + /* Parse number */ + u8 seenDP = 0; + u8 seenE = 0; + assert( '-' < '0' ); + if( c<='0' ){ + j = c=='-' ? i+1 : i; + if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; + } + j = i+1; + for(;; j++){ + c = z[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return -1; + if( seenDP ) return -1; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return -1; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return -1; + continue; + } + break; + } + if( z[j-1]<'0' ) return -1; + jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, + j - i, &z[i]); + return j; + }else if( c=='}' ){ + return -2; /* End of {...} */ + }else if( c==']' ){ + return -3; /* End of [...] */ + }else if( c==0 ){ + return 0; /* End of file */ + }else{ + return -1; /* Syntax error */ + } +} + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory associated with +** pParse. +** +** pParse is uninitialized when this routine is called. +*/ +static int jsonParse( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx, /* Report errors here */ + const char *zJson /* Input JSON text to be parsed */ +){ + int i; + memset(pParse, 0, sizeof(*pParse)); + if( zJson==0 ) return 1; + pParse->zJson = zJson; + i = jsonParseValue(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ + assert( pParse->iDepth==0 ); + while( safe_isspace(zJson[i]) ) i++; + if( zJson[i] ) i = -1; + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* Mark node i of pParse as being a child of iParent. Call recursively +** to fill in all the descendants of node i. +*/ +static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ + JsonNode *pNode = &pParse->aNode[i]; + u32 j; + pParse->aUp[i] = iParent; + switch( pNode->eType ){ + case JSON_ARRAY: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ + jsonParseFillInParentage(pParse, i+j, i); + } + break; + } + case JSON_OBJECT: { + for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ + pParse->aUp[i+j] = i; + jsonParseFillInParentage(pParse, i+j+1, i); + } + break; + } + default: { + break; + } + } +} + +/* +** Compute the parentage of all nodes in a completed parse. +*/ +static int jsonParseFindParents(JsonParse *pParse){ + u32 *aUp; + assert( pParse->aUp==0 ); + aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); + if( aUp==0 ){ + pParse->oom = 1; + return SQLITE_NOMEM; + } + jsonParseFillInParentage(pParse, 0, 0); + return SQLITE_OK; +} + +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* First cache entry */ +#define JSON_CACHE_SZ 4 /* Max number of cache entries */ + +/* +** Obtain a complete parse of the JSON found in the first argument +** of the argv array. Use the sqlite3_get_auxdata() cache for this +** parse if it is available. If the cache is not available or if it +** is no longer valid, parse the JSON again and return the new parse, +** and also register the new parse so that it will be available for +** future sqlite3_get_auxdata() calls. +*/ +static JsonParse *jsonParseCached( + sqlite3_context *pCtx, + sqlite3_value **argv, + sqlite3_context *pErrCtx +){ + const char *zJson = (const char*)sqlite3_value_text(argv[0]); + int nJson = sqlite3_value_bytes(argv[0]); + JsonParse *p; + JsonParse *pMatch = 0; + int iKey; + int iMinKey = 0; + u32 iMinHold = 0xffffffff; + u32 iMaxHold = 0; + if( zJson==0 ) return 0; + for(iKey=0; iKeynJson==nJson + && memcmp(p->zJson,zJson,nJson)==0 + ){ + p->nErr = 0; + pMatch = p; + }else if( p->iHoldiHold; + iMinKey = iKey; + } + if( p->iHold>iMaxHold ){ + iMaxHold = p->iHold; + } + } + if( pMatch ){ + pMatch->nErr = 0; + pMatch->iHold = iMaxHold+1; + return pMatch; + } + p = sqlite3_malloc( sizeof(*p) + nJson + 1 ); + if( p==0 ){ + sqlite3_result_error_nomem(pCtx); + return 0; + } + memset(p, 0, sizeof(*p)); + p->zJson = (char*)&p[1]; + memcpy((char*)p->zJson, zJson, nJson+1); + if( jsonParse(p, pErrCtx, p->zJson) ){ + sqlite3_free(p); + return 0; + } + p->nJson = nJson; + p->iHold = iMaxHold+1; + sqlite3_set_auxdata(pCtx, JSON_CACHE_ID+iMinKey, p, + (void(*)(void*))jsonParseFree); + return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID+iMinKey); +} + +/* +** Compare the OBJECT label at pNode against zKey,nKey. Return true on +** a match. +*/ +static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ + if( pNode->jnFlags & JNODE_RAW ){ + if( pNode->n!=nKey ) return 0; + return strncmp(pNode->u.zJContent, zKey, nKey)==0; + }else{ + if( pNode->n!=nKey+2 ) return 0; + return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; + } +} + +/* forward declaration */ +static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); + +/* +** Search along zPath to find the node specified. Return a pointer +** to that node, or NULL if zPath is malformed or if there is no such +** node. +** +** If pApnd!=0, then try to append new nodes to complete zPath if it is +** possible to do so and if no existing node corresponds to zPath. If +** new nodes are appended *pApnd is set to 1. +*/ +static JsonNode *jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this node */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + const char **pzErr /* Make *pzErr point to any syntax error in zPath */ +){ + u32 i, j, nKey; + const char *zKey; + JsonNode *pRoot = &pParse->aNode[iRoot]; + if( zPath[0]==0 ) return pRoot; + if( zPath[0]=='.' ){ + if( pRoot->eType!=JSON_OBJECT ) return 0; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){} + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + *pzErr = zPath; + return 0; + } + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + } + if( nKey==0 ){ + *pzErr = zPath; + return 0; + } + j = 1; + for(;;){ + while( j<=pRoot->n ){ + if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ + return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); + } + j++; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( pApnd ){ + u32 iStart, iLabel; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); + zPath += i; + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + pParse->aNode[iLabel].jnFlags |= JNODE_RAW; + } + return pNode; + } + }else if( zPath[0]=='[' && safe_isdigit(zPath[1]) ){ + if( pRoot->eType!=JSON_ARRAY ) return 0; + i = 0; + j = 1; + while( safe_isdigit(zPath[j]) ){ + i = i*10 + zPath[j] - '0'; + j++; + } + if( zPath[j]!=']' ){ + *pzErr = zPath; + return 0; + } + zPath += j + 1; + j = 1; + for(;;){ + while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ + if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; + j += jsonNodeSize(&pRoot[j]); + } + if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; + iRoot += pRoot->u.iAppend; + pRoot = &pParse->aNode[iRoot]; + j = 1; + } + if( j<=pRoot->n ){ + return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); + } + if( i==0 && pApnd ){ + u32 iStart; + JsonNode *pNode; + iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); + pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); + if( pParse->oom ) return 0; + if( pNode ){ + pRoot = &pParse->aNode[iRoot]; + pRoot->u.iAppend = iStart - iRoot; + pRoot->jnFlags |= JNODE_APPEND; + } + return pNode; + } + }else{ + *pzErr = zPath; + } + return 0; +} + +/* +** Append content to pParse that will complete zPath. Return a pointer +** to the inserted node, or return NULL if the append fails. +*/ +static JsonNode *jsonLookupAppend( + JsonParse *pParse, /* Append content to the JSON parse */ + const char *zPath, /* Description of content to append */ + int *pApnd, /* Set this flag to 1 */ + const char **pzErr /* Make this point to any syntax error */ +){ + *pApnd = 1; + if( zPath[0]==0 ){ + jsonParseAddNode(pParse, JSON_NULL, 0, 0); + return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; + } + if( zPath[0]=='.' ){ + jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); + }else if( strncmp(zPath,"[0]",3)==0 ){ + jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); + }else{ + return 0; + } + if( pParse->oom ) return 0; + return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); +} + +/* +** Return the text of a syntax error message on a JSON path. Space is +** obtained from sqlite3_malloc(). +*/ +static char *jsonPathSyntaxError(const char *zErr){ + return sqlite3_mprintf("JSON path error near '%q'", zErr); +} + +/* +** Do a node lookup using zPath. Return a pointer to the node on success. +** Return NULL if not found or if there is an error. +** +** On an error, write an error message into pCtx and increment the +** pParse->nErr counter. +** +** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if +** nodes are appended. +*/ +static JsonNode *jsonLookup( + JsonParse *pParse, /* The JSON to search */ + const char *zPath, /* The path to search */ + int *pApnd, /* Append nodes to complete path if not NULL */ + sqlite3_context *pCtx /* Report errors here, if not NULL */ +){ + const char *zErr = 0; + JsonNode *pNode = 0; + char *zMsg; + + if( zPath==0 ) return 0; + if( zPath[0]!='$' ){ + zErr = zPath; + goto lookup_err; + } + zPath++; + pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); + if( zErr==0 ) return pNode; + +lookup_err: + pParse->nErr++; + assert( zErr!=0 && pCtx!=0 ); + zMsg = jsonPathSyntaxError(zErr); + if( zMsg ){ + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(pCtx); + } + return 0; +} + + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/* +** Mark all NULL entries in the Object passed in as JNODE_REMOVE. +*/ +static void jsonRemoveAllNulls(JsonNode *pNode){ + int i, n; + assert( pNode->eType==JSON_OBJECT ); + n = pNode->n; + for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ + switch( pNode[i].eType ){ + case JSON_NULL: + pNode[i].jnFlags |= JNODE_REMOVE; + break; + case JSON_OBJECT: + jsonRemoveAllNulls(&pNode[i]); + break; + } + } +} + + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#ifdef SQLITE_DEBUG +/* +** The json_parse(JSON) function returns a string which describes +** a parse of the JSON provided. Or it returns NULL if JSON is not +** well-formed. +*/ +static void jsonParseFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* Output string - not real JSON */ + JsonParse x; /* The parse */ + u32 i; + + assert( argc==1 ); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + jsonParseFindParents(&x); + jsonInit(&s, ctx); + for(i=0; inNode ); + if( argc==2 ){ + const char *zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; + } + if( pNode==0 ){ + return; + } + if( pNode->eType==JSON_ARRAY ){ + assert( (pNode->jnFlags & JNODE_APPEND)==0 ); + for(i=1; i<=pNode->n; n++){ + i += jsonNodeSize(&pNode[i]); + } + } + sqlite3_result_int64(ctx, n); +} + +/* +** json_extract(JSON, PATH, ...) +** +** Return the element described by PATH. Return NULL if there is no +** PATH element. If there are multiple PATHs, then return a JSON array +** with the result from each path. Throw an error if the JSON or any PATH +** is malformed. +*/ +static void jsonExtractFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + JsonNode *pNode; + const char *zPath; + JsonString jx; + int i; + + if( argc<2 ) return; + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=1; inErr ) break; + if( argc>2 ){ + jsonAppendSeparator(&jx); + if( pNode ){ + jsonRenderNode(pNode, &jx, 0); + }else{ + jsonAppendRaw(&jx, "null", 4); + } + }else if( pNode ){ + jsonReturn(pNode, ctx, 0); + } + } + if( argc>2 && i==argc ){ + jsonAppendChar(&jx, ']'); + jsonResult(&jx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + jsonReset(&jx); +} + +/* This is the RFC 7396 MergePatch algorithm. +*/ +static JsonNode *jsonMergePatch( + JsonParse *pParse, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Node of the TARGET in pParse */ + JsonNode *pPatch /* The PATCH */ +){ + u32 i, j; + u32 iRoot; + JsonNode *pTarget; + if( pPatch->eType!=JSON_OBJECT ){ + return pPatch; + } + assert( iTarget>=0 && iTargetnNode ); + pTarget = &pParse->aNode[iTarget]; + assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); + if( pTarget->eType!=JSON_OBJECT ){ + jsonRemoveAllNulls(pPatch); + return pPatch; + } + iRoot = iTarget; + for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ + u32 nKey; + const char *zKey; + assert( pPatch[i].eType==JSON_STRING ); + assert( pPatch[i].jnFlags & JNODE_LABEL ); + nKey = pPatch[i].n; + zKey = pPatch[i].u.zJContent; + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ + assert( pTarget[j].eType==JSON_STRING ); + assert( pTarget[j].jnFlags & JNODE_LABEL ); + assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); + if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ + if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; + if( pPatch[i+1].eType==JSON_NULL ){ + pTarget[j+1].jnFlags |= JNODE_REMOVE; + }else{ + JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); + if( pNew==0 ) return 0; + pTarget = &pParse->aNode[iTarget]; + if( pNew!=&pTarget[j+1] ){ + pTarget[j+1].u.pPatch = pNew; + pTarget[j+1].jnFlags |= JNODE_PATCH; + } + } + break; + } + } + if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ + int iStart, iPatch; + iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); + jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); + iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); + if( pParse->oom ) return 0; + jsonRemoveAllNulls(pPatch); + pTarget = &pParse->aNode[iTarget]; + pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; + pParse->aNode[iRoot].u.iAppend = iStart - iRoot; + iRoot = iStart; + pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; + pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; + } + } + return pTarget; +} + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The JSON that is being patched */ + JsonParse y; /* The patch */ + JsonNode *pResult; /* The result of the merge */ + + UNUSED_PARAM(argc); + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ + jsonParseReset(&x); + return; + } + pResult = jsonMergePatch(&x, 0, y.aNode); + assert( pResult!=0 || x.oom ); + if( pResult ){ + jsonReturnJson(pResult, ctx, 0); + }else{ + sqlite3_result_error_nomem(ctx); + } + jsonParseReset(&x); + jsonParseReset(&y); +} + + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(ctx, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonInit(&jx, ctx); + jsonAppendChar(&jx, '{'); + for(i=0; ijnFlags |= JNODE_REMOVE; + } + if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ + jsonReturnJson(x.aNode, ctx, 0); + } +remove_done: + jsonParseReset(&x); +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonReplaceFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, "replace"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + pNode = jsonLookup(&x, zPath, 0, ctx); + if( x.nErr ) goto replace_err; + if( pNode ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +replace_err: + jsonParseReset(&x); +} + +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON or PATH is malformed, throw an error. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonSetFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse x; /* The parse */ + JsonNode *pNode; + const char *zPath; + u32 i; + int bApnd; + int bIsSet = *(int*)sqlite3_user_data(ctx); + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + return; + } + if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; + assert( x.nNode ); + for(i=1; i<(u32)argc; i+=2){ + zPath = (const char*)sqlite3_value_text(argv[i]); + bApnd = 0; + pNode = jsonLookup(&x, zPath, &bApnd, ctx); + if( x.oom ){ + sqlite3_result_error_nomem(ctx); + goto jsonSetDone; + }else if( x.nErr ){ + goto jsonSetDone; + }else if( pNode && (bApnd || bIsSet) ){ + pNode->jnFlags |= (u8)JNODE_REPLACE; + pNode->u.iReplace = i + 1; + } + } + if( x.aNode[0].jnFlags & JNODE_REPLACE ){ + sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); + }else{ + jsonReturnJson(x.aNode, ctx, argv); + } +jsonSetDone: + jsonParseReset(&x); +} + +/* +** json_type(JSON) +** json_type(JSON, PATH) +** +** Return the top-level "type" of a JSON string. Throw an error if +** either the JSON or PATH inputs are not well-formed. +*/ +static void jsonTypeFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + const char *zPath; + JsonNode *pNode; + + p = jsonParseCached(ctx, argv, ctx); + if( p==0 ) return; + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + pNode = jsonLookup(p, zPath, 0, ctx); + }else{ + pNode = p->aNode; + } + if( pNode ){ + sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); + } +} + +/* +** json_valid(JSON) +** +** Return 1 if JSON is a well-formed JSON string according to RFC-7159. +** Return 0 otherwise. +*/ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + UNUSED_PARAM(argc); + p = jsonParseCached(ctx, argv, 0); + sqlite3_result_int(ctx, p!=0); +} + + +/**************************************************************************** +** Aggregate SQL function implementations +****************************************************************************/ +/* +** json_group_array(VALUE) +** +** Return a JSON array composed of all values in the aggregate. +*/ +static void jsonArrayStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '['); + }else{ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + jsonAppendValue(pStr, argv[0]); + } +} +static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + pStr->pCtx = ctx; + jsonAppendChar(pStr, ']'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonArrayValue(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 0); +} +static void jsonArrayFinal(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 1); +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** This method works for both json_group_array() and json_group_object(). +** It works by removing the first element of the group by searching forward +** to the first comma (",") that is not within a string and deleting all +** text through that comma. +*/ +static void jsonGroupInverse( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + int inStr = 0; + char *z; + JsonString *pStr; + UNUSED_PARAM(argc); + UNUSED_PARAM(argv); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); +#ifdef NEVER + /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will + ** always have been called to initalize it */ + if( NEVER(!pStr) ) return; +#endif + z = pStr->zBuf; + for(i=1; z[i]!=',' || inStr; i++){ + assert( inUsed ); + if( z[i]=='"' ){ + inStr = !inStr; + }else if( z[i]=='\\' ){ + i++; + } + } + pStr->nUsed -= i; + memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1); +} +#else +# define jsonGroupInverse 0 +#endif + + +/* +** json_group_obj(NAME,VALUE) +** +** Return a JSON object composed of all names and values in the aggregate. +*/ +static void jsonObjectStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + const char *z; + u32 n; + UNUSED_PARAM(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonInit(pStr, ctx); + jsonAppendChar(pStr, '{'); + }else{ + jsonAppendChar(pStr, ','); + pStr->pCtx = ctx; + } + z = (const char*)sqlite3_value_text(argv[0]); + n = (u32)sqlite3_value_bytes(argv[0]); + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendValue(pStr, argv[1]); + } +} +static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + jsonAppendChar(pStr, '}'); + if( pStr->bErr ){ + if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); + assert( pStr->bStatic ); + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + pStr->nUsed--; + } + }else{ + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonObjectValue(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 0); +} +static void jsonObjectFinal(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 1); +} + + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 iBegin; /* The first node of the scan */ + u32 i; /* Index in sParse.aNode[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u8 eType; /* Type of top-level element */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + char *zJson; /* Input JSON */ + char *zRoot; /* Path by which to filter zJson */ + JsonParse sParse; /* Parse of the input JSON */ +}; + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_PATH 7 +#define JEACH_JSON 8 +#define JEACH_ROOT 9 + + UNUSED_PARAM(pzErr); + UNUSED_PARAM(argv); + UNUSED_PARAM(argc); + UNUSED_PARAM(pAux); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachCursor *pCur; + + UNUSED_PARAM(p); + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + sqlite3_free(p->zJson); + sqlite3_free(p->zRoot); + jsonParseReset(&p->sParse); + p->iRowid = 0; + p->i = 0; + p->iEnd = 0; + p->eType = 0; + p->zJson = 0; + p->zRoot = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + if( p->bRecursive ){ + if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; + p->i++; + p->iRowid++; + if( p->iiEnd ){ + u32 iUp = p->sParse.aUp[p->i]; + JsonNode *pUp = &p->sParse.aNode[iUp]; + p->eType = pUp->eType; + if( pUp->eType==JSON_ARRAY ){ + if( iUp==p->i-1 ){ + pUp->u.iKey = 0; + }else{ + pUp->u.iKey++; + } + } + } + }else{ + switch( p->eType ){ + case JSON_ARRAY: { + p->i += jsonNodeSize(&p->sParse.aNode[p->i]); + p->iRowid++; + break; + } + case JSON_OBJECT: { + p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); + p->iRowid++; + break; + } + default: { + p->i = p->iEnd; + break; + } + } + } + return SQLITE_OK; +} + +/* Append the name of the path for element i to pStr +*/ +static void jsonEachComputePath( + JsonEachCursor *p, /* The cursor */ + JsonString *pStr, /* Write the path here */ + u32 i /* Path to this element */ +){ + JsonNode *pNode, *pUp; + u32 iUp; + if( i==0 ){ + jsonAppendChar(pStr, '$'); + return; + } + iUp = p->sParse.aUp[i]; + jsonEachComputePath(p, pStr, iUp); + pNode = &p->sParse.aNode[i]; + pUp = &p->sParse.aNode[iUp]; + if( pUp->eType==JSON_ARRAY ){ + jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); + }else{ + assert( pUp->eType==JSON_OBJECT ); + if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; + assert( pNode->eType==JSON_STRING ); + assert( pNode->jnFlags & JNODE_LABEL ); + jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); + } +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + JsonNode *pThis = &p->sParse.aNode[p->i]; + switch( i ){ + case JEACH_KEY: { + if( p->i==0 ) break; + if( p->eType==JSON_OBJECT ){ + jsonReturn(pThis, ctx, 0); + }else if( p->eType==JSON_ARRAY ){ + u32 iKey; + if( p->bRecursive ){ + if( p->iRowid==0 ) break; + iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; + }else{ + iKey = p->iRowid; + } + sqlite3_result_int64(ctx, (sqlite3_int64)iKey); + } + break; + } + case JEACH_VALUE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_TYPE: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + if( pThis->jnFlags & JNODE_LABEL ) pThis++; + if( pThis->eType>=JSON_ARRAY ) break; + jsonReturn(pThis, ctx, 0); + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, + (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); + break; + } + case JEACH_PARENT: { + if( p->i>p->iBegin && p->bRecursive ){ + sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); + } + break; + } + case JEACH_FULLKEY: { + JsonString x; + jsonInit(&x, ctx); + if( p->bRecursive ){ + jsonEachComputePath(p, &x, p->i); + }else{ + if( p->zRoot ){ + jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); + }else{ + jsonAppendChar(&x, '$'); + } + if( p->eType==JSON_ARRAY ){ + jsonPrintf(30, &x, "[%d]", p->iRowid); + }else if( p->eType==JSON_OBJECT ){ + jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); + } + } + jsonResult(&x); + break; + } + case JEACH_PATH: { + if( p->bRecursive ){ + JsonString x; + jsonInit(&x, ctx); + jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); + jsonResult(&x); + break; + } + /* For json_each() path and root are the same so fall through + ** into the root case */ + } + default: { + const char *zRoot = p->zRoot; + if( zRoot==0 ) zRoot = "$"; + sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + assert( i==JEACH_JSON ); + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zRoot are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int jsonIdx = -1; + int rootIdx = -1; + const struct sqlite3_index_constraint *pConstraint; + + UNUSED_PARAM(tab); + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case JEACH_JSON: jsonIdx = i; break; + case JEACH_ROOT: rootIdx = i; break; + default: /* no-op */ break; + } + } + if( jsonIdx<0 ){ + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = 1e99; + }else{ + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; + if( rootIdx<0 ){ + pIdxInfo->idxNum = 1; + }else{ + pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[rootIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *z; + const char *zRoot = 0; + sqlite3_int64 n; + + UNUSED_PARAM(idxStr); + UNUSED_PARAM(argc); + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + z = (const char*)sqlite3_value_text(argv[0]); + if( z==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[0]); + p->zJson = sqlite3_malloc64( n+1 ); + if( p->zJson==0 ) return SQLITE_NOMEM; + memcpy(p->zJson, z, (size_t)n+1); + if( jsonParse(&p->sParse, 0, p->zJson) ){ + int rc = SQLITE_NOMEM; + if( p->sParse.oom==0 ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; + } + jsonEachCursorReset(p); + return rc; + }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ + jsonEachCursorReset(p); + return SQLITE_NOMEM; + }else{ + JsonNode *pNode = 0; + if( idxNum==3 ){ + const char *zErr = 0; + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + n = sqlite3_value_bytes(argv[1]); + p->zRoot = sqlite3_malloc64( n+1 ); + if( p->zRoot==0 ) return SQLITE_NOMEM; + memcpy(p->zRoot, zRoot, (size_t)n+1); + if( zRoot[0]!='$' ){ + zErr = zRoot; + }else{ + pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); + } + if( zErr ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + }else if( pNode==0 ){ + return SQLITE_OK; + } + }else{ + pNode = p->sParse.aNode; + } + p->iBegin = p->i = (int)(pNode - p->sParse.aNode); + p->eType = pNode->eType; + if( p->eType>=JSON_ARRAY ){ + pNode->u.iKey = 0; + p->iEnd = p->i + pNode->n + 1; + if( p->bRecursive ){ + p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; + if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ + p->i--; + } + }else{ + p->i++; + } + }else{ + p->iEnd = p->i+1; + } + } + return SQLITE_OK; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; + +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0 /* xRollbackTo */ +}; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/**************************************************************************** +** The following routines are the only publically visible identifiers in this +** file. Call the following routines in order to register the various SQL +** functions and the virtual table implemented by this file. +****************************************************************************/ + +SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ + int rc = SQLITE_OK; + unsigned int i; + static const struct { + const char *zName; + int nArg; + int flag; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "json", 1, 0, jsonRemoveFunc }, + { "json_array", -1, 0, jsonArrayFunc }, + { "json_array_length", 1, 0, jsonArrayLengthFunc }, + { "json_array_length", 2, 0, jsonArrayLengthFunc }, + { "json_extract", -1, 0, jsonExtractFunc }, + { "json_insert", -1, 0, jsonSetFunc }, + { "json_object", -1, 0, jsonObjectFunc }, + { "json_patch", 2, 0, jsonPatchFunc }, + { "json_quote", 1, 0, jsonQuoteFunc }, + { "json_remove", -1, 0, jsonRemoveFunc }, + { "json_replace", -1, 0, jsonReplaceFunc }, + { "json_set", -1, 1, jsonSetFunc }, + { "json_type", 1, 0, jsonTypeFunc }, + { "json_type", 2, 0, jsonTypeFunc }, + { "json_valid", 1, 0, jsonValidFunc }, + +#if SQLITE_DEBUG + /* DEBUG and TESTING functions */ + { "json_parse", 1, 0, jsonParseFunc }, + { "json_test1", 1, 0, jsonTest1Func }, +#endif + }; + static const struct { + const char *zName; + int nArg; + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + void (*xValue)(sqlite3_context*); + } aAgg[] = { + { "json_group_array", 1, + jsonArrayStep, jsonArrayFinal, jsonArrayValue }, + { "json_group_object", 2, + jsonObjectStep, jsonObjectFinal, jsonObjectValue }, + }; +#ifndef SQLITE_OMIT_VIRTUALTABLE + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; +#endif + for(i=0; ipWriteRowid, 1); sqlite3_bind_null(pRtree->pWriteRowid, 2); @@ -173701,7 +181276,7 @@ static int rtreeUpdate( /* Figure out the rowid of the new row. */ if( bHaveRowid==0 ){ - rc = newRowid(pRtree, &cell.iRowid); + rc = rtreeNewRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; @@ -173793,7 +181368,7 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ */ static int rtreeSavepoint(sqlite3_vtab *pVtab, int iSavepoint){ Rtree *pRtree = (Rtree *)pVtab; - int iwt = pRtree->inWrTrans; + u8 iwt = pRtree->inWrTrans; UNUSED_PARAMETER(iSavepoint); pRtree->inWrTrans = 0; nodeBlobReset(pRtree); @@ -173974,7 +181549,11 @@ static int rtreeSqlInit( sqlite3_str_appendf(p, "UPDATE \"%w\".\"%w_rowid\"SET ", zDb, zPrefix); for(ii=0; iinAux; ii++){ if( ii ) sqlite3_str_append(p, ",", 1); - sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + if( iinAuxNotNull ){ + sqlite3_str_appendf(p,"a%d=coalesce(?%d,a%d)",ii,ii+2,ii); + }else{ + sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + } } sqlite3_str_appendf(p, " WHERE rowid=?1"); zSql = sqlite3_str_finish(p); @@ -174743,6 +182322,1673 @@ static void rtreecheck( } } +/* Conditionally include the geopoly code */ +#ifdef SQLITE_ENABLE_GEOPOLY +/************** Include geopoly.c in the middle of rtree.c *******************/ +/************** Begin file geopoly.c *****************************************/ +/* +** 2018-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an alternative R-Tree virtual table that +** uses polygons to express the boundaries of 2-dimensional objects. +** +** This file is #include-ed onto the end of "rtree.c" so that it has +** access to all of the R-Tree internals. +*/ +/* #include */ + +/* Enable -DGEOPOLY_ENABLE_DEBUG for debugging facilities */ +#ifdef GEOPOLY_ENABLE_DEBUG + static int geo_debug = 0; +# define GEODEBUG(X) if(geo_debug)printf X +#else +# define GEODEBUG(X) +#endif + +#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ +/* +** Versions of isspace(), isalnum() and isdigit() to which it is safe +** to pass signed char values. +*/ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function. +*/ +static const char geopolyIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define safe_isspace(x) (geopolyIsSpace[(unsigned char)x]) +#endif /* JSON NULL - back to original code */ + +/* Compiler and version */ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* Datatype for coordinates +*/ +typedef float GeoCoord; + +/* +** Internal representation of a polygon. +** +** The polygon consists of a sequence of vertexes. There is a line +** segment between each pair of vertexes, and one final segment from +** the last vertex back to the first. (This differs from the GeoJSON +** standard in which the final vertex is a repeat of the first.) +** +** The polygon follows the right-hand rule. The area to the right of +** each segment is "outside" and the area to the left is "inside". +** +** The on-disk representation consists of a 4-byte header followed by +** the values. The 4-byte header is: +** +** encoding (1 byte) 0=big-endian, 1=little-endian +** nvertex (3 bytes) Number of vertexes as a big-endian integer +*/ +typedef struct GeoPoly GeoPoly; +struct GeoPoly { + int nVertex; /* Number of vertexes */ + unsigned char hdr[4]; /* Header for on-disk representation */ + GeoCoord a[2]; /* 2*nVertex values. X (longitude) first, then Y */ +}; + +/* +** State of a parse of a GeoJSON input. +*/ +typedef struct GeoParse GeoParse; +struct GeoParse { + const unsigned char *z; /* Unparsed input */ + int nVertex; /* Number of vertexes in a[] */ + int nAlloc; /* Space allocated to a[] */ + int nErr; /* Number of errors encountered */ + GeoCoord *a; /* Array of vertexes. From sqlite3_malloc64() */ +}; + +/* Do a 4-byte byte swap */ +static void geopolySwab32(unsigned char *a){ + unsigned char t = a[0]; + a[0] = a[3]; + a[3] = t; + t = a[1]; + a[1] = a[2]; + a[2] = t; +} + +/* Skip whitespace. Return the next non-whitespace character. */ +static char geopolySkipSpace(GeoParse *p){ + while( p->z[0] && safe_isspace(p->z[0]) ) p->z++; + return p->z[0]; +} + +/* Parse out a number. Write the value into *pVal if pVal!=0. +** return non-zero on success and zero if the next token is not a number. +*/ +static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ + char c = geopolySkipSpace(p); + const unsigned char *z = p->z; + int j = 0; + int seenDP = 0; + int seenE = 0; + if( c=='-' ){ + j = 1; + c = z[j]; + } + if( c=='0' && z[j+1]>='0' && z[j+1]<='9' ) return 0; + for(;; j++){ + c = z[j]; + if( c>='0' && c<='9' ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return 0; + if( seenDP ) return 0; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return 0; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return 0; + continue; + } + break; + } + if( z[j-1]<'0' ) return 0; + if( pVal ) *pVal = (GeoCoord)atof((const char*)p->z); + p->z += j; + return 1; +} + +/* +** If the input is a well-formed JSON array of coordinates with at least +** four coordinates and where each coordinate is itself a two-value array, +** then convert the JSON into a GeoPoly object and return a pointer to +** that object. +** +** If any error occurs, return NULL. +*/ +static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ + GeoParse s; + int rc = SQLITE_OK; + memset(&s, 0, sizeof(s)); + s.z = z; + if( geopolySkipSpace(&s)=='[' ){ + s.z++; + while( geopolySkipSpace(&s)=='[' ){ + int ii = 0; + char c; + s.z++; + if( s.nVertex>=s.nAlloc ){ + GeoCoord *aNew; + s.nAlloc = s.nAlloc*2 + 16; + aNew = sqlite3_realloc64(s.a, s.nAlloc*sizeof(GeoCoord)*2 ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + s.nErr++; + break; + } + s.a = aNew; + } + while( geopolyParseNumber(&s, ii<=1 ? &s.a[s.nVertex*2+ii] : 0) ){ + ii++; + if( ii==2 ) s.nVertex++; + c = geopolySkipSpace(&s); + s.z++; + if( c==',' ) continue; + if( c==']' && ii>=2 ) break; + s.nErr++; + rc = SQLITE_ERROR; + goto parse_json_err; + } + if( geopolySkipSpace(&s)==',' ){ + s.z++; + continue; + } + break; + } + if( geopolySkipSpace(&s)==']' + && s.nVertex>=4 + && s.a[0]==s.a[s.nVertex*2-2] + && s.a[1]==s.a[s.nVertex*2-1] + && (s.z++, geopolySkipSpace(&s)==0) + ){ + int nByte; + GeoPoly *pOut; + int x = 1; + s.nVertex--; /* Remove the redundant vertex at the end */ + nByte = sizeof(GeoPoly) * s.nVertex*2*sizeof(GeoCoord); + pOut = sqlite3_malloc64( nByte ); + x = 1; + if( pOut==0 ) goto parse_json_err; + pOut->nVertex = s.nVertex; + memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord)); + pOut->hdr[0] = *(unsigned char*)&x; + pOut->hdr[1] = (s.nVertex>>16)&0xff; + pOut->hdr[2] = (s.nVertex>>8)&0xff; + pOut->hdr[3] = s.nVertex&0xff; + sqlite3_free(s.a); + if( pRc ) *pRc = SQLITE_OK; + return pOut; + }else{ + s.nErr++; + rc = SQLITE_ERROR; + } + } +parse_json_err: + if( pRc ) *pRc = rc; + sqlite3_free(s.a); + return 0; +} + +/* +** Given a function parameter, try to interpret it as a polygon, either +** in the binary format or JSON text. Compute a GeoPoly object and +** return a pointer to that object. Or if the input is not a well-formed +** polygon, put an error message in sqlite3_context and return NULL. +*/ +static GeoPoly *geopolyFuncParam( + sqlite3_context *pCtx, /* Context for error messages */ + sqlite3_value *pVal, /* The value to decode */ + int *pRc /* Write error here */ +){ + GeoPoly *p = 0; + int nByte; + if( sqlite3_value_type(pVal)==SQLITE_BLOB + && (nByte = sqlite3_value_bytes(pVal))>=(4+6*sizeof(GeoCoord)) + ){ + const unsigned char *a = sqlite3_value_blob(pVal); + int nVertex; + nVertex = (a[1]<<16) + (a[2]<<8) + a[3]; + if( (a[0]==0 || a[0]==1) + && (nVertex*2*sizeof(GeoCoord) + 4)==(unsigned int)nByte + ){ + p = sqlite3_malloc64( sizeof(*p) + (nVertex-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + if( pCtx ) sqlite3_result_error_nomem(pCtx); + }else{ + int x = 1; + p->nVertex = nVertex; + memcpy(p->hdr, a, nByte); + if( a[0] != *(unsigned char*)&x ){ + int ii; + for(ii=0; iia[ii]); + } + p->hdr[0] ^= 1; + } + } + } + if( pRc ) *pRc = SQLITE_OK; + return p; + }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ + const unsigned char *zJson = sqlite3_value_text(pVal); + if( zJson==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + return geopolyParseJson(zJson, pRc); + }else{ + if( pRc ) *pRc = SQLITE_ERROR; + return 0; + } +} + +/* +** Implementation of the geopoly_blob(X) function. +** +** If the input is a well-formed Geopoly BLOB or JSON string +** then return the BLOB representation of the polygon. Otherwise +** return NULL. +*/ +static void geopolyBlobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_json(X) +** +** Interpret X as a polygon and render it as a JSON array +** of coordinates. Or, if X is not a valid polygon, return NULL. +*/ +static void geopolyJsonFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + sqlite3_str_append(x, "[", 1); + for(i=0; inVertex; i++){ + sqlite3_str_appendf(x, "[%!g,%!g],", p->a[i*2], p->a[i*2+1]); + } + sqlite3_str_appendf(x, "[%!g,%!g]]", p->a[0], p->a[1]); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_svg(X, ....) +** +** Interpret X as a polygon and render it as a SVG . +** Additional arguments are added as attributes to the . +*/ +static void geopolySvgFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + char cSep = '\''; + sqlite3_str_appendf(x, "a[i*2], p->a[i*2+1]); + cSep = ' '; + } + sqlite3_str_appendf(x, " %g,%g'", p->a[0], p->a[1]); + for(i=1; i"); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL Function: geopoly_xform(poly, A, B, C, D, E, F) +** +** Transform and/or translate a polygon as follows: +** +** x1 = A*x0 + B*y0 + E +** y1 = C*x0 + D*y0 + F +** +** For a translation: +** +** geopoly_xform(poly, 1, 0, 0, 1, x-offset, y-offset) +** +** Rotate by R around the point (0,0): +** +** geopoly_xform(poly, cos(R), sin(R), -sin(R), cos(R), 0, 0) +*/ +static void geopolyXformFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + double A = sqlite3_value_double(argv[1]); + double B = sqlite3_value_double(argv[2]); + double C = sqlite3_value_double(argv[3]); + double D = sqlite3_value_double(argv[4]); + double E = sqlite3_value_double(argv[5]); + double F = sqlite3_value_double(argv[6]); + GeoCoord x1, y1, x0, y0; + int ii; + if( p ){ + for(ii=0; iinVertex; ii++){ + x0 = p->a[ii*2]; + y0 = p->a[ii*2+1]; + x1 = (GeoCoord)(A*x0 + B*y0 + E); + y1 = (GeoCoord)(C*x0 + D*y0 + F); + p->a[ii*2] = x1; + p->a[ii*2+1] = y1; + } + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** Implementation of the geopoly_area(X) function. +** +** If the input is a well-formed Geopoly BLOB then return the area +** enclosed by the polygon. If the polygon circulates clockwise instead +** of counterclockwise (as it should) then return the negative of the +** enclosed area. Otherwise return NULL. +*/ +static void geopolyAreaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + double rArea = 0.0; + int ii; + for(ii=0; iinVertex-1; ii++){ + rArea += (p->a[ii*2] - p->a[ii*2+2]) /* (x0 - x1) */ + * (p->a[ii*2+1] + p->a[ii*2+3]) /* (y0 + y1) */ + * 0.5; + } + rArea += (p->a[ii*2] - p->a[0]) /* (xN - x0) */ + * (p->a[ii*2+1] + p->a[1]) /* (yN + y0) */ + * 0.5; + sqlite3_result_double(context, rArea); + sqlite3_free(p); + } +} + +/* +** If pPoly is a polygon, compute its bounding box. Then: +** +** (1) if aCoord!=0 store the bounding box in aCoord, returning NULL +** (2) otherwise, compute a GeoPoly for the bounding box and return the +** new GeoPoly +** +** If pPoly is NULL but aCoord is not NULL, then compute a new GeoPoly from +** the bounding box in aCoord and return a pointer to that GeoPoly. +*/ +static GeoPoly *geopolyBBox( + sqlite3_context *context, /* For recording the error */ + sqlite3_value *pPoly, /* The polygon */ + RtreeCoord *aCoord, /* Results here */ + int *pRc /* Error code here */ +){ + GeoPoly *pOut = 0; + GeoPoly *p; + float mnX, mxX, mnY, mxY; + if( pPoly==0 && aCoord!=0 ){ + p = 0; + mnX = aCoord[0].f; + mxX = aCoord[1].f; + mnY = aCoord[2].f; + mxY = aCoord[3].f; + goto geopolyBboxFill; + }else{ + p = geopolyFuncParam(context, pPoly, pRc); + } + if( p ){ + int ii; + mnX = mxX = p->a[0]; + mnY = mxY = p->a[1]; + for(ii=1; iinVertex; ii++){ + double r = p->a[ii*2]; + if( rmxX ) mxX = (float)r; + r = p->a[ii*2+1]; + if( rmxY ) mxY = (float)r; + } + if( pRc ) *pRc = SQLITE_OK; + if( aCoord==0 ){ + geopolyBboxFill: + pOut = sqlite3_realloc(p, sizeof(GeoPoly)+sizeof(GeoCoord)*6); + if( pOut==0 ){ + sqlite3_free(p); + if( context ) sqlite3_result_error_nomem(context); + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + pOut->nVertex = 4; + ii = 1; + pOut->hdr[0] = *(unsigned char*)ⅈ + pOut->hdr[1] = 0; + pOut->hdr[2] = 0; + pOut->hdr[3] = 4; + pOut->a[0] = mnX; + pOut->a[1] = mnY; + pOut->a[2] = mxX; + pOut->a[3] = mnY; + pOut->a[4] = mxX; + pOut->a[5] = mxY; + pOut->a[6] = mnX; + pOut->a[7] = mxY; + }else{ + sqlite3_free(p); + aCoord[0].f = mnX; + aCoord[1].f = mxX; + aCoord[2].f = mnY; + aCoord[3].f = mxY; + } + } + return pOut; +} + +/* +** Implementation of the geopoly_bbox(X) SQL function. +*/ +static void geopolyBBoxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** State vector for the geopoly_group_bbox() aggregate function. +*/ +typedef struct GeoBBox GeoBBox; +struct GeoBBox { + int isInit; + RtreeCoord a[4]; +}; + + +/* +** Implementation of the geopoly_group_bbox(X) aggregate SQL function. +*/ +static void geopolyBBoxStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + RtreeCoord a[4]; + int rc = SQLITE_OK; + (void)geopolyBBox(context, argv[0], a, &rc); + if( rc==SQLITE_OK ){ + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox)); + if( pBBox==0 ) return; + if( pBBox->isInit==0 ){ + pBBox->isInit = 1; + memcpy(pBBox->a, a, sizeof(RtreeCoord)*4); + }else{ + if( a[0].f < pBBox->a[0].f ) pBBox->a[0] = a[0]; + if( a[1].f > pBBox->a[1].f ) pBBox->a[1] = a[1]; + if( a[2].f < pBBox->a[2].f ) pBBox->a[2] = a[2]; + if( a[3].f > pBBox->a[3].f ) pBBox->a[3] = a[3]; + } + } +} +static void geopolyBBoxFinal( + sqlite3_context *context +){ + GeoPoly *p; + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, 0); + if( pBBox==0 ) return; + p = geopolyBBox(context, 0, pBBox->a, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + + +/* +** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2). +** Returns: +** +** +2 x0,y0 is on the line segement +** +** +1 x0,y0 is beneath line segment +** +** 0 x0,y0 is not on or beneath the line segment or the line segment +** is vertical and x0,y0 is not on the line segment +** +** The left-most coordinate min(x1,x2) is not considered to be part of +** the line segment for the purposes of this analysis. +*/ +static int pointBeneathLine( + double x0, double y0, + double x1, double y1, + double x2, double y2 +){ + double y; + if( x0==x1 && y0==y1 ) return 2; + if( x1x2 ) return 0; + }else if( x1>x2 ){ + if( x0<=x2 || x0>x1 ) return 0; + }else{ + /* Vertical line segment */ + if( x0!=x1 ) return 0; + if( y0y1 && y0>y2 ) return 0; + return 2; + } + y = y1 + (y2-y1)*(x0-x1)/(x2-x1); + if( y0==y ) return 2; + if( y0nVertex-1; ii++){ + v = pointBeneathLine(x0,y0,p1->a[ii*2],p1->a[ii*2+1], + p1->a[ii*2+2],p1->a[ii*2+3]); + if( v==2 ) break; + cnt += v; + } + if( v!=2 ){ + v = pointBeneathLine(x0,y0,p1->a[ii*2],p1->a[ii*2+1], + p1->a[0],p1->a[1]); + } + if( v==2 ){ + sqlite3_result_int(context, 1); + }else if( ((v+cnt)&1)==0 ){ + sqlite3_result_int(context, 0); + }else{ + sqlite3_result_int(context, 2); + } + sqlite3_free(p1); +} + +/* Forward declaration */ +static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2); + +/* +** SQL function: geopoly_within(P1,P2) +** +** Return +2 if P1 and P2 are the same polygon +** Return +1 if P2 is contained within P1 +** Return 0 if any part of P2 is on the outside of P1 +** +*/ +static void geopolyWithinFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x==2 ? 1 : x==4 ? 2 : 0); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* Objects used by the overlap algorihm. */ +typedef struct GeoEvent GeoEvent; +typedef struct GeoSegment GeoSegment; +typedef struct GeoOverlap GeoOverlap; +struct GeoEvent { + double x; /* X coordinate at which event occurs */ + int eType; /* 0 for ADD, 1 for REMOVE */ + GeoSegment *pSeg; /* The segment to be added or removed */ + GeoEvent *pNext; /* Next event in the sorted list */ +}; +struct GeoSegment { + double C, B; /* y = C*x + B */ + double y; /* Current y value */ + float y0; /* Initial y value */ + unsigned char side; /* 1 for p1, 2 for p2 */ + unsigned int idx; /* Which segment within the side */ + GeoSegment *pNext; /* Next segment in a list sorted by y */ +}; +struct GeoOverlap { + GeoEvent *aEvent; /* Array of all events */ + GeoSegment *aSegment; /* Array of all segments */ + int nEvent; /* Number of events */ + int nSegment; /* Number of segments */ +}; + +/* +** Add a single segment and its associated events. +*/ +static void geopolyAddOneSegment( + GeoOverlap *p, + GeoCoord x0, + GeoCoord y0, + GeoCoord x1, + GeoCoord y1, + unsigned char side, + unsigned int idx +){ + GeoSegment *pSeg; + GeoEvent *pEvent; + if( x0==x1 ) return; /* Ignore vertical segments */ + if( x0>x1 ){ + GeoCoord t = x0; + x0 = x1; + x1 = t; + t = y0; + y0 = y1; + y1 = t; + } + pSeg = p->aSegment + p->nSegment; + p->nSegment++; + pSeg->C = (y1-y0)/(x1-x0); + pSeg->B = y1 - x1*pSeg->C; + pSeg->y0 = y0; + pSeg->side = side; + pSeg->idx = idx; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x0; + pEvent->eType = 0; + pEvent->pSeg = pSeg; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x1; + pEvent->eType = 1; + pEvent->pSeg = pSeg; +} + + + +/* +** Insert all segments and events for polygon pPoly. +*/ +static void geopolyAddSegments( + GeoOverlap *p, /* Add segments to this Overlap object */ + GeoPoly *pPoly, /* Take all segments from this polygon */ + unsigned char side /* The side of pPoly */ +){ + unsigned int i; + GeoCoord *x; + for(i=0; i<(unsigned)pPoly->nVertex-1; i++){ + x = pPoly->a + (i*2); + geopolyAddOneSegment(p, x[0], x[1], x[2], x[3], side, i); + } + x = pPoly->a + (i*2); + geopolyAddOneSegment(p, x[0], x[1], pPoly->a[0], pPoly->a[1], side, i); +} + +/* +** Merge two lists of sorted events by X coordinate +*/ +static GeoEvent *geopolyEventMerge(GeoEvent *pLeft, GeoEvent *pRight){ + GeoEvent head, *pLast; + head.pNext = 0; + pLast = &head; + while( pRight && pLeft ){ + if( pRight->x <= pLeft->x ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort an array of nEvent event objects into a list. +*/ +static GeoEvent *geopolySortEventsByX(GeoEvent *aEvent, int nEvent){ + int mx = 0; + int i, j; + GeoEvent *p; + GeoEvent *a[50]; + for(i=0; ipNext = 0; + for(j=0; j=mx ) mx = j+1; + } + p = 0; + for(i=0; iy - pLeft->y; + if( r==0.0 ) r = pRight->C - pLeft->C; + if( r<0.0 ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort a list of GeoSegments in order of increasing Y and in the event of +** a tie, increasing C (slope). +*/ +static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){ + int mx = 0; + int i; + GeoSegment *p; + GeoSegment *a[50]; + while( pList ){ + p = pList; + pList = pList->pNext; + p->pNext = 0; + for(i=0; i=mx ) mx = i+1; + } + p = 0; + for(i=0; inVertex + p2->nVertex + 2; + GeoOverlap *p; + int nByte; + GeoEvent *pThisEvent; + double rX; + int rc = 0; + int needSort = 0; + GeoSegment *pActive = 0; + GeoSegment *pSeg; + unsigned char aOverlap[4]; + + nByte = sizeof(GeoEvent)*nVertex*2 + + sizeof(GeoSegment)*nVertex + + sizeof(GeoOverlap); + p = sqlite3_malloc( nByte ); + if( p==0 ) return -1; + p->aEvent = (GeoEvent*)&p[1]; + p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; + p->nEvent = p->nSegment = 0; + geopolyAddSegments(p, p1, 1); + geopolyAddSegments(p, p2, 2); + pThisEvent = geopolySortEventsByX(p->aEvent, p->nEvent); + rX = pThisEvent->x==0.0 ? -1.0 : 0.0; + memset(aOverlap, 0, sizeof(aOverlap)); + while( pThisEvent ){ + if( pThisEvent->x!=rX ){ + GeoSegment *pPrev = 0; + int iMask = 0; + GEODEBUG(("Distinct X: %g\n", pThisEvent->x)); + rX = pThisEvent->x; + if( needSort ){ + GEODEBUG(("SORT\n")); + pActive = geopolySortSegmentsByYAndC(pActive); + needSort = 0; + } + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pPrev ){ + if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + pPrev = 0; + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + double y = pSeg->C*rX + pSeg->B; + GEODEBUG(("Segment %d.%d %g->%g\n", pSeg->side, pSeg->idx, pSeg->y, y)); + pSeg->y = y; + if( pPrev ){ + if( pPrev->y>pSeg->y && pPrev->side!=pSeg->side ){ + rc = 1; + GEODEBUG(("Crossing: %d.%d and %d.%d\n", + pPrev->side, pPrev->idx, + pSeg->side, pSeg->idx)); + goto geopolyOverlapDone; + }else if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + } + GEODEBUG(("%s %d.%d C=%g B=%g\n", + pThisEvent->eType ? "RM " : "ADD", + pThisEvent->pSeg->side, pThisEvent->pSeg->idx, + pThisEvent->pSeg->C, + pThisEvent->pSeg->B)); + if( pThisEvent->eType==0 ){ + /* Add a segment */ + pSeg = pThisEvent->pSeg; + pSeg->y = pSeg->y0; + pSeg->pNext = pActive; + pActive = pSeg; + needSort = 1; + }else{ + /* Remove a segment */ + if( pActive==pThisEvent->pSeg ){ + pActive = pActive->pNext; + }else{ + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pSeg->pNext==pThisEvent->pSeg ){ + pSeg->pNext = pSeg->pNext->pNext; + break; + } + } + } + } + pThisEvent = pThisEvent->pNext; + } + if( aOverlap[3]==0 ){ + rc = 0; + }else if( aOverlap[1]!=0 && aOverlap[2]==0 ){ + rc = 3; + }else if( aOverlap[1]==0 && aOverlap[2]!=0 ){ + rc = 2; + }else if( aOverlap[1]==0 && aOverlap[2]==0 ){ + rc = 4; + }else{ + rc = 1; + } + +geopolyOverlapDone: + sqlite3_free(p); + return rc; +} + +/* +** SQL function: geopoly_overlap(P1,P2) +** +** Determine whether or not P1 and P2 overlap. Return value: +** +** 0 The two polygons are disjoint +** 1 They overlap +** 2 P1 is completely contained within P2 +** 3 P2 is completely contained within P1 +** 4 P1 and P2 are the same polygon +** NULL Either P1 or P2 or both are not valid polygons +*/ +static void geopolyOverlapFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* +** Enable or disable debugging output +*/ +static void geopolyDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ +#ifdef GEOPOLY_ENABLE_DEBUG + geo_debug = sqlite3_value_int(argv[0]); +#endif +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the geopoly virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int geopolyInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + int nDb; /* Length of string argv[1] */ + int nName; /* Length of string argv[2] */ + sqlite3_str *pSql; + char *zSql; + int ii; + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + + /* Allocate the sqlite3_vtab structure */ + nDb = (int)strlen(argv[1]); + nName = (int)strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->eCoordType = RTREE_COORD_REAL32; + pRtree->nDim = 2; + pRtree->nDim2 = 4; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(_shape"); + pRtree->nAux = 1; /* Add one for _shape */ + pRtree->nAuxNotNull = 1; /* The _shape column is always not-null */ + for(ii=3; iinAux++; + sqlite3_str_appendf(pSql, ",%s", argv[ii]); + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + sqlite3_free(zSql); + if( rc ) goto geopolyInit_fail; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto geopolyInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto geopolyInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +geopolyInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** GEOPOLY virtual table module xCreate method. +*/ +static int geopolyCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** GEOPOLY virtual table module xConnect method. +*/ +static int geopolyConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + + +/* +** GEOPOLY virtual table module xFilter method. +** +** Query plans: +** +** 1 rowid lookup +** 2 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 3 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 4 full table scan +*/ +static int geopolyFilter( + sqlite3_vtab_cursor *pVtabCursor, /* The cursor to initialize */ + int idxNum, /* Query plan */ + const char *idxStr, /* Not Used */ + int argc, sqlite3_value **argv /* Parameters to the query plan */ +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int rc = SQLITE_OK; + int iCell = 0; + sqlite3_stmt *pStmt; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + freeCursorConstraints(pCsr); + sqlite3_free(pCsr->aPoint); + pStmt = pCsr->pReadAux; + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->pReadAux = pStmt; + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && idxNum<=3 ){ + RtreeCoord bbox[4]; + RtreeConstraint *p; + assert( argc==1 ); + geopolyBBox(0, argv[0], bbox, &rc); + if( rc ){ + goto geopoly_filter_end; + } + pCsr->aConstraint = p = sqlite3_malloc(sizeof(RtreeConstraint)*4); + pCsr->nConstraint = 4; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*4); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + if( idxNum==2 ){ + /* Overlap query */ + p->op = 'B'; + p->iCoord = 0; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 1; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 2; + p->u.rValue = bbox[3].f; + p++; + p->op = 'D'; + p->iCoord = 3; + p->u.rValue = bbox[2].f; + }else{ + /* Within query */ + p->op = 'D'; + p->iCoord = 0; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 1; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 2; + p->u.rValue = bbox[2].f; + p++; + p->op = 'B'; + p->iCoord = 3; + p->u.rValue = bbox[3].f; + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto geopoly_filter_end; + } + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + +geopoly_filter_end: + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 "rowid" Direct lookup by rowid. +** 2 "rtree" R-tree overlap query using geopoly_overlap() +** 3 "rtree" R-tree within query using geopoly_within() +** 4 "fullscan" full-table scan. +** ------------------------------------------------ +*/ +static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + int iRowidTerm = -1; + int iFuncTerm = -1; + int idxNum = 0; + + for(ii=0; iinConstraint; ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + if( !p->usable ) continue; + if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + iRowidTerm = ii; + break; + } + if( p->iColumn==0 && p->op>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + /* p->op==SQLITE_INDEX_CONSTRAINT_FUNCTION for geopoly_overlap() + ** p->op==(SQLITE_INDEX_CONTRAINT_FUNCTION+1) for geopoly_within(). + ** See geopolyFindFunction() */ + iFuncTerm = ii; + idxNum = p->op - SQLITE_INDEX_CONSTRAINT_FUNCTION + 2; + } + } + + if( iRowidTerm>=0 ){ + pIdxInfo->idxNum = 1; + pIdxInfo->idxStr = "rowid"; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + if( iFuncTerm>=0 ){ + pIdxInfo->idxNum = idxNum; + pIdxInfo->idxStr = "rtree"; + pIdxInfo->aConstraintUsage[iFuncTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iFuncTerm].omit = 0; + pIdxInfo->estimatedCost = 300.0; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->idxNum = 4; + pIdxInfo->idxStr = "fullscan"; + pIdxInfo->estimatedCost = 3000000.0; + pIdxInfo->estimatedRows = 100000; + return SQLITE_OK; +} + + +/* +** GEOPOLY virtual table module xColumn method. +*/ +static int geopolyColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 && sqlite3_vtab_nochange(ctx) ) return SQLITE_OK; + if( i<=pRtree->nAux ){ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pReadAux, i+2)); + } + return SQLITE_OK; +} + + +/* +** The xUpdate method for GEOPOLY module virtual tables. +** +** For DELETE: +** +** argv[0] = the rowid to be deleted +** +** For INSERT: +** +** argv[0] = SQL NULL +** argv[1] = rowid to insert, or an SQL NULL to select automatically +** argv[2] = _shape column +** argv[3] = first application-defined column.... +** +** For UPDATE: +** +** argv[0] = rowid to modify. Never NULL +** argv[1] = rowid after the change. Never NULL +** argv[2] = new value for _shape +** argv[3] = new value for first application-defined column.... +*/ +static int geopolyUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + i64 oldRowid; /* The old rowid */ + int oldRowidValid; /* True if oldRowid is valid */ + i64 newRowid; /* The new rowid */ + int newRowidValid; /* True if newRowid is valid */ + int coordChange = 0; /* Change in coordinates */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + oldRowidValid = sqlite3_value_type(aData[0])!=SQLITE_NULL;; + oldRowid = oldRowidValid ? sqlite3_value_int64(aData[0]) : 0; + newRowidValid = nData>1 && sqlite3_value_type(aData[1])!=SQLITE_NULL; + newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0; + cell.iRowid = newRowid; + + if( nData>1 /* not a DELETE */ + && (!oldRowidValid /* INSERT */ + || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */ + || oldRowid!=newRowid) /* Rowid change */ + ){ + geopolyBBox(0, aData[2], cell.aCoord, &rc); + if( rc ){ + if( rc==SQLITE_ERROR ){ + pVtab->zErrMsg = + sqlite3_mprintf("_shape does not contain a valid polygon"); + } + goto geopoly_update_end; + } + coordChange = 1; + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( newRowidValid && (!oldRowidValid || oldRowid!=newRowid) ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + } + } + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( rc==SQLITE_OK && (nData==1 || (coordChange && oldRowidValid)) ){ + rc = rtreeDeleteRowid(pRtree, oldRowid); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 && coordChange ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + if( !newRowidValid ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + pRtree->iReinsertHeight = -1; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + /* Change the data */ + if( rc==SQLITE_OK && nData>1 ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + int nChange = 0; + sqlite3_bind_int64(pUp, 1, cell.iRowid); + assert( pRtree->nAux>=1 ); + if( sqlite3_value_nochange(aData[2]) ){ + sqlite3_bind_null(pUp, 2); + }else{ + sqlite3_bind_value(pUp, 2, aData[2]); + nChange = 1; + } + for(jj=1; jjnAux; jj++){ + nChange++; + sqlite3_bind_value(pUp, jj+2, aData[jj+2]); + } + if( nChange ){ + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } + } + +geopoly_update_end: + rtreeRelease(pRtree); + return rc; +} + +/* +** Report that geopoly_overlap() is an overloaded function suitable +** for use in xBestIndex. +*/ +static int geopolyFindFunction( + sqlite3_vtab *pVtab, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + if( sqlite3_stricmp(zName, "geopoly_overlap")==0 ){ + *pxFunc = geopolyOverlapFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION; + } + if( sqlite3_stricmp(zName, "geopoly_within")==0 ){ + *pxFunc = geopolyWithinFunc; + *ppArg = 0; + return SQLITE_INDEX_CONSTRAINT_FUNCTION+1; + } + return 0; +} + + +static sqlite3_module geopolyModule = { + 2, /* iVersion */ + geopolyCreate, /* xCreate - create a table */ + geopolyConnect, /* xConnect - connect to an existing table */ + geopolyBestIndex, /* xBestIndex - Determine search strategy */ + rtreeDisconnect, /* xDisconnect - Disconnect from a table */ + rtreeDestroy, /* xDestroy - Drop a table */ + rtreeOpen, /* xOpen - open a cursor */ + rtreeClose, /* xClose - close a cursor */ + geopolyFilter, /* xFilter - configure scan constraints */ + rtreeNext, /* xNext - advance a cursor */ + rtreeEof, /* xEof */ + geopolyColumn, /* xColumn - read data */ + rtreeRowid, /* xRowid - read data */ + geopolyUpdate, /* xUpdate - write data */ + rtreeBeginTransaction, /* xBegin - begin transaction */ + rtreeEndTransaction, /* xSync - sync transaction */ + rtreeEndTransaction, /* xCommit - commit transaction */ + rtreeEndTransaction, /* xRollback - rollback transaction */ + geopolyFindFunction, /* xFindFunction - function overloading */ + rtreeRename, /* xRename - rename the table */ + rtreeSavepoint, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ +}; + +static int sqlite3_geopoly_init(sqlite3 *db){ + int rc = SQLITE_OK; + static const struct { + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + int nArg; + const char *zName; + } aFunc[] = { + { geopolyAreaFunc, 1, "geopoly_area" }, + { geopolyBlobFunc, 1, "geopoly_blob" }, + { geopolyJsonFunc, 1, "geopoly_json" }, + { geopolySvgFunc, -1, "geopoly_svg" }, + { geopolyWithinFunc, 2, "geopoly_within" }, + { geopolyContainsPointFunc, 3, "geopoly_contains_point" }, + { geopolyOverlapFunc, 2, "geopoly_overlap" }, + { geopolyDebugFunc, 1, "geopoly_debug" }, + { geopolyBBoxFunc, 1, "geopoly_bbox" }, + { geopolyXformFunc, 7, "geopoly_xform" }, + }; + static const struct { + void (*xStep)(sqlite3_context*,int,sqlite3_value**); + void (*xFinal)(sqlite3_context*); + const char *zName; + } aAgg[] = { + { geopolyBBoxStep, geopolyBBoxFinal, "geopoly_group_bbox" }, + }; + int i; + for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt ) sqlite3BtreeBeginTrans(pBt, 1); + if( pBt ) sqlite3BtreeBeginTrans(pBt, 1, 0); } return SQLITE_OK; } @@ -187734,2436 +196985,6 @@ SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p){ #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ /************** End of sqlite3session.c **************************************/ -/************** Begin file json1.c *******************************************/ -/* -** 2015-08-12 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This SQLite extension implements JSON functions. The interface is -** modeled after MySQL JSON functions: -** -** https://dev.mysql.com/doc/refman/5.7/en/json.html -** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) -*/ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) -#if !defined(SQLITEINT_H) -/* #include "sqlite3ext.h" */ -#endif -SQLITE_EXTENSION_INIT1 -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* Mark a function parameter as unused, to suppress nuisance compiler -** warnings. */ -#ifndef UNUSED_PARAM -# define UNUSED_PARAM(X) (void)(X) -#endif - -#ifndef LARGEST_INT64 -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - -/* -** Versions of isspace(), isalnum() and isdigit() to which it is safe -** to pass signed char values. -*/ -#ifdef sqlite3Isdigit - /* Use the SQLite core versions if this routine is part of the - ** SQLite amalgamation */ -# define safe_isdigit(x) sqlite3Isdigit(x) -# define safe_isalnum(x) sqlite3Isalnum(x) -# define safe_isxdigit(x) sqlite3Isxdigit(x) -#else - /* Use the standard library for separate compilation */ -#include /* amalgamator: keep */ -# define safe_isdigit(x) isdigit((unsigned char)(x)) -# define safe_isalnum(x) isalnum((unsigned char)(x)) -# define safe_isxdigit(x) isxdigit((unsigned char)(x)) -#endif - -/* -** Growing our own isspace() routine this way is twice as fast as -** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). -*/ -static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; -#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) - -#ifndef SQLITE_AMALGAMATION - /* Unsigned integer types. These are already defined in the sqliteInt.h, - ** but the definitions need to be repeated for separate compilation. */ - typedef sqlite3_uint64 u64; - typedef unsigned int u32; - typedef unsigned short int u16; - typedef unsigned char u8; -#endif - -/* Objects */ -typedef struct JsonString JsonString; -typedef struct JsonNode JsonNode; -typedef struct JsonParse JsonParse; - -/* An instance of this object represents a JSON string -** under construction. Really, this is a generic string accumulator -** that can be and is used to create strings other than JSON. -*/ -struct JsonString { - sqlite3_context *pCtx; /* Function context - put error messages here */ - char *zBuf; /* Append JSON content here */ - u64 nAlloc; /* Bytes of storage available in zBuf[] */ - u64 nUsed; /* Bytes of zBuf[] currently used */ - u8 bStatic; /* True if zBuf is static space */ - u8 bErr; /* True if an error has been encountered */ - char zSpace[100]; /* Initial static space */ -}; - -/* JSON type values -*/ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 - -/* The "subtype" set for JSON values */ -#define JSON_SUBTYPE 74 /* Ascii for "J" */ - -/* -** Names of the various JSON types: -*/ -static const char * const jsonType[] = { - "null", "true", "false", "integer", "real", "text", "array", "object" -}; - -/* Bit values for the JsonNode.jnFlag field -*/ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.u.iReplace */ -#define JNODE_PATCH 0x10 /* Patch with JsonNode.u.pPatch */ -#define JNODE_APPEND 0x20 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x40 /* Is a label of an object */ - - -/* A single node of parsed JSON -*/ -struct JsonNode { - u8 eType; /* One of the JSON_ type values */ - u8 jnFlags; /* JNODE flags */ - u32 n; /* Bytes of content, or number of sub-nodes */ - union { - const char *zJContent; /* Content for INT, REAL, and STRING */ - u32 iAppend; /* More terms for ARRAY and OBJECT */ - u32 iKey; /* Key for ARRAY objects in json_tree() */ - u32 iReplace; /* Replacement content for JNODE_REPLACE */ - JsonNode *pPatch; /* Node chain of patch for JNODE_PATCH */ - } u; -}; - -/* A completely parsed JSON string -*/ -struct JsonParse { - u32 nNode; /* Number of slots of aNode[] used */ - u32 nAlloc; /* Number of slots of aNode[] allocated */ - JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ - u32 *aUp; /* Index of parent of each node */ - u8 oom; /* Set to true if out of memory */ - u8 nErr; /* Number of errors seen */ - u16 iDepth; /* Nesting depth */ - int nJson; /* Length of the zJson string in bytes */ -}; - -/* -** Maximum nesting depth of JSON for this implementation. -** -** This limit is needed to avoid a stack overflow in the recursive -** descent parser. A depth of 2000 is far deeper than any sane JSON -** should go. -*/ -#define JSON_MAX_DEPTH 2000 - -/************************************************************************** -** Utility routines for dealing with JsonString objects -**************************************************************************/ - -/* Set the JsonString object to an empty string -*/ -static void jsonZero(JsonString *p){ - p->zBuf = p->zSpace; - p->nAlloc = sizeof(p->zSpace); - p->nUsed = 0; - p->bStatic = 1; -} - -/* Initialize the JsonString object -*/ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ - p->pCtx = pCtx; - p->bErr = 0; - jsonZero(p); -} - - -/* Free all allocated memory and reset the JsonString object back to its -** initial state. -*/ -static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); - jsonZero(p); -} - - -/* Report an out-of-memory (OOM) condition -*/ -static void jsonOom(JsonString *p){ - p->bErr = 1; - sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); -} - -/* Enlarge pJson->zBuf so that it can hold at least N more bytes. -** Return zero on success. Return non-zero on an OOM error -*/ -static int jsonGrow(JsonString *p, u32 N){ - u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; - char *zNew; - if( p->bStatic ){ - if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); - if( zNew==0 ){ - jsonOom(p); - return SQLITE_NOMEM; - } - memcpy(zNew, p->zBuf, (size_t)p->nUsed); - p->zBuf = zNew; - p->bStatic = 0; - }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); - return SQLITE_NOMEM; - } - p->zBuf = zNew; - } - p->nAlloc = nTotal; - return SQLITE_OK; -} - -/* Append N bytes from zIn onto the end of the JsonString string. -*/ -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; - memcpy(p->zBuf+p->nUsed, zIn, N); - p->nUsed += N; -} - -/* Append formatted text (not to exceed N bytes) to the JsonString. -*/ -static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ - va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; - va_start(ap, zFormat); - sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); - va_end(ap); - p->nUsed += (int)strlen(p->zBuf+p->nUsed); -} - -/* Append a single character -*/ -static void jsonAppendChar(JsonString *p, char c){ - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; - p->zBuf[p->nUsed++] = c; -} - -/* Append a comma separator to the output buffer, if the previous -** character is not '[' or '{'. -*/ -static void jsonAppendSeparator(JsonString *p){ - char c; - if( p->nUsed==0 ) return; - c = p->zBuf[p->nUsed-1]; - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); -} - -/* Append the N-byte string in zIn to the end of the JsonString string -** under construction. Enclose the string in "..." and escape -** any double-quotes or backslash characters contained within the -** string. -*/ -static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; - p->zBuf[p->nUsed++] = '"'; - for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - }else if( c<=0x1f ){ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - if( aSpecial[c] ){ - c = aSpecial[c]; - goto json_simple_escape; - } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0' + (c>>4); - c = "0123456789abcdef"[c&0xf]; - } - p->zBuf[p->nUsed++] = c; - } - p->zBuf[p->nUsed++] = '"'; - assert( p->nUsednAlloc ); -} - -/* -** Append a function parameter value to the JSON string under -** construction. -*/ -static void jsonAppendValue( - JsonString *p, /* Append to this JSON string */ - sqlite3_value *pValue /* Value to append */ -){ - switch( sqlite3_value_type(pValue) ){ - case SQLITE_NULL: { - jsonAppendRaw(p, "null", 4); - break; - } - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - const char *z = (const char*)sqlite3_value_text(pValue); - u32 n = (u32)sqlite3_value_bytes(pValue); - jsonAppendRaw(p, z, n); - break; - } - case SQLITE_TEXT: { - const char *z = (const char*)sqlite3_value_text(pValue); - u32 n = (u32)sqlite3_value_bytes(pValue); - if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ - jsonAppendRaw(p, z, n); - }else{ - jsonAppendString(p, z, n); - } - break; - } - default: { - if( p->bErr==0 ){ - sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 2; - jsonReset(p); - } - break; - } - } -} - - -/* Make the JSON in p the result of the SQL function. -*/ -static void jsonResult(JsonString *p){ - if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); - } - assert( p->bStatic ); -} - -/************************************************************************** -** Utility routines for dealing with JsonNode and JsonParse objects -**************************************************************************/ - -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonNodeSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} - -/* -** Reclaim all memory allocated by a JsonParse object. But do not -** delete the JsonParse object itself. -*/ -static void jsonParseReset(JsonParse *pParse){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - pParse->nNode = 0; - pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; -} - -/* -** Free a JsonParse object that was obtained from sqlite3_malloc(). -*/ -static void jsonParseFree(JsonParse *pParse){ - jsonParseReset(pParse); - sqlite3_free(pParse); -} - -/* -** Convert the JsonNode pNode into a pure JSON string and -** append to pOut. Subsubstructure is also included. Return -** the number of JsonNode objects that are encoded. -*/ -static void jsonRenderNode( - JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ -){ - if( pNode->jnFlags & (JNODE_REPLACE|JNODE_PATCH) ){ - if( pNode->jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode->u.iReplace]); - return; - } - pNode = pNode->u.pPatch; - } - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - jsonAppendRaw(pOut, "null", 4); - break; - } - case JSON_TRUE: { - jsonAppendRaw(pOut, "true", 4); - break; - } - case JSON_FALSE: { - jsonAppendRaw(pOut, "false", 5); - break; - } - case JSON_STRING: { - if( pNode->jnFlags & JNODE_RAW ){ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - break; - } - /* Fall through into the next case */ - } - case JSON_REAL: - case JSON_INT: { - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - break; - } - case JSON_ARRAY: { - u32 j = 1; - jsonAppendChar(pOut, '['); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, ']'); - break; - } - case JSON_OBJECT: { - u32 j = 1; - jsonAppendChar(pOut, '{'); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - jsonAppendChar(pOut, ':'); - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, '}'); - break; - } - } -} - -/* -** Return a JsonNode and all its descendents as a JSON string. -*/ -static void jsonReturnJson( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); -} - -/* -** Make the JsonNode the return value of the function. -*/ -static void jsonReturn( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - sqlite3_result_null(pCtx); - break; - } - case JSON_TRUE: { - sqlite3_result_int(pCtx, 1); - break; - } - case JSON_FALSE: { - sqlite3_result_int(pCtx, 0); - break; - } - case JSON_INT: { - sqlite3_int64 i = 0; - const char *z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ - unsigned v = *(z++) - '0'; - if( i>=LARGEST_INT64/10 ){ - if( i>LARGEST_INT64/10 ) goto int_as_real; - if( z[0]>='0' && z[0]<='9' ) goto int_as_real; - if( v==9 ) goto int_as_real; - if( v==8 ){ - if( pNode->u.zJContent[0]=='-' ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - goto int_done; - }else{ - goto int_as_real; - } - } - } - i = i*10 + v; - } - if( pNode->u.zJContent[0]=='-' ){ i = -i; } - sqlite3_result_int64(pCtx, i); - int_done: - break; - int_as_real: /* fall through to real */; - } - case JSON_REAL: { - double r; -#ifdef SQLITE_AMALGAMATION - const char *z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); -#else - r = strtod(pNode->u.zJContent, 0); -#endif - sqlite3_result_double(pCtx, r); - break; - } - case JSON_STRING: { -#if 0 /* Never happens because JNODE_RAW is only set by json_set(), - ** json_insert() and json_replace() and those routines do not - ** call jsonReturn() */ - if( pNode->jnFlags & JNODE_RAW ){ - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else -#endif - assert( (pNode->jnFlags & JNODE_RAW)==0 ); - if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ - /* JSON formatted without any backslash-escapes */ - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, - SQLITE_TRANSIENT); - }else{ - /* Translate JSON formatted string into raw text */ - u32 i; - u32 n = pNode->n; - const char *z = pNode->u.zJContent; - char *zOut; - u32 j; - zOut = sqlite3_malloc( n+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } - for(i=1, j=0; i>6)); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - zOut[j++] = (char)(0xe0 | (v>>12)); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - } - }else{ - if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - } - zOut[j++] = c; - } - } - } - zOut[j] = 0; - sqlite3_result_text(pCtx, zOut, j, sqlite3_free); - } - break; - } - case JSON_ARRAY: - case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); - break; - } - } -} - -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - -/* -** A macro to hint to the compiler that a function should not be -** inlined. -*/ -#if defined(__GNUC__) -# define JSON_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define JSON_NOINLINE __declspec(noinline) -#else -# define JSON_NOINLINE -#endif - - -static JSON_NOINLINE int jsonParseAddNodeExpand( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - u32 nNew; - JsonNode *pNew; - assert( pParse->nNode>=pParse->nAlloc ); - if( pParse->oom ) return -1; - nNew = pParse->nAlloc*2 + 10; - pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); - if( pNew==0 ){ - pParse->oom = 1; - return -1; - } - pParse->nAlloc = nNew; - pParse->aNode = pNew; - assert( pParse->nNodenAlloc ); - return jsonParseAddNode(pParse, eType, n, zContent); -} - -/* -** Create a new JsonNode instance based on the arguments and append that -** instance to the JsonParse. Return the index in pParse->aNode[] of the -** new node, or -1 if a memory allocation fails. -*/ -static int jsonParseAddNode( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - JsonNode *p; - if( pParse->nNode>=pParse->nAlloc ){ - return jsonParseAddNodeExpand(pParse, eType, n, zContent); - } - p = &pParse->aNode[pParse->nNode]; - p->eType = (u8)eType; - p->jnFlags = 0; - p->n = n; - p->u.zJContent = zContent; - return pParse->nNode++; -} - -/* -** Return true if z[] begins with 4 (or more) hexadecimal digits -*/ -static int jsonIs4Hex(const char *z){ - int i; - for(i=0; i<4; i++) if( !safe_isxdigit(z[i]) ) return 0; - return 1; -} - -/* -** Parse a single JSON value which begins at pParse->zJson[i]. Return the -** index of the first character past the end of the value parsed. -** -** Return negative for a syntax error. Special cases: return -2 if the -** first non-whitespace character is '}' and return -3 if the first -** non-whitespace character is ']'. -*/ -static int jsonParseValue(JsonParse *pParse, u32 i){ - char c; - u32 j; - int iThis; - int x; - JsonNode *pNode; - const char *z = pParse->zJson; - while( safe_isspace(z[i]) ){ i++; } - if( (c = z[i])=='{' ){ - /* Parse object */ - iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; - for(j=i+1;;j++){ - while( safe_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; - x = jsonParseValue(pParse, j); - if( x<0 ){ - pParse->iDepth--; - if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; - } - if( pParse->oom ) return -1; - pNode = &pParse->aNode[pParse->nNode-1]; - if( pNode->eType!=JSON_STRING ) return -1; - pNode->jnFlags |= JNODE_LABEL; - j = x; - while( safe_isspace(z[j]) ){ j++; } - if( z[j]!=':' ) return -1; - j++; - x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ) return -1; - j = x; - while( safe_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!='}' ) return -1; - break; - } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - return j+1; - }else if( c=='[' ){ - /* Parse array */ - iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; - for(j=i+1;;j++){ - while( safe_isspace(z[j]) ){ j++; } - if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1; - x = jsonParseValue(pParse, j); - pParse->iDepth--; - if( x<0 ){ - if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; - } - j = x; - while( safe_isspace(z[j]) ){ j++; } - c = z[j]; - if( c==',' ) continue; - if( c!=']' ) return -1; - break; - } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - return j+1; - }else if( c=='"' ){ - /* Parse string */ - u8 jnFlags = 0; - j = i+1; - for(;;){ - c = z[j]; - if( (c & ~0x1f)==0 ){ - /* Control characters are not allowed in strings */ - return -1; - } - if( c=='\\' ){ - c = z[++j]; - if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' - || c=='n' || c=='r' || c=='t' - || (c=='u' && jsonIs4Hex(z+j+1)) ){ - jnFlags = JNODE_ESCAPE; - }else{ - return -1; - } - }else if( c=='"' ){ - break; - } - j++; - } - jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]); - if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; - return j+1; - }else if( c=='n' - && strncmp(z+i,"null",4)==0 - && !safe_isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - }else if( c=='t' - && strncmp(z+i,"true",4)==0 - && !safe_isalnum(z[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - }else if( c=='f' - && strncmp(z+i,"false",5)==0 - && !safe_isalnum(z[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - }else if( c=='-' || (c>='0' && c<='9') ){ - /* Parse number */ - u8 seenDP = 0; - u8 seenE = 0; - assert( '-' < '0' ); - if( c<='0' ){ - j = c=='-' ? i+1 : i; - if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1; - } - j = i+1; - for(;; j++){ - c = z[j]; - if( c>='0' && c<='9' ) continue; - if( c=='.' ){ - if( z[j-1]=='-' ) return -1; - if( seenDP ) return -1; - seenDP = 1; - continue; - } - if( c=='e' || c=='E' ){ - if( z[j-1]<'0' ) return -1; - if( seenE ) return -1; - seenDP = seenE = 1; - c = z[j+1]; - if( c=='+' || c=='-' ){ - j++; - c = z[j+1]; - } - if( c<'0' || c>'9' ) return -1; - continue; - } - break; - } - if( z[j-1]<'0' ) return -1; - jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, - j - i, &z[i]); - return j; - }else if( c=='}' ){ - return -2; /* End of {...} */ - }else if( c==']' ){ - return -3; /* End of [...] */ - }else if( c==0 ){ - return 0; /* End of file */ - }else{ - return -1; /* Syntax error */ - } -} - -/* -** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. -** -** pParse is uninitialized when this routine is called. -*/ -static int jsonParse( - JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ -){ - int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; - i = jsonParseValue(pParse, 0); - if( pParse->oom ) i = -1; - if( i>0 ){ - assert( pParse->iDepth==0 ); - while( safe_isspace(zJson[i]) ) i++; - if( zJson[i] ) i = -1; - } - if( i<=0 ){ - if( pCtx!=0 ){ - if( pParse->oom ){ - sqlite3_result_error_nomem(pCtx); - }else{ - sqlite3_result_error(pCtx, "malformed JSON", -1); - } - } - jsonParseReset(pParse); - return 1; - } - return 0; -} - -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. -*/ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); - } - break; - } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); - } - break; - } - default: { - break; - } - } -} - -/* -** Compute the parentage of all nodes in a completed parse. -*/ -static int jsonParseFindParents(JsonParse *pParse){ - u32 *aUp; - assert( pParse->aUp==0 ); - aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); - if( aUp==0 ){ - pParse->oom = 1; - return SQLITE_NOMEM; - } - jsonParseFillInParentage(pParse, 0, 0); - return SQLITE_OK; -} - -/* -** Magic number used for the JSON parse cache in sqlite3_get_auxdata() -*/ -#define JSON_CACHE_ID (-429938) - -/* -** Obtain a complete parse of the JSON found in the first argument -** of the argv array. Use the sqlite3_get_auxdata() cache for this -** parse if it is available. If the cache is not available or if it -** is no longer valid, parse the JSON again and return the new parse, -** and also register the new parse so that it will be available for -** future sqlite3_get_auxdata() calls. -*/ -static JsonParse *jsonParseCached( - sqlite3_context *pCtx, - sqlite3_value **argv -){ - const char *zJson = (const char*)sqlite3_value_text(argv[0]); - int nJson = sqlite3_value_bytes(argv[0]); - JsonParse *p; - if( zJson==0 ) return 0; - p = (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID); - if( p && p->nJson==nJson && memcmp(p->zJson,zJson,nJson)==0 ){ - p->nErr = 0; - return p; /* The cached entry matches, so return it */ - } - p = sqlite3_malloc( sizeof(*p) + nJson + 1 ); - if( p==0 ){ - sqlite3_result_error_nomem(pCtx); - return 0; - } - memset(p, 0, sizeof(*p)); - p->zJson = (char*)&p[1]; - memcpy((char*)p->zJson, zJson, nJson+1); - if( jsonParse(p, pCtx, p->zJson) ){ - sqlite3_free(p); - return 0; - } - p->nJson = nJson; - sqlite3_set_auxdata(pCtx, JSON_CACHE_ID, p, (void(*)(void*))jsonParseFree); - return (JsonParse*)sqlite3_get_auxdata(pCtx, JSON_CACHE_ID); -} - -/* -** Compare the OBJECT label at pNode against zKey,nKey. Return true on -** a match. -*/ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; - } -} - -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); - -/* -** Search along zPath to find the node specified. Return a pointer -** to that node, or NULL if zPath is malformed or if there is no such -** node. -** -** If pApnd!=0, then try to append new nodes to complete zPath if it is -** possible to do so and if no existing node corresponds to zPath. If -** new nodes are appended *pApnd is set to 1. -*/ -static JsonNode *jsonLookupStep( - JsonParse *pParse, /* The JSON to search */ - u32 iRoot, /* Begin the search at this node */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ -){ - u32 i, j, nKey; - const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; - if( zPath[0]==0 ) return pRoot; - if( zPath[0]=='.' ){ - if( pRoot->eType!=JSON_OBJECT ) return 0; - zPath++; - if( zPath[0]=='"' ){ - zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} - nKey = i-1; - if( zPath[i] ){ - i++; - }else{ - *pzErr = zPath; - return 0; - } - }else{ - zKey = zPath; - for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} - nKey = i; - } - if( nKey==0 ){ - *pzErr = zPath; - return 0; - } - j = 1; - for(;;){ - while( j<=pRoot->n ){ - if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ - return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); - } - j++; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( pApnd ){ - u32 iStart, iLabel; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); - zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - pParse->aNode[iLabel].jnFlags |= JNODE_RAW; - } - return pNode; - } - }else if( zPath[0]=='[' && safe_isdigit(zPath[1]) ){ - if( pRoot->eType!=JSON_ARRAY ) return 0; - i = 0; - j = 1; - while( safe_isdigit(zPath[j]) ){ - i = i*10 + zPath[j] - '0'; - j++; - } - if( zPath[j]!=']' ){ - *pzErr = zPath; - return 0; - } - zPath += j + 1; - j = 1; - for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( j<=pRoot->n ){ - return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); - } - if( i==0 && pApnd ){ - u32 iStart; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - } - return pNode; - } - }else{ - *pzErr = zPath; - } - return 0; -} - -/* -** Append content to pParse that will complete zPath. Return a pointer -** to the inserted node, or return NULL if the append fails. -*/ -static JsonNode *jsonLookupAppend( - JsonParse *pParse, /* Append content to the JSON parse */ - const char *zPath, /* Description of content to append */ - int *pApnd, /* Set this flag to 1 */ - const char **pzErr /* Make this point to any syntax error */ -){ - *pApnd = 1; - if( zPath[0]==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; - } - if( zPath[0]=='.' ){ - jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - }else if( strncmp(zPath,"[0]",3)==0 ){ - jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - }else{ - return 0; - } - if( pParse->oom ) return 0; - return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); -} - -/* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). -*/ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); -} - -/* -** Do a node lookup using zPath. Return a pointer to the node on success. -** Return NULL if not found or if there is an error. -** -** On an error, write an error message into pCtx and increment the -** pParse->nErr counter. -** -** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if -** nodes are appended. -*/ -static JsonNode *jsonLookup( - JsonParse *pParse, /* The JSON to search */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx /* Report errors here, if not NULL */ -){ - const char *zErr = 0; - JsonNode *pNode = 0; - char *zMsg; - - if( zPath==0 ) return 0; - if( zPath[0]!='$' ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); - if( zErr==0 ) return pNode; - -lookup_err: - pParse->nErr++; - assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); - if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); - }else{ - sqlite3_result_error_nomem(pCtx); - } - return 0; -} - - -/* -** Report the wrong number of arguments for json_insert(), json_replace() -** or json_set(). -*/ -static void jsonWrongNumArgs( - sqlite3_context *pCtx, - const char *zFuncName -){ - char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", - zFuncName); - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); -} - -/* -** Mark all NULL entries in the Object passed in as JNODE_REMOVE. -*/ -static void jsonRemoveAllNulls(JsonNode *pNode){ - int i, n; - assert( pNode->eType==JSON_OBJECT ); - n = pNode->n; - for(i=2; i<=n; i += jsonNodeSize(&pNode[i])+1){ - switch( pNode[i].eType ){ - case JSON_NULL: - pNode[i].jnFlags |= JNODE_REMOVE; - break; - case JSON_OBJECT: - jsonRemoveAllNulls(&pNode[i]); - break; - } - } -} - - -/**************************************************************************** -** SQL functions used for testing and debugging -****************************************************************************/ - -#ifdef SQLITE_DEBUG -/* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. -*/ -static void jsonParseFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; - - assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; inNode ); - if( argc==2 ){ - const char *zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(p, zPath, 0, ctx); - }else{ - pNode = p->aNode; - } - if( pNode==0 ){ - return; - } - if( pNode->eType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); - } - } - sqlite3_result_int64(ctx, n); -} - -/* -** json_extract(JSON, PATH, ...) -** -** Return the element described by PATH. Return NULL if there is no -** PATH element. If there are multiple PATHs, then return a JSON array -** with the result from each path. Throw an error if the JSON or any PATH -** is malformed. -*/ -static void jsonExtractFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse *p; /* The parse */ - JsonNode *pNode; - const char *zPath; - JsonString jx; - int i; - - if( argc<2 ) return; - p = jsonParseCached(ctx, argv); - if( p==0 ) return; - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '['); - for(i=1; inErr ) break; - if( argc>2 ){ - jsonAppendSeparator(&jx); - if( pNode ){ - jsonRenderNode(pNode, &jx, 0); - }else{ - jsonAppendRaw(&jx, "null", 4); - } - }else if( pNode ){ - jsonReturn(pNode, ctx, 0); - } - } - if( argc>2 && i==argc ){ - jsonAppendChar(&jx, ']'); - jsonResult(&jx); - sqlite3_result_subtype(ctx, JSON_SUBTYPE); - } - jsonReset(&jx); -} - -/* This is the RFC 7396 MergePatch algorithm. -*/ -static JsonNode *jsonMergePatch( - JsonParse *pParse, /* The JSON parser that contains the TARGET */ - u32 iTarget, /* Node of the TARGET in pParse */ - JsonNode *pPatch /* The PATCH */ -){ - u32 i, j; - u32 iRoot; - JsonNode *pTarget; - if( pPatch->eType!=JSON_OBJECT ){ - return pPatch; - } - assert( iTarget>=0 && iTargetnNode ); - pTarget = &pParse->aNode[iTarget]; - assert( (pPatch->jnFlags & JNODE_APPEND)==0 ); - if( pTarget->eType!=JSON_OBJECT ){ - jsonRemoveAllNulls(pPatch); - return pPatch; - } - iRoot = iTarget; - for(i=1; in; i += jsonNodeSize(&pPatch[i+1])+1){ - u32 nKey; - const char *zKey; - assert( pPatch[i].eType==JSON_STRING ); - assert( pPatch[i].jnFlags & JNODE_LABEL ); - nKey = pPatch[i].n; - zKey = pPatch[i].u.zJContent; - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - for(j=1; jn; j += jsonNodeSize(&pTarget[j+1])+1 ){ - assert( pTarget[j].eType==JSON_STRING ); - assert( pTarget[j].jnFlags & JNODE_LABEL ); - assert( (pPatch[i].jnFlags & JNODE_RAW)==0 ); - if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){ - if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break; - if( pPatch[i+1].eType==JSON_NULL ){ - pTarget[j+1].jnFlags |= JNODE_REMOVE; - }else{ - JsonNode *pNew = jsonMergePatch(pParse, iTarget+j+1, &pPatch[i+1]); - if( pNew==0 ) return 0; - pTarget = &pParse->aNode[iTarget]; - if( pNew!=&pTarget[j+1] ){ - pTarget[j+1].u.pPatch = pNew; - pTarget[j+1].jnFlags |= JNODE_PATCH; - } - } - break; - } - } - if( j>=pTarget->n && pPatch[i+1].eType!=JSON_NULL ){ - int iStart, iPatch; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - jsonParseAddNode(pParse, JSON_STRING, nKey, zKey); - iPatch = jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - if( pParse->oom ) return 0; - jsonRemoveAllNulls(pPatch); - pTarget = &pParse->aNode[iTarget]; - pParse->aNode[iRoot].jnFlags |= JNODE_APPEND; - pParse->aNode[iRoot].u.iAppend = iStart - iRoot; - iRoot = iStart; - pParse->aNode[iPatch].jnFlags |= JNODE_PATCH; - pParse->aNode[iPatch].u.pPatch = &pPatch[i+1]; - } - } - return pTarget; -} - -/* -** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON -** object that is the result of running the RFC 7396 MergePatch() algorithm -** on the two arguments. -*/ -static void jsonPatchFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The JSON that is being patched */ - JsonParse y; /* The patch */ - JsonNode *pResult; /* The result of the merge */ - - UNUSED_PARAM(argc); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - if( jsonParse(&y, ctx, (const char*)sqlite3_value_text(argv[1])) ){ - jsonParseReset(&x); - return; - } - pResult = jsonMergePatch(&x, 0, y.aNode); - assert( pResult!=0 || x.oom ); - if( pResult ){ - jsonReturnJson(pResult, ctx, 0); - }else{ - sqlite3_result_error_nomem(ctx); - } - jsonParseReset(&x); - jsonParseReset(&y); -} - - -/* -** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON -** object that contains all name/value given in arguments. Or if any name -** is not a string or if any value is a BLOB, throw an error. -*/ -static void jsonObjectFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - int i; - JsonString jx; - const char *z; - u32 n; - - if( argc&1 ){ - sqlite3_result_error(ctx, "json_object() requires an even number " - "of arguments", -1); - return; - } - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '{'); - for(i=0; ijnFlags |= JNODE_REMOVE; - } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); - } -remove_done: - jsonParseReset(&x); -} - -/* -** json_replace(JSON, PATH, VALUE, ...) -** -** Replace the value at PATH with VALUE. If PATH does not already exist, -** this routine is a no-op. If JSON or PATH is malformed, throw an error. -*/ -static void jsonReplaceFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - - if( argc<1 ) return; - if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, "replace"); - return; - } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; - if( pNode ){ - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -replace_err: - jsonParseReset(&x); -} - -/* -** json_set(JSON, PATH, VALUE, ...) -** -** Set the value at PATH to VALUE. Create the PATH if it does not already -** exist. Overwrite existing values that do exist. -** If JSON or PATH is malformed, throw an error. -** -** json_insert(JSON, PATH, VALUE, ...) -** -** Create PATH and initialize it to VALUE. If PATH already exists, this -** routine is a no-op. If JSON or PATH is malformed, throw an error. -*/ -static void jsonSetFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - int bApnd; - int bIsSet = *(int*)sqlite3_user_data(ctx); - - if( argc<1 ) return; - if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); - return; - } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ - sqlite3_result_error_nomem(ctx); - goto jsonSetDone; - }else if( x.nErr ){ - goto jsonSetDone; - }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->u.iReplace = i + 1; - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].u.iReplace]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -jsonSetDone: - jsonParseReset(&x); -} - -/* -** json_type(JSON) -** json_type(JSON, PATH) -** -** Return the top-level "type" of a JSON string. Throw an error if -** either the JSON or PATH inputs are not well-formed. -*/ -static void jsonTypeFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - const char *zPath; - JsonNode *pNode; - - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - if( argc==2 ){ - zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(&x, zPath, 0, ctx); - }else{ - pNode = x.aNode; - } - if( pNode ){ - sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); - } - jsonParseReset(&x); -} - -/* -** json_valid(JSON) -** -** Return 1 if JSON is a well-formed JSON string according to RFC-7159. -** Return 0 otherwise. -*/ -static void jsonValidFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - int rc = 0; - - UNUSED_PARAM(argc); - if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){ - rc = 1; - } - jsonParseReset(&x); - sqlite3_result_int(ctx, rc); -} - - -/**************************************************************************** -** Aggregate SQL function implementations -****************************************************************************/ -/* -** json_group_array(VALUE) -** -** Return a JSON array composed of all values in the aggregate. -*/ -static void jsonArrayStep( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonString *pStr; - UNUSED_PARAM(argc); - pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); - if( pStr ){ - if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); - jsonAppendChar(pStr, '['); - }else{ - jsonAppendChar(pStr, ','); - pStr->pCtx = ctx; - } - jsonAppendValue(pStr, argv[0]); - } -} -static void jsonArrayFinal(sqlite3_context *ctx){ - JsonString *pStr; - pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); - if( pStr ){ - pStr->pCtx = ctx; - jsonAppendChar(pStr, ']'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); - }else{ - sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); - pStr->bStatic = 1; - } - }else{ - sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); - } - sqlite3_result_subtype(ctx, JSON_SUBTYPE); -} - -/* -** json_group_obj(NAME,VALUE) -** -** Return a JSON object composed of all names and values in the aggregate. -*/ -static void jsonObjectStep( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonString *pStr; - const char *z; - u32 n; - UNUSED_PARAM(argc); - pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); - if( pStr ){ - if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); - jsonAppendChar(pStr, '{'); - }else{ - jsonAppendChar(pStr, ','); - pStr->pCtx = ctx; - } - z = (const char*)sqlite3_value_text(argv[0]); - n = (u32)sqlite3_value_bytes(argv[0]); - jsonAppendString(pStr, z, n); - jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); - } -} -static void jsonObjectFinal(sqlite3_context *ctx){ - JsonString *pStr; - pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); - if( pStr ){ - jsonAppendChar(pStr, '}'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); - }else{ - sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); - pStr->bStatic = 1; - } - }else{ - sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); - } - sqlite3_result_subtype(ctx, JSON_SUBTYPE); -} - - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/**************************************************************************** -** The json_each virtual table -****************************************************************************/ -typedef struct JsonEachCursor JsonEachCursor; -struct JsonEachCursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - u32 iRowid; /* The rowid */ - u32 iBegin; /* The first node of the scan */ - u32 i; /* Index in sParse.aNode[] of current row */ - u32 iEnd; /* EOF when i equals or exceeds this value */ - u8 eType; /* Type of top-level element */ - u8 bRecursive; /* True for json_tree(). False for json_each() */ - char *zJson; /* Input JSON */ - char *zRoot; /* Path by which to filter zJson */ - JsonParse sParse; /* Parse of the input JSON */ -}; - -/* Constructor for the json_each virtual table */ -static int jsonEachConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - sqlite3_vtab *pNew; - int rc; - -/* Column numbers */ -#define JEACH_KEY 0 -#define JEACH_VALUE 1 -#define JEACH_TYPE 2 -#define JEACH_ATOM 3 -#define JEACH_ID 4 -#define JEACH_PARENT 5 -#define JEACH_FULLKEY 6 -#define JEACH_PATH 7 -#define JEACH_JSON 8 -#define JEACH_ROOT 9 - - UNUSED_PARAM(pzErr); - UNUSED_PARAM(argv); - UNUSED_PARAM(argc); - UNUSED_PARAM(pAux); - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," - "json HIDDEN,root HIDDEN)"); - if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); - } - return rc; -} - -/* destructor for json_each virtual table */ -static int jsonEachDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); - return SQLITE_OK; -} - -/* constructor for a JsonEachCursor object for json_each(). */ -static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - JsonEachCursor *pCur; - - UNUSED_PARAM(p); - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - *ppCursor = &pCur->base; - return SQLITE_OK; -} - -/* constructor for a JsonEachCursor object for json_tree(). */ -static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - int rc = jsonEachOpenEach(p, ppCursor); - if( rc==SQLITE_OK ){ - JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; - pCur->bRecursive = 1; - } - return rc; -} - -/* Reset a JsonEachCursor back to its original state. Free any memory -** held. */ -static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); - sqlite3_free(p->zRoot); - jsonParseReset(&p->sParse); - p->iRowid = 0; - p->i = 0; - p->iEnd = 0; - p->eType = 0; - p->zJson = 0; - p->zRoot = 0; -} - -/* Destructor for a jsonEachCursor object */ -static int jsonEachClose(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - jsonEachCursorReset(p); - sqlite3_free(cur); - return SQLITE_OK; -} - -/* Return TRUE if the jsonEachCursor object has been advanced off the end -** of the JSON object */ -static int jsonEachEof(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - return p->i >= p->iEnd; -} - -/* Advance the cursor to the next element for json_tree() */ -static int jsonEachNext(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->bRecursive ){ - if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; - p->i++; - p->iRowid++; - if( p->iiEnd ){ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ){ - if( iUp==p->i-1 ){ - pUp->u.iKey = 0; - }else{ - pUp->u.iKey++; - } - } - } - }else{ - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; - } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; - } - } - } - return SQLITE_OK; -} - -/* Append the name of the path for element i to pStr -*/ -static void jsonEachComputePath( - JsonEachCursor *p, /* The cursor */ - JsonString *pStr, /* Write the path here */ - u32 i /* Path to this element */ -){ - JsonNode *pNode, *pUp; - u32 iUp; - if( i==0 ){ - jsonAppendChar(pStr, '$'); - return; - } - iUp = p->sParse.aUp[i]; - jsonEachComputePath(p, pStr, iUp); - pNode = &p->sParse.aNode[i]; - pUp = &p->sParse.aNode[iUp]; - if( pUp->eType==JSON_ARRAY ){ - jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); - }else{ - assert( pUp->eType==JSON_OBJECT ); - if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - assert( pNode->eType==JSON_STRING ); - assert( pNode->jnFlags & JNODE_LABEL ); - jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); - } -} - -/* Return the value of a column */ -static int jsonEachColumn( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ -){ - JsonEachCursor *p = (JsonEachCursor*)cur; - JsonNode *pThis = &p->sParse.aNode[p->i]; - switch( i ){ - case JEACH_KEY: { - if( p->i==0 ) break; - if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); - }else if( p->eType==JSON_ARRAY ){ - u32 iKey; - if( p->bRecursive ){ - if( p->iRowid==0 ) break; - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; - }else{ - iKey = p->iRowid; - } - sqlite3_result_int64(ctx, (sqlite3_int64)iKey); - } - break; - } - case JEACH_VALUE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); - break; - } - case JEACH_TYPE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); - break; - } - case JEACH_ATOM: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); - break; - } - case JEACH_ID: { - sqlite3_result_int64(ctx, - (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); - break; - } - case JEACH_PARENT: { - if( p->i>p->iBegin && p->bRecursive ){ - sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); - } - break; - } - case JEACH_FULLKEY: { - JsonString x; - jsonInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else if( p->eType==JSON_OBJECT ){ - jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); - } - } - jsonResult(&x); - break; - } - case JEACH_PATH: { - if( p->bRecursive ){ - JsonString x; - jsonInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); - break; - } - /* For json_each() path and root are the same so fall through - ** into the root case */ - } - default: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); - break; - } - case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); - break; - } - } - return SQLITE_OK; -} - -/* Return the current rowid value */ -static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - JsonEachCursor *p = (JsonEachCursor*)cur; - *pRowid = p->iRowid; - return SQLITE_OK; -} - -/* The query strategy is to look for an equality constraint on the json -** column. Without such a constraint, the table cannot operate. idxNum is -** 1 if the constraint is found, 3 if the constraint and zRoot are found, -** and 0 otherwise. -*/ -static int jsonEachBestIndex( - sqlite3_vtab *tab, - sqlite3_index_info *pIdxInfo -){ - int i; - int jsonIdx = -1; - int rootIdx = -1; - const struct sqlite3_index_constraint *pConstraint; - - UNUSED_PARAM(tab); - pConstraint = pIdxInfo->aConstraint; - for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - switch( pConstraint->iColumn ){ - case JEACH_JSON: jsonIdx = i; break; - case JEACH_ROOT: rootIdx = i; break; - default: /* no-op */ break; - } - } - if( jsonIdx<0 ){ - pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = 1e99; - }else{ - pIdxInfo->estimatedCost = 1.0; - pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; - pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; - if( rootIdx<0 ){ - pIdxInfo->idxNum = 1; - }else{ - pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2; - pIdxInfo->aConstraintUsage[rootIdx].omit = 1; - pIdxInfo->idxNum = 3; - } - } - return SQLITE_OK; -} - -/* Start a search on a new JSON string */ -static int jsonEachFilter( - sqlite3_vtab_cursor *cur, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - JsonEachCursor *p = (JsonEachCursor*)cur; - const char *z; - const char *zRoot = 0; - sqlite3_int64 n; - - UNUSED_PARAM(idxStr); - UNUSED_PARAM(argc); - jsonEachCursorReset(p); - if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); - if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ - int rc = SQLITE_NOMEM; - if( p->sParse.oom==0 ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); - if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; - } - jsonEachCursorReset(p); - return rc; - }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ - jsonEachCursorReset(p); - return SQLITE_NOMEM; - }else{ - JsonNode *pNode = 0; - if( idxNum==3 ){ - const char *zErr = 0; - zRoot = (const char*)sqlite3_value_text(argv[1]); - if( zRoot==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[1]); - p->zRoot = sqlite3_malloc64( n+1 ); - if( p->zRoot==0 ) return SQLITE_NOMEM; - memcpy(p->zRoot, zRoot, (size_t)n+1); - if( zRoot[0]!='$' ){ - zErr = zRoot; - }else{ - pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); - } - if( zErr ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); - jsonEachCursorReset(p); - return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; - }else if( pNode==0 ){ - return SQLITE_OK; - } - }else{ - pNode = p->sParse.aNode; - } - p->iBegin = p->i = (int)(pNode - p->sParse.aNode); - p->eType = pNode->eType; - if( p->eType>=JSON_ARRAY ){ - pNode->u.iKey = 0; - p->iEnd = p->i + pNode->n + 1; - if( p->bRecursive ){ - p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; - if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ - p->i--; - } - }else{ - p->i++; - } - }else{ - p->iEnd = p->i+1; - } - } - return SQLITE_OK; -} - -/* The methods of the json_each virtual table */ -static sqlite3_module jsonEachModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenEach, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ -}; - -/* The methods of the json_tree virtual table. */ -static sqlite3_module jsonTreeModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenTree, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ -}; -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/**************************************************************************** -** The following routines are the only publically visible identifiers in this -** file. Call the following routines in order to register the various SQL -** functions and the virtual table implemented by this file. -****************************************************************************/ - -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ - int rc = SQLITE_OK; - unsigned int i; - static const struct { - const char *zName; - int nArg; - int flag; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aFunc[] = { - { "json", 1, 0, jsonRemoveFunc }, - { "json_array", -1, 0, jsonArrayFunc }, - { "json_array_length", 1, 0, jsonArrayLengthFunc }, - { "json_array_length", 2, 0, jsonArrayLengthFunc }, - { "json_extract", -1, 0, jsonExtractFunc }, - { "json_insert", -1, 0, jsonSetFunc }, - { "json_object", -1, 0, jsonObjectFunc }, - { "json_patch", 2, 0, jsonPatchFunc }, - { "json_quote", 1, 0, jsonQuoteFunc }, - { "json_remove", -1, 0, jsonRemoveFunc }, - { "json_replace", -1, 0, jsonReplaceFunc }, - { "json_set", -1, 1, jsonSetFunc }, - { "json_type", 1, 0, jsonTypeFunc }, - { "json_type", 2, 0, jsonTypeFunc }, - { "json_valid", 1, 0, jsonValidFunc }, - -#if SQLITE_DEBUG - /* DEBUG and TESTING functions */ - { "json_parse", 1, 0, jsonParseFunc }, - { "json_test1", 1, 0, jsonTest1Func }, -#endif - }; - static const struct { - const char *zName; - int nArg; - void (*xStep)(sqlite3_context*,int,sqlite3_value**); - void (*xFinal)(sqlite3_context*); - } aAgg[] = { - { "json_group_array", 1, jsonArrayStep, jsonArrayFinal }, - { "json_group_object", 2, jsonObjectStep, jsonObjectFinal }, - }; -#ifndef SQLITE_OMIT_VIRTUALTABLE - static const struct { - const char *zName; - sqlite3_module *pModule; - } aMod[] = { - { "json_each", &jsonEachModule }, - { "json_tree", &jsonTreeModule }, - }; -#endif - for(i=0; i ** @@ -190650,7 +197471,7 @@ struct Fts5ExtensionApi { ** extra data to the FTS index or require FTS5 to query for multiple terms, ** so it is efficient in terms of disk space and query speed. However, it ** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: +** token "first" is substituted for "1st" by the tokenizer, then the query: ** ** ** ... MATCH '1s*' @@ -191542,9 +198363,12 @@ static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); /************************************************************************** ** Interface to automatically generated code in fts5_unicode2.c. */ -static int sqlite3Fts5UnicodeIsalnum(int c); static int sqlite3Fts5UnicodeIsdiacritic(int c); static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); + +static int sqlite3Fts5UnicodeCatParse(const char*, u8*); +static int sqlite3Fts5UnicodeCategory(int iCode); +static void sqlite3Fts5UnicodeAscii(u8*, u8*); /* ** End of interface to code in fts5_unicode2.c. **************************************************************************/ @@ -191720,6 +198544,7 @@ typedef union { #define fts5YY_MIN_REDUCE 83 #define fts5YY_MAX_REDUCE 110 /************* End control #defines *******************************************/ +#define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) /* Define the fts5yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -192279,11 +199104,11 @@ static fts5YYACTIONTYPE fts5yy_find_shift_action( do{ i = fts5yy_shift_ofst[stateno]; assert( i>=0 ); - assert( i+fts5YYNFTS5TOKEN<=(int)sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]) ); + /* assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD ); */ assert( iLookAhead!=fts5YYNOCODE ); assert( iLookAhead < fts5YYNFTS5TOKEN ); i += iLookAhead; - if( fts5yy_lookahead[i]!=iLookAhead ){ + if( i>=fts5YY_NLOOKAHEAD || fts5yy_lookahead[i]!=iLookAhead ){ #ifdef fts5YYFALLBACK fts5YYCODETYPE iFallback; /* Fallback token */ if( iLookAhead=fts5YY_ACTTAB_COUNT j0 ){ #ifndef NDEBUG @@ -192333,7 +199159,7 @@ static fts5YYACTIONTYPE fts5yy_find_shift_action( ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. */ -static int fts5yy_find_reduce_action( +static fts5YYACTIONTYPE fts5yy_find_reduce_action( fts5YYACTIONTYPE stateno, /* Current state number */ fts5YYCODETYPE iLookAhead /* The look-ahead token */ ){ @@ -192501,7 +199327,7 @@ static fts5YYACTIONTYPE fts5yy_reduce( sqlite3Fts5ParserCTX_PDECL /* %extra_context */ ){ int fts5yygoto; /* The next state */ - int fts5yyact; /* The next action */ + fts5YYACTIONTYPE fts5yyact; /* The next action */ fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */ int fts5yysize; /* Amount to pop the stack */ sqlite3Fts5ParserARG_FETCH @@ -192857,12 +199683,12 @@ static void sqlite3Fts5Parser( do{ assert( fts5yyact==fts5yypParser->fts5yytos->stateno ); - fts5yyact = fts5yy_find_shift_action(fts5yymajor,fts5yyact); + fts5yyact = fts5yy_find_shift_action((fts5YYCODETYPE)fts5yymajor,fts5yyact); if( fts5yyact >= fts5YY_MIN_REDUCE ){ fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyact-fts5YY_MIN_REDUCE,fts5yymajor, fts5yyminor sqlite3Fts5ParserCTX_PARAM); }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ - fts5yy_shift(fts5yypParser,fts5yyact,fts5yymajor,fts5yyminor); + fts5yy_shift(fts5yypParser,fts5yyact,(fts5YYCODETYPE)fts5yymajor,fts5yyminor); #ifndef fts5YYNOERRORRECOVERY fts5yypParser->fts5yyerrcnt--; #endif @@ -192990,6 +199816,21 @@ static void sqlite3Fts5Parser( return; } +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +static int sqlite3Fts5ParserFallback(int iToken){ +#ifdef fts5YYFALLBACK + if( iToken<(int)(sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0])) ){ + return fts5yyFallback[iToken]; + } +#else + (void)iToken; +#endif + return 0; +} + /* ** 2014 May 31 ** @@ -195100,6 +201941,7 @@ static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); /* #include */ static void sqlite3Fts5ParserTrace(FILE*, char*); #endif +static int sqlite3Fts5ParserFallback(int); struct Fts5Expr { @@ -197604,14 +204446,19 @@ static void fts5ExprIsAlnum( sqlite3_value **apVal /* Function arguments */ ){ int iCode; + u8 aArr[32]; if( nArg!=1 ){ sqlite3_result_error(pCtx, "wrong number of arguments to function fts5_isalnum", -1 ); return; } + memset(aArr, 0, sizeof(aArr)); + sqlite3Fts5UnicodeCatParse("L*", aArr); + sqlite3Fts5UnicodeCatParse("N*", aArr); + sqlite3Fts5UnicodeCatParse("Co", aArr); iCode = sqlite3_value_int(apVal[0]); - sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode)); + sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory(iCode)]); } static void fts5ExprFold( @@ -197655,10 +204502,12 @@ static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } - /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ + /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and + ** sqlite3Fts5ParserFallback() are unused */ #ifndef NDEBUG (void)sqlite3Fts5ParserTrace; #endif + (void)sqlite3Fts5ParserFallback; return rc; } @@ -203706,7 +210555,10 @@ static int sqlite3Fts5IndexCharlenToBytelen( for(i=0; i=nByte ) return 0; /* Input contains fewer than nChar chars */ if( (unsigned char)p[n++]>=0xc0 ){ - while( (p[n] & 0xc0)==0x80 ) n++; + while( (p[n] & 0xc0)==0x80 ){ + n++; + if( n>=nByte ) break; + } } } return n; @@ -205231,7 +212083,7 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ case FTS5_SAVEPOINT: assert( p->ts.eState==1 ); assert( iSavepoint>=0 ); - assert( iSavepoint>p->ts.iSavepoint ); + assert( iSavepoint>=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint; break; @@ -206156,6 +213008,13 @@ static int fts5FilterMethod( assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); + if( pTab->pSortCsr->bDesc ){ + pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid; + }else{ + pCsr->iLastRowid = pTab->pSortCsr->iLastRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid; + } pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); @@ -207586,7 +214445,7 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2018-06-04 19:24:41 c7ee0833225bfd8c5ec2f9bf62b97c4e04d03bd9566366d5221ac8fb199a87ca", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2018-09-25 19:08:10 fb90e7189ae6d62e77ba3a308ca5d683f90bbe633cf681865365b8e92792d1c7", -1, SQLITE_TRANSIENT); } static int fts5Init(sqlite3 *db){ @@ -209074,6 +215933,8 @@ struct Unicode61Tokenizer { int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ int nException; int *aiException; + + unsigned char aCategory[32]; /* True for token char categories */ }; static int fts5UnicodeAddExceptions( @@ -209098,7 +215959,7 @@ static int fts5UnicodeAddExceptions( if( iCode<128 ){ p->aTokenChar[iCode] = (unsigned char)bTokenChars; }else{ - bToken = sqlite3Fts5UnicodeIsalnum(iCode); + bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]; assert( (bToken==0 || bToken==1) ); assert( (bTokenChars==0 || bTokenChars==1) ); if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ @@ -209159,6 +216020,21 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){ return; } +static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){ + const char *z = zCat; + + while( *z ){ + while( *z==' ' || *z=='\t' ) z++; + if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){ + return SQLITE_ERROR; + } + while( *z!=' ' && *z!='\t' && *z!='\0' ) z++; + } + + sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar); + return SQLITE_OK; +} + /* ** Create a "unicode61" tokenizer. */ @@ -209177,15 +216053,28 @@ static int fts5UnicodeCreate( }else{ p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); if( p ){ + const char *zCat = "L* N* Co"; int i; memset(p, 0, sizeof(Unicode61Tokenizer)); - memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); + p->bRemoveDiacritic = 1; p->nFold = 64; p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } + + /* Search for a "categories" argument */ + for(i=0; rc==SQLITE_OK && iaCategory[sqlite3Fts5UnicodeCategory(iCode)] + ^ fts5UnicodeIsException(p, iCode) + ); } static int fts5UnicodeTokenize( @@ -210098,135 +216993,6 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ /* #include */ -/* -** Return true if the argument corresponds to a unicode codepoint -** classified as either a letter or a number. Otherwise false. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -static int sqlite3Fts5UnicodeIsalnum(int c){ - /* Each unsigned integer in the following array corresponds to a contiguous - ** range of unicode codepoints that are not either letters or numbers (i.e. - ** codepoints for which this function should return 0). - ** - ** The most significant 22 bits in each 32-bit value contain the first - ** codepoint in the range. The least significant 10 bits are used to store - ** the size of the range (always at least 1). In other words, the value - ** ((C<<22) + N) represents a range of N codepoints starting with codepoint - ** C. It is not possible to represent a range larger than 1023 codepoints - ** using this format. - */ - static const unsigned int aEntry[] = { - 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, - 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, - 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, - 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, - 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, - 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, - 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, - 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, - 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, - 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, - 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, - 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, - 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, - 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, - 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, - 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, - 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, - 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, - 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, - 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, - 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, - 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, - 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, - 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, - 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, - 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, - 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, - 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, - 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, - 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, - 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, - 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, - 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, - 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, - 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, - 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, - 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, - 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, - 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, - 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, - 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, - 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, - 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, - 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, - 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, - 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, - 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, - 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, - 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, - 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, - 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, - 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, - 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, - 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, - 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, - 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, - 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, - 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, - 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, - 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, - 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, - 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, - 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, - 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, - 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, - 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, - 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, - 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, - 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, - 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, - 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, - 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, - 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, - 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, - 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, - 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, - 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, - 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, - 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, - 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, - 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, - 0x380400F0, - }; - static const unsigned int aAscii[4] = { - 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, - }; - - if( (unsigned int)c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( (unsigned int)c<(1<<22) ){ - unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes = 0; - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aEntry[iTest] ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - assert( aEntry[0]=aEntry[iRes] ); - return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); - } - return 1; -} /* @@ -210439,6 +217205,539 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ return ret; } + +#if 0 +static int sqlite3Fts5UnicodeNCat(void) { + return 32; +} +#endif + +static int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ + aArray[0] = 1; + switch( zCat[0] ){ + case 'C': + switch( zCat[1] ){ + case 'c': aArray[1] = 1; break; + case 'f': aArray[2] = 1; break; + case 'n': aArray[3] = 1; break; + case 's': aArray[4] = 1; break; + case 'o': aArray[31] = 1; break; + case '*': + aArray[1] = 1; + aArray[2] = 1; + aArray[3] = 1; + aArray[4] = 1; + aArray[31] = 1; + break; + default: return 1; } + break; + + case 'L': + switch( zCat[1] ){ + case 'l': aArray[5] = 1; break; + case 'm': aArray[6] = 1; break; + case 'o': aArray[7] = 1; break; + case 't': aArray[8] = 1; break; + case 'u': aArray[9] = 1; break; + case 'C': aArray[30] = 1; break; + case '*': + aArray[5] = 1; + aArray[6] = 1; + aArray[7] = 1; + aArray[8] = 1; + aArray[9] = 1; + aArray[30] = 1; + break; + default: return 1; } + break; + + case 'M': + switch( zCat[1] ){ + case 'c': aArray[10] = 1; break; + case 'e': aArray[11] = 1; break; + case 'n': aArray[12] = 1; break; + case '*': + aArray[10] = 1; + aArray[11] = 1; + aArray[12] = 1; + break; + default: return 1; } + break; + + case 'N': + switch( zCat[1] ){ + case 'd': aArray[13] = 1; break; + case 'l': aArray[14] = 1; break; + case 'o': aArray[15] = 1; break; + case '*': + aArray[13] = 1; + aArray[14] = 1; + aArray[15] = 1; + break; + default: return 1; } + break; + + case 'P': + switch( zCat[1] ){ + case 'c': aArray[16] = 1; break; + case 'd': aArray[17] = 1; break; + case 'e': aArray[18] = 1; break; + case 'f': aArray[19] = 1; break; + case 'i': aArray[20] = 1; break; + case 'o': aArray[21] = 1; break; + case 's': aArray[22] = 1; break; + case '*': + aArray[16] = 1; + aArray[17] = 1; + aArray[18] = 1; + aArray[19] = 1; + aArray[20] = 1; + aArray[21] = 1; + aArray[22] = 1; + break; + default: return 1; } + break; + + case 'S': + switch( zCat[1] ){ + case 'c': aArray[23] = 1; break; + case 'k': aArray[24] = 1; break; + case 'm': aArray[25] = 1; break; + case 'o': aArray[26] = 1; break; + case '*': + aArray[23] = 1; + aArray[24] = 1; + aArray[25] = 1; + aArray[26] = 1; + break; + default: return 1; } + break; + + case 'Z': + switch( zCat[1] ){ + case 'l': aArray[27] = 1; break; + case 'p': aArray[28] = 1; break; + case 's': aArray[29] = 1; break; + case '*': + aArray[27] = 1; + aArray[28] = 1; + aArray[29] = 1; + break; + default: return 1; } + break; + + } + return 0; +} + +static u16 aFts5UnicodeBlock[] = { + 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760, + 1760, 1760, 1760, 1760, 1760, 1763, 1765, + }; +static u16 aFts5UnicodeMap[] = { + 0, 32, 33, 36, 37, 40, 41, 42, 43, 44, + 45, 46, 48, 58, 60, 63, 65, 91, 92, 93, + 94, 95, 96, 97, 123, 124, 125, 126, 127, 160, + 161, 162, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 180, 181, 182, 184, 185, + 186, 187, 188, 191, 192, 215, 216, 223, 247, 248, + 256, 312, 313, 329, 330, 377, 383, 385, 387, 388, + 391, 394, 396, 398, 402, 403, 405, 406, 409, 412, + 414, 415, 417, 418, 423, 427, 428, 431, 434, 436, + 437, 440, 442, 443, 444, 446, 448, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 477, 478, 496, + 497, 498, 499, 500, 503, 505, 506, 564, 570, 572, + 573, 575, 577, 580, 583, 584, 592, 660, 661, 688, + 706, 710, 722, 736, 741, 748, 749, 750, 751, 768, + 880, 884, 885, 886, 890, 891, 894, 900, 902, 903, + 904, 908, 910, 912, 913, 931, 940, 975, 977, 978, + 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072, + 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369, + 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473, + 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545, + 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611, + 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758, + 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791, + 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984, + 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075, + 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210, + 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369, + 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416, + 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482, + 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519, + 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561, + 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622, + 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677, + 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749, + 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790, + 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869, + 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902, + 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947, + 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006, + 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059, + 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134, + 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199, + 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263, + 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302, + 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402, + 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458, + 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544, + 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655, + 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737, + 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773, + 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860, + 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896, + 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967, + 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046, + 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153, + 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190, + 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229, + 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295, + 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704, + 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888, + 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743, + 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906, + 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068, + 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107, + 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160, + 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435, + 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480, + 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679, + 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754, + 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824, + 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978, + 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043, + 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098, + 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168, + 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288, + 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406, + 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616, + 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976, + 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033, + 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118, + 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141, + 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184, + 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219, + 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249, + 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275, + 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317, + 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413, + 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459, + 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484, + 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500, + 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523, + 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597, + 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623, + 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972, + 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180, + 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665, + 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091, + 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, + 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, + 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, + 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, + 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, + 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, + 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, + 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, + 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, + 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, + 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, + 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, + 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, + 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, + 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, + 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, + 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, + 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, + 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, + 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, + 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, + 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, + 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, + 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, + 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, + 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, + 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, + 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, + 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, + 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, + 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, + 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, + 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, + 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, + 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, + 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, + 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, + 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, + 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, + 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, + 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, + 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, + 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, + 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, + 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, + 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, + 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, + 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, + 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, + 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, + 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263, + 311, 320, 373, 377, 394, 400, 464, 509, 640, 672, + 768, 800, 816, 833, 834, 842, 896, 927, 928, 968, + 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103, + 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432, + 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623, + 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912, + 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178, + 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285, + 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416, + 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760, + 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216, + 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, + 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, + 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, + 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, + 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, + 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, + 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, + 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, + 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, + 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, + 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, + 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, + 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, + 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, + 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, + 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, + 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488, + 1, 32, 256, 0, 65533, + }; +static u16 aFts5UnicodeData[] = { + 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53, + 49, 85, 333, 85, 121, 85, 841, 54, 53, 50, + 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61, + 53, 151, 58, 53, 56, 58, 39, 52, 57, 34, + 58, 56, 58, 57, 79, 56, 37, 85, 56, 47, + 39, 51, 111, 53, 745, 57, 233, 773, 57, 261, + 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126, + 126, 73, 69, 137, 37, 73, 37, 105, 101, 73, + 37, 73, 37, 190, 158, 37, 126, 126, 73, 37, + 126, 94, 37, 39, 94, 69, 135, 41, 40, 37, + 41, 40, 37, 41, 40, 37, 542, 37, 606, 37, + 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37, + 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582, + 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596, + 158, 38, 56, 94, 38, 101, 53, 88, 41, 53, + 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105, + 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541, + 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38, + 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76, + 53, 76, 53, 44, 871, 103, 85, 162, 121, 85, + 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684, + 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58, + 204, 70, 76, 58, 140, 71, 333, 103, 90, 39, + 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333, + 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300, + 38, 108, 38, 172, 501, 807, 108, 53, 39, 359, + 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268, + 138, 44, 74, 39, 236, 327, 76, 85, 333, 53, + 38, 199, 231, 44, 74, 263, 71, 711, 231, 39, + 135, 44, 39, 106, 140, 74, 74, 44, 39, 42, + 71, 103, 76, 333, 71, 87, 207, 58, 55, 76, + 42, 199, 71, 711, 231, 71, 71, 71, 44, 106, + 76, 76, 108, 44, 135, 39, 333, 76, 103, 44, + 76, 42, 295, 103, 711, 231, 71, 167, 44, 39, + 106, 172, 76, 42, 74, 44, 39, 71, 76, 333, + 53, 55, 44, 74, 263, 71, 711, 231, 71, 167, + 44, 39, 42, 44, 42, 140, 74, 74, 44, 44, + 42, 71, 103, 76, 333, 58, 39, 207, 44, 39, + 199, 103, 135, 71, 39, 71, 71, 103, 391, 74, + 44, 74, 106, 106, 44, 39, 42, 333, 111, 218, + 55, 58, 106, 263, 103, 743, 327, 167, 39, 108, + 138, 108, 140, 76, 71, 71, 76, 333, 239, 58, + 74, 263, 103, 743, 327, 167, 44, 39, 42, 44, + 170, 44, 74, 74, 76, 74, 39, 71, 76, 333, + 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106, + 44, 39, 42, 71, 76, 333, 207, 58, 199, 74, + 583, 775, 295, 39, 231, 44, 106, 108, 44, 266, + 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268, + 53, 333, 85, 71, 39, 71, 39, 39, 135, 231, + 103, 39, 39, 71, 135, 44, 71, 204, 76, 39, + 167, 38, 204, 333, 135, 39, 122, 501, 58, 53, + 122, 76, 218, 333, 335, 58, 44, 58, 44, 58, + 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42, + 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90, + 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76, + 74, 76, 39, 333, 213, 199, 74, 76, 135, 108, + 39, 106, 71, 234, 103, 140, 423, 44, 74, 76, + 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41, + 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319, + 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151, + 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551, + 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108, + 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76, + 42, 236, 266, 44, 74, 364, 117, 38, 117, 55, + 39, 44, 333, 335, 213, 49, 149, 108, 61, 333, + 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138, + 76, 106, 74, 44, 202, 108, 58, 85, 333, 967, + 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76, + 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44, + 74, 268, 202, 332, 44, 333, 333, 245, 38, 213, + 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44, + 74, 231, 333, 245, 346, 300, 314, 76, 42, 967, + 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415, + 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159, + 266, 268, 74, 76, 181, 333, 103, 333, 967, 198, + 85, 277, 108, 53, 428, 42, 236, 135, 44, 135, + 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260, + 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265, + 261, 265, 197, 201, 261, 41, 41, 41, 94, 229, + 265, 453, 261, 264, 261, 264, 261, 264, 165, 69, + 137, 40, 56, 37, 120, 101, 69, 137, 40, 120, + 133, 69, 137, 120, 261, 169, 120, 101, 69, 137, + 40, 88, 381, 162, 209, 85, 52, 51, 54, 84, + 51, 54, 52, 277, 59, 60, 162, 61, 309, 52, + 51, 149, 80, 117, 57, 54, 50, 373, 57, 53, + 48, 341, 61, 162, 194, 47, 38, 207, 121, 54, + 50, 38, 335, 121, 54, 50, 422, 855, 428, 139, + 44, 107, 396, 90, 41, 154, 41, 90, 37, 105, + 69, 105, 37, 58, 41, 90, 57, 169, 218, 41, + 58, 41, 58, 41, 58, 137, 58, 37, 137, 37, + 135, 37, 90, 69, 73, 185, 94, 101, 58, 57, + 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186, + 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018, + 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666, + 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217, + 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57, + 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281, + 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69, + 254, 105, 37, 94, 37, 94, 165, 70, 105, 37, + 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221, + 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231, + 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52, + 51, 117, 52, 51, 53, 52, 51, 309, 49, 85, + 49, 53, 52, 51, 85, 52, 51, 54, 50, 54, + 50, 54, 50, 54, 50, 181, 38, 341, 81, 858, + 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 90, + 54, 50, 54, 50, 54, 50, 54, 50, 49, 54, + 82, 58, 302, 140, 74, 49, 166, 90, 110, 38, + 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887, + 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178, + 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274, + 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38, + 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333, + 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798, + 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69, + 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382, + 70, 37, 231, 44, 103, 44, 135, 44, 743, 74, + 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74, + 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333, + 903, 268, 85, 743, 364, 74, 53, 935, 108, 42, + 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333, + 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263, + 44, 42, 333, 149, 519, 38, 199, 122, 39, 42, + 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44, + 39, 71, 38, 85, 359, 42, 76, 74, 85, 39, + 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74, + 44, 74, 44, 74, 53, 42, 44, 333, 39, 39, + 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399, + 229, 165, 39, 44, 327, 57, 423, 167, 39, 71, + 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55, + 58, 524, 245, 54, 50, 53, 236, 53, 81, 80, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 85, 54, 50, 149, + 112, 117, 149, 49, 54, 50, 54, 50, 54, 50, + 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34, + 117, 55, 117, 54, 50, 53, 57, 53, 49, 85, + 333, 85, 121, 85, 841, 54, 53, 50, 56, 48, + 56, 837, 54, 57, 50, 57, 54, 50, 53, 54, + 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199, + 103, 87, 57, 56, 58, 87, 58, 153, 90, 98, + 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455, + 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575, + 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263, + 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71, + 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799, + 71, 39, 108, 76, 140, 135, 103, 871, 108, 44, + 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615, + 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655, + 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34, + 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149, + 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383, + 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182, + 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898, + 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236, + 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837, + 841, 229, 581, 841, 837, 41, 73, 41, 73, 137, + 265, 133, 37, 229, 357, 841, 837, 73, 137, 265, + 233, 837, 73, 137, 169, 41, 233, 837, 841, 837, + 841, 837, 841, 837, 841, 837, 841, 837, 841, 901, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71, + 39, 39, 327, 135, 39, 39, 39, 39, 39, 39, + 103, 71, 39, 39, 39, 39, 39, 39, 71, 39, + 135, 231, 135, 135, 39, 327, 551, 103, 167, 551, + 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946, + 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210, + 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266, + 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351, + 34, 3074, 7692, 63, 63, + }; + +static int sqlite3Fts5UnicodeCategory(int iCode) { + int iRes = -1; + int iHi; + int iLo; + int ret; + u16 iKey; + + if( iCode>=(1<<20) ){ + return 0; + } + iLo = aFts5UnicodeBlock[(iCode>>16)]; + iHi = aFts5UnicodeBlock[1+(iCode>>16)]; + iKey = (iCode & 0xFFFF); + while( iHi>iLo ){ + int iTest = (iHi + iLo) / 2; + assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest; + } + } + + if( iRes<0 ) return 0; + if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0; + ret = aFts5UnicodeData[iRes] & 0x1F; + if( ret!=30 ) return ret; + return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9; +} + +static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ + int i = 0; + int iTbl = 0; + while( i<128 ){ + int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; + int n = (aFts5UnicodeData[iTbl] >> 5) + i; + for(; i<128 && iauth.nAuthPW = nPW; rc = sqlite3UserAuthCheckLogin(db, "main", &authLevel); db->auth.authLevel = authLevel; - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); if( rc ){ return rc; /* OOM error, I/O error, etc. */ } diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.h b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.h index 4bf489a..05d11a8 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.h +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3-binding.h @@ -124,9 +124,9 @@ extern "C" { ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.24.0" -#define SQLITE_VERSION_NUMBER 3024000 -#define SQLITE_SOURCE_ID "2018-06-04 19:24:41 c7ee0833225bfd8c5ec2f9bf62b97c4e04d03bd9566366d5221ac8fb199a87ca" +#define SQLITE_VERSION "3.25.2" +#define SQLITE_VERSION_NUMBER 3025002 +#define SQLITE_SOURCE_ID "2018-09-25 19:08:10 fb90e7189ae6d62e77ba3a308ca5d683f90bbe633cf681865365b8e92792d1c7" /* ** CAPI3REF: Run-Time Library Version Numbers @@ -473,6 +473,7 @@ SQLITE_API int sqlite3_exec( */ #define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) #define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -512,6 +513,7 @@ SQLITE_API int sqlite3_exec( #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) #define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) @@ -887,7 +889,8 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the ** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary -** write ahead log and shared memory files used for transaction control +** write ahead log ([WAL file]) and shared memory +** files used for transaction control ** are automatically deleted when the latest connection to the database ** closes. Setting persistent WAL mode causes those files to persist after ** close. Persisting the files is useful when other processes that do not @@ -1073,6 +1076,26 @@ struct sqlite3_io_methods { ** a file lock using the xLock or xShmLock methods of the VFS to wait ** for up to M milliseconds before failing, where M is the single ** unsigned integer parameter. +** +**
  • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provide a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1108,6 +1131,7 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 #define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 #define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -2122,6 +2146,12 @@ struct sqlite3_mem_methods { ** with no schema and no content. The following process works even for ** a badly corrupted database file: **
      +**
    1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. **
    2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); **
    3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); **
    4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); @@ -2270,12 +2300,17 @@ SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); ** program, the value returned reflects the number of rows modified by the ** previous INSERT, UPDATE or DELETE statement within the same trigger. ** -** See also the [sqlite3_total_changes()] interface, the -** [count_changes pragma], and the [changes() SQL function]. -** ** If a separate thread makes changes on the same database connection ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. +** +** See also: +**
        +**
      • the [sqlite3_total_changes()] interface +**
      • the [count_changes pragma] +**
      • the [changes() SQL function] +**
      • the [data_version pragma] +**
      */ SQLITE_API int sqlite3_changes(sqlite3*); @@ -2293,13 +2328,26 @@ SQLITE_API int sqlite3_changes(sqlite3*); ** count, but those made as part of REPLACE constraint resolution are ** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. -** -** See also the [sqlite3_changes()] interface, the -** [count_changes pragma], and the [total_changes() SQL function]. ** +** This the [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. +** ** If a separate thread makes changes on the same database connection ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. +** +** See also: +**
        +**
      • the [sqlite3_changes()] interface +**
      • the [count_changes pragma] +**
      • the [changes() SQL function] +**
      • the [data_version pragma] +**
      • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
      */ SQLITE_API int sqlite3_total_changes(sqlite3*); @@ -3355,13 +3403,24 @@ SQLITE_API sqlite3_int64 sqlite3_uri_int64(const char*, const char*, sqlite3_int ** [database connection] D failed, then the sqlite3_errcode(D) interface ** returns the numeric [result code] or [extended result code] for that ** API call. -** If the most recent API call was successful, -** then the return value from sqlite3_errcode() is undefined. ** ^The sqlite3_extended_errcode() ** interface is the same except that it always returns the ** [extended result code] even when extended result codes are ** disabled. ** +** The values returned by sqlite3_errcode() and/or +** sqlite3_extended_errcode() might change with each API call. +** Except, there are some interfaces that are guaranteed to never +** change the value of the error code. The error-code preserving +** interfaces are: +** +**
        +**
      • sqlite3_errcode() +**
      • sqlite3_extended_errcode() +**
      • sqlite3_errmsg() +**
      • sqlite3_errmsg16() +**
      +** ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language ** text that describes the error, as either UTF-8 or UTF-16 respectively. ** ^(Memory to hold the error message string is managed internally. @@ -4515,11 +4574,25 @@ SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** -** ^(If a memory allocation error occurs during the evaluation of any -** of these routines, a default value is returned. The default value -** is either the integer 0, the floating point number 0.0, or a NULL -** pointer. Subsequent calls to [sqlite3_errcode()] will return -** [SQLITE_NOMEM].)^ +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
        +**
      • sqlite3_column_blob() +**
      • sqlite3_column_text() +**
      • sqlite3_column_text16() +**
      • sqlite3_column_bytes() +**
      • sqlite3_column_bytes16() +**
      +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); @@ -4596,11 +4669,13 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only differences between -** these routines are the text encoding expected for -** the second parameter (the name of the function being created) -** and the presence or absence of a destructor callback for -** the application data pointer. +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database @@ -4646,7 +4721,8 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc ** callback only; NULL pointers must be passed as the xStep and xFinal @@ -4655,15 +4731,24 @@ SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); ** SQL function or aggregate, pass NULL pointers for all three function ** callbacks. ** -** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, -** then it is destructor for the application data pointer. -** The destructor is invoked when the function is deleted, either by being -** overloaded or when the database connection closes.)^ -** ^The destructor is also invoked if the call to -** sqlite3_create_function_v2() fails. -** ^When the destructor callback of the tenth parameter is invoked, it -** is passed a single argument which is a copy of the application data -** pointer which was the fifth parameter to sqlite3_create_function_v2(). +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -4716,6 +4801,18 @@ SQLITE_API int sqlite3_create_function_v2( void (*xFinal)(sqlite3_context*), void(*xDestroy)(void*) ); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -4858,6 +4955,28 @@ SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int6 ** ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
        +**
      • sqlite3_value_blob() +**
      • sqlite3_value_text() +**
      • sqlite3_value_text16() +**
      • sqlite3_value_text16le() +**
      • sqlite3_value_text16be() +**
      • sqlite3_value_bytes() +**
      • sqlite3_value_bytes16() +**
      +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); SQLITE_API double sqlite3_value_double(sqlite3_value*); @@ -6324,6 +6443,7 @@ struct sqlite3_index_info { #define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 #define SQLITE_INDEX_CONSTRAINT_ISNULL 71 #define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 /* ** CAPI3REF: Register A Virtual Table Implementation @@ -7000,6 +7120,7 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files ** METHOD: sqlite3 +** KEYWORDS: {file control} ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -7014,11 +7135,18 @@ SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. ** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes ** a pointer to the underlying [sqlite3_file] object to be written into -** the space pointed to by the 4th parameter. ^The [SQLITE_FCNTL_FILE_POINTER] -** case is a short-circuit path which does not actually invoke the -** underlying sqlite3_io_methods.xFileControl method. +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. ** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error @@ -8837,7 +8965,6 @@ SQLITE_API int sqlite3_system_errno(sqlite3*); /* ** CAPI3REF: Database Snapshot ** KEYWORDS: {snapshot} {sqlite3_snapshot} -** EXPERIMENTAL ** ** An instance of the snapshot object records the state of a [WAL mode] ** database for some specific point in history. @@ -8854,11 +8981,6 @@ SQLITE_API int sqlite3_system_errno(sqlite3*); ** version of the database file so that it is possible to later open a new read ** transaction that sees that historical version of the database rather than ** the most recent version. -** -** The constructor for this object is [sqlite3_snapshot_get()]. The -** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer -** to an historical snapshot (if possible). The destructor for -** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. */ typedef struct sqlite3_snapshot { unsigned char hidden[48]; @@ -8866,7 +8988,7 @@ typedef struct sqlite3_snapshot { /* ** CAPI3REF: Record A Database Snapshot -** EXPERIMENTAL +** CONSTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a ** new [sqlite3_snapshot] object that records the current state of @@ -8882,7 +9004,7 @@ typedef struct sqlite3_snapshot { ** in this case. ** **
        -**
      • The database handle must be in [autocommit mode]. +**
      • The database handle must not be in [autocommit mode]. ** **
      • Schema S of [database connection] D must be a [WAL mode] database. ** @@ -8905,7 +9027,7 @@ typedef struct sqlite3_snapshot { ** to avoid a memory leak. ** ** The [sqlite3_snapshot_get()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( sqlite3 *db, @@ -8915,24 +9037,35 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( /* ** CAPI3REF: Start a read transaction on an historical snapshot -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** -** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a -** read transaction for schema S of -** [database connection] D such that the read transaction -** refers to historical [snapshot] P, rather than the most -** recent change to the database. -** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success -** or an appropriate [error code] if it fails. +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. ** -** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be -** the first operation following the [BEGIN] that takes the schema S -** out of [autocommit mode]. -** ^In other words, schema S must not currently be in -** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the -** database connection D must be out of [autocommit mode]. -** ^A [snapshot] will fail to open if it has been overwritten by a -** [checkpoint]. ** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the ** database connection D does not know that the database file for ** schema S is in [WAL mode]. A database connection might not know @@ -8943,7 +9076,7 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( ** database connection in order to make it ready to use snapshots.) ** ** The [sqlite3_snapshot_open()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( sqlite3 *db, @@ -8953,20 +9086,20 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( /* ** CAPI3REF: Destroy a snapshot -** EXPERIMENTAL +** DESTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. ** The application must eventually free every [sqlite3_snapshot] object ** using this routine to avoid a memory leak. ** ** The [sqlite3_snapshot_free()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** ** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages ** of two valid snapshot handles. @@ -8985,6 +9118,9 @@ SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); ** Otherwise, this API returns a negative value if P1 refers to an older ** snapshot than P2, zero if the two handles refer to the same database ** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, @@ -8993,23 +9129,26 @@ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( /* ** CAPI3REF: Recover snapshots from a wal file -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** -** If all connections disconnect from a database file but do not perform -** a checkpoint, the existing wal file is opened along with the database -** file the next time the database is opened. At this point it is only -** possible to successfully call sqlite3_snapshot_open() to open the most -** recent snapshot of the database (the one at the head of the wal file), -** even though the wal file may contain other valid snapshots for which -** clients have sqlite3_snapshot handles. +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. ** -** This function attempts to scan the wal file associated with database zDb +** This function attempts to scan the WAL file associated with database zDb ** of database handle db and make all valid snapshots available to ** sqlite3_snapshot_open(). It is an error if there is already a read -** transaction open on the database, or if the database is not a wal mode +** transaction open on the database, or if the database is not a WAL mode ** database. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); @@ -9120,7 +9259,7 @@ SQLITE_API int sqlite3_deserialize( ** in the P argument is held in memory obtained from [sqlite3_malloc64()] ** and that SQLite should take ownership of this memory and automatically ** free it when it has finished using it. Without this flag, the caller -** is resposible for freeing any dynamically allocated memory. +** is responsible for freeing any dynamically allocated memory. ** ** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to ** grow the size of the database using calls to [sqlite3_realloc64()]. This @@ -11298,7 +11437,7 @@ struct Fts5ExtensionApi { ** This way, even if the tokenizer does not provide synonyms ** when tokenizing query text (it should not - to do would be ** inefficient), it doesn't matter if the user queries for -** 'first + place' or '1st + place', as there are entires in the +** 'first + place' or '1st + place', as there are entries in the ** FTS index corresponding to both forms of the first token. **
    ** @@ -11326,7 +11465,7 @@ struct Fts5ExtensionApi { ** extra data to the FTS index or require FTS5 to query for multiple terms, ** so it is efficient in terms of disk space and query speed. However, it ** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: +** token "first" is substituted for "1st" by the tokenizer, then the query: ** ** ** ... MATCH '1s*' diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3.go b/vendor/github.com/mattn/go-sqlite3/sqlite3.go index d8fe4b0..942b0b5 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3.go +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3.go @@ -78,8 +78,38 @@ _sqlite3_exec(sqlite3* db, const char* pcmd, long long* rowid, long long* change return rv; } +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY +extern int _sqlite3_step_blocking(sqlite3_stmt *stmt); +extern int _sqlite3_step_row_blocking(sqlite3_stmt* stmt, long long* rowid, long long* changes); +extern int _sqlite3_prepare_v2_blocking(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, const char **pzTail); + static int -_sqlite3_step(sqlite3_stmt* stmt, long long* rowid, long long* changes) +_sqlite3_step_internal(sqlite3_stmt *stmt) +{ + return _sqlite3_step_blocking(stmt); +} + +static int +_sqlite3_step_row_internal(sqlite3_stmt* stmt, long long* rowid, long long* changes) +{ + return _sqlite3_step_row_blocking(stmt, rowid, changes); +} + +static int +_sqlite3_prepare_v2_internal(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, const char **pzTail) +{ + return _sqlite3_prepare_v2_blocking(db, zSql, nBytes, ppStmt, pzTail); +} + +#else +static int +_sqlite3_step_internal(sqlite3_stmt *stmt) +{ + return sqlite3_step(stmt); +} + +static int +_sqlite3_step_row_internal(sqlite3_stmt* stmt, long long* rowid, long long* changes) { int rv = sqlite3_step(stmt); sqlite3* db = sqlite3_db_handle(stmt); @@ -88,6 +118,13 @@ _sqlite3_step(sqlite3_stmt* stmt, long long* rowid, long long* changes) return rv; } +static int +_sqlite3_prepare_v2_internal(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, const char **pzTail) +{ + return sqlite3_prepare_v2(db, zSql, nBytes, ppStmt, pzTail); +} +#endif + void _sqlite3_result_text(sqlite3_context* ctx, const char* s) { sqlite3_result_text(ctx, s, -1, &free); } @@ -119,6 +156,8 @@ int commitHookTrampoline(void*); void rollbackHookTrampoline(void*); void updateHookTrampoline(void*, int, char*, char*, sqlite3_int64); +int authorizerTrampoline(void*, int, char*, char*, char*, char*); + #ifdef SQLITE_LIMIT_WORKER_THREADS # define _SQLITE_HAS_LIMIT # define SQLITE_LIMIT_LENGTH 0 @@ -200,18 +239,57 @@ func Version() (libVersion string, libVersionNumber int, sourceID string) { } const ( + // used by authorizer and pre_update_hook SQLITE_DELETE = C.SQLITE_DELETE SQLITE_INSERT = C.SQLITE_INSERT SQLITE_UPDATE = C.SQLITE_UPDATE + + // used by authorzier - as return value + SQLITE_OK = C.SQLITE_OK + SQLITE_IGNORE = C.SQLITE_IGNORE + SQLITE_DENY = C.SQLITE_DENY + + // different actions query tries to do - passed as argument to authorizer + SQLITE_CREATE_INDEX = C.SQLITE_CREATE_INDEX + SQLITE_CREATE_TABLE = C.SQLITE_CREATE_TABLE + SQLITE_CREATE_TEMP_INDEX = C.SQLITE_CREATE_TEMP_INDEX + SQLITE_CREATE_TEMP_TABLE = C.SQLITE_CREATE_TEMP_TABLE + SQLITE_CREATE_TEMP_TRIGGER = C.SQLITE_CREATE_TEMP_TRIGGER + SQLITE_CREATE_TEMP_VIEW = C.SQLITE_CREATE_TEMP_VIEW + SQLITE_CREATE_TRIGGER = C.SQLITE_CREATE_TRIGGER + SQLITE_CREATE_VIEW = C.SQLITE_CREATE_VIEW + SQLITE_CREATE_VTABLE = C.SQLITE_CREATE_VTABLE + SQLITE_DROP_INDEX = C.SQLITE_DROP_INDEX + SQLITE_DROP_TABLE = C.SQLITE_DROP_TABLE + SQLITE_DROP_TEMP_INDEX = C.SQLITE_DROP_TEMP_INDEX + SQLITE_DROP_TEMP_TABLE = C.SQLITE_DROP_TEMP_TABLE + SQLITE_DROP_TEMP_TRIGGER = C.SQLITE_DROP_TEMP_TRIGGER + SQLITE_DROP_TEMP_VIEW = C.SQLITE_DROP_TEMP_VIEW + SQLITE_DROP_TRIGGER = C.SQLITE_DROP_TRIGGER + SQLITE_DROP_VIEW = C.SQLITE_DROP_VIEW + SQLITE_DROP_VTABLE = C.SQLITE_DROP_VTABLE + SQLITE_PRAGMA = C.SQLITE_PRAGMA + SQLITE_READ = C.SQLITE_READ + SQLITE_SELECT = C.SQLITE_SELECT + SQLITE_TRANSACTION = C.SQLITE_TRANSACTION + SQLITE_ATTACH = C.SQLITE_ATTACH + SQLITE_DETACH = C.SQLITE_DETACH + SQLITE_ALTER_TABLE = C.SQLITE_ALTER_TABLE + SQLITE_REINDEX = C.SQLITE_REINDEX + SQLITE_ANALYZE = C.SQLITE_ANALYZE + SQLITE_FUNCTION = C.SQLITE_FUNCTION + SQLITE_SAVEPOINT = C.SQLITE_SAVEPOINT + SQLITE_COPY = C.SQLITE_COPY + /*SQLITE_RECURSIVE = C.SQLITE_RECURSIVE*/ ) -// SQLiteDriver implement sql.Driver. +// SQLiteDriver implements driver.Driver. type SQLiteDriver struct { Extensions []string ConnectHook func(*SQLiteConn) error } -// SQLiteConn implement sql.Conn. +// SQLiteConn implements driver.Conn. type SQLiteConn struct { mu sync.Mutex db *C.sqlite3 @@ -221,12 +299,12 @@ type SQLiteConn struct { aggregators []*aggInfo } -// SQLiteTx implemen sql.Tx. +// SQLiteTx implements driver.Tx. type SQLiteTx struct { c *SQLiteConn } -// SQLiteStmt implement sql.Stmt. +// SQLiteStmt implements driver.Stmt. type SQLiteStmt struct { mu sync.Mutex c *SQLiteConn @@ -236,13 +314,13 @@ type SQLiteStmt struct { cls bool } -// SQLiteResult implement sql.Result. +// SQLiteResult implements sql.Result. type SQLiteResult struct { id int64 changes int64 } -// SQLiteRows implement sql.Rows. +// SQLiteRows implements driver.Rows. type SQLiteRows struct { s *SQLiteStmt nc int @@ -440,6 +518,20 @@ func (c *SQLiteConn) RegisterUpdateHook(callback func(int, string, string, int64 } } +// RegisterAuthorizer sets the authorizer for connection. +// +// The parameters to the callback are the operation (one of the constants +// SQLITE_INSERT, SQLITE_DELETE, or SQLITE_UPDATE), and 1 to 3 arguments, +// depending on operation. More details see: +// https://www.sqlite.org/c3ref/c_alter_table.html +func (c *SQLiteConn) RegisterAuthorizer(callback func(int, string, string, string) int) { + if callback == nil { + C.sqlite3_set_authorizer(c.db, nil, nil) + } else { + C.sqlite3_set_authorizer(c.db, (*[0]byte)(C.authorizerTrampoline), unsafe.Pointer(newHandle(c, callback))) + } +} + // RegisterFunc makes a Go function available as a SQLite function. // // The Go function can have arguments of the following types: any @@ -1582,7 +1674,7 @@ func (c *SQLiteConn) prepare(ctx context.Context, query string) (driver.Stmt, er defer C.free(unsafe.Pointer(pquery)) var s *C.sqlite3_stmt var tail *C.char - rv := C.sqlite3_prepare_v2(c.db, pquery, -1, &s, &tail) + rv := C._sqlite3_prepare_v2_internal(c.db, pquery, -1, &s, &tail) if rv != C.SQLITE_OK { return nil, c.lastError() } @@ -1816,7 +1908,7 @@ func (s *SQLiteStmt) exec(ctx context.Context, args []namedValue) (driver.Result } var rowid, changes C.longlong - rv := C._sqlite3_step(s.s, &rowid, &changes) + rv := C._sqlite3_step_row_internal(s.s, &rowid, &changes) if rv != C.SQLITE_ROW && rv != C.SQLITE_OK && rv != C.SQLITE_DONE { err := s.c.lastError() C.sqlite3_reset(s.s) @@ -1883,12 +1975,12 @@ func (rc *SQLiteRows) DeclTypes() []string { // Next move cursor to next. func (rc *SQLiteRows) Next(dest []driver.Value) error { + rc.s.mu.Lock() + defer rc.s.mu.Unlock() if rc.s.closed { return io.EOF } - rc.s.mu.Lock() - defer rc.s.mu.Unlock() - rv := C.sqlite3_step(rc.s.s) + rv := C._sqlite3_step_internal(rc.s.s) if rv == C.SQLITE_DONE { return io.EOF } diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_func_crypt.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_func_crypt.go index 3774a97..d397c8c 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3_func_crypt.go +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_func_crypt.go @@ -83,13 +83,13 @@ func CryptEncoderSSHA256(salt string) func(pass []byte, hash interface{}) []byte } } -// CryptEncoderSHA384 encodes a password with SHA256 +// CryptEncoderSHA384 encodes a password with SHA384 func CryptEncoderSHA384(pass []byte, hash interface{}) []byte { h := sha512.Sum384(pass) return h[:] } -// CryptEncoderSSHA384 encodes a password with SHA256 +// CryptEncoderSSHA384 encodes a password with SHA384 // with the configured salt func CryptEncoderSSHA384(salt string) func(pass []byte, hash interface{}) []byte { return func(pass []byte, hash interface{}) []byte { @@ -100,13 +100,13 @@ func CryptEncoderSSHA384(salt string) func(pass []byte, hash interface{}) []byte } } -// CryptEncoderSHA512 encodes a password with SHA256 +// CryptEncoderSHA512 encodes a password with SHA512 func CryptEncoderSHA512(pass []byte, hash interface{}) []byte { h := sha512.Sum512(pass) return h[:] } -// CryptEncoderSSHA512 encodes a password with SHA256 +// CryptEncoderSSHA512 encodes a password with SHA512 // with the configured salt func CryptEncoderSSHA512(salt string) func(pass []byte, hash interface{}) []byte { return func(pass []byte, hash interface{}) []byte { diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.c b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.c new file mode 100644 index 0000000..1af1726 --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.c @@ -0,0 +1,85 @@ +// Copyright (C) 2018 Yasuhiro Matsumoto . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY +#include +#include + +extern int unlock_notify_wait(sqlite3 *db); + +int +_sqlite3_step_blocking(sqlite3_stmt *stmt) +{ + int rv; + sqlite3* db; + + db = sqlite3_db_handle(stmt); + for (;;) { + rv = sqlite3_step(stmt); + if (rv != SQLITE_LOCKED) { + break; + } + if (sqlite3_extended_errcode(db) != SQLITE_LOCKED_SHAREDCACHE) { + break; + } + rv = unlock_notify_wait(db); + if (rv != SQLITE_OK) { + break; + } + sqlite3_reset(stmt); + } + + return rv; +} + +int +_sqlite3_step_row_blocking(sqlite3_stmt* stmt, long long* rowid, long long* changes) +{ + int rv; + sqlite3* db; + + db = sqlite3_db_handle(stmt); + for (;;) { + rv = sqlite3_step(stmt); + if (rv!=SQLITE_LOCKED) { + break; + } + if (sqlite3_extended_errcode(db) != SQLITE_LOCKED_SHAREDCACHE) { + break; + } + rv = unlock_notify_wait(db); + if (rv != SQLITE_OK) { + break; + } + sqlite3_reset(stmt); + } + + *rowid = (long long) sqlite3_last_insert_rowid(db); + *changes = (long long) sqlite3_changes(db); + return rv; +} + +int +_sqlite3_prepare_v2_blocking(sqlite3 *db, const char *zSql, int nBytes, sqlite3_stmt **ppStmt, const char **pzTail) +{ + int rv; + + for (;;) { + rv = sqlite3_prepare_v2(db, zSql, nBytes, ppStmt, pzTail); + if (rv!=SQLITE_LOCKED) { + break; + } + if (sqlite3_extended_errcode(db) != SQLITE_LOCKED_SHAREDCACHE) { + break; + } + rv = unlock_notify_wait(db); + if (rv != SQLITE_OK) { + break; + } + } + + return rv; +} +#endif diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.go new file mode 100644 index 0000000..5dde027 --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify.go @@ -0,0 +1,93 @@ +// Copyright (C) 2018 Yasuhiro Matsumoto . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build cgo +// +build sqlite_unlock_notify + +package sqlite3 + +/* +#cgo CFLAGS: -DSQLITE_ENABLE_UNLOCK_NOTIFY + +#include +#include + +extern void unlock_notify_callback(void *arg, int argc); +*/ +import "C" +import ( + "fmt" + "math" + "sync" + "unsafe" +) + +type unlock_notify_table struct { + sync.Mutex + seqnum uint + table map[uint]chan struct{} +} + +var unt unlock_notify_table = unlock_notify_table{table: make(map[uint]chan struct{})} + +func (t *unlock_notify_table) add(c chan struct{}) uint { + t.Lock() + defer t.Unlock() + h := t.seqnum + t.table[h] = c + t.seqnum++ + return h +} + +func (t *unlock_notify_table) remove(h uint) { + t.Lock() + defer t.Unlock() + delete(t.table, h) +} + +func (t *unlock_notify_table) get(h uint) chan struct{} { + t.Lock() + defer t.Unlock() + c, ok := t.table[h] + if !ok { + panic(fmt.Sprintf("Non-existent key for unlcok-notify channel: %d", h)) + } + return c +} + +//export unlock_notify_callback +func unlock_notify_callback(argv unsafe.Pointer, argc C.int) { + for i := 0; i < int(argc); i++ { + parg := ((*(*[(math.MaxInt32 - 1) / unsafe.Sizeof((*C.uint)(nil))]*[1]uint)(argv))[i]) + arg := *parg + h := arg[0] + c := unt.get(h) + c <- struct{}{} + } +} + +//export unlock_notify_wait +func unlock_notify_wait(db *C.sqlite3) C.int { + // It has to be a bufferred channel to not block in sqlite_unlock_notify + // as sqlite_unlock_notify could invoke the callback before it returns. + c := make(chan struct{}, 1) + defer close(c) + + h := unt.add(c) + defer unt.remove(h) + + pargv := C.malloc(C.sizeof_uint) + defer C.free(pargv) + + argv := (*[1]uint)(pargv) + argv[0] = h + if rv := C.sqlite3_unlock_notify(db, (*[0]byte)(C.unlock_notify_callback), unsafe.Pointer(pargv)); rv != C.SQLITE_OK { + return rv + } + + <-c + + return C.SQLITE_OK +} diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify_test.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify_test.go new file mode 100644 index 0000000..812c543 --- /dev/null +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_opt_unlock_notify_test.go @@ -0,0 +1,222 @@ +// Copyright (C) 2018 Yasuhiro Matsumoto . +// +// Use of this source code is governed by an MIT-style +// license that can be found in the LICENSE file. + +// +build sqlite_unlock_notify + +package sqlite3 + +import ( + "database/sql" + "fmt" + "os" + "sync" + "testing" + "time" +) + +func TestUnlockNotify(t *testing.T) { + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + dsn := fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", tempFilename, 500) + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal("Failed to open database:", err) + } + defer db.Close() + + _, err = db.Exec("CREATE TABLE foo(id INTEGER, status INTEGER)") + if err != nil { + t.Fatal("Failed to create table:", err) + } + + tx, err := db.Begin() + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + + _, err = tx.Exec("INSERT INTO foo(id, status) VALUES(1, 100)") + if err != nil { + t.Fatal("Failed to insert null:", err) + } + + _, err = tx.Exec("UPDATE foo SET status = 200 WHERE id = 1") + if err != nil { + t.Fatal("Failed to update table:", err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + timer := time.NewTimer(500 * time.Millisecond) + go func() { + <-timer.C + err := tx.Commit() + if err != nil { + t.Fatal("Failed to commit transaction:", err) + } + wg.Done() + }() + + rows, err := db.Query("SELECT count(*) from foo") + if err != nil { + t.Fatal("Unable to query foo table:", err) + } + + if rows.Next() { + var count int + if err := rows.Scan(&count); err != nil { + t.Fatal("Failed to Scan rows", err) + } + } + if err := rows.Err(); err != nil { + t.Fatal("Failed at the call to Next:", err) + } + wg.Wait() + +} + +func TestUnlockNotifyMany(t *testing.T) { + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + dsn := fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", tempFilename, 500) + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal("Failed to open database:", err) + } + defer db.Close() + + _, err = db.Exec("CREATE TABLE foo(id INTEGER, status INTEGER)") + if err != nil { + t.Fatal("Failed to create table:", err) + } + + tx, err := db.Begin() + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + + _, err = tx.Exec("INSERT INTO foo(id, status) VALUES(1, 100)") + if err != nil { + t.Fatal("Failed to insert null:", err) + } + + _, err = tx.Exec("UPDATE foo SET status = 200 WHERE id = 1") + if err != nil { + t.Fatal("Failed to update table:", err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + timer := time.NewTimer(500 * time.Millisecond) + go func() { + <-timer.C + err := tx.Commit() + if err != nil { + t.Fatal("Failed to commit transaction:", err) + } + wg.Done() + }() + + const concurrentQueries = 1000 + wg.Add(concurrentQueries) + for i := 0; i < concurrentQueries; i++ { + go func() { + rows, err := db.Query("SELECT count(*) from foo") + if err != nil { + t.Fatal("Unable to query foo table:", err) + } + + if rows.Next() { + var count int + if err := rows.Scan(&count); err != nil { + t.Fatal("Failed to Scan rows", err) + } + } + if err := rows.Err(); err != nil { + t.Fatal("Failed at the call to Next:", err) + } + wg.Done() + }() + } + wg.Wait() +} + +func TestUnlockNotifyDeadlock(t *testing.T) { + tempFilename := TempFilename(t) + defer os.Remove(tempFilename) + dsn := fmt.Sprintf("file:%s?cache=shared&mode=rwc&_busy_timeout=%d", tempFilename, 500) + db, err := sql.Open("sqlite3", dsn) + if err != nil { + t.Fatal("Failed to open database:", err) + } + defer db.Close() + + _, err = db.Exec("CREATE TABLE foo(id INTEGER, status INTEGER)") + if err != nil { + t.Fatal("Failed to create table:", err) + } + + tx, err := db.Begin() + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + + _, err = tx.Exec("INSERT INTO foo(id, status) VALUES(1, 100)") + if err != nil { + t.Fatal("Failed to insert null:", err) + } + + _, err = tx.Exec("UPDATE foo SET status = 200 WHERE id = 1") + if err != nil { + t.Fatal("Failed to update table:", err) + } + + wg := sync.WaitGroup{} + wg.Add(1) + timer := time.NewTimer(500 * time.Millisecond) + go func() { + <-timer.C + err := tx.Commit() + if err != nil { + t.Fatal("Failed to commit transaction:", err) + } + wg.Done() + }() + + wg.Add(1) + go func() { + tx2, err := db.Begin() + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer tx2.Rollback() + + _, err = tx2.Exec("DELETE FROM foo") + if err != nil { + t.Fatal("Failed to delete table:", err) + } + err = tx2.Commit() + if err != nil { + t.Fatal("Failed to commit transaction:", err) + } + wg.Done() + }() + + rows, err := tx.Query("SELECT count(*) from foo") + if err != nil { + t.Fatal("Unable to query foo table:", err) + } + + if rows.Next() { + var count int + if err := rows.Scan(&count); err != nil { + t.Fatal("Failed to Scan rows", err) + } + } + if err := rows.Err(); err != nil { + t.Fatal("Failed at the call to Next:", err) + } + + wg.Wait() +} diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_other.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_other.go index 086ed43..71778fc 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3_other.go +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_other.go @@ -10,5 +10,8 @@ package sqlite3 /* #cgo CFLAGS: -I. #cgo linux LDFLAGS: -ldl +#cgo linux,ppc LDFLAGS: -lpthread +#cgo linux,ppc64 LDFLAGS: -lpthread +#cgo linux,ppc64le LDFLAGS: -lpthread */ import "C" diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3_test.go b/vendor/github.com/mattn/go-sqlite3/sqlite3_test.go index 75d8f52..b295ddd 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3_test.go +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3_test.go @@ -1574,6 +1574,47 @@ func TestUpdateAndTransactionHooks(t *testing.T) { } } +func TestAuthorizer(t *testing.T) { + var authorizerReturn = 0 + + sql.Register("sqlite3_Authorizer", &SQLiteDriver{ + ConnectHook: func(conn *SQLiteConn) error { + conn.RegisterAuthorizer(func(op int, arg1, arg2, arg3 string) int { + return authorizerReturn + }) + return nil + }, + }) + db, err := sql.Open("sqlite3_Authorizer", ":memory:") + if err != nil { + t.Fatal("Failed to open database:", err) + } + defer db.Close() + + statements := []string{ + "create table foo (id integer primary key, name varchar)", + "insert into foo values (9, 'test9')", + "update foo set name = 'test99' where id = 9", + "select * from foo", + } + + authorizerReturn = SQLITE_OK + for _, statement := range statements { + _, err = db.Exec(statement) + if err != nil { + t.Fatalf("No error expected [%v]: %v", statement, err) + } + } + + authorizerReturn = SQLITE_DENY + for _, statement := range statements { + _, err = db.Exec(statement) + if err == nil { + t.Fatalf("Authorizer didn't worked - nil received, but error expected: [%v]", statement) + } + } +} + func TestNilAndEmptyBytes(t *testing.T) { db, err := sql.Open("sqlite3", ":memory:") if err != nil { @@ -1690,7 +1731,10 @@ func TestSuite(t *testing.T) { defer d.Close() db = &TestDB{t, d, SQLITE, sync.Once{}} - testing.RunTests(func(string, string) (bool, error) { return true, nil }, tests) + ok := testing.RunTests(func(string, string) (bool, error) { return true, nil }, tests) + if !ok { + t.Fatal("A subtest failed") + } if !testing.Short() { for _, b := range benchmarks { diff --git a/vendor/github.com/mattn/go-sqlite3/sqlite3ext.h b/vendor/github.com/mattn/go-sqlite3/sqlite3ext.h index 913e72d..50e6866 100644 --- a/vendor/github.com/mattn/go-sqlite3/sqlite3ext.h +++ b/vendor/github.com/mattn/go-sqlite3/sqlite3ext.h @@ -311,6 +311,12 @@ struct sqlite3_api_routines { int (*str_errcode)(sqlite3_str*); int (*str_length)(sqlite3_str*); char *(*str_value)(sqlite3_str*); + int (*create_window_function)(sqlite3*,const char*,int,int,void*, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInv)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*)); }; /* @@ -596,6 +602,8 @@ typedef int (*sqlite3_loadext_entry)( #define sqlite3_str_errcode sqlite3_api->str_errcode #define sqlite3_str_length sqlite3_api->str_length #define sqlite3_str_value sqlite3_api->str_value +/* Version 3.25.0 and later */ +#define sqlite3_create_window_function sqlite3_api->create_window_function #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) diff --git a/vendor/github.com/miekg/dns/.travis.yml b/vendor/github.com/miekg/dns/.travis.yml index 542dd68..1825937 100644 --- a/vendor/github.com/miekg/dns/.travis.yml +++ b/vendor/github.com/miekg/dns/.travis.yml @@ -1,12 +1,10 @@ language: go sudo: false -go: - - 1.9.x - - tip -env: - - TESTS="-race -v -bench=. -coverprofile=coverage.txt -covermode=atomic" - - TESTS="-race -v ./..." +go: + - 1.10.x + - 1.11.x + - tip before_install: # don't use the miekg/dns when testing forks @@ -14,7 +12,7 @@ before_install: - ln -s $TRAVIS_BUILD_DIR $GOPATH/src/github.com/miekg/ || true script: - - go test $TESTS + - go test -race -v -bench=. -coverprofile=coverage.txt -covermode=atomic ./... after_success: - bash <(curl -s https://codecov.io/bash) diff --git a/vendor/github.com/miekg/dns/Gopkg.lock b/vendor/github.com/miekg/dns/Gopkg.lock index 0c73a64..6866322 100644 --- a/vendor/github.com/miekg/dns/Gopkg.lock +++ b/vendor/github.com/miekg/dns/Gopkg.lock @@ -3,19 +3,55 @@ [[projects]] branch = "master" + digest = "1:6914c49eed986dfb8dffb33516fa129c49929d4d873f41e073c83c11c372b870" name = "golang.org/x/crypto" - packages = ["ed25519","ed25519/internal/edwards25519"] - revision = "b080dc9a8c480b08e698fb1219160d598526310f" + packages = [ + "ed25519", + "ed25519/internal/edwards25519", + ] + pruneopts = "" + revision = "e3636079e1a4c1f337f212cc5cd2aca108f6c900" [[projects]] branch = "master" + digest = "1:08e41d63f8dac84d83797368b56cf0b339e42d0224e5e56668963c28aec95685" name = "golang.org/x/net" - packages = ["bpf","internal/iana","internal/socket","ipv4","ipv6"] - revision = "894f8ed5849b15b810ae41e9590a0d05395bba27" + packages = [ + "bpf", + "context", + "internal/iana", + "internal/socket", + "ipv4", + "ipv6", + ] + pruneopts = "" + revision = "4dfa2610cdf3b287375bbba5b8f2a14d3b01d8de" + +[[projects]] + branch = "master" + digest = "1:b2ea75de0ccb2db2ac79356407f8a4cd8f798fe15d41b381c00abf3ae8e55ed1" + name = "golang.org/x/sync" + packages = ["errgroup"] + pruneopts = "" + revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + +[[projects]] + branch = "master" + digest = "1:149a432fabebb8221a80f77731b1cd63597197ded4f14af606ebe3a0959004ec" + name = "golang.org/x/sys" + packages = ["unix"] + pruneopts = "" + revision = "e4b3c5e9061176387e7cea65e4dc5853801f3fb7" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "c4abc38abaeeeeb9be92455c9c02cae32841122b8982aaa067ef25bb8e86ff9d" + input-imports = [ + "golang.org/x/crypto/ed25519", + "golang.org/x/net/ipv4", + "golang.org/x/net/ipv6", + "golang.org/x/sync/errgroup", + "golang.org/x/sys/unix", + ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/vendor/github.com/miekg/dns/Gopkg.toml b/vendor/github.com/miekg/dns/Gopkg.toml index 2f655b2..85e6ff3 100644 --- a/vendor/github.com/miekg/dns/Gopkg.toml +++ b/vendor/github.com/miekg/dns/Gopkg.toml @@ -24,3 +24,15 @@ [[constraint]] branch = "master" name = "golang.org/x/crypto" + +[[constraint]] + branch = "master" + name = "golang.org/x/net" + +[[constraint]] + branch = "master" + name = "golang.org/x/sys" + +[[constraint]] + branch = "master" + name = "golang.org/x/sync" diff --git a/vendor/github.com/miekg/dns/README.md b/vendor/github.com/miekg/dns/README.md index 1ad23c7..39737c7 100644 --- a/vendor/github.com/miekg/dns/README.md +++ b/vendor/github.com/miekg/dns/README.md @@ -7,10 +7,10 @@ > Less is more. -Complete and usable DNS library. All widely used Resource Records are supported, including the -DNSSEC types. It follows a lean and mean philosophy. If there is stuff you should know as a DNS -programmer there isn't a convenience function for it. Server side and client side programming is -supported, i.e. you can build servers and resolvers with it. +Complete and usable DNS library. All Resource Records are supported, including the DNSSEC types. +It follows a lean and mean philosophy. If there is stuff you should know as a DNS programmer there +isn't a convenience function for it. Server side and client side programming is supported, i.e. you +can build servers and resolvers with it. We try to keep the "master" branch as sane as possible and at the bleeding edge of standards, avoiding breaking changes wherever reasonable. We support the last two versions of Go. @@ -42,10 +42,9 @@ A not-so-up-to-date-list-that-may-be-actually-current: * https://github.com/tianon/rawdns * https://mesosphere.github.io/mesos-dns/ * https://pulse.turbobytes.com/ -* https://play.google.com/store/apps/details?id=com.turbobytes.dig * https://github.com/fcambus/statzone * https://github.com/benschw/dns-clb-go -* https://github.com/corny/dnscheck for http://public-dns.info/ +* https://github.com/corny/dnscheck for * https://namesmith.io * https://github.com/miekg/unbound * https://github.com/miekg/exdns @@ -56,7 +55,7 @@ A not-so-up-to-date-list-that-may-be-actually-current: * https://github.com/bamarni/dockness * https://github.com/fffaraz/microdns * http://kelda.io -* https://github.com/ipdcode/hades (JD.COM) +* https://github.com/ipdcode/hades * https://github.com/StackExchange/dnscontrol/ * https://www.dnsperf.com/ * https://dnssectest.net/ @@ -64,29 +63,32 @@ A not-so-up-to-date-list-that-may-be-actually-current: * https://github.com/oif/apex * https://github.com/jedisct1/dnscrypt-proxy * https://github.com/jedisct1/rpdns +* https://github.com/xor-gate/sshfp +* https://github.com/rs/dnstrace +* https://blitiri.com.ar/p/dnss ([github mirror](https://github.com/albertito/dnss)) +* https://github.com/semihalev/sdns +* https://render.com Send pull request if you want to be listed here. # Features -* UDP/TCP queries, IPv4 and IPv6; -* RFC 1035 zone file parsing ($INCLUDE, $ORIGIN, $TTL and $GENERATE (for all record types) are supported; -* Fast: - * Reply speed around ~ 80K qps (faster hardware results in more qps); - * Parsing RRs ~ 100K RR/s, that's 5M records in about 50 seconds; -* Server side programming (mimicking the net/http package); -* Client side programming; -* DNSSEC: signing, validating and key generation for DSA, RSA, ECDSA and Ed25519; -* EDNS0, NSID, Cookies; -* AXFR/IXFR; -* TSIG, SIG(0); -* DNS over TLS: optional encrypted connection between client and server; -* DNS name compression; -* Depends only on the standard library. +* UDP/TCP queries, IPv4 and IPv6 +* RFC 1035 zone file parsing ($INCLUDE, $ORIGIN, $TTL and $GENERATE (for all record types) are supported +* Fast +* Server side programming (mimicking the net/http package) +* Client side programming +* DNSSEC: signing, validating and key generation for DSA, RSA, ECDSA and Ed25519 +* EDNS0, NSID, Cookies +* AXFR/IXFR +* TSIG, SIG(0) +* DNS over TLS (DoT): encrypted connection between client and server over TCP +* DNS name compression Have fun! Miek Gieben - 2010-2012 - +DNS Authors 2012- # Building @@ -98,8 +100,8 @@ work: ## Examples -A short "how to use the API" is at the beginning of doc.go (this also will show -when you call `godoc github.com/miekg/dns`). +A short "how to use the API" is at the beginning of doc.go (this also will show when you call `godoc +github.com/miekg/dns`). Example programs can be found in the `github.com/miekg/exdns` repository. @@ -157,12 +159,13 @@ Example programs can be found in the `github.com/miekg/exdns` repository. * 7553 - URI record * 7858 - DNS over TLS: Initiation and Performance Considerations * 7871 - EDNS0 Client Subnet -* 7873 - Domain Name System (DNS) Cookies (draft-ietf-dnsop-cookies) +* 7873 - Domain Name System (DNS) Cookies * 8080 - EdDSA for DNSSEC +* 8499 - DNS Terminology -## Loosely based upon +## Loosely Based Upon -* `ldns` -* `NSD` -* `Net::DNS` -* `GRONG` +* ldns - +* NSD - +* Net::DNS - +* GRONG - diff --git a/vendor/github.com/miekg/dns/acceptfunc.go b/vendor/github.com/miekg/dns/acceptfunc.go new file mode 100644 index 0000000..78c076c --- /dev/null +++ b/vendor/github.com/miekg/dns/acceptfunc.go @@ -0,0 +1,56 @@ +package dns + +// MsgAcceptFunc is used early in the server code to accept or reject a message with RcodeFormatError. +// It returns a MsgAcceptAction to indicate what should happen with the message. +type MsgAcceptFunc func(dh Header) MsgAcceptAction + +// DefaultMsgAcceptFunc checks the request and will reject if: +// +// * isn't a request (don't respond in that case). +// * opcode isn't OpcodeQuery or OpcodeNotify +// * Zero bit isn't zero +// * has more than 1 question in the question section +// * has more than 1 RR in the Answer section +// * has more than 0 RRs in the Authority section +// * has more than 2 RRs in the Additional section +var DefaultMsgAcceptFunc MsgAcceptFunc = defaultMsgAcceptFunc + +// MsgAcceptAction represents the action to be taken. +type MsgAcceptAction int + +const ( + MsgAccept MsgAcceptAction = iota // Accept the message + MsgReject // Reject the message with a RcodeFormatError + MsgIgnore // Ignore the error and send nothing back. +) + +func defaultMsgAcceptFunc(dh Header) MsgAcceptAction { + if isResponse := dh.Bits&_QR != 0; isResponse { + return MsgIgnore + } + + // Don't allow dynamic updates, because then the sections can contain a whole bunch of RRs. + opcode := int(dh.Bits>>11) & 0xF + if opcode != OpcodeQuery && opcode != OpcodeNotify { + return MsgReject + } + + if isZero := dh.Bits&_Z != 0; isZero { + return MsgReject + } + if dh.Qdcount != 1 { + return MsgReject + } + // NOTIFY requests can have a SOA in the ANSWER section. See RFC 1996 Section 3.7 and 3.11. + if dh.Ancount > 1 { + return MsgReject + } + // IXFR request could have one SOA RR in the NS section. See RFC 1995, section 3. + if dh.Nscount > 1 { + return MsgReject + } + if dh.Arcount > 2 { + return MsgReject + } + return MsgAccept +} diff --git a/vendor/github.com/miekg/dns/acceptfunc_test.go b/vendor/github.com/miekg/dns/acceptfunc_test.go new file mode 100644 index 0000000..2d784a6 --- /dev/null +++ b/vendor/github.com/miekg/dns/acceptfunc_test.go @@ -0,0 +1,35 @@ +package dns + +import ( + "testing" +) + +func TestAcceptNotify(t *testing.T) { + HandleFunc("example.org.", handleNotify) + s, addrstr, err := RunLocalUDPServer(":0") + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + defer s.Shutdown() + + m := new(Msg) + m.SetNotify("example.org.") + // Set a SOA hint in the answer section, this is allowed according to RFC 1996. + soa, _ := NewRR("example.org. IN SOA sns.dns.icann.org. noc.dns.icann.org. 2018112827 7200 3600 1209600 3600") + m.Answer = []RR{soa} + + c := new(Client) + resp, _, err := c.Exchange(m, addrstr) + if err != nil { + t.Errorf("failed to exchange: %v", err) + } + if resp.Rcode != RcodeSuccess { + t.Errorf("expected %s, got %s", RcodeToString[RcodeSuccess], RcodeToString[resp.Rcode]) + } +} + +func handleNotify(w ResponseWriter, req *Msg) { + m := new(Msg) + m.SetReply(req) + w.WriteMsg(m) +} diff --git a/vendor/github.com/miekg/dns/client.go b/vendor/github.com/miekg/dns/client.go index dd6b512..2393564 100644 --- a/vendor/github.com/miekg/dns/client.go +++ b/vendor/github.com/miekg/dns/client.go @@ -7,11 +7,8 @@ import ( "context" "crypto/tls" "encoding/binary" - "fmt" "io" - "io/ioutil" "net" - "net/http" "strings" "time" ) @@ -19,8 +16,6 @@ import ( const ( dnsTimeout time.Duration = 2 * time.Second tcpIdleTimeout time.Duration = 8 * time.Second - - dohMimeType = "application/dns-message" ) // A Conn represents a connection to a DNS server. @@ -44,7 +39,6 @@ type Client struct { DialTimeout time.Duration // net.DialTimeout, defaults to 2 seconds, or net.Dialer.Timeout if expiring earlier - overridden by Timeout when that value is non-zero ReadTimeout time.Duration // net.Conn.SetReadTimeout value for connections, defaults to 2 seconds - overridden by Timeout when that value is non-zero WriteTimeout time.Duration // net.Conn.SetWriteTimeout value for connections, defaults to 2 seconds - overridden by Timeout when that value is non-zero - HTTPClient *http.Client // The http.Client to use for DNS-over-HTTPS TsigSecret map[string]string // secret(s) for Tsig map[], zonename must be in canonical form (lowercase, fqdn, see RFC 4034 Section 6.2) SingleInflight bool // if true suppress multiple outstanding queries for the same Qname, Qtype and Qclass group singleflight @@ -89,32 +83,22 @@ func (c *Client) Dial(address string) (conn *Conn, err error) { // create a new dialer with the appropriate timeout var d net.Dialer if c.Dialer == nil { - d = net.Dialer{Timeout:c.getTimeoutForRequest(c.dialTimeout())} + d = net.Dialer{Timeout: c.getTimeoutForRequest(c.dialTimeout())} } else { - d = net.Dialer(*c.Dialer) + d = *c.Dialer } - network := "udp" - useTLS := false - - switch c.Net { - case "tcp-tls": - network = "tcp" - useTLS = true - case "tcp4-tls": - network = "tcp4" - useTLS = true - case "tcp6-tls": - network = "tcp6" - useTLS = true - default: - if c.Net != "" { - network = c.Net - } + network := c.Net + if network == "" { + network = "udp" } + useTLS := strings.HasPrefix(network, "tcp") && strings.HasSuffix(network, "-tls") + conn = new(Conn) if useTLS { + network = strings.TrimSuffix(network, "-tls") + conn.Conn, err = tls.DialWithDialer(&d, network, address, c.TLSConfig) } else { conn.Conn, err = d.Dial(network, address) @@ -122,6 +106,7 @@ func (c *Client) Dial(address string) (conn *Conn, err error) { if err != nil { return nil, err } + return conn, nil } @@ -141,11 +126,6 @@ func (c *Client) Dial(address string) (conn *Conn, err error) { // attribute appropriately func (c *Client) Exchange(m *Msg, address string) (r *Msg, rtt time.Duration, err error) { if !c.SingleInflight { - if c.Net == "https" { - // TODO(tmthrgd): pipe timeouts into exchangeDOH - return c.exchangeDOH(context.TODO(), m, address) - } - return c.exchange(m, address) } @@ -158,11 +138,6 @@ func (c *Client) Exchange(m *Msg, address string) (r *Msg, rtt time.Duration, er cl = cl1 } r, rtt, err, shared := c.group.Do(m.Question[0].Name+t+cl, func() (*Msg, time.Duration, error) { - if c.Net == "https" { - // TODO(tmthrgd): pipe timeouts into exchangeDOH - return c.exchangeDOH(context.TODO(), m, address) - } - return c.exchange(m, address) }) if r != nil && shared { @@ -208,67 +183,6 @@ func (c *Client) exchange(m *Msg, a string) (r *Msg, rtt time.Duration, err erro return r, rtt, err } -func (c *Client) exchangeDOH(ctx context.Context, m *Msg, a string) (r *Msg, rtt time.Duration, err error) { - p, err := m.Pack() - if err != nil { - return nil, 0, err - } - - req, err := http.NewRequest(http.MethodPost, a, bytes.NewReader(p)) - if err != nil { - return nil, 0, err - } - - req.Header.Set("Content-Type", dohMimeType) - req.Header.Set("Accept", dohMimeType) - - hc := http.DefaultClient - if c.HTTPClient != nil { - hc = c.HTTPClient - } - - if ctx != context.Background() && ctx != context.TODO() { - req = req.WithContext(ctx) - } - - t := time.Now() - - resp, err := hc.Do(req) - if err != nil { - return nil, 0, err - } - defer closeHTTPBody(resp.Body) - - if resp.StatusCode != http.StatusOK { - return nil, 0, fmt.Errorf("dns: server returned HTTP %d error: %q", resp.StatusCode, resp.Status) - } - - if ct := resp.Header.Get("Content-Type"); ct != dohMimeType { - return nil, 0, fmt.Errorf("dns: unexpected Content-Type %q; expected %q", ct, dohMimeType) - } - - p, err = ioutil.ReadAll(resp.Body) - if err != nil { - return nil, 0, err - } - - rtt = time.Since(t) - - r = new(Msg) - if err := r.Unpack(p); err != nil { - return r, 0, err - } - - // TODO: TSIG? Is it even supported over DoH? - - return r, rtt, nil -} - -func closeHTTPBody(r io.ReadCloser) error { - io.Copy(ioutil.Discard, io.LimitReader(r, 8<<20)) - return r.Close() -} - // ReadMsg reads a message from the connection co. // If the received message contains a TSIG record the transaction signature // is verified. This method always tries to return the message, however if an @@ -406,16 +320,12 @@ func (co *Conn) Read(p []byte) (n int, err error) { return 0, err } if l > len(p) { - return int(l), io.ErrShortBuffer + return l, io.ErrShortBuffer } return tcpRead(r, p[:l]) } // UDP connection - n, err = co.Conn.Read(p) - if err != nil { - return n, err - } - return n, err + return co.Conn.Read(p) } // WriteMsg sends a message through the connection co. @@ -437,10 +347,8 @@ func (co *Conn) WriteMsg(m *Msg) (err error) { if err != nil { return err } - if _, err = co.Write(out); err != nil { - return err - } - return nil + _, err = co.Write(out) + return err } // Write implements the net.Conn Write method. @@ -462,8 +370,7 @@ func (co *Conn) Write(p []byte) (n int, err error) { n, err := io.Copy(w, bytes.NewReader(p)) return int(n), err } - n, err = co.Conn.Write(p) - return n, err + return co.Conn.Write(p) } // Return the appropriate timeout for a specific request @@ -530,11 +437,7 @@ func ExchangeConn(c net.Conn, m *Msg) (r *Msg, err error) { // DialTimeout acts like Dial but takes a timeout. func DialTimeout(network, address string, timeout time.Duration) (conn *Conn, err error) { client := Client{Net: network, Dialer: &net.Dialer{Timeout: timeout}} - conn, err = client.Dial(address) - if err != nil { - return nil, err - } - return conn, nil + return client.Dial(address) } // DialWithTLS connects to the address on the named network with TLS. @@ -543,12 +446,7 @@ func DialWithTLS(network, address string, tlsConfig *tls.Config) (conn *Conn, er network += "-tls" } client := Client{Net: network, TLSConfig: tlsConfig} - conn, err = client.Dial(address) - - if err != nil { - return nil, err - } - return conn, nil + return client.Dial(address) } // DialTimeoutWithTLS acts like DialWithTLS but takes a timeout. @@ -557,30 +455,22 @@ func DialTimeoutWithTLS(network, address string, tlsConfig *tls.Config, timeout network += "-tls" } client := Client{Net: network, Dialer: &net.Dialer{Timeout: timeout}, TLSConfig: tlsConfig} - conn, err = client.Dial(address) - if err != nil { - return nil, err - } - return conn, nil + return client.Dial(address) } // ExchangeContext acts like Exchange, but honors the deadline on the provided // context, if present. If there is both a context deadline and a configured // timeout on the client, the earliest of the two takes effect. func (c *Client) ExchangeContext(ctx context.Context, m *Msg, a string) (r *Msg, rtt time.Duration, err error) { - if !c.SingleInflight && c.Net == "https" { - return c.exchangeDOH(ctx, m, a) - } - var timeout time.Duration if deadline, ok := ctx.Deadline(); !ok { timeout = 0 } else { - timeout = deadline.Sub(time.Now()) + timeout = time.Until(deadline) } // not passing the context to the underlying calls, as the API does not support // context. For timeouts you should set up Client.Dialer and call Client.Exchange. - // TODO(tmthrgd): this is a race condition + // TODO(tmthrgd,miekg): this is a race condition. c.Dialer = &net.Dialer{Timeout: timeout} return c.Exchange(m, a) } diff --git a/vendor/github.com/miekg/dns/client_test.go b/vendor/github.com/miekg/dns/client_test.go index 020e141..e7bb8ba 100644 --- a/vendor/github.com/miekg/dns/client_test.go +++ b/vendor/github.com/miekg/dns/client_test.go @@ -7,7 +7,6 @@ import ( "net" "strconv" "strings" - "sync" "testing" "time" ) @@ -93,7 +92,7 @@ func TestClientLocalAddress(t *testing.T) { t.Errorf("failed to get an valid answer\n%v", r) } if len(r.Extra) != 1 { - t.Errorf("failed to get additional answers\n%v", r) + t.Fatalf("failed to get additional answers\n%v", r) } txt := r.Extra[0].(*TXT) if txt == nil { @@ -373,12 +372,12 @@ func TestTruncatedMsg(t *testing.T) { m.Truncated = true buf, err = m.Pack() if err != nil { - t.Errorf("failed to pack truncated: %v", err) + t.Errorf("failed to pack truncated message: %v", err) } r = new(Msg) - if err = r.Unpack(buf); err != nil && err != ErrTruncated { - t.Errorf("unable to unpack truncated message: %v", err) + if err = r.Unpack(buf); err != nil { + t.Errorf("failed to unpack truncated message: %v", err) } if !r.Truncated { t.Errorf("truncated message wasn't unpacked as truncated") @@ -407,9 +406,10 @@ func TestTruncatedMsg(t *testing.T) { buf1 = buf[:len(buf)-off] r = new(Msg) - if err = r.Unpack(buf1); err != nil && err != ErrTruncated { - t.Errorf("unable to unpack cutoff message: %v", err) + if err = r.Unpack(buf1); err == nil { + t.Error("cutoff message should have failed to unpack") } + // r's header might be still usable. if !r.Truncated { t.Error("truncated cutoff message wasn't unpacked as truncated") } @@ -438,8 +438,8 @@ func TestTruncatedMsg(t *testing.T) { buf1 = buf[:len(buf)-off] r = new(Msg) - if err = r.Unpack(buf1); err != nil && err != ErrTruncated { - t.Errorf("unable to unpack cutoff message: %v", err) + if err = r.Unpack(buf1); err == nil { + t.Error("cutoff message should have failed to unpack") } if !r.Truncated { t.Error("truncated cutoff message wasn't unpacked as truncated") @@ -454,8 +454,8 @@ func TestTruncatedMsg(t *testing.T) { r = new(Msg) err = r.Unpack(buf1) - if err == nil || err == ErrTruncated { - t.Errorf("error should not be ErrTruncated from question cutoff unpack: %v", err) + if err == nil { + t.Errorf("error should be nil after question cutoff unpack: %v", err) } // Finally, if we only have the header, we don't return an error. @@ -484,49 +484,34 @@ func TestTimeout(t *testing.T) { m := new(Msg) m.SetQuestion("miek.nl.", TypeTXT) - // Use a channel + timeout to ensure we don't get stuck if the - // Client Timeout is not working properly - done := make(chan struct{}, 2) + runTest := func(name string, exchange func(m *Msg, addr string, timeout time.Duration) (*Msg, time.Duration, error)) { + t.Run(name, func(t *testing.T) { + start := time.Now() - timeout := time.Millisecond - allowable := timeout + (10 * time.Millisecond) - abortAfter := timeout + (100 * time.Millisecond) + timeout := time.Millisecond + allowable := timeout + 10*time.Millisecond - start := time.Now() + _, _, err := exchange(m, addrstr, timeout) + if err == nil { + t.Errorf("no timeout using Client.%s", name) + } - go func() { + length := time.Since(start) + if length > allowable { + t.Errorf("exchange took longer %v than specified Timeout %v", length, allowable) + } + }) + } + runTest("Exchange", func(m *Msg, addr string, timeout time.Duration) (*Msg, time.Duration, error) { c := &Client{Timeout: timeout} - _, _, err := c.Exchange(m, addrstr) - if err == nil { - t.Error("no timeout using Client.Exchange") - } - done <- struct{}{} - }() - - go func() { + return c.Exchange(m, addr) + }) + runTest("ExchangeContext", func(m *Msg, addr string, timeout time.Duration) (*Msg, time.Duration, error) { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - c := &Client{} - _, _, err := c.ExchangeContext(ctx, m, addrstr) - if err == nil { - t.Error("no timeout using Client.ExchangeContext") - } - done <- struct{}{} - }() - // Wait for both the Exchange and ExchangeContext tests to be done. - for i := 0; i < 2; i++ { - select { - case <-done: - case <-time.After(abortAfter): - } - } - - length := time.Since(start) - - if length > allowable { - t.Errorf("exchange took longer %v than specified Timeout %v", length, allowable) - } + return new(Client).ExchangeContext(ctx, m, addrstr) + }) } // Check that responses from deduplicated requests aren't shared between callers @@ -535,16 +520,13 @@ func TestConcurrentExchanges(t *testing.T) { cases[0] = new(Msg) cases[1] = new(Msg) cases[1].Truncated = true - for _, m := range cases { - block := make(chan struct{}) - waiting := make(chan struct{}) + for _, m := range cases { + mm := m // redeclare m so as not to trip the race detector handler := func(w ResponseWriter, req *Msg) { - r := m.Copy() + r := mm.Copy() r.SetReply(req) - waiting <- struct{}{} - <-block w.WriteMsg(r) } @@ -559,54 +541,27 @@ func TestConcurrentExchanges(t *testing.T) { m := new(Msg) m.SetQuestion("miek.nl.", TypeSRV) + c := &Client{ SingleInflight: true, } - r := make([]*Msg, 2) + // Force this client to always return the same request, + // even though we're querying sequentially. Running the + // Exchange calls below concurrently can fail due to + // goroutine scheduling, but this simulates the same + // outcome. + c.group.dontDeleteForTesting = true - var wg sync.WaitGroup - wg.Add(len(r)) - for i := 0; i < len(r); i++ { - go func(i int) { - defer wg.Done() - r[i], _, _ = c.Exchange(m.Copy(), addrstr) - if r[i] == nil { - t.Errorf("response %d is nil", i) - } - }(i) + r := make([]*Msg, 2) + for i := range r { + r[i], _, _ = c.Exchange(m.Copy(), addrstr) + if r[i] == nil { + t.Errorf("response %d is nil", i) + } } - select { - case <-waiting: - case <-time.After(time.Second): - t.FailNow() - } - close(block) - wg.Wait() if r[0] == r[1] { t.Errorf("got same response, expected non-shared responses") } } } - -func TestDoHExchange(t *testing.T) { - const addrstr = "https://dns.cloudflare.com/dns-query" - - m := new(Msg) - m.SetQuestion("miek.nl.", TypeSOA) - - cl := &Client{Net: "https"} - - r, _, err := cl.Exchange(m, addrstr) - if err != nil { - t.Fatalf("failed to exchange: %v", err) - } - - if r == nil || r.Rcode != RcodeSuccess { - t.Errorf("failed to get an valid answer\n%v", r) - } - - t.Log(r) - - // TODO: proper tests for this -} diff --git a/vendor/github.com/miekg/dns/compress_generate.go b/vendor/github.com/miekg/dns/compress_generate.go deleted file mode 100644 index 9a136c4..0000000 --- a/vendor/github.com/miekg/dns/compress_generate.go +++ /dev/null @@ -1,198 +0,0 @@ -//+build ignore - -// compression_generate.go is meant to run with go generate. It will use -// go/{importer,types} to track down all the RR struct types. Then for each type -// it will look to see if there are (compressible) names, if so it will add that -// type to compressionLenHelperType and comressionLenSearchType which "fake" the -// compression so that Len() is fast. -package main - -import ( - "bytes" - "fmt" - "go/format" - "go/importer" - "go/types" - "log" - "os" -) - -var packageHdr = ` -// Code generated by "go run compress_generate.go"; DO NOT EDIT. - -package dns - -` - -// getTypeStruct will take a type and the package scope, and return the -// (innermost) struct if the type is considered a RR type (currently defined as -// those structs beginning with a RR_Header, could be redefined as implementing -// the RR interface). The bool return value indicates if embedded structs were -// resolved. -func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { - st, ok := t.Underlying().(*types.Struct) - if !ok { - return nil, false - } - if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { - return st, false - } - if st.Field(0).Anonymous() { - st, _ := getTypeStruct(st.Field(0).Type(), scope) - return st, true - } - return nil, false -} - -func main() { - // Import and type-check the package - pkg, err := importer.Default().Import("github.com/miekg/dns") - fatalIfErr(err) - scope := pkg.Scope() - - var domainTypes []string // Types that have a domain name in them (either compressible or not). - var cdomainTypes []string // Types that have a compressible domain name in them (subset of domainType) -Names: - for _, name := range scope.Names() { - o := scope.Lookup(name) - if o == nil || !o.Exported() { - continue - } - st, _ := getTypeStruct(o.Type(), scope) - if st == nil { - continue - } - if name == "PrivateRR" { - continue - } - - if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { - log.Fatalf("Constant Type%s does not exist.", o.Name()) - } - - for i := 1; i < st.NumFields(); i++ { - if _, ok := st.Field(i).Type().(*types.Slice); ok { - if st.Tag(i) == `dns:"domain-name"` { - domainTypes = append(domainTypes, o.Name()) - continue Names - } - if st.Tag(i) == `dns:"cdomain-name"` { - cdomainTypes = append(cdomainTypes, o.Name()) - domainTypes = append(domainTypes, o.Name()) - continue Names - } - continue - } - - switch { - case st.Tag(i) == `dns:"domain-name"`: - domainTypes = append(domainTypes, o.Name()) - continue Names - case st.Tag(i) == `dns:"cdomain-name"`: - cdomainTypes = append(cdomainTypes, o.Name()) - domainTypes = append(domainTypes, o.Name()) - continue Names - } - } - } - - b := &bytes.Buffer{} - b.WriteString(packageHdr) - - // compressionLenHelperType - all types that have domain-name/cdomain-name can be used for compressing names - - fmt.Fprint(b, "func compressionLenHelperType(c map[string]int, r RR, initLen int) int {\n") - fmt.Fprint(b, "currentLen := initLen\n") - fmt.Fprint(b, "switch x := r.(type) {\n") - for _, name := range domainTypes { - o := scope.Lookup(name) - st, _ := getTypeStruct(o.Type(), scope) - - fmt.Fprintf(b, "case *%s:\n", name) - for i := 1; i < st.NumFields(); i++ { - out := func(s string) { - fmt.Fprintf(b, "currentLen -= len(x.%s) + 1\n", st.Field(i).Name()) - fmt.Fprintf(b, "currentLen += compressionLenHelper(c, x.%s, currentLen)\n", st.Field(i).Name()) - } - - if _, ok := st.Field(i).Type().(*types.Slice); ok { - switch st.Tag(i) { - case `dns:"domain-name"`: - fallthrough - case `dns:"cdomain-name"`: - // For HIP we need to slice over the elements in this slice. - fmt.Fprintf(b, `for i := range x.%s { - currentLen -= len(x.%s[i]) + 1 -} -`, st.Field(i).Name(), st.Field(i).Name()) - fmt.Fprintf(b, `for i := range x.%s { - currentLen += compressionLenHelper(c, x.%s[i], currentLen) -} -`, st.Field(i).Name(), st.Field(i).Name()) - } - continue - } - - switch { - case st.Tag(i) == `dns:"cdomain-name"`: - fallthrough - case st.Tag(i) == `dns:"domain-name"`: - out(st.Field(i).Name()) - } - } - } - fmt.Fprintln(b, "}\nreturn currentLen - initLen\n}\n\n") - - // compressionLenSearchType - search cdomain-tags types for compressible names. - - fmt.Fprint(b, "func compressionLenSearchType(c map[string]int, r RR) (int, bool, int) {\n") - fmt.Fprint(b, "switch x := r.(type) {\n") - for _, name := range cdomainTypes { - o := scope.Lookup(name) - st, _ := getTypeStruct(o.Type(), scope) - - fmt.Fprintf(b, "case *%s:\n", name) - j := 1 - for i := 1; i < st.NumFields(); i++ { - out := func(s string, j int) { - fmt.Fprintf(b, "k%d, ok%d, sz%d := compressionLenSearch(c, x.%s)\n", j, j, j, st.Field(i).Name()) - } - - // There are no slice types with names that can be compressed. - - switch { - case st.Tag(i) == `dns:"cdomain-name"`: - out(st.Field(i).Name(), j) - j++ - } - } - k := "k1" - ok := "ok1" - sz := "sz1" - for i := 2; i < j; i++ { - k += fmt.Sprintf(" + k%d", i) - ok += fmt.Sprintf(" && ok%d", i) - sz += fmt.Sprintf(" + sz%d", i) - } - fmt.Fprintf(b, "return %s, %s, %s\n", k, ok, sz) - } - fmt.Fprintln(b, "}\nreturn 0, false, 0\n}\n\n") - - // gofmt - res, err := format.Source(b.Bytes()) - if err != nil { - b.WriteTo(os.Stderr) - log.Fatal(err) - } - - f, err := os.Create("zcompress.go") - fatalIfErr(err) - defer f.Close() - f.Write(res) -} - -func fatalIfErr(err error) { - if err != nil { - log.Fatal(err) - } -} diff --git a/vendor/github.com/miekg/dns/defaults.go b/vendor/github.com/miekg/dns/defaults.go index 14e18b0..391d67a 100644 --- a/vendor/github.com/miekg/dns/defaults.go +++ b/vendor/github.com/miekg/dns/defaults.go @@ -4,6 +4,7 @@ import ( "errors" "net" "strconv" + "strings" ) const hexDigit = "0123456789abcdef" @@ -163,11 +164,72 @@ func (dns *Msg) IsEdns0() *OPT { // the number of labels. When false is returned the number of labels is not // defined. Also note that this function is extremely liberal; almost any // string is a valid domain name as the DNS is 8 bit protocol. It checks if each -// label fits in 63 characters, but there is no length check for the entire -// string s. I.e. a domain name longer than 255 characters is considered valid. +// label fits in 63 characters and that the entire name will fit into the 255 +// octet wire format limit. func IsDomainName(s string) (labels int, ok bool) { - _, labels, err := packDomainName(s, nil, 0, nil, false) - return labels, err == nil + // XXX: The logic in this function was copied from packDomainName and + // should be kept in sync with that function. + + const lenmsg = 256 + + if len(s) == 0 { // Ok, for instance when dealing with update RR without any rdata. + return 0, false + } + + s = Fqdn(s) + + // Each dot ends a segment of the name. Except for escaped dots (\.), which + // are normal dots. + + var ( + off int + begin int + wasDot bool + ) + for i := 0; i < len(s); i++ { + switch s[i] { + case '\\': + if off+1 > lenmsg { + return labels, false + } + + // check for \DDD + if i+3 < len(s) && isDigit(s[i+1]) && isDigit(s[i+2]) && isDigit(s[i+3]) { + i += 3 + begin += 3 + } else { + i++ + begin++ + } + + wasDot = false + case '.': + if wasDot { + // two dots back to back is not legal + return labels, false + } + wasDot = true + + labelLen := i - begin + if labelLen >= 1<<6 { // top two bits of length must be clear + return labels, false + } + + // off can already (we're in a loop) be bigger than lenmsg + // this happens when a name isn't fully qualified + off += 1 + labelLen + if off > lenmsg { + return labels, false + } + + labels++ + begin = i + 1 + default: + wasDot = false + } + } + + return labels, true } // IsSubDomain checks if child is indeed a child of the parent. If child and parent @@ -181,7 +243,7 @@ func IsSubDomain(parent, child string) bool { // The checking is performed on the binary payload. func IsMsg(buf []byte) error { // Header - if len(buf) < 12 { + if len(buf) < headerSize { return errors.New("dns: bad message header") } // Header: Opcode @@ -191,11 +253,18 @@ func IsMsg(buf []byte) error { // IsFqdn checks if a domain name is fully qualified. func IsFqdn(s string) bool { - l := len(s) - if l == 0 { + s2 := strings.TrimSuffix(s, ".") + if s == s2 { return false } - return s[l-1] == '.' + + i := strings.LastIndexFunc(s2, func(r rune) bool { + return r != '\\' + }) + + // Test whether we have an even number of escape sequences before + // the dot or none. + return (len(s2)-i)%2 != 0 } // IsRRset checks if a set of RRs is a valid RRset as defined by RFC 2181. @@ -244,12 +313,19 @@ func ReverseAddr(addr string) (arpa string, err error) { if ip == nil { return "", &Error{err: "unrecognized address: " + addr} } - if ip.To4() != nil { - return strconv.Itoa(int(ip[15])) + "." + strconv.Itoa(int(ip[14])) + "." + strconv.Itoa(int(ip[13])) + "." + - strconv.Itoa(int(ip[12])) + ".in-addr.arpa.", nil + if v4 := ip.To4(); v4 != nil { + buf := make([]byte, 0, net.IPv4len*4+len("in-addr.arpa.")) + // Add it, in reverse, to the buffer + for i := len(v4) - 1; i >= 0; i-- { + buf = strconv.AppendInt(buf, int64(v4[i]), 10) + buf = append(buf, '.') + } + // Append "in-addr.arpa." and return (buf already has the final .) + buf = append(buf, "in-addr.arpa."...) + return string(buf), nil } // Must be IPv6 - buf := make([]byte, 0, len(ip)*4+len("ip6.arpa.")) + buf := make([]byte, 0, net.IPv6len*4+len("ip6.arpa.")) // Add it, in reverse, to the buffer for i := len(ip) - 1; i >= 0; i-- { v := ip[i] diff --git a/vendor/github.com/miekg/dns/dns.go b/vendor/github.com/miekg/dns/dns.go index e7557f5..f57337b 100644 --- a/vendor/github.com/miekg/dns/dns.go +++ b/vendor/github.com/miekg/dns/dns.go @@ -34,10 +34,30 @@ type RR interface { // copy returns a copy of the RR copy() RR - // len returns the length (in octets) of the uncompressed RR in wire format. - len() int - // pack packs an RR into wire format. - pack([]byte, int, map[string]int, bool) (int, error) + + // len returns the length (in octets) of the compressed or uncompressed RR in wire format. + // + // If compression is nil, the uncompressed size will be returned, otherwise the compressed + // size will be returned and domain names will be added to the map for future compression. + len(off int, compression map[string]struct{}) int + + // pack packs the records RDATA into wire format. The header will + // already have been packed into msg. + pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) + + // unpack unpacks an RR from wire format. + // + // This will only be called on a new and empty RR type with only the header populated. It + // will only be called if the record's RDATA is non-empty. + unpack(msg []byte, off int) (off1 int, err error) + + // parse parses an RR from zone file format. + // + // This will only be called on a new and empty RR type with only the header populated. + parse(c *zlexer, origin, file string) *ParseError + + // isDuplicate returns whether the two RRs are duplicates. + isDuplicate(r2 RR) bool } // RR_Header is the header all DNS resource records share. @@ -70,28 +90,45 @@ func (h *RR_Header) String() string { return s } -func (h *RR_Header) len() int { - l := len(h.Name) + 1 +func (h *RR_Header) len(off int, compression map[string]struct{}) int { + l := domainNameLen(h.Name, off, compression, true) l += 10 // rrtype(2) + class(2) + ttl(4) + rdlength(2) return l } +func (h *RR_Header) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + // RR_Header has no RDATA to pack. + return off, nil +} + +func (h *RR_Header) unpack(msg []byte, off int) (int, error) { + panic("dns: internal error: unpack should never be called on RR_Header") +} + +func (h *RR_Header) parse(c *zlexer, origin, file string) *ParseError { + panic("dns: internal error: parse should never be called on RR_Header") +} + // ToRFC3597 converts a known RR to the unknown RR representation from RFC 3597. func (rr *RFC3597) ToRFC3597(r RR) error { - buf := make([]byte, r.len()*2) - off, err := PackRR(r, buf, 0, nil, false) + buf := make([]byte, Len(r)*2) + headerEnd, off, err := packRR(r, buf, 0, compressionMap{}, false) if err != nil { return err } buf = buf[:off] - if int(r.Header().Rdlength) > off { - return ErrBuf + + *rr = RFC3597{Hdr: *r.Header()} + rr.Hdr.Rdlength = uint16(off - headerEnd) + + if noRdata(rr.Hdr) { + return nil } - rfc3597, _, err := unpackRFC3597(*r.Header(), buf, off-int(r.Header().Rdlength)) + _, err = rr.unpack(buf, headerEnd) if err != nil { return err } - *rr = *rfc3597.(*RFC3597) + return nil } diff --git a/vendor/github.com/miekg/dns/dns_bench_test.go b/vendor/github.com/miekg/dns/dns_bench_test.go index 7bf8bd2..a34fe99 100644 --- a/vendor/github.com/miekg/dns/dns_bench_test.go +++ b/vendor/github.com/miekg/dns/dns_bench_test.go @@ -1,6 +1,7 @@ package dns import ( + "fmt" "net" "testing" ) @@ -63,6 +64,50 @@ func BenchmarkMsgLengthPack(b *testing.B) { } } +func BenchmarkMsgLengthMassive(b *testing.B) { + makeMsg := func(question string, ans, ns, e []RR) *Msg { + msg := new(Msg) + msg.SetQuestion(Fqdn(question), TypeANY) + msg.Answer = append(msg.Answer, ans...) + msg.Ns = append(msg.Ns, ns...) + msg.Extra = append(msg.Extra, e...) + msg.Compress = true + return msg + } + const name1 = "12345678901234567890123456789012345.12345678.123." + rrMx := testRR(name1 + " 3600 IN MX 10 " + name1) + answer := []RR{rrMx, rrMx} + for i := 0; i < 128; i++ { + rrA := testRR(fmt.Sprintf("example%03d.something%03delse.org. 2311 IN A 127.0.0.1", i/32, i%32)) + answer = append(answer, rrA) + } + answer = append(answer, rrMx, rrMx) + msg := makeMsg(name1, answer, nil, nil) + b.ResetTimer() + for i := 0; i < b.N; i++ { + msg.Len() + } +} + +func BenchmarkMsgLengthOnlyQuestion(b *testing.B) { + msg := new(Msg) + msg.SetQuestion(Fqdn("12345678901234567890123456789012345.12345678.123."), TypeANY) + msg.Compress = true + b.ResetTimer() + for i := 0; i < b.N; i++ { + msg.Len() + } +} + +func BenchmarkMsgLengthEscapedName(b *testing.B) { + msg := new(Msg) + msg.SetQuestion(`\1\2\3\4\5\6\7\8\9\0\1\2\3\4\5\6\7\8\9\0\1\2\3\4\5\6\7\8\9\0\1\2\3\4\5.\1\2\3\4\5\6\7\8.\1\2\3.`, TypeANY) + b.ResetTimer() + for i := 0; i < b.N; i++ { + msg.Len() + } +} + func BenchmarkPackDomainName(b *testing.B) { name1 := "12345678901234567890123456789012345.12345678.123." buf := make([]byte, len(name1)+1) @@ -92,6 +137,36 @@ func BenchmarkUnpackDomainNameUnprintable(b *testing.B) { } } +func BenchmarkUnpackDomainNameLongest(b *testing.B) { + buf := make([]byte, len(longestDomain)+1) + n, err := PackDomainName(longestDomain, buf, 0, nil, false) + if err != nil { + b.Fatal(err) + } + if n != maxDomainNameWireOctets { + b.Fatalf("name wrong size in wire format, expected %d, got %d", maxDomainNameWireOctets, n) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, _ = UnpackDomainName(buf, 0) + } +} + +func BenchmarkUnpackDomainNameLongestUnprintable(b *testing.B) { + buf := make([]byte, len(longestUnprintableDomain)+1) + n, err := PackDomainName(longestUnprintableDomain, buf, 0, nil, false) + if err != nil { + b.Fatal(err) + } + if n != maxDomainNameWireOctets { + b.Fatalf("name wrong size in wire format, expected %d, got %d", maxDomainNameWireOctets, n) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _, _ = UnpackDomainName(buf, 0) + } +} + func BenchmarkCopy(b *testing.B) { b.ReportAllocs() m := new(Msg) @@ -112,7 +187,7 @@ func BenchmarkCopy(b *testing.B) { func BenchmarkPackA(b *testing.B) { a := &A{Hdr: RR_Header{Name: ".", Rrtype: TypeA, Class: ClassANY}, A: net.IPv4(127, 0, 0, 1)} - buf := make([]byte, a.len()) + buf := make([]byte, Len(a)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -123,7 +198,7 @@ func BenchmarkPackA(b *testing.B) { func BenchmarkUnpackA(b *testing.B) { a := &A{Hdr: RR_Header{Name: ".", Rrtype: TypeA, Class: ClassANY}, A: net.IPv4(127, 0, 0, 1)} - buf := make([]byte, a.len()) + buf := make([]byte, Len(a)) PackRR(a, buf, 0, nil, false) a = nil b.ReportAllocs() @@ -136,7 +211,7 @@ func BenchmarkUnpackA(b *testing.B) { func BenchmarkPackMX(b *testing.B) { m := &MX{Hdr: RR_Header{Name: ".", Rrtype: TypeA, Class: ClassANY}, Mx: "mx.miek.nl."} - buf := make([]byte, m.len()) + buf := make([]byte, Len(m)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -147,7 +222,7 @@ func BenchmarkPackMX(b *testing.B) { func BenchmarkUnpackMX(b *testing.B) { m := &MX{Hdr: RR_Header{Name: ".", Rrtype: TypeA, Class: ClassANY}, Mx: "mx.miek.nl."} - buf := make([]byte, m.len()) + buf := make([]byte, Len(m)) PackRR(m, buf, 0, nil, false) m = nil b.ReportAllocs() @@ -160,7 +235,7 @@ func BenchmarkUnpackMX(b *testing.B) { func BenchmarkPackAAAAA(b *testing.B) { aaaa := testRR(". IN A ::1") - buf := make([]byte, aaaa.len()) + buf := make([]byte, Len(aaaa)) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { @@ -171,7 +246,7 @@ func BenchmarkPackAAAAA(b *testing.B) { func BenchmarkUnpackAAAA(b *testing.B) { aaaa := testRR(". IN A ::1") - buf := make([]byte, aaaa.len()) + buf := make([]byte, Len(aaaa)) PackRR(aaaa, buf, 0, nil, false) aaaa = nil b.ReportAllocs() @@ -202,6 +277,45 @@ func BenchmarkPackMsg(b *testing.B) { } } +func BenchmarkPackMsgMassive(b *testing.B) { + makeMsg := func(question string, ans, ns, e []RR) *Msg { + msg := new(Msg) + msg.SetQuestion(Fqdn(question), TypeANY) + msg.Answer = append(msg.Answer, ans...) + msg.Ns = append(msg.Ns, ns...) + msg.Extra = append(msg.Extra, e...) + msg.Compress = true + return msg + } + const name1 = "12345678901234567890123456789012345.12345678.123." + rrMx := testRR(name1 + " 3600 IN MX 10 " + name1) + answer := []RR{rrMx, rrMx} + for i := 0; i < 128; i++ { + rrA := testRR(fmt.Sprintf("example%03d.something%03delse.org. 2311 IN A 127.0.0.1", i/32, i%32)) + answer = append(answer, rrA) + } + answer = append(answer, rrMx, rrMx) + msg := makeMsg(name1, answer, nil, nil) + buf := make([]byte, 512) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = msg.PackBuffer(buf) + } +} + +func BenchmarkPackMsgOnlyQuestion(b *testing.B) { + msg := new(Msg) + msg.SetQuestion(Fqdn("12345678901234567890123456789012345.12345678.123."), TypeANY) + msg.Compress = true + buf := make([]byte, 512) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = msg.PackBuffer(buf) + } +} + func BenchmarkUnpackMsg(b *testing.B) { makeMsg := func(question string, ans, ns, e []RR) *Msg { msg := new(Msg) @@ -228,3 +342,29 @@ func BenchmarkIdGeneration(b *testing.B) { _ = id() } } + +func BenchmarkReverseAddr(b *testing.B) { + b.Run("IP4", func(b *testing.B) { + for n := 0; n < b.N; n++ { + addr, err := ReverseAddr("192.0.2.1") + if err != nil { + b.Fatal(err) + } + if expect := "1.2.0.192.in-addr.arpa."; addr != expect { + b.Fatalf("invalid reverse address, expected %q, got %q", expect, addr) + } + } + }) + + b.Run("IP6", func(b *testing.B) { + for n := 0; n < b.N; n++ { + addr, err := ReverseAddr("2001:db8::68") + if err != nil { + b.Fatal(err) + } + if expect := "8.6.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa."; addr != expect { + b.Fatalf("invalid reverse address, expected %q, got %q", expect, addr) + } + } + }) +} diff --git a/vendor/github.com/miekg/dns/dns_test.go b/vendor/github.com/miekg/dns/dns_test.go index 3c9d910..f1dbf23 100644 --- a/vendor/github.com/miekg/dns/dns_test.go +++ b/vendor/github.com/miekg/dns/dns_test.go @@ -10,8 +10,7 @@ import ( func TestPackUnpack(t *testing.T) { out := new(Msg) out.Answer = make([]RR, 1) - key := new(DNSKEY) - key = &DNSKEY{Flags: 257, Protocol: 3, Algorithm: RSASHA1} + key := &DNSKEY{Flags: 257, Protocol: 3, Algorithm: RSASHA1} key.Hdr = RR_Header{Name: "miek.nl.", Rrtype: TypeDNSKEY, Class: ClassINET, Ttl: 3600} key.PublicKey = "AwEAAaHIwpx3w4VHKi6i1LHnTaWeHCL154Jug0Rtc9ji5qwPXpBo6A5sRv7cSsPQKPIwxLpyCrbJ4mr2L0EPOdvP6z6YfljK2ZmTbogU9aSU2fiq/4wjxbdkLyoDVgtO+JsxNN4bjr4WcWhsmk1Hg93FV9ZpkWb0Tbad8DFqNDzr//kZ" @@ -25,8 +24,7 @@ func TestPackUnpack(t *testing.T) { t.Error("failed to unpack msg with DNSKEY") } - sig := new(RRSIG) - sig = &RRSIG{TypeCovered: TypeDNSKEY, Algorithm: RSASHA1, Labels: 2, + sig := &RRSIG{TypeCovered: TypeDNSKEY, Algorithm: RSASHA1, Labels: 2, OrigTtl: 3600, Expiration: 4000, Inception: 4000, KeyTag: 34641, SignerName: "miek.nl.", Signature: "AwEAAaHIwpx3w4VHKi6i1LHnTaWeHCL154Jug0Rtc9ji5qwPXpBo6A5sRv7cSsPQKPIwxLpyCrbJ4mr2L0EPOdvP6z6YfljK2ZmTbogU9aSU2fiq/4wjxbdkLyoDVgtO+JsxNN4bjr4WcWhsmk1Hg93FV9ZpkWb0Tbad8DFqNDzr//kZ"} sig.Hdr = RR_Header{Name: "miek.nl.", Rrtype: TypeRRSIG, Class: ClassINET, Ttl: 3600} @@ -134,10 +132,10 @@ func TestPackNAPTR(t *testing.T) { `apple.com. IN NAPTR 50 50 "se" "SIPS+D2T" "" _sips._tcp.apple.com.`, } { rr := testRR(n) - msg := make([]byte, rr.len()) + msg := make([]byte, Len(rr)) if off, err := PackRR(rr, msg, 0, nil, false); err != nil { t.Errorf("packing failed: %v", err) - t.Errorf("length %d, need more than %d", rr.len(), off) + t.Errorf("length %d, need more than %d", Len(rr), off) } } } @@ -281,8 +279,8 @@ func TestTKEY(t *testing.T) { t.Fatal("Unable to decode TKEY") } // Make sure we get back the same length - if rr.len() != len(tkeyBytes) { - t.Fatalf("Lengths don't match %d != %d", rr.len(), len(tkeyBytes)) + if Len(rr) != len(tkeyBytes) { + t.Fatalf("Lengths don't match %d != %d", Len(rr), len(tkeyBytes)) } // make space for it with some fudge room msg := make([]byte, tkeyLen+1000) @@ -293,10 +291,10 @@ func TestTKEY(t *testing.T) { if offset != len(tkeyBytes) { t.Fatalf("mismatched TKEY RR size %d != %d", len(tkeyBytes), offset) } - if bytes.Compare(tkeyBytes, msg[0:offset]) != 0 { + if !bytes.Equal(tkeyBytes, msg[0:offset]) { t.Fatal("mismatched TKEY data after rewriting bytes") } - t.Logf("got TKEY of: " + rr.String()) + // Now add some bytes to this and make sure we can encode OtherData properly tkey := rr.(*TKEY) tkey.OtherData = "abcd" @@ -305,16 +303,14 @@ func TestTKEY(t *testing.T) { if packErr != nil { t.Fatal("unable to pack TKEY RR after modification", packErr) } - if offset != (len(tkeyBytes) + 2) { + if offset != len(tkeyBytes)+2 { t.Fatalf("mismatched TKEY RR size %d != %d", offset, len(tkeyBytes)+2) } - t.Logf("modified to TKEY of: " + rr.String()) // Make sure we can parse our string output tkey.Hdr.Class = ClassINET // https://github.com/miekg/dns/issues/577 - newRR, newError := NewRR(tkey.String()) + _, newError := NewRR(tkey.String()) if newError != nil { t.Fatalf("unable to parse TKEY string: %s", newError) } - t.Log("got reparsed TKEY of newRR: " + newRR.String()) } diff --git a/vendor/github.com/miekg/dns/dnssec.go b/vendor/github.com/miekg/dns/dnssec.go index 7e6bac4..3954d41 100644 --- a/vendor/github.com/miekg/dns/dnssec.go +++ b/vendor/github.com/miekg/dns/dnssec.go @@ -67,9 +67,6 @@ var AlgorithmToString = map[uint8]string{ PRIVATEOID: "PRIVATEOID", } -// StringToAlgorithm is the reverse of AlgorithmToString. -var StringToAlgorithm = reverseInt8(AlgorithmToString) - // AlgorithmToHash is a map of algorithm crypto hash IDs to crypto.Hash's. var AlgorithmToHash = map[uint8]crypto.Hash{ RSAMD5: crypto.MD5, // Deprecated in RFC 6725 @@ -102,9 +99,6 @@ var HashToString = map[uint8]string{ SHA512: "SHA512", } -// StringToHash is a map of names to hash IDs. -var StringToHash = reverseInt8(HashToString) - // DNSKEY flag values. const ( SEP = 1 @@ -173,7 +167,7 @@ func (k *DNSKEY) KeyTag() uint16 { keytag += int(v) << 8 } } - keytag += (keytag >> 16) & 0xFFFF + keytag += keytag >> 16 & 0xFFFF keytag &= 0xFFFF } return uint16(keytag) @@ -268,16 +262,17 @@ func (rr *RRSIG) Sign(k crypto.Signer, rrset []RR) error { return ErrKey } + h0 := rrset[0].Header() rr.Hdr.Rrtype = TypeRRSIG - rr.Hdr.Name = rrset[0].Header().Name - rr.Hdr.Class = rrset[0].Header().Class + rr.Hdr.Name = h0.Name + rr.Hdr.Class = h0.Class if rr.OrigTtl == 0 { // If set don't override - rr.OrigTtl = rrset[0].Header().Ttl + rr.OrigTtl = h0.Ttl } - rr.TypeCovered = rrset[0].Header().Rrtype - rr.Labels = uint8(CountLabel(rrset[0].Header().Name)) + rr.TypeCovered = h0.Rrtype + rr.Labels = uint8(CountLabel(h0.Name)) - if strings.HasPrefix(rrset[0].Header().Name, "*") { + if strings.HasPrefix(h0.Name, "*") { rr.Labels-- // wildcard, remove from label count } @@ -401,7 +396,7 @@ func (rr *RRSIG) Verify(k *DNSKEY, rrset []RR) error { if rr.Algorithm != k.Algorithm { return ErrKey } - if strings.ToLower(rr.SignerName) != strings.ToLower(k.Hdr.Name) { + if !strings.EqualFold(rr.SignerName, k.Hdr.Name) { return ErrKey } if k.Protocol != 3 { @@ -411,10 +406,7 @@ func (rr *RRSIG) Verify(k *DNSKEY, rrset []RR) error { // IsRRset checked that we have at least one RR and that the RRs in // the set have consistent type, class, and name. Also check that type and // class matches the RRSIG record. - if rrset[0].Header().Class != rr.Hdr.Class { - return ErrRRset - } - if rrset[0].Header().Rrtype != rr.TypeCovered { + if h0 := rrset[0].Header(); h0.Class != rr.Hdr.Class || h0.Rrtype != rr.TypeCovered { return ErrRRset } @@ -512,8 +504,8 @@ func (rr *RRSIG) ValidityPeriod(t time.Time) bool { } modi := (int64(rr.Inception) - utc) / year68 mode := (int64(rr.Expiration) - utc) / year68 - ti := int64(rr.Inception) + (modi * year68) - te := int64(rr.Expiration) + (mode * year68) + ti := int64(rr.Inception) + modi*year68 + te := int64(rr.Expiration) + mode*year68 return ti <= utc && utc <= te } @@ -533,6 +525,11 @@ func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey { return nil } + if len(keybuf) < 1+1+64 { + // Exponent must be at least 1 byte and modulus at least 64 + return nil + } + // RFC 2537/3110, section 2. RSA Public KEY Resource Records // Length is in the 0th byte, unless its zero, then it // it in bytes 1 and 2 and its a 16 bit number @@ -542,14 +539,23 @@ func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey { explen = uint16(keybuf[1])<<8 | uint16(keybuf[2]) keyoff = 3 } - if explen > 4 { - // Larger exponent than supported by the crypto package. + + if explen > 4 || explen == 0 || keybuf[keyoff] == 0 { + // Exponent larger than supported by the crypto package, + // empty, or contains prohibited leading zero. return nil } + + modoff := keyoff + int(explen) + modlen := len(keybuf) - modoff + if modlen < 64 || modlen > 512 || keybuf[modoff] == 0 { + // Modulus is too small, large, or contains prohibited leading zero. + return nil + } + pubkey := new(rsa.PublicKey) - pubkey.N = big.NewInt(0) - expo := uint64(0) + var expo uint64 for i := 0; i < int(explen); i++ { expo <<= 8 expo |= uint64(keybuf[keyoff+i]) @@ -560,7 +566,9 @@ func (k *DNSKEY) publicKeyRSA() *rsa.PublicKey { } pubkey.E = int(expo) - pubkey.N.SetBytes(keybuf[keyoff+int(explen):]) + pubkey.N = big.NewInt(0) + pubkey.N.SetBytes(keybuf[modoff:]) + return pubkey } @@ -642,15 +650,16 @@ func rawSignatureData(rrset []RR, s *RRSIG) (buf []byte, err error) { wires := make(wireSlice, len(rrset)) for i, r := range rrset { r1 := r.copy() - r1.Header().Ttl = s.OrigTtl - labels := SplitDomainName(r1.Header().Name) + h := r1.Header() + h.Ttl = s.OrigTtl + labels := SplitDomainName(h.Name) // 6.2. Canonical RR Form. (4) - wildcards if len(labels) > int(s.Labels) { // Wildcard - r1.Header().Name = "*." + strings.Join(labels[len(labels)-int(s.Labels):], ".") + "." + h.Name = "*." + strings.Join(labels[len(labels)-int(s.Labels):], ".") + "." } // RFC 4034: 6.2. Canonical RR Form. (2) - domain name to lowercase - r1.Header().Name = strings.ToLower(r1.Header().Name) + h.Name = strings.ToLower(h.Name) // 6.2. Canonical RR Form. (3) - domain rdata to lowercase. // NS, MD, MF, CNAME, SOA, MB, MG, MR, PTR, // HINFO, MINFO, MX, RP, AFSDB, RT, SIG, PX, NXT, NAPTR, KX, @@ -708,7 +717,7 @@ func rawSignatureData(rrset []RR, s *RRSIG) (buf []byte, err error) { x.Target = strings.ToLower(x.Target) } // 6.2. Canonical RR Form. (5) - origTTL - wire := make([]byte, r1.len()+1) // +1 to be safe(r) + wire := make([]byte, Len(r1)+1) // +1 to be safe(r) off, err1 := PackRR(r1, wire, 0, nil, false) if err1 != nil { return nil, err1 diff --git a/vendor/github.com/miekg/dns/dnssec_keyscan.go b/vendor/github.com/miekg/dns/dnssec_keyscan.go index e2d9d8f..5e65422 100644 --- a/vendor/github.com/miekg/dns/dnssec_keyscan.go +++ b/vendor/github.com/miekg/dns/dnssec_keyscan.go @@ -1,7 +1,7 @@ package dns import ( - "bytes" + "bufio" "crypto" "crypto/dsa" "crypto/ecdsa" @@ -181,22 +181,10 @@ func readPrivateKeyED25519(m map[string]string) (ed25519.PrivateKey, error) { if err != nil { return nil, err } - if len(p1) != 32 { + if len(p1) != ed25519.SeedSize { return nil, ErrPrivKey } - // RFC 8080 and Golang's x/crypto/ed25519 differ as to how the - // private keys are represented. RFC 8080 specifies that private - // keys be stored solely as the seed value (p1 above) while the - // ed25519 package represents them as the seed value concatenated - // to the public key, which is derived from the seed value. - // - // ed25519.GenerateKey reads exactly 32 bytes from the passed in - // io.Reader and uses them as the seed. It also derives the - // public key and produces a compatible private key. - _, p, err = ed25519.GenerateKey(bytes.NewReader(p1)) - if err != nil { - return nil, err - } + p = ed25519.NewKeyFromSeed(p1) case "created", "publish", "activate": /* not used in Go (yet) */ } @@ -207,23 +195,12 @@ func readPrivateKeyED25519(m map[string]string) (ed25519.PrivateKey, error) { // parseKey reads a private key from r. It returns a map[string]string, // with the key-value pairs, or an error when the file is not correct. func parseKey(r io.Reader, file string) (map[string]string, error) { - s, cancel := scanInit(r) m := make(map[string]string) - c := make(chan lex) - k := "" - defer func() { - cancel() - // zlexer can send up to two tokens, the next one and possibly 1 remainders. - // Do a non-blocking read. - _, ok := <-c - _, ok = <-c - if !ok { - // too bad - } - }() - // Start the lexer - go klexer(s, c) - for l := range c { + var k string + + c := newKLexer(r) + + for l, ok := c.Next(); ok; l, ok = c.Next() { // It should alternate switch l.value { case zKey: @@ -232,41 +209,111 @@ func parseKey(r io.Reader, file string) (map[string]string, error) { if k == "" { return nil, &ParseError{file, "no private key seen", l} } - //println("Setting", strings.ToLower(k), "to", l.token, "b") + m[strings.ToLower(k)] = l.token k = "" } } + + // Surface any read errors from r. + if err := c.Err(); err != nil { + return nil, &ParseError{file: file, err: err.Error()} + } + return m, nil } -// klexer scans the sourcefile and returns tokens on the channel c. -func klexer(s *scan, c chan lex) { - var l lex - str := "" // Hold the current read text - commt := false - key := true - x, err := s.tokenText() - defer close(c) - for err == nil { - l.column = s.position.Column - l.line = s.position.Line +type klexer struct { + br io.ByteReader + + readErr error + + line int + column int + + key bool + + eol bool // end-of-line +} + +func newKLexer(r io.Reader) *klexer { + br, ok := r.(io.ByteReader) + if !ok { + br = bufio.NewReaderSize(r, 1024) + } + + return &klexer{ + br: br, + + line: 1, + + key: true, + } +} + +func (kl *klexer) Err() error { + if kl.readErr == io.EOF { + return nil + } + + return kl.readErr +} + +// readByte returns the next byte from the input +func (kl *klexer) readByte() (byte, bool) { + if kl.readErr != nil { + return 0, false + } + + c, err := kl.br.ReadByte() + if err != nil { + kl.readErr = err + return 0, false + } + + // delay the newline handling until the next token is delivered, + // fixes off-by-one errors when reporting a parse error. + if kl.eol { + kl.line++ + kl.column = 0 + kl.eol = false + } + + if c == '\n' { + kl.eol = true + } else { + kl.column++ + } + + return c, true +} + +func (kl *klexer) Next() (lex, bool) { + var ( + l lex + + str strings.Builder + + commt bool + ) + + for x, ok := kl.readByte(); ok; x, ok = kl.readByte() { + l.line, l.column = kl.line, kl.column + switch x { case ':': - if commt { + if commt || !kl.key { break } - l.token = str - if key { - l.value = zKey - c <- l - // Next token is a space, eat it - s.tokenText() - key = false - str = "" - } else { - l.value = zValue - } + + kl.key = false + + // Next token is a space, eat it + kl.readByte() + + l.value = zKey + l.token = str.String() + return l, true case ';': commt = true case '\n': @@ -274,24 +321,32 @@ func klexer(s *scan, c chan lex) { // Reset a comment commt = false } + + kl.key = true + l.value = zValue - l.token = str - c <- l - str = "" - commt = false - key = true + l.token = str.String() + return l, true default: if commt { break } - str += string(x) + + str.WriteByte(x) } - x, err = s.tokenText() } - if len(str) > 0 { + + if kl.readErr != nil && kl.readErr != io.EOF { + // Don't return any tokens after a read error occurs. + return lex{value: zEOF}, false + } + + if str.Len() > 0 { // Send remainder - l.token = str l.value = zValue - c <- l + l.token = str.String() + return l, true } + + return lex{value: zEOF}, false } diff --git a/vendor/github.com/miekg/dns/dnssec_privkey.go b/vendor/github.com/miekg/dns/dnssec_privkey.go index 46f3215..0c65be1 100644 --- a/vendor/github.com/miekg/dns/dnssec_privkey.go +++ b/vendor/github.com/miekg/dns/dnssec_privkey.go @@ -82,7 +82,7 @@ func (r *DNSKEY) PrivateKeyString(p crypto.PrivateKey) string { "Public_value(y): " + pub + "\n" case ed25519.PrivateKey: - private := toBase64(p[:32]) + private := toBase64(p.Seed()) return format + "Algorithm: " + algorithm + "\n" + "PrivateKey: " + private + "\n" diff --git a/vendor/github.com/miekg/dns/dnssec_test.go b/vendor/github.com/miekg/dns/dnssec_test.go index 2dea4e5..32c195d 100644 --- a/vendor/github.com/miekg/dns/dnssec_test.go +++ b/vendor/github.com/miekg/dns/dnssec_test.go @@ -12,18 +12,6 @@ import ( "golang.org/x/crypto/ed25519" ) -func getKey() *DNSKEY { - key := new(DNSKEY) - key.Hdr.Name = "miek.nl." - key.Hdr.Class = ClassINET - key.Hdr.Ttl = 14400 - key.Flags = 256 - key.Protocol = 3 - key.Algorithm = RSASHA256 - key.PublicKey = "AwEAAcNEU67LJI5GEgF9QLNqLO1SMq1EdoQ6E9f85ha0k0ewQGCblyW2836GiVsm6k8Kr5ECIoMJ6fZWf3CQSQ9ycWfTyOHfmI3eQ/1Covhb2y4bAmL/07PhrL7ozWBW3wBfM335Ft9xjtXHPy7ztCbV9qZ4TVDTW/Iyg0PiwgoXVesz" - return key -} - func getSoa() *SOA { soa := new(SOA) soa.Hdr = RR_Header{"miek.nl.", TypeSOA, ClassINET, 14400, 0} @@ -858,3 +846,13 @@ func TestRsaExponentUnpack(t *testing.T) { t.Fatalf("cannot verify RRSIG with keytag [%d]. Cause [%s]", ksk.KeyTag(), e.Error()) } } + +func TestParseKeyReadError(t *testing.T) { + m, err := parseKey(errReader{}, "") + if err == nil || !strings.Contains(err.Error(), errTestReadError.Error()) { + t.Errorf("expected error to contain %q, but got %v", errTestReadError, err) + } + if m != nil { + t.Errorf("expected a nil map, but got %v", m) + } +} diff --git a/vendor/github.com/miekg/dns/doc.go b/vendor/github.com/miekg/dns/doc.go index 0389d72..d3d7cec 100644 --- a/vendor/github.com/miekg/dns/doc.go +++ b/vendor/github.com/miekg/dns/doc.go @@ -1,20 +1,20 @@ /* Package dns implements a full featured interface to the Domain Name System. -Server- and client-side programming is supported. -The package allows complete control over what is sent out to the DNS. The package -API follows the less-is-more principle, by presenting a small, clean interface. +Both server- and client-side programming is supported. The package allows +complete control over what is sent out to the DNS. The API follows the +less-is-more principle, by presenting a small, clean interface. -The package dns supports (asynchronous) querying/replying, incoming/outgoing zone transfers, +It supports (asynchronous) querying/replying, incoming/outgoing zone transfers, TSIG, EDNS0, dynamic updates, notifies and DNSSEC validation/signing. -Note that domain names MUST be fully qualified, before sending them, unqualified + +Note that domain names MUST be fully qualified before sending them, unqualified names in a message will result in a packing failure. -Resource records are native types. They are not stored in wire format. -Basic usage pattern for creating a new resource record: +Resource records are native types. They are not stored in wire format. Basic +usage pattern for creating a new resource record: r := new(dns.MX) - r.Hdr = dns.RR_Header{Name: "miek.nl.", Rrtype: dns.TypeMX, - Class: dns.ClassINET, Ttl: 3600} + r.Hdr = dns.RR_Header{Name: "miek.nl.", Rrtype: dns.TypeMX, Class: dns.ClassINET, Ttl: 3600} r.Preference = 10 r.Mx = "mx.miek.nl." @@ -30,8 +30,8 @@ Or even: mx, err := dns.NewRR("$ORIGIN nl.\nmiek 1H IN MX 10 mx.miek") -In the DNS messages are exchanged, these messages contain resource -records (sets). Use pattern for creating a message: +In the DNS messages are exchanged, these messages contain resource records +(sets). Use pattern for creating a message: m := new(dns.Msg) m.SetQuestion("miek.nl.", dns.TypeMX) @@ -40,8 +40,8 @@ Or when not certain if the domain name is fully qualified: m.SetQuestion(dns.Fqdn("miek.nl"), dns.TypeMX) -The message m is now a message with the question section set to ask -the MX records for the miek.nl. zone. +The message m is now a message with the question section set to ask the MX +records for the miek.nl. zone. The following is slightly more verbose, but more flexible: @@ -51,9 +51,8 @@ The following is slightly more verbose, but more flexible: m1.Question = make([]dns.Question, 1) m1.Question[0] = dns.Question{"miek.nl.", dns.TypeMX, dns.ClassINET} -After creating a message it can be sent. -Basic use pattern for synchronous querying the DNS at a -server configured on 127.0.0.1 and port 53: +After creating a message it can be sent. Basic use pattern for synchronous +querying the DNS at a server configured on 127.0.0.1 and port 53: c := new(dns.Client) in, rtt, err := c.Exchange(m1, "127.0.0.1:53") @@ -99,25 +98,24 @@ the Answer section: Domain Name and TXT Character String Representations -Both domain names and TXT character strings are converted to presentation -form both when unpacked and when converted to strings. +Both domain names and TXT character strings are converted to presentation form +both when unpacked and when converted to strings. For TXT character strings, tabs, carriage returns and line feeds will be -converted to \t, \r and \n respectively. Back slashes and quotations marks -will be escaped. Bytes below 32 and above 127 will be converted to \DDD -form. +converted to \t, \r and \n respectively. Back slashes and quotations marks will +be escaped. Bytes below 32 and above 127 will be converted to \DDD form. -For domain names, in addition to the above rules brackets, periods, -spaces, semicolons and the at symbol are escaped. +For domain names, in addition to the above rules brackets, periods, spaces, +semicolons and the at symbol are escaped. DNSSEC -DNSSEC (DNS Security Extension) adds a layer of security to the DNS. It -uses public key cryptography to sign resource records. The -public keys are stored in DNSKEY records and the signatures in RRSIG records. +DNSSEC (DNS Security Extension) adds a layer of security to the DNS. It uses +public key cryptography to sign resource records. The public keys are stored in +DNSKEY records and the signatures in RRSIG records. -Requesting DNSSEC information for a zone is done by adding the DO (DNSSEC OK) bit -to a request. +Requesting DNSSEC information for a zone is done by adding the DO (DNSSEC OK) +bit to a request. m := new(dns.Msg) m.SetEdns0(4096, true) @@ -126,9 +124,9 @@ Signature generation, signature verification and key generation are all supporte DYNAMIC UPDATES -Dynamic updates reuses the DNS message format, but renames three of -the sections. Question is Zone, Answer is Prerequisite, Authority is -Update, only the Additional is not renamed. See RFC 2136 for the gory details. +Dynamic updates reuses the DNS message format, but renames three of the +sections. Question is Zone, Answer is Prerequisite, Authority is Update, only +the Additional is not renamed. See RFC 2136 for the gory details. You can set a rather complex set of rules for the existence of absence of certain resource records or names in a zone to specify if resource records @@ -145,10 +143,9 @@ DNS function shows which functions exist to specify the prerequisites. NONE rrset empty RRset does not exist dns.RRsetNotUsed zone rrset rr RRset exists (value dep) dns.Used -The prerequisite section can also be left empty. -If you have decided on the prerequisites you can tell what RRs should -be added or deleted. The next table shows the options you have and -what functions to call. +The prerequisite section can also be left empty. If you have decided on the +prerequisites you can tell what RRs should be added or deleted. The next table +shows the options you have and what functions to call. 3.4.2.6 - Table Of Metavalues Used In Update Section @@ -181,10 +178,10 @@ changes to the RRset after calling SetTsig() the signature will be incorrect. ... // When sending the TSIG RR is calculated and filled in before sending -When requesting an zone transfer (almost all TSIG usage is when requesting zone transfers), with -TSIG, this is the basic use pattern. In this example we request an AXFR for -miek.nl. with TSIG key named "axfr." and secret "so6ZGir4GPAqINNh9U5c3A==" -and using the server 176.58.119.54: +When requesting an zone transfer (almost all TSIG usage is when requesting zone +transfers), with TSIG, this is the basic use pattern. In this example we +request an AXFR for miek.nl. with TSIG key named "axfr." and secret +"so6ZGir4GPAqINNh9U5c3A==" and using the server 176.58.119.54: t := new(dns.Transfer) m := new(dns.Msg) @@ -194,8 +191,8 @@ and using the server 176.58.119.54: c, err := t.In(m, "176.58.119.54:53") for r := range c { ... } -You can now read the records from the transfer as they come in. Each envelope is checked with TSIG. -If something is not correct an error is returned. +You can now read the records from the transfer as they come in. Each envelope +is checked with TSIG. If something is not correct an error is returned. Basic use pattern validating and replying to a message that has TSIG set. @@ -220,29 +217,30 @@ Basic use pattern validating and replying to a message that has TSIG set. PRIVATE RRS -RFC 6895 sets aside a range of type codes for private use. This range -is 65,280 - 65,534 (0xFF00 - 0xFFFE). When experimenting with new Resource Records these +RFC 6895 sets aside a range of type codes for private use. This range is 65,280 +- 65,534 (0xFF00 - 0xFFFE). When experimenting with new Resource Records these can be used, before requesting an official type code from IANA. -see http://miek.nl/2014/September/21/idn-and-private-rr-in-go-dns/ for more +See https://miek.nl/2014/September/21/idn-and-private-rr-in-go-dns/ for more information. EDNS0 -EDNS0 is an extension mechanism for the DNS defined in RFC 2671 and updated -by RFC 6891. It defines an new RR type, the OPT RR, which is then completely +EDNS0 is an extension mechanism for the DNS defined in RFC 2671 and updated by +RFC 6891. It defines an new RR type, the OPT RR, which is then completely abused. + Basic use pattern for creating an (empty) OPT RR: o := new(dns.OPT) o.Hdr.Name = "." // MUST be the root zone, per definition. o.Hdr.Rrtype = dns.TypeOPT -The rdata of an OPT RR consists out of a slice of EDNS0 (RFC 6891) -interfaces. Currently only a few have been standardized: EDNS0_NSID -(RFC 5001) and EDNS0_SUBNET (draft-vandergaast-edns-client-subnet-02). Note -that these options may be combined in an OPT RR. -Basic use pattern for a server to check if (and which) options are set: +The rdata of an OPT RR consists out of a slice of EDNS0 (RFC 6891) interfaces. +Currently only a few have been standardized: EDNS0_NSID (RFC 5001) and +EDNS0_SUBNET (draft-vandergaast-edns-client-subnet-02). Note that these options +may be combined in an OPT RR. Basic use pattern for a server to check if (and +which) options are set: // o is a dns.OPT for _, s := range o.Option { @@ -262,10 +260,9 @@ From RFC 2931: ... protection for glue records, DNS requests, protection for message headers on requests and responses, and protection of the overall integrity of a response. -It works like TSIG, except that SIG(0) uses public key cryptography, instead of the shared -secret approach in TSIG. -Supported algorithms: DSA, ECDSAP256SHA256, ECDSAP384SHA384, RSASHA1, RSASHA256 and -RSASHA512. +It works like TSIG, except that SIG(0) uses public key cryptography, instead of +the shared secret approach in TSIG. Supported algorithms: DSA, ECDSAP256SHA256, +ECDSAP384SHA384, RSASHA1, RSASHA256 and RSASHA512. Signing subsequent messages in multi-message sessions is not implemented. */ diff --git a/vendor/github.com/miekg/dns/duplicate.go b/vendor/github.com/miekg/dns/duplicate.go new file mode 100644 index 0000000..05c14aa --- /dev/null +++ b/vendor/github.com/miekg/dns/duplicate.go @@ -0,0 +1,38 @@ +package dns + +//go:generate go run duplicate_generate.go + +// IsDuplicate checks of r1 and r2 are duplicates of each other, excluding the TTL. +// So this means the header data is equal *and* the RDATA is the same. Return true +// is so, otherwise false. +// It's is a protocol violation to have identical RRs in a message. +func IsDuplicate(r1, r2 RR) bool { + // Check whether the record header is identical. + if !r1.Header().isDuplicate(r2.Header()) { + return false + } + + // Check whether the RDATA is identical. + return r1.isDuplicate(r2) +} + +func (r1 *RR_Header) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*RR_Header) + if !ok { + return false + } + if r1.Class != r2.Class { + return false + } + if r1.Rrtype != r2.Rrtype { + return false + } + if !isDulicateName(r1.Name, r2.Name) { + return false + } + // ignore TTL + return true +} + +// isDulicateName checks if the domain names s1 and s2 are equal. +func isDulicateName(s1, s2 string) bool { return equal(s1, s2) } diff --git a/vendor/github.com/miekg/dns/duplicate_generate.go b/vendor/github.com/miekg/dns/duplicate_generate.go new file mode 100644 index 0000000..69f15eb --- /dev/null +++ b/vendor/github.com/miekg/dns/duplicate_generate.go @@ -0,0 +1,144 @@ +//+build ignore + +// types_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will generate conversion tables (TypeToRR and TypeToString) and banal +// methods (len, Header, copy) based on the struct tags. The generated source is +// written to ztypes.go, and is meant to be checked into git. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" +) + +var packageHdr = ` +// Code generated by "go run duplicate_generate.go"; DO NOT EDIT. + +package dns + +` + +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + // Collect actual types (*X) + var namedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + + if st, _ := getTypeStruct(o.Type(), scope); st == nil { + continue + } + + if name == "PrivateRR" || name == "OPT" { + continue + } + + namedTypes = append(namedTypes, o.Name()) + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + // Generate the duplicate check for each type. + fmt.Fprint(b, "// isDuplicate() functions\n\n") + for _, name := range namedTypes { + + o := scope.Lookup(name) + st, isEmbedded := getTypeStruct(o.Type(), scope) + if isEmbedded { + continue + } + fmt.Fprintf(b, "func (r1 *%s) isDuplicate(_r2 RR) bool {\n", name) + fmt.Fprintf(b, "r2, ok := _r2.(*%s)\n", name) + fmt.Fprint(b, "if !ok { return false }\n") + fmt.Fprint(b, "_ = r2\n") + for i := 1; i < st.NumFields(); i++ { + field := st.Field(i).Name() + o2 := func(s string) { fmt.Fprintf(b, s+"\n", field, field) } + o3 := func(s string) { fmt.Fprintf(b, s+"\n", field, field, field) } + + // For some reason, a and aaaa don't pop up as *types.Slice here (mostly like because the are + // *indirectly* defined as a slice in the net package). + if _, ok := st.Field(i).Type().(*types.Slice); ok { + o2("if len(r1.%s) != len(r2.%s) {\nreturn false\n}") + + if st.Tag(i) == `dns:"cdomain-name"` || st.Tag(i) == `dns:"domain-name"` { + o3(`for i := 0; i < len(r1.%s); i++ { + if !isDulicateName(r1.%s[i], r2.%s[i]) { + return false + } + }`) + + continue + } + + o3(`for i := 0; i < len(r1.%s); i++ { + if r1.%s[i] != r2.%s[i] { + return false + } + }`) + + continue + } + + switch st.Tag(i) { + case `dns:"-"`: + // ignored + case `dns:"a"`, `dns:"aaaa"`: + o2("if !r1.%s.Equal(r2.%s) {\nreturn false\n}") + case `dns:"cdomain-name"`, `dns:"domain-name"`: + o2("if !isDulicateName(r1.%s, r2.%s) {\nreturn false\n}") + default: + o2("if r1.%s != r2.%s {\nreturn false\n}") + } + } + fmt.Fprintf(b, "return true\n}\n\n") + } + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + // write result + f, err := os.Create("zduplicate.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/miekg/dns/duplicate_test.go b/vendor/github.com/miekg/dns/duplicate_test.go new file mode 100644 index 0000000..f8b5345 --- /dev/null +++ b/vendor/github.com/miekg/dns/duplicate_test.go @@ -0,0 +1,84 @@ +package dns + +import "testing" + +func TestDuplicateA(t *testing.T) { + a1, _ := NewRR("www.example.org. 2700 IN A 127.0.0.1") + a2, _ := NewRR("www.example.org. IN A 127.0.0.1") + if !IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s to be duplicates, but got false", a1.String(), a2.String()) + } + + a2, _ = NewRR("www.example.org. IN A 127.0.0.2") + if IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s not to be duplicates, but got true", a1.String(), a2.String()) + } +} + +func TestDuplicateTXT(t *testing.T) { + a1, _ := NewRR("www.example.org. IN TXT \"aa\"") + a2, _ := NewRR("www.example.org. IN TXT \"aa\"") + + if !IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s to be duplicates, but got false", a1.String(), a2.String()) + } + + a2, _ = NewRR("www.example.org. IN TXT \"aa\" \"bb\"") + if IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s not to be duplicates, but got true", a1.String(), a2.String()) + } + + a1, _ = NewRR("www.example.org. IN TXT \"aa\" \"bc\"") + if IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s not to be duplicates, but got true", a1.String(), a2.String()) + } +} + +func TestDuplicateOwner(t *testing.T) { + a1, _ := NewRR("www.example.org. IN A 127.0.0.1") + a2, _ := NewRR("www.example.org. IN A 127.0.0.1") + if !IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s to be duplicates, but got false", a1.String(), a2.String()) + } + + a2, _ = NewRR("WWw.exaMPle.org. IN A 127.0.0.2") + if IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s to be duplicates, but got false", a1.String(), a2.String()) + } +} + +func TestDuplicateDomain(t *testing.T) { + a1, _ := NewRR("www.example.org. IN CNAME example.org.") + a2, _ := NewRR("www.example.org. IN CNAME example.org.") + if !IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s to be duplicates, but got false", a1.String(), a2.String()) + } + + a2, _ = NewRR("www.example.org. IN CNAME exAMPLe.oRG.") + if !IsDuplicate(a1, a2) { + t.Errorf("expected %s/%s to be duplicates, but got false", a1.String(), a2.String()) + } +} + +func TestDuplicateWrongRrtype(t *testing.T) { + // Test that IsDuplicate won't panic for a record that's lying about + // it's Rrtype. + + r1 := &A{Hdr: RR_Header{Rrtype: TypeA}} + r2 := &AAAA{Hdr: RR_Header{Rrtype: TypeA}} + if IsDuplicate(r1, r2) { + t.Errorf("expected %s/%s not to be duplicates, but got true", r1.String(), r2.String()) + } + + r3 := &AAAA{Hdr: RR_Header{Rrtype: TypeA}} + r4 := &A{Hdr: RR_Header{Rrtype: TypeA}} + if IsDuplicate(r3, r4) { + t.Errorf("expected %s/%s not to be duplicates, but got true", r3.String(), r4.String()) + } + + r5 := &AAAA{Hdr: RR_Header{Rrtype: TypeA}} + r6 := &AAAA{Hdr: RR_Header{Rrtype: TypeA}} + if !IsDuplicate(r5, r6) { + t.Errorf("expected %s/%s to be duplicates, but got false", r5.String(), r6.String()) + } +} diff --git a/vendor/github.com/miekg/dns/edns.go b/vendor/github.com/miekg/dns/edns.go index 55059eb..805641b 100644 --- a/vendor/github.com/miekg/dns/edns.go +++ b/vendor/github.com/miekg/dns/edns.go @@ -78,8 +78,8 @@ func (rr *OPT) String() string { return s } -func (rr *OPT) len() int { - l := rr.Hdr.len() +func (rr *OPT) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) for i := 0; i < len(rr.Option); i++ { l += 4 // Account for 2-byte option code and 2-byte option length. lo, _ := rr.Option[i].pack() @@ -88,29 +88,34 @@ func (rr *OPT) len() int { return l } +func (rr *OPT) parse(c *zlexer, origin, file string) *ParseError { + panic("dns: internal error: parse should never be called on OPT") +} + +func (r1 *OPT) isDuplicate(r2 RR) bool { return false } + // return the old value -> delete SetVersion? // Version returns the EDNS version used. Only zero is defined. func (rr *OPT) Version() uint8 { - return uint8((rr.Hdr.Ttl & 0x00FF0000) >> 16) + return uint8(rr.Hdr.Ttl & 0x00FF0000 >> 16) } // SetVersion sets the version of EDNS. This is usually zero. func (rr *OPT) SetVersion(v uint8) { - rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | (uint32(v) << 16) + rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | uint32(v)<<16 } // ExtendedRcode returns the EDNS extended RCODE field (the upper 8 bits of the TTL). func (rr *OPT) ExtendedRcode() int { - return int((rr.Hdr.Ttl&0xFF000000)>>24) + 15 + return int(rr.Hdr.Ttl&0xFF000000>>24) << 4 } // SetExtendedRcode sets the EDNS extended RCODE field. -func (rr *OPT) SetExtendedRcode(v uint8) { - if v < RcodeBadVers { // Smaller than 16.. Use the 4 bits you have! - return - } - rr.Hdr.Ttl = rr.Hdr.Ttl&0x00FFFFFF | (uint32(v-15) << 24) +// +// If the RCODE is not an extended RCODE, will reset the extended RCODE field to 0. +func (rr *OPT) SetExtendedRcode(v uint16) { + rr.Hdr.Ttl = rr.Hdr.Ttl&0x00FFFFFF | uint32(v>>4)<<24 } // UDPSize returns the UDP buffer size. @@ -154,6 +159,8 @@ type EDNS0 interface { unpack([]byte) error // String returns the string representation of the option. String() string + // copy returns a deep-copy of the option. + copy() EDNS0 } // EDNS0_NSID option is used to retrieve a nameserver @@ -184,7 +191,8 @@ func (e *EDNS0_NSID) pack() ([]byte, error) { // Option implements the EDNS0 interface. func (e *EDNS0_NSID) Option() uint16 { return EDNS0NSID } // Option returns the option code. func (e *EDNS0_NSID) unpack(b []byte) error { e.Nsid = hex.EncodeToString(b); return nil } -func (e *EDNS0_NSID) String() string { return string(e.Nsid) } +func (e *EDNS0_NSID) String() string { return e.Nsid } +func (e *EDNS0_NSID) copy() EDNS0 { return &EDNS0_NSID{e.Code, e.Nsid} } // EDNS0_SUBNET is the subnet option that is used to give the remote nameserver // an idea of where the client lives. See RFC 7871. It can then give back a different @@ -274,22 +282,16 @@ func (e *EDNS0_SUBNET) unpack(b []byte) error { if e.SourceNetmask > net.IPv4len*8 || e.SourceScope > net.IPv4len*8 { return errors.New("dns: bad netmask") } - addr := make([]byte, net.IPv4len) - for i := 0; i < net.IPv4len && 4+i < len(b); i++ { - addr[i] = b[4+i] - } - e.Address = net.IPv4(addr[0], addr[1], addr[2], addr[3]) + addr := make(net.IP, net.IPv4len) + copy(addr, b[4:]) + e.Address = addr.To16() case 2: if e.SourceNetmask > net.IPv6len*8 || e.SourceScope > net.IPv6len*8 { return errors.New("dns: bad netmask") } - addr := make([]byte, net.IPv6len) - for i := 0; i < net.IPv6len && 4+i < len(b); i++ { - addr[i] = b[4+i] - } - e.Address = net.IP{addr[0], addr[1], addr[2], addr[3], addr[4], - addr[5], addr[6], addr[7], addr[8], addr[9], addr[10], - addr[11], addr[12], addr[13], addr[14], addr[15]} + addr := make(net.IP, net.IPv6len) + copy(addr, b[4:]) + e.Address = addr default: return errors.New("dns: bad address family") } @@ -308,6 +310,16 @@ func (e *EDNS0_SUBNET) String() (s string) { return } +func (e *EDNS0_SUBNET) copy() EDNS0 { + return &EDNS0_SUBNET{ + e.Code, + e.Family, + e.SourceNetmask, + e.SourceScope, + e.Address, + } +} + // The EDNS0_COOKIE option is used to add a DNS Cookie to a message. // // o := new(dns.OPT) @@ -343,6 +355,7 @@ func (e *EDNS0_COOKIE) pack() ([]byte, error) { func (e *EDNS0_COOKIE) Option() uint16 { return EDNS0COOKIE } func (e *EDNS0_COOKIE) unpack(b []byte) error { e.Cookie = hex.EncodeToString(b); return nil } func (e *EDNS0_COOKIE) String() string { return e.Cookie } +func (e *EDNS0_COOKIE) copy() EDNS0 { return &EDNS0_COOKIE{e.Code, e.Cookie} } // The EDNS0_UL (Update Lease) (draft RFC) option is used to tell the server to set // an expiration on an update RR. This is helpful for clients that cannot clean @@ -364,6 +377,7 @@ type EDNS0_UL struct { // Option implements the EDNS0 interface. func (e *EDNS0_UL) Option() uint16 { return EDNS0UL } func (e *EDNS0_UL) String() string { return strconv.FormatUint(uint64(e.Lease), 10) } +func (e *EDNS0_UL) copy() EDNS0 { return &EDNS0_UL{e.Code, e.Lease} } // Copied: http://golang.org/src/pkg/net/dnsmsg.go func (e *EDNS0_UL) pack() ([]byte, error) { @@ -418,10 +432,13 @@ func (e *EDNS0_LLQ) unpack(b []byte) error { func (e *EDNS0_LLQ) String() string { s := strconv.FormatUint(uint64(e.Version), 10) + " " + strconv.FormatUint(uint64(e.Opcode), 10) + - " " + strconv.FormatUint(uint64(e.Error), 10) + " " + strconv.FormatUint(uint64(e.Id), 10) + + " " + strconv.FormatUint(uint64(e.Error), 10) + " " + strconv.FormatUint(e.Id, 10) + " " + strconv.FormatUint(uint64(e.LeaseLife), 10) return s } +func (e *EDNS0_LLQ) copy() EDNS0 { + return &EDNS0_LLQ{e.Code, e.Version, e.Opcode, e.Error, e.Id, e.LeaseLife} +} // EDNS0_DUA implements the EDNS0 "DNSSEC Algorithm Understood" option. See RFC 6975. type EDNS0_DAU struct { @@ -445,6 +462,7 @@ func (e *EDNS0_DAU) String() string { } return s } +func (e *EDNS0_DAU) copy() EDNS0 { return &EDNS0_DAU{e.Code, e.AlgCode} } // EDNS0_DHU implements the EDNS0 "DS Hash Understood" option. See RFC 6975. type EDNS0_DHU struct { @@ -468,6 +486,7 @@ func (e *EDNS0_DHU) String() string { } return s } +func (e *EDNS0_DHU) copy() EDNS0 { return &EDNS0_DHU{e.Code, e.AlgCode} } // EDNS0_N3U implements the EDNS0 "NSEC3 Hash Understood" option. See RFC 6975. type EDNS0_N3U struct { @@ -492,6 +511,7 @@ func (e *EDNS0_N3U) String() string { } return s } +func (e *EDNS0_N3U) copy() EDNS0 { return &EDNS0_N3U{e.Code, e.AlgCode} } // EDNS0_EXPIRE implementes the EDNS0 option as described in RFC 7314. type EDNS0_EXPIRE struct { @@ -502,13 +522,11 @@ type EDNS0_EXPIRE struct { // Option implements the EDNS0 interface. func (e *EDNS0_EXPIRE) Option() uint16 { return EDNS0EXPIRE } func (e *EDNS0_EXPIRE) String() string { return strconv.FormatUint(uint64(e.Expire), 10) } +func (e *EDNS0_EXPIRE) copy() EDNS0 { return &EDNS0_EXPIRE{e.Code, e.Expire} } func (e *EDNS0_EXPIRE) pack() ([]byte, error) { b := make([]byte, 4) - b[0] = byte(e.Expire >> 24) - b[1] = byte(e.Expire >> 16) - b[2] = byte(e.Expire >> 8) - b[3] = byte(e.Expire) + binary.BigEndian.PutUint32(b, e.Expire) return b, nil } @@ -543,6 +561,11 @@ func (e *EDNS0_LOCAL) Option() uint16 { return e.Code } func (e *EDNS0_LOCAL) String() string { return strconv.FormatInt(int64(e.Code), 10) + ":0x" + hex.EncodeToString(e.Data) } +func (e *EDNS0_LOCAL) copy() EDNS0 { + b := make([]byte, len(e.Data)) + copy(b, e.Data) + return &EDNS0_LOCAL{e.Code, b} +} func (e *EDNS0_LOCAL) pack() ([]byte, error) { b := make([]byte, len(e.Data)) @@ -615,6 +638,7 @@ func (e *EDNS0_TCP_KEEPALIVE) String() (s string) { } return } +func (e *EDNS0_TCP_KEEPALIVE) copy() EDNS0 { return &EDNS0_TCP_KEEPALIVE{e.Code, e.Length, e.Timeout} } // EDNS0_PADDING option is used to add padding to a request/response. The default // value of padding SHOULD be 0x0 but other values MAY be used, for instance if @@ -628,3 +652,8 @@ func (e *EDNS0_PADDING) Option() uint16 { return EDNS0PADDING } func (e *EDNS0_PADDING) pack() ([]byte, error) { return e.Padding, nil } func (e *EDNS0_PADDING) unpack(b []byte) error { e.Padding = b; return nil } func (e *EDNS0_PADDING) String() string { return fmt.Sprintf("%0X", e.Padding) } +func (e *EDNS0_PADDING) copy() EDNS0 { + b := make([]byte, len(e.Padding)) + copy(b, e.Padding) + return &EDNS0_PADDING{b} +} diff --git a/vendor/github.com/miekg/dns/edns_test.go b/vendor/github.com/miekg/dns/edns_test.go index f7cf157..69682a5 100644 --- a/vendor/github.com/miekg/dns/edns_test.go +++ b/vendor/github.com/miekg/dns/edns_test.go @@ -1,6 +1,9 @@ package dns -import "testing" +import ( + "net" + "testing" +) func TestOPTTtl(t *testing.T) { e := &OPT{} @@ -62,7 +65,47 @@ func TestOPTTtl(t *testing.T) { } e.SetExtendedRcode(42) - if e.ExtendedRcode() != 42 { - t.Errorf("set 42, expected %d, got %d", 42, e.ExtendedRcode()) + // ExtendedRcode has the last 4 bits set to 0. + if e.ExtendedRcode() != 42&0xFFFFFFF0 { + t.Errorf("set 42, expected %d, got %d", 42&0xFFFFFFF0, e.ExtendedRcode()) + } + + // This will reset the 8 upper bits of the extended rcode + e.SetExtendedRcode(RcodeNotAuth) + if e.ExtendedRcode() != 0 { + t.Errorf("Setting a non-extended rcode is expected to set extended rcode to 0, got: %d", e.ExtendedRcode()) + } +} + +func TestEDNS0_SUBNETUnpack(t *testing.T) { + for _, ip := range []net.IP{ + net.IPv4(0xde, 0xad, 0xbe, 0xef), + net.ParseIP("192.0.2.1"), + net.ParseIP("2001:db8::68"), + } { + var s1 EDNS0_SUBNET + s1.Address = ip + + if ip.To4() == nil { + s1.Family = 2 + s1.SourceNetmask = net.IPv6len * 8 + } else { + s1.Family = 1 + s1.SourceNetmask = net.IPv4len * 8 + } + + b, err := s1.pack() + if err != nil { + t.Fatalf("failed to pack: %v", err) + } + + var s2 EDNS0_SUBNET + if err := s2.unpack(b); err != nil { + t.Fatalf("failed to unpack: %v", err) + } + + if !ip.Equal(s2.Address) { + t.Errorf("address different after unpacking; expected %s, got %s", ip, s2.Address) + } } } diff --git a/vendor/github.com/miekg/dns/format.go b/vendor/github.com/miekg/dns/format.go index 3f5303c..86057f9 100644 --- a/vendor/github.com/miekg/dns/format.go +++ b/vendor/github.com/miekg/dns/format.go @@ -20,7 +20,7 @@ func Field(r RR, i int) string { return "" } d := reflect.ValueOf(r).Elem().Field(i) - switch k := d.Kind(); k { + switch d.Kind() { case reflect.String: return d.String() case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: diff --git a/vendor/github.com/miekg/dns/generate.go b/vendor/github.com/miekg/dns/generate.go index e4481a4..97bc39f 100644 --- a/vendor/github.com/miekg/dns/generate.go +++ b/vendor/github.com/miekg/dns/generate.go @@ -2,8 +2,8 @@ package dns import ( "bytes" - "errors" "fmt" + "io" "strconv" "strings" ) @@ -18,142 +18,225 @@ import ( // * rhs (rdata) // But we are lazy here, only the range is parsed *all* occurrences // of $ after that are interpreted. -// Any error are returned as a string value, the empty string signals -// "no error". -func generate(l lex, c chan lex, t chan *Token, o string) string { +func (zp *ZoneParser) generate(l lex) (RR, bool) { + token := l.token step := 1 - if i := strings.IndexAny(l.token, "/"); i != -1 { - if i+1 == len(l.token) { - return "bad step in $GENERATE range" + if i := strings.IndexByte(token, '/'); i >= 0 { + if i+1 == len(token) { + return zp.setParseError("bad step in $GENERATE range", l) } - if s, err := strconv.Atoi(l.token[i+1:]); err == nil { - if s < 0 { - return "bad step in $GENERATE range" - } - step = s - } else { - return "bad step in $GENERATE range" + + s, err := strconv.Atoi(token[i+1:]) + if err != nil || s <= 0 { + return zp.setParseError("bad step in $GENERATE range", l) } - l.token = l.token[:i] + + step = s + token = token[:i] } - sx := strings.SplitN(l.token, "-", 2) + + sx := strings.SplitN(token, "-", 2) if len(sx) != 2 { - return "bad start-stop in $GENERATE range" + return zp.setParseError("bad start-stop in $GENERATE range", l) } + start, err := strconv.Atoi(sx[0]) if err != nil { - return "bad start in $GENERATE range" + return zp.setParseError("bad start in $GENERATE range", l) } + end, err := strconv.Atoi(sx[1]) if err != nil { - return "bad stop in $GENERATE range" + return zp.setParseError("bad stop in $GENERATE range", l) } if end < 0 || start < 0 || end < start { - return "bad range in $GENERATE range" + return zp.setParseError("bad range in $GENERATE range", l) } - <-c // _BLANK + zp.c.Next() // _BLANK + // Create a complete new string, which we then parse again. - s := "" -BuildRR: - l = <-c - if l.value != zNewline && l.value != zEOF { - s += l.token - goto BuildRR - } - for i := start; i <= end; i += step { - var ( - escape bool - dom bytes.Buffer - mod string - err error - offset int - ) + var s string + for l, ok := zp.c.Next(); ok; l, ok = zp.c.Next() { + if l.err { + return zp.setParseError("bad data in $GENERATE directive", l) + } + if l.value == zNewline { + break + } - for j := 0; j < len(s); j++ { // No 'range' because we need to jump around - switch s[j] { - case '\\': - if escape { - dom.WriteByte('\\') - escape = false - continue - } - escape = true - case '$': - mod = "%d" - offset = 0 - if escape { - dom.WriteByte('$') - escape = false - continue - } - escape = false - if j+1 >= len(s) { // End of the string - dom.WriteString(fmt.Sprintf(mod, i+offset)) - continue - } else { - if s[j+1] == '$' { - dom.WriteByte('$') - j++ - continue - } - } - // Search for { and } - if s[j+1] == '{' { // Modifier block - sep := strings.Index(s[j+2:], "}") - if sep == -1 { - return "bad modifier in $GENERATE" - } - mod, offset, err = modToPrintf(s[j+2 : j+2+sep]) - if err != nil { - return err.Error() - } - j += 2 + sep // Jump to it - } - dom.WriteString(fmt.Sprintf(mod, i+offset)) - default: - if escape { // Pretty useless here - escape = false - continue - } - dom.WriteByte(s[j]) - } - } - // Re-parse the RR and send it on the current channel t - rx, err := NewRR("$ORIGIN " + o + "\n" + dom.String()) - if err != nil { - return err.Error() - } - t <- &Token{RR: rx} - // Its more efficient to first built the rrlist and then parse it in - // one go! But is this a problem? + s += l.token + } + + r := &generateReader{ + s: s, + + cur: start, + start: start, + end: end, + step: step, + + file: zp.file, + lex: &l, + } + zp.sub = NewZoneParser(r, zp.origin, zp.file) + zp.sub.includeDepth, zp.sub.includeAllowed = zp.includeDepth, zp.includeAllowed + zp.sub.SetDefaultTTL(defaultTtl) + return zp.subNext() +} + +type generateReader struct { + s string + si int + + cur int + start int + end int + step int + + mod bytes.Buffer + + escape bool + + eof bool + + file string + lex *lex +} + +func (r *generateReader) parseError(msg string, end int) *ParseError { + r.eof = true // Make errors sticky. + + l := *r.lex + l.token = r.s[r.si-1 : end] + l.column += r.si // l.column starts one zBLANK before r.s + + return &ParseError{r.file, msg, l} +} + +func (r *generateReader) Read(p []byte) (int, error) { + // NewZLexer, through NewZoneParser, should use ReadByte and + // not end up here. + + panic("not implemented") +} + +func (r *generateReader) ReadByte() (byte, error) { + if r.eof { + return 0, io.EOF + } + if r.mod.Len() > 0 { + return r.mod.ReadByte() + } + + if r.si >= len(r.s) { + r.si = 0 + r.cur += r.step + + r.eof = r.cur > r.end || r.cur < 0 + return '\n', nil + } + + si := r.si + r.si++ + + switch r.s[si] { + case '\\': + if r.escape { + r.escape = false + return '\\', nil + } + + r.escape = true + return r.ReadByte() + case '$': + if r.escape { + r.escape = false + return '$', nil + } + + mod := "%d" + + if si >= len(r.s)-1 { + // End of the string + fmt.Fprintf(&r.mod, mod, r.cur) + return r.mod.ReadByte() + } + + if r.s[si+1] == '$' { + r.si++ + return '$', nil + } + + var offset int + + // Search for { and } + if r.s[si+1] == '{' { + // Modifier block + sep := strings.Index(r.s[si+2:], "}") + if sep < 0 { + return 0, r.parseError("bad modifier in $GENERATE", len(r.s)) + } + + var errMsg string + mod, offset, errMsg = modToPrintf(r.s[si+2 : si+2+sep]) + if errMsg != "" { + return 0, r.parseError(errMsg, si+3+sep) + } + if r.start+offset < 0 || r.end+offset > 1<<31-1 { + return 0, r.parseError("bad offset in $GENERATE", si+3+sep) + } + + r.si += 2 + sep // Jump to it + } + + fmt.Fprintf(&r.mod, mod, r.cur+offset) + return r.mod.ReadByte() + default: + if r.escape { // Pretty useless here + r.escape = false + return r.ReadByte() + } + + return r.s[si], nil } - return "" } // Convert a $GENERATE modifier 0,0,d to something Printf can deal with. -func modToPrintf(s string) (string, int, error) { - xs := strings.SplitN(s, ",", 3) - if len(xs) != 3 { - return "", 0, errors.New("bad modifier in $GENERATE") +func modToPrintf(s string) (string, int, string) { + // Modifier is { offset [ ,width [ ,base ] ] } - provide default + // values for optional width and type, if necessary. + var offStr, widthStr, base string + switch xs := strings.Split(s, ","); len(xs) { + case 1: + offStr, widthStr, base = xs[0], "0", "d" + case 2: + offStr, widthStr, base = xs[0], xs[1], "d" + case 3: + offStr, widthStr, base = xs[0], xs[1], xs[2] + default: + return "", 0, "bad modifier in $GENERATE" } - // xs[0] is offset, xs[1] is width, xs[2] is base - if xs[2] != "o" && xs[2] != "d" && xs[2] != "x" && xs[2] != "X" { - return "", 0, errors.New("bad base in $GENERATE") + + switch base { + case "o", "d", "x", "X": + default: + return "", 0, "bad base in $GENERATE" } - offset, err := strconv.Atoi(xs[0]) - if err != nil || offset > 255 { - return "", 0, errors.New("bad offset in $GENERATE") + + offset, err := strconv.Atoi(offStr) + if err != nil { + return "", 0, "bad offset in $GENERATE" } - width, err := strconv.Atoi(xs[1]) - if err != nil || width > 255 { - return "", offset, errors.New("bad width in $GENERATE") + + width, err := strconv.Atoi(widthStr) + if err != nil || width < 0 || width > 255 { + return "", 0, "bad width in $GENERATE" } - switch { - case width < 0: - return "", offset, errors.New("bad width in $GENERATE") - case width == 0: - return "%" + xs[1] + xs[2], offset, nil + + if width == 0 { + return "%" + base, offset, "" } - return "%0" + xs[1] + xs[2], offset, nil + + return "%0" + widthStr + base, offset, "" } diff --git a/vendor/github.com/miekg/dns/generate_test.go b/vendor/github.com/miekg/dns/generate_test.go new file mode 100644 index 0000000..6ba78aa --- /dev/null +++ b/vendor/github.com/miekg/dns/generate_test.go @@ -0,0 +1,206 @@ +package dns + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestGenerateRangeGuard(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "dns") + if err != nil { + t.Fatalf("could not create tmpdir for test: %v", err) + } + defer os.RemoveAll(tmpdir) + + for i := 0; i <= 1; i++ { + path := filepath.Join(tmpdir, fmt.Sprintf("%04d.conf", i)) + data := []byte(fmt.Sprintf("dhcp-%04d A 10.0.0.%d", i, i)) + + if err := ioutil.WriteFile(path, data, 0644); err != nil { + t.Fatalf("could not create tmpfile for test: %v", err) + } + } + + var tests = [...]struct { + zone string + fail bool + }{ + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 dhcp-${0,4,d} A 10.0.0.$ +`, false}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 dhcp-${0,0,x} A 10.0.0.$ +`, false}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 128-129 dhcp-${-128,4,d} A 10.0.0.$ +`, false}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 128-129 dhcp-${-129,4,d} A 10.0.0.$ +`, true}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-2 dhcp-${2147483647,4,d} A 10.0.0.$ +`, true}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 dhcp-${2147483646,4,d} A 10.0.0.$ +`, false}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1/step dhcp-${0,4,d} A 10.0.0.$ +`, true}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1/ dhcp-${0,4,d} A 10.0.0.$ +`, true}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-10/2 dhcp-${0,4,d} A 10.0.0.$ +`, false}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1/0 dhcp-${0,4,d} A 10.0.0.$ +`, true}, + {`@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 $$INCLUDE ` + tmpdir + string(filepath.Separator) + `${0,4,d}.conf +`, false}, + } +Outer: + for i := range tests { + for tok := range ParseZone(strings.NewReader(tests[i].zone), "test.", "test") { + if tok.Error != nil { + if !tests[i].fail { + t.Errorf("expected \n\n%s\nto be parsed, but got %v", tests[i].zone, tok.Error) + } + continue Outer + } + } + if tests[i].fail { + t.Errorf("expected \n\n%s\nto fail, but got no error", tests[i].zone) + } + } +} + +func TestGenerateIncludeDepth(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dns") + if err != nil { + t.Fatalf("could not create tmpfile for test: %v", err) + } + defer os.Remove(tmpfile.Name()) + + zone := `@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 $$INCLUDE ` + tmpfile.Name() + ` +` + if _, err := tmpfile.WriteString(zone); err != nil { + t.Fatalf("could not write to tmpfile for test: %v", err) + } + if err := tmpfile.Close(); err != nil { + t.Fatalf("could not close tmpfile for test: %v", err) + } + + zp := NewZoneParser(strings.NewReader(zone), ".", tmpfile.Name()) + zp.SetIncludeAllowed(true) + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + const expected = "too deeply nested $INCLUDE" + if err := zp.Err(); err == nil || !strings.Contains(err.Error(), expected) { + t.Errorf("expected error to include %q, got %v", expected, err) + } +} + +func TestGenerateIncludeDisallowed(t *testing.T) { + const zone = `@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 $$INCLUDE test.conf +` + zp := NewZoneParser(strings.NewReader(zone), ".", "") + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + const expected = "$INCLUDE directive not allowed" + if err := zp.Err(); err == nil || !strings.Contains(err.Error(), expected) { + t.Errorf("expected error to include %q, got %v", expected, err) + } +} + +func TestGenerateSurfacesErrors(t *testing.T) { + const zone = `@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 dhcp-${0,4,dd} A 10.0.0.$ +` + zp := NewZoneParser(strings.NewReader(zone), ".", "test") + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + const expected = `test: dns: bad base in $GENERATE: "${0,4,dd}" at line: 2:20` + if err := zp.Err(); err == nil || err.Error() != expected { + t.Errorf("expected specific error, wanted %q, got %v", expected, err) + } +} + +func TestGenerateSurfacesLexerErrors(t *testing.T) { + const zone = `@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 0-1 dhcp-${0,4,d} A 10.0.0.$ ) +` + zp := NewZoneParser(strings.NewReader(zone), ".", "test") + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + const expected = `test: dns: bad data in $GENERATE directive: "extra closing brace" at line: 2:40` + if err := zp.Err(); err == nil || err.Error() != expected { + t.Errorf("expected specific error, wanted %q, got %v", expected, err) + } +} + +func TestGenerateModToPrintf(t *testing.T) { + tests := []struct { + mod string + wantFmt string + wantOffset int + wantErr bool + }{ + {"0,0,d", "%d", 0, false}, + {"0,0", "%d", 0, false}, + {"0", "%d", 0, false}, + {"3,2,d", "%02d", 3, false}, + {"3,2", "%02d", 3, false}, + {"3", "%d", 3, false}, + {"0,0,o", "%o", 0, false}, + {"0,0,x", "%x", 0, false}, + {"0,0,X", "%X", 0, false}, + {"0,0,z", "", 0, true}, + {"0,0,0,d", "", 0, true}, + {"-100,0,d", "%d", -100, false}, + } + for _, test := range tests { + gotFmt, gotOffset, errMsg := modToPrintf(test.mod) + switch { + case errMsg != "" && !test.wantErr: + t.Errorf("modToPrintf(%q) - expected empty-error, but got %v", test.mod, errMsg) + case errMsg == "" && test.wantErr: + t.Errorf("modToPrintf(%q) - expected error, but got empty-error", test.mod) + case gotFmt != test.wantFmt: + t.Errorf("modToPrintf(%q) - expected format %q, but got %q", test.mod, test.wantFmt, gotFmt) + case gotOffset != test.wantOffset: + t.Errorf("modToPrintf(%q) - expected offset %d, but got %d", test.mod, test.wantOffset, gotOffset) + } + } +} + +func BenchmarkGenerate(b *testing.B) { + const zone = `@ IN SOA ns.test. hostmaster.test. ( 1 8h 2h 7d 1d ) +$GENERATE 32-158 dhcp-${-32,4,d} A 10.0.0.$ +` + + for n := 0; n < b.N; n++ { + zp := NewZoneParser(strings.NewReader(zone), ".", "") + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + if err := zp.Err(); err != nil { + b.Fatal(err) + } + } +} diff --git a/vendor/github.com/miekg/dns/labels.go b/vendor/github.com/miekg/dns/labels.go index 760b89e..ca8c204 100644 --- a/vendor/github.com/miekg/dns/labels.go +++ b/vendor/github.com/miekg/dns/labels.go @@ -16,7 +16,7 @@ func SplitDomainName(s string) (labels []string) { fqdnEnd := 0 // offset of the final '.' or the length of the name idx := Split(s) begin := 0 - if s[len(s)-1] == '.' { + if IsFqdn(s) { fqdnEnd = len(s) - 1 } else { fqdnEnd = len(s) @@ -36,8 +36,7 @@ func SplitDomainName(s string) (labels []string) { } } - labels = append(labels, s[begin:fqdnEnd]) - return labels + return append(labels, s[begin:fqdnEnd]) } // CompareDomainName compares the names s1 and s2 and @@ -178,10 +177,10 @@ func equal(a, b string) bool { ai := a[i] bi := b[i] if ai >= 'A' && ai <= 'Z' { - ai |= ('a' - 'A') + ai |= 'a' - 'A' } if bi >= 'A' && bi <= 'Z' { - bi |= ('a' - 'A') + bi |= 'a' - 'A' } if ai != bi { return false diff --git a/vendor/github.com/miekg/dns/labels_test.go b/vendor/github.com/miekg/dns/labels_test.go index d9bb556..3f666df 100644 --- a/vendor/github.com/miekg/dns/labels_test.go +++ b/vendor/github.com/miekg/dns/labels_test.go @@ -155,13 +155,15 @@ func TestIsDomainName(t *testing.T) { lab int } names := map[string]*ret{ - "..": {false, 1}, - "@.": {true, 1}, - "www.example.com": {true, 3}, - "www.e%ample.com": {true, 3}, - "www.example.com.": {true, 3}, - "mi\\k.nl.": {true, 2}, - "mi\\k.nl": {true, 2}, + "..": {false, 1}, + "@.": {true, 1}, + "www.example.com": {true, 3}, + "www.e%ample.com": {true, 3}, + "www.example.com.": {true, 3}, + "mi\\k.nl.": {true, 2}, + "mi\\k.nl": {true, 2}, + longestDomain: {true, 4}, + longestUnprintableDomain: {true, 4}, } for d, ok := range names { l, k := IsDomainName(d) @@ -172,6 +174,42 @@ func TestIsDomainName(t *testing.T) { } } +func TestIsFqdnEscaped(t *testing.T) { + for s, expect := range map[string]bool{ + ".": true, + "\\.": false, + "\\\\.": true, + "\\\\\\.": false, + "\\\\\\\\.": true, + "a.": true, + "a\\.": false, + "a\\\\.": true, + "a\\\\\\.": false, + "ab.": true, + "ab\\.": false, + "ab\\\\.": true, + "ab\\\\\\.": false, + "..": true, + ".\\.": false, + ".\\\\.": true, + ".\\\\\\.": false, + "example.org.": true, + "example.org\\.": false, + "example.org\\\\.": true, + "example.org\\\\\\.": false, + "example\\.org.": true, + "example\\\\.org.": true, + "example\\\\\\.org.": true, + "\\example.org.": true, + "\\\\example.org.": true, + "\\\\\\example.org.": true, + } { + if got := IsFqdn(s); got != expect { + t.Errorf("IsFqdn(%q) = %t, expected %t", s, got, expect) + } + } +} + func BenchmarkSplitLabels(b *testing.B) { for i := 0; i < b.N; i++ { Split("www.example.com.") diff --git a/vendor/github.com/miekg/dns/leak_test.go b/vendor/github.com/miekg/dns/leak_test.go index ff83ac7..af37011 100644 --- a/vendor/github.com/miekg/dns/leak_test.go +++ b/vendor/github.com/miekg/dns/leak_test.go @@ -29,7 +29,6 @@ func interestingGoroutines() (gs []string) { strings.Contains(stack, "closeWriteAndWait") || strings.Contains(stack, "testing.Main(") || strings.Contains(stack, "testing.(*T).Run(") || - strings.Contains(stack, "created by net/http.(*http2Transport).newClientConn") || // These only show up with GOTRACEBACK=2; Issue 5005 (comment 28) strings.Contains(stack, "runtime.goexit") || strings.Contains(stack, "created by runtime.gc") || diff --git a/vendor/github.com/miekg/dns/length_test.go b/vendor/github.com/miekg/dns/length_test.go index 66521a5..7a2b79a 100644 --- a/vendor/github.com/miekg/dns/length_test.go +++ b/vendor/github.com/miekg/dns/length_test.go @@ -4,14 +4,13 @@ import ( "encoding/hex" "fmt" "net" - "reflect" "strings" "testing" ) func TestCompressLength(t *testing.T) { m := new(Msg) - m.SetQuestion("miek.nl", TypeMX) + m.SetQuestion("miek.nl.", TypeMX) ul := m.Len() m.Compress = true if ul != m.Len() { @@ -82,89 +81,89 @@ func TestMsgLength(t *testing.T) { } } -func TestCompressionLenHelper(t *testing.T) { - c := make(map[string]int) - compressionLenHelper(c, "example.com", 12) - if c["example.com"] != 12 { - t.Errorf("bad %d", c["example.com"]) +func TestCompressionLenSearchInsert(t *testing.T) { + c := make(map[string]struct{}) + compressionLenSearch(c, "example.com", 12) + if _, ok := c["example.com"]; !ok { + t.Errorf("bad example.com") } - if c["com"] != 20 { - t.Errorf("bad %d", c["com"]) + if _, ok := c["com"]; !ok { + t.Errorf("bad com") } // Test boundaries - c = make(map[string]int) + c = make(map[string]struct{}) // foo label starts at 16379 // com label starts at 16384 - compressionLenHelper(c, "foo.com", 16379) - if c["foo.com"] != 16379 { - t.Errorf("bad %d", c["foo.com"]) + compressionLenSearch(c, "foo.com", 16379) + if _, ok := c["foo.com"]; !ok { + t.Errorf("bad foo.com") } // com label is accessible - if c["com"] != 16383 { - t.Errorf("bad %d", c["com"]) + if _, ok := c["com"]; !ok { + t.Errorf("bad com") } - c = make(map[string]int) + c = make(map[string]struct{}) // foo label starts at 16379 // com label starts at 16385 => outside range - compressionLenHelper(c, "foo.com", 16380) - if c["foo.com"] != 16380 { - t.Errorf("bad %d", c["foo.com"]) + compressionLenSearch(c, "foo.com", 16380) + if _, ok := c["foo.com"]; !ok { + t.Errorf("bad foo.com") } // com label is NOT accessible - if c["com"] != 0 { - t.Errorf("bad %d", c["com"]) + if _, ok := c["com"]; ok { + t.Errorf("bad com") } - c = make(map[string]int) - compressionLenHelper(c, "example.com", 16375) - if c["example.com"] != 16375 { - t.Errorf("bad %d", c["example.com"]) + c = make(map[string]struct{}) + compressionLenSearch(c, "example.com", 16375) + if _, ok := c["example.com"]; !ok { + t.Errorf("bad example.com") } // com starts AFTER 16384 - if c["com"] != 16383 { - t.Errorf("bad %d", c["com"]) + if _, ok := c["com"]; !ok { + t.Errorf("bad com") } - c = make(map[string]int) - compressionLenHelper(c, "example.com", 16376) - if c["example.com"] != 16376 { - t.Errorf("bad %d", c["example.com"]) + c = make(map[string]struct{}) + compressionLenSearch(c, "example.com", 16376) + if _, ok := c["example.com"]; !ok { + t.Errorf("bad example.com") } // com starts AFTER 16384 - if c["com"] != 0 { - t.Errorf("bad %d", c["com"]) + if _, ok := c["com"]; ok { + t.Errorf("bad com") } } func TestCompressionLenSearch(t *testing.T) { - c := make(map[string]int) - compressed, ok, fullSize := compressionLenSearch(c, "a.b.org.") - if compressed != 0 || ok || fullSize != 14 { - panic(fmt.Errorf("Failed: compressed:=%d, ok:=%v, fullSize:=%d", compressed, ok, fullSize)) + c := make(map[string]struct{}) + compressed, ok := compressionLenSearch(c, "a.b.org.", maxCompressionOffset) + if compressed != 0 || ok { + t.Errorf("Failed: compressed:=%d, ok:=%v", compressed, ok) } - c["org."] = 3 - compressed, ok, fullSize = compressionLenSearch(c, "a.b.org.") - if compressed != 4 || !ok || fullSize != 8 { - panic(fmt.Errorf("Failed: compressed:=%d, ok:=%v, fullSize:=%d", compressed, ok, fullSize)) + c["org."] = struct{}{} + compressed, ok = compressionLenSearch(c, "a.b.org.", maxCompressionOffset) + if compressed != 4 || !ok { + t.Errorf("Failed: compressed:=%d, ok:=%v", compressed, ok) } - c["b.org."] = 5 - compressed, ok, fullSize = compressionLenSearch(c, "a.b.org.") - if compressed != 6 || !ok || fullSize != 4 { - panic(fmt.Errorf("Failed: compressed:=%d, ok:=%v, fullSize:=%d", compressed, ok, fullSize)) + c["b.org."] = struct{}{} + compressed, ok = compressionLenSearch(c, "a.b.org.", maxCompressionOffset) + if compressed != 2 || !ok { + t.Errorf("Failed: compressed:=%d, ok:=%v", compressed, ok) } // Not found long compression - c["x.b.org."] = 5 - compressed, ok, fullSize = compressionLenSearch(c, "a.b.org.") - if compressed != 6 || !ok || fullSize != 4 { - panic(fmt.Errorf("Failed: compressed:=%d, ok:=%v, fullSize:=%d", compressed, ok, fullSize)) + c["x.b.org."] = struct{}{} + compressed, ok = compressionLenSearch(c, "a.b.org.", maxCompressionOffset) + if compressed != 2 || !ok { + t.Errorf("Failed: compressed:=%d, ok:=%v", compressed, ok) } // Found long compression - c["a.b.org."] = 5 - compressed, ok, fullSize = compressionLenSearch(c, "a.b.org.") - if compressed != 8 || !ok || fullSize != 0 { - panic(fmt.Errorf("Failed: compressed:=%d, ok:=%v, fullSize:=%d", compressed, ok, fullSize)) + c["a.b.org."] = struct{}{} + compressed, ok = compressionLenSearch(c, "a.b.org.", maxCompressionOffset) + if compressed != 0 || !ok { + t.Errorf("Failed: compressed:=%d, ok:=%v", compressed, ok) } } @@ -174,8 +173,7 @@ func TestMsgLength2(t *testing.T) { // google.com. IN A? "064e81800001000b0004000506676f6f676c6503636f6d0000010001c00c00010001000000050004adc22986c00c00010001000000050004adc22987c00c00010001000000050004adc22988c00c00010001000000050004adc22989c00c00010001000000050004adc2298ec00c00010001000000050004adc22980c00c00010001000000050004adc22981c00c00010001000000050004adc22982c00c00010001000000050004adc22983c00c00010001000000050004adc22984c00c00010001000000050004adc22985c00c00020001000000050006036e7331c00cc00c00020001000000050006036e7332c00cc00c00020001000000050006036e7333c00cc00c00020001000000050006036e7334c00cc0d800010001000000050004d8ef200ac0ea00010001000000050004d8ef220ac0fc00010001000000050004d8ef240ac10e00010001000000050004d8ef260a0000290500000000050000", // amazon.com. IN A? (reply has no EDNS0 record) - // TODO(miek): this one is off-by-one, need to find out why - //"6de1818000010004000a000806616d617a6f6e03636f6d0000010001c00c000100010000000500044815c2d4c00c000100010000000500044815d7e8c00c00010001000000050004b02062a6c00c00010001000000050004cdfbf236c00c000200010000000500140570646e733408756c747261646e73036f726700c00c000200010000000500150570646e733508756c747261646e7304696e666f00c00c000200010000000500160570646e733608756c747261646e7302636f02756b00c00c00020001000000050014036e7331037033310664796e656374036e657400c00c00020001000000050006036e7332c0cfc00c00020001000000050006036e7333c0cfc00c00020001000000050006036e7334c0cfc00c000200010000000500110570646e733108756c747261646e73c0dac00c000200010000000500080570646e7332c127c00c000200010000000500080570646e7333c06ec0cb00010001000000050004d04e461fc0eb00010001000000050004cc0dfa1fc0fd00010001000000050004d04e471fc10f00010001000000050004cc0dfb1fc12100010001000000050004cc4a6c01c121001c000100000005001020010502f3ff00000000000000000001c13e00010001000000050004cc4a6d01c13e001c0001000000050010261000a1101400000000000000000001", + "6de1818000010004000a000806616d617a6f6e03636f6d0000010001c00c000100010000000500044815c2d4c00c000100010000000500044815d7e8c00c00010001000000050004b02062a6c00c00010001000000050004cdfbf236c00c000200010000000500140570646e733408756c747261646e73036f726700c00c000200010000000500150570646e733508756c747261646e7304696e666f00c00c000200010000000500160570646e733608756c747261646e7302636f02756b00c00c00020001000000050014036e7331037033310664796e656374036e657400c00c00020001000000050006036e7332c0cfc00c00020001000000050006036e7333c0cfc00c00020001000000050006036e7334c0cfc00c000200010000000500110570646e733108756c747261646e73c0dac00c000200010000000500080570646e7332c127c00c000200010000000500080570646e7333c06ec0cb00010001000000050004d04e461fc0eb00010001000000050004cc0dfa1fc0fd00010001000000050004d04e471fc10f00010001000000050004cc0dfb1fc12100010001000000050004cc4a6c01c121001c000100000005001020010502f3ff00000000000000000001c13e00010001000000050004cc4a6d01c13e001c0001000000050010261000a1101400000000000000000001", // yahoo.com. IN A? "fc2d81800001000300070008057961686f6f03636f6d0000010001c00c00010001000000050004628afd6dc00c00010001000000050004628bb718c00c00010001000000050004cebe242dc00c00020001000000050006036e7336c00cc00c00020001000000050006036e7338c00cc00c00020001000000050006036e7331c00cc00c00020001000000050006036e7332c00cc00c00020001000000050006036e7333c00cc00c00020001000000050006036e7334c00cc00c00020001000000050006036e7335c00cc07b0001000100000005000444b48310c08d00010001000000050004448eff10c09f00010001000000050004cb54dd35c0b100010001000000050004628a0b9dc0c30001000100000005000477a0f77cc05700010001000000050004ca2bdfaac06900010001000000050004caa568160000290500000000050000", // microsoft.com. IN A? @@ -200,10 +198,10 @@ func TestMsgLength2(t *testing.T) { lenUnComp := m.Len() b, _ = m.Pack() pacUnComp := len(b) - if pacComp+1 != lenComp { + if pacComp != lenComp { t.Errorf("msg.Len(compressed)=%d actual=%d for test %d", lenComp, pacComp, i) } - if pacUnComp+1 != lenUnComp { + if pacUnComp != lenUnComp { t.Errorf("msg.Len(uncompressed)=%d actual=%d for test %d", lenUnComp, pacUnComp, i) } } @@ -262,6 +260,52 @@ func TestMsgCompressLengthLargeRecords(t *testing.T) { } } +func compressionMapsEqual(a map[string]struct{}, b map[string]int) bool { + if len(a) != len(b) { + return false + } + + for k := range a { + if _, ok := b[k]; !ok { + return false + } + } + + return true +} + +func compressionMapsDifference(a map[string]struct{}, b map[string]int) string { + var s strings.Builder + + var c int + fmt.Fprintf(&s, "length compression map (%d):", len(a)) + for k := range b { + if _, ok := a[k]; !ok { + if c > 0 { + s.WriteString(",") + } + + fmt.Fprintf(&s, " missing %q", k) + c++ + } + } + + c = 0 + fmt.Fprintf(&s, "\npack compression map (%d):", len(b)) + for k := range a { + if _, ok := b[k]; !ok { + if c > 0 { + s.WriteString(",") + } + + fmt.Fprintf(&s, " missing %q", k) + c++ + } + } + + return s.String() +} + func TestCompareCompressionMapsForANY(t *testing.T) { msg := new(Msg) msg.Compress = true @@ -278,19 +322,19 @@ func TestCompareCompressionMapsForANY(t *testing.T) { for labelSize := 0; labelSize < 63; labelSize++ { msg.SetQuestion(fmt.Sprintf("a%s.service.acme.", strings.Repeat("x", labelSize)), TypeANY) - compressionFake := make(map[string]int) - lenFake := compressedLenWithCompressionMap(msg, compressionFake) + compressionFake := make(map[string]struct{}) + lenFake := msgLenWithCompressionMap(msg, compressionFake) compressionReal := make(map[string]int) - buf, err := msg.packBufferWithCompressionMap(nil, compressionReal) + buf, err := msg.packBufferWithCompressionMap(nil, compressionMap{ext: compressionReal}, true) if err != nil { t.Fatal(err) } if lenFake != len(buf) { t.Fatalf("padding= %d ; Predicted len := %d != real:= %d", labelSize, lenFake, len(buf)) } - if !reflect.DeepEqual(compressionFake, compressionReal) { - t.Fatalf("padding= %d ; Fake Compression Map != Real Compression Map\n*** Real:= %v\n\n***Fake:= %v", labelSize, compressionReal, compressionFake) + if !compressionMapsEqual(compressionFake, compressionReal) { + t.Fatalf("padding= %d ; Fake Compression Map != Real Compression Map\n%s", labelSize, compressionMapsDifference(compressionFake, compressionReal)) } } } @@ -311,19 +355,19 @@ func TestCompareCompressionMapsForSRV(t *testing.T) { for labelSize := 0; labelSize < 63; labelSize++ { msg.SetQuestion(fmt.Sprintf("a%s.service.acme.", strings.Repeat("x", labelSize)), TypeAAAA) - compressionFake := make(map[string]int) - lenFake := compressedLenWithCompressionMap(msg, compressionFake) + compressionFake := make(map[string]struct{}) + lenFake := msgLenWithCompressionMap(msg, compressionFake) compressionReal := make(map[string]int) - buf, err := msg.packBufferWithCompressionMap(nil, compressionReal) + buf, err := msg.packBufferWithCompressionMap(nil, compressionMap{ext: compressionReal}, true) if err != nil { t.Fatal(err) } if lenFake != len(buf) { t.Fatalf("padding= %d ; Predicted len := %d != real:= %d", labelSize, lenFake, len(buf)) } - if !reflect.DeepEqual(compressionFake, compressionReal) { - t.Fatalf("padding= %d ; Fake Compression Map != Real Compression Map\n*** Real:= %v\n\n***Fake:= %v", labelSize, compressionReal, compressionFake) + if !compressionMapsEqual(compressionFake, compressionReal) { + t.Fatalf("padding= %d ; Fake Compression Map != Real Compression Map\n%s", labelSize, compressionMapsDifference(compressionFake, compressionReal)) } } } @@ -369,3 +413,96 @@ func TestMsgCompressLengthLargeRecordsAllValues(t *testing.T) { } } } + +func TestMsgCompressionMultipleQuestions(t *testing.T) { + msg := new(Msg) + msg.Compress = true + msg.SetQuestion("www.example.org.", TypeA) + msg.Question = append(msg.Question, Question{"other.example.org.", TypeA, ClassINET}) + + predicted := msg.Len() + buf, err := msg.Pack() + if err != nil { + t.Error(err) + } + if predicted != len(buf) { + t.Fatalf("predicted compressed length is wrong: predicted %d, actual %d", predicted, len(buf)) + } +} + +func TestMsgCompressMultipleCompressedNames(t *testing.T) { + msg := new(Msg) + msg.Compress = true + msg.SetQuestion("www.example.com.", TypeSRV) + msg.Answer = append(msg.Answer, &MINFO{ + Hdr: RR_Header{Name: "www.example.com.", Class: 1, Rrtype: TypeSRV, Ttl: 0x3c}, + Rmail: "mail.example.org.", + Email: "mail.example.org.", + }) + msg.Answer = append(msg.Answer, &SOA{ + Hdr: RR_Header{Name: "www.example.com.", Class: 1, Rrtype: TypeSRV, Ttl: 0x3c}, + Ns: "ns.example.net.", + Mbox: "mail.example.net.", + }) + + predicted := msg.Len() + buf, err := msg.Pack() + if err != nil { + t.Error(err) + } + if predicted != len(buf) { + t.Fatalf("predicted compressed length is wrong: predicted %d, actual %d", predicted, len(buf)) + } +} + +func TestMsgCompressLengthEscapingMatch(t *testing.T) { + // Although slightly non-optimal, "example.org." and "ex\\097mple.org." + // are not considered equal in the compression map, even though \097 is + // a valid escaping of a. This test ensures that the Len code and the + // Pack code don't disagree on this. + + msg := new(Msg) + msg.Compress = true + msg.SetQuestion("www.example.org.", TypeA) + msg.Answer = append(msg.Answer, &NS{Hdr: RR_Header{Name: "ex\\097mple.org.", Rrtype: TypeNS, Class: ClassINET}, Ns: "ns.example.org."}) + + predicted := msg.Len() + buf, err := msg.Pack() + if err != nil { + t.Error(err) + } + if predicted != len(buf) { + t.Fatalf("predicted compressed length is wrong: predicted %d, actual %d", predicted, len(buf)) + } +} + +func TestMsgLengthEscaped(t *testing.T) { + msg := new(Msg) + msg.SetQuestion(`\000\001\002.\003\004\005\006\007\008\009.\a\b\c.`, TypeA) + + predicted := msg.Len() + buf, err := msg.Pack() + if err != nil { + t.Error(err) + } + if predicted != len(buf) { + t.Fatalf("predicted compressed length is wrong: predicted %d, actual %d", predicted, len(buf)) + } +} + +func TestMsgCompressLengthEscaped(t *testing.T) { + msg := new(Msg) + msg.Compress = true + msg.SetQuestion("www.example.org.", TypeA) + msg.Answer = append(msg.Answer, &NS{Hdr: RR_Header{Name: `\000\001\002.example.org.`, Rrtype: TypeNS, Class: ClassINET}, Ns: `ns.\e\x\a\m\p\l\e.org.`}) + msg.Answer = append(msg.Answer, &NS{Hdr: RR_Header{Name: `www.\e\x\a\m\p\l\e.org.`, Rrtype: TypeNS, Class: ClassINET}, Ns: "ns.example.org."}) + + predicted := msg.Len() + buf, err := msg.Pack() + if err != nil { + t.Error(err) + } + if predicted != len(buf) { + t.Fatalf("predicted compressed length is wrong: predicted %d, actual %d", predicted, len(buf)) + } +} diff --git a/vendor/github.com/miekg/dns/listen_go111.go b/vendor/github.com/miekg/dns/listen_go111.go new file mode 100644 index 0000000..fad195c --- /dev/null +++ b/vendor/github.com/miekg/dns/listen_go111.go @@ -0,0 +1,44 @@ +// +build go1.11 +// +build aix darwin dragonfly freebsd linux netbsd openbsd + +package dns + +import ( + "context" + "net" + "syscall" + + "golang.org/x/sys/unix" +) + +const supportsReusePort = true + +func reuseportControl(network, address string, c syscall.RawConn) error { + var opErr error + err := c.Control(func(fd uintptr) { + opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1) + }) + if err != nil { + return err + } + + return opErr +} + +func listenTCP(network, addr string, reuseport bool) (net.Listener, error) { + var lc net.ListenConfig + if reuseport { + lc.Control = reuseportControl + } + + return lc.Listen(context.Background(), network, addr) +} + +func listenUDP(network, addr string, reuseport bool) (net.PacketConn, error) { + var lc net.ListenConfig + if reuseport { + lc.Control = reuseportControl + } + + return lc.ListenPacket(context.Background(), network, addr) +} diff --git a/vendor/github.com/miekg/dns/listen_go_not111.go b/vendor/github.com/miekg/dns/listen_go_not111.go new file mode 100644 index 0000000..b920141 --- /dev/null +++ b/vendor/github.com/miekg/dns/listen_go_not111.go @@ -0,0 +1,23 @@ +// +build !go1.11 !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd + +package dns + +import "net" + +const supportsReusePort = false + +func listenTCP(network, addr string, reuseport bool) (net.Listener, error) { + if reuseport { + // TODO(tmthrgd): return an error? + } + + return net.Listen(network, addr) +} + +func listenUDP(network, addr string, reuseport bool) (net.PacketConn, error) { + if reuseport { + // TODO(tmthrgd): return an error? + } + + return net.ListenPacket(network, addr) +} diff --git a/vendor/github.com/miekg/dns/msg.go b/vendor/github.com/miekg/dns/msg.go index dcd3b6a..5191fc0 100644 --- a/vendor/github.com/miekg/dns/msg.go +++ b/vendor/github.com/miekg/dns/msg.go @@ -9,7 +9,6 @@ package dns //go:generate go run msg_generate.go -//go:generate go run compress_generate.go import ( crand "crypto/rand" @@ -18,12 +17,35 @@ import ( "math/big" "math/rand" "strconv" + "strings" "sync" ) const ( maxCompressionOffset = 2 << 13 // We have 14 bits for the compression pointer maxDomainNameWireOctets = 255 // See RFC 1035 section 2.3.4 + + // This is the maximum number of compression pointers that should occur in a + // semantically valid message. Each label in a domain name must be at least one + // octet and is separated by a period. The root label won't be represented by a + // compression pointer to a compression pointer, hence the -2 to exclude the + // smallest valid root label. + // + // It is possible to construct a valid message that has more compression pointers + // than this, and still doesn't loop, by pointing to a previous pointer. This is + // not something a well written implementation should ever do, so we leave them + // to trip the maximum compression pointer check. + maxCompressionPointers = (maxDomainNameWireOctets+1)/2 - 2 + + // This is the maximum length of a domain name in presentation format. The + // maximum wire length of a domain name is 255 octets (see above), with the + // maximum label length being 63. The wire format requires one extra byte over + // the presentation format, reducing the number of octets by 1. Each label in + // the name will be separated by a single period, with each octet in the label + // expanding to at most 4 bytes (\DDD). If all other labels are of the maximum + // length, then the final label can only be 61 octets long to not exceed the + // maximum allowed wire length. + maxDomainNamePresentationLength = 61*4 + 1 + 63*4 + 1 + 63*4 + 1 + 63*4 + 1 ) // Errors defined in this package. @@ -46,10 +68,9 @@ var ( ErrRRset error = &Error{err: "bad rrset"} ErrSecret error = &Error{err: "no secrets defined"} ErrShortRead error = &Error{err: "short read"} - ErrSig error = &Error{err: "bad signature"} // ErrSig indicates that a signature can not be cryptographically validated. - ErrSoa error = &Error{err: "no SOA"} // ErrSOA indicates that no SOA RR was seen when doing zone transfers. - ErrTime error = &Error{err: "bad time"} // ErrTime indicates a timing error in TSIG authentication. - ErrTruncated error = &Error{err: "failed to unpack truncated message"} // ErrTruncated indicates that we failed to unpack a truncated message. We unpacked as much as we had so Msg can still be used, if desired. + ErrSig error = &Error{err: "bad signature"} // ErrSig indicates that a signature can not be cryptographically validated. + ErrSoa error = &Error{err: "no SOA"} // ErrSOA indicates that no SOA RR was seen when doing zone transfers. + ErrTime error = &Error{err: "bad time"} // ErrTime indicates a timing error in TSIG authentication. ) // Id by default, returns a 16 bits random number to be used as a @@ -151,7 +172,7 @@ var RcodeToString = map[int]string{ RcodeFormatError: "FORMERR", RcodeServerFailure: "SERVFAIL", RcodeNameError: "NXDOMAIN", - RcodeNotImplemented: "NOTIMPL", + RcodeNotImplemented: "NOTIMP", RcodeRefused: "REFUSED", RcodeYXDomain: "YXDOMAIN", // See RFC 2136 RcodeYXRrset: "YXRRSET", @@ -169,6 +190,39 @@ var RcodeToString = map[int]string{ RcodeBadCookie: "BADCOOKIE", } +// compressionMap is used to allow a more efficient compression map +// to be used for internal packDomainName calls without changing the +// signature or functionality of public API. +// +// In particular, map[string]uint16 uses 25% less per-entry memory +// than does map[string]int. +type compressionMap struct { + ext map[string]int // external callers + int map[string]uint16 // internal callers +} + +func (m compressionMap) valid() bool { + return m.int != nil || m.ext != nil +} + +func (m compressionMap) insert(s string, pos int) { + if m.ext != nil { + m.ext[s] = pos + } else { + m.int[s] = uint16(pos) + } +} + +func (m compressionMap) find(s string) (int, bool) { + if m.ext != nil { + pos, ok := m.ext[s] + return pos, ok + } + + pos, ok := m.int[s] + return int(pos), ok +} + // Domain names are a sequence of counted strings // split at the dots. They end with a zero-length string. @@ -177,143 +231,156 @@ var RcodeToString = map[int]string{ // map needs to hold a mapping between domain names and offsets // pointing into msg. func PackDomainName(s string, msg []byte, off int, compression map[string]int, compress bool) (off1 int, err error) { - off1, _, err = packDomainName(s, msg, off, compression, compress) - return + return packDomainName(s, msg, off, compressionMap{ext: compression}, compress) } -func packDomainName(s string, msg []byte, off int, compression map[string]int, compress bool) (off1 int, labels int, err error) { - // special case if msg == nil - lenmsg := 256 - if msg != nil { - lenmsg = len(msg) - } +func packDomainName(s string, msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + // XXX: A logical copy of this function exists in IsDomainName and + // should be kept in sync with this function. + ls := len(s) if ls == 0 { // Ok, for instance when dealing with update RR without any rdata. - return off, 0, nil + return off, nil } - // If not fully qualified, error out, but only if msg == nil #ugly - switch { - case msg == nil: - if s[ls-1] != '.' { - s += "." - ls++ - } - case msg != nil: - if s[ls-1] != '.' { - return lenmsg, 0, ErrFqdn - } + + // If not fully qualified, error out. + if !IsFqdn(s) { + return len(msg), ErrFqdn } + // Each dot ends a segment of the name. // We trade each dot byte for a length byte. // Except for escaped dots (\.), which are normal dots. // There is also a trailing zero. // Compression - nameoffset := -1 pointer := -1 + // Emit sequence of counted strings, chopping at dots. - begin := 0 - bs := []byte(s) - roBs, bsFresh, escapedDot := s, true, false + var ( + begin int + compBegin int + compOff int + bs []byte + wasDot bool + ) +loop: for i := 0; i < ls; i++ { - if bs[i] == '\\' { - for j := i; j < ls-1; j++ { - bs[j] = bs[j+1] - } - ls-- - if off+1 > lenmsg { - return lenmsg, labels, ErrBuf - } - // check for \DDD - if i+2 < ls && isDigit(bs[i]) && isDigit(bs[i+1]) && isDigit(bs[i+2]) { - bs[i] = dddToByte(bs[i:]) - for j := i + 1; j < ls-2; j++ { - bs[j] = bs[j+2] - } - ls -= 2 - } - escapedDot = bs[i] == '.' - bsFresh = false - continue + var c byte + if bs == nil { + c = s[i] + } else { + c = bs[i] } - if bs[i] == '.' { - if i > 0 && bs[i-1] == '.' && !escapedDot { + switch c { + case '\\': + if off+1 > len(msg) { + return len(msg), ErrBuf + } + + if bs == nil { + bs = []byte(s) + } + + // check for \DDD + if i+3 < ls && isDigit(bs[i+1]) && isDigit(bs[i+2]) && isDigit(bs[i+3]) { + bs[i] = dddToByte(bs[i+1:]) + copy(bs[i+1:ls-3], bs[i+4:]) + ls -= 3 + compOff += 3 + } else { + copy(bs[i:ls-1], bs[i+1:]) + ls-- + compOff++ + } + + wasDot = false + case '.': + if wasDot { // two dots back to back is not legal - return lenmsg, labels, ErrRdata + return len(msg), ErrRdata } - if i-begin >= 1<<6 { // top two bits of length must be clear - return lenmsg, labels, ErrRdata + wasDot = true + + labelLen := i - begin + if labelLen >= 1<<6 { // top two bits of length must be clear + return len(msg), ErrRdata } + // off can already (we're in a loop) be bigger than len(msg) // this happens when a name isn't fully qualified - if off+1 > lenmsg { - return lenmsg, labels, ErrBuf - } - if msg != nil { - msg[off] = byte(i - begin) - } - offset := off - off++ - for j := begin; j < i; j++ { - if off+1 > lenmsg { - return lenmsg, labels, ErrBuf - } - if msg != nil { - msg[off] = bs[j] - } - off++ - } - if compress && !bsFresh { - roBs = string(bs) - bsFresh = true + if off+1+labelLen > len(msg) { + return len(msg), ErrBuf } + // Don't try to compress '.' - // We should only compress when compress it true, but we should also still pick + // We should only compress when compress is true, but we should also still pick // up names that can be used for *future* compression(s). - if compression != nil && roBs[begin:] != "." { - if p, ok := compression[roBs[begin:]]; !ok { - // Only offsets smaller than this can be used. - if offset < maxCompressionOffset { - compression[roBs[begin:]] = offset - } - } else { + if compression.valid() && !isRootLabel(s, bs, begin, ls) { + if p, ok := compression.find(s[compBegin:]); ok { // The first hit is the longest matching dname // keep the pointer offset we get back and store // the offset of the current name, because that's // where we need to insert the pointer later // If compress is true, we're allowed to compress this dname - if pointer == -1 && compress { - pointer = p // Where to point to - nameoffset = offset // Where to point from - break + if compress { + pointer = p // Where to point to + break loop } + } else if off < maxCompressionOffset { + // Only offsets smaller than maxCompressionOffset can be used. + compression.insert(s[compBegin:], off) } } - labels++ + + // The following is covered by the length check above. + msg[off] = byte(labelLen) + + if bs == nil { + copy(msg[off+1:], s[begin:i]) + } else { + copy(msg[off+1:], bs[begin:i]) + } + off += 1 + labelLen + begin = i + 1 + compBegin = begin + compOff + default: + wasDot = false } - escapedDot = false } + // Root label is special - if len(bs) == 1 && bs[0] == '.' { - return off, labels, nil + if isRootLabel(s, bs, 0, ls) { + return off, nil } + // If we did compression and we find something add the pointer here if pointer != -1 { // We have two bytes (14 bits) to put the pointer in - // if msg == nil, we will never do compression - binary.BigEndian.PutUint16(msg[nameoffset:], uint16(pointer^0xC000)) - off = nameoffset + 1 - goto End + binary.BigEndian.PutUint16(msg[off:], uint16(pointer^0xC000)) + return off + 2, nil } - if msg != nil && off < len(msg) { + + if off < len(msg) { msg[off] = 0 } -End: - off++ - return off, labels, nil + + return off + 1, nil +} + +// isRootLabel returns whether s or bs, from off to end, is the root +// label ".". +// +// If bs is nil, s will be checked, otherwise bs will be checked. +func isRootLabel(s string, bs []byte, off, end int) bool { + if bs == nil { + return s[off:end] == "." + } + + return end-off == 1 && bs[off] == '.' } // Unpack a domain name. @@ -330,12 +397,16 @@ End: // In theory, the pointers are only allowed to jump backward. // We let them jump anywhere and stop jumping after a while. -// UnpackDomainName unpacks a domain name into a string. +// UnpackDomainName unpacks a domain name into a string. It returns +// the name, the new offset into msg and any error that occurred. +// +// When an error is encountered, the unpacked name will be discarded +// and len(msg) will be returned as the offset. func UnpackDomainName(msg []byte, off int) (string, int, error) { - s := make([]byte, 0, 64) + s := make([]byte, 0, maxDomainNamePresentationLength) off1 := 0 lenmsg := len(msg) - maxLen := maxDomainNameWireOctets + budget := maxDomainNameWireOctets ptr := 0 // number of pointers followed Loop: for { @@ -354,27 +425,19 @@ Loop: if off+c > lenmsg { return "", lenmsg, ErrBuf } + budget -= c + 1 // +1 for the label separator + if budget <= 0 { + return "", lenmsg, ErrLongDomain + } for j := off; j < off+c; j++ { switch b := msg[j]; b { case '.', '(', ')', ';', ' ', '@': fallthrough case '"', '\\': s = append(s, '\\', b) - // presentation-format \X escapes add an extra byte - maxLen++ default: - if b < 32 || b >= 127 { // unprintable, use \DDD - var buf [3]byte - bufs := strconv.AppendInt(buf[:0], int64(b), 10) - s = append(s, '\\') - for i := 0; i < 3-len(bufs); i++ { - s = append(s, '0') - } - for _, r := range bufs { - s = append(s, r) - } - // presentation-format \DDD escapes add 3 extra bytes - maxLen += 3 + if b < ' ' || b > '~' { // unprintable, use \DDD + s = append(s, escapeByte(b)...) } else { s = append(s, b) } @@ -396,7 +459,7 @@ Loop: if ptr == 0 { off1 = off } - if ptr++; ptr > 10 { + if ptr++; ptr > maxCompressionPointers { return "", lenmsg, &Error{err: "too many compression pointers"} } // pointer should guarantee that it advances and points forwards at least @@ -412,10 +475,7 @@ Loop: off1 = off } if len(s) == 0 { - s = []byte(".") - } else if len(s) >= maxLen { - // error if the name is too long, but don't throw it away - return string(s), lenmsg, ErrLongDomain + return ".", off1, nil } return string(s), off1, nil } @@ -512,7 +572,7 @@ func unpackTxt(msg []byte, off0 int) (ss []string, off int, err error) { off = off0 var s string for off < len(msg) && err == nil { - s, off, err = unpackTxtString(msg, off) + s, off, err = unpackString(msg, off) if err == nil { ss = append(ss, s) } @@ -520,43 +580,16 @@ func unpackTxt(msg []byte, off0 int) (ss []string, off int, err error) { return } -func unpackTxtString(msg []byte, offset int) (string, int, error) { - if offset+1 > len(msg) { - return "", offset, &Error{err: "overflow unpacking txt"} - } - l := int(msg[offset]) - if offset+l+1 > len(msg) { - return "", offset, &Error{err: "overflow unpacking txt"} - } - s := make([]byte, 0, l) - for _, b := range msg[offset+1 : offset+1+l] { - switch b { - case '"', '\\': - s = append(s, '\\', b) - default: - if b < 32 || b > 127 { // unprintable - var buf [3]byte - bufs := strconv.AppendInt(buf[:0], int64(b), 10) - s = append(s, '\\') - for i := 0; i < 3-len(bufs); i++ { - s = append(s, '0') - } - for _, r := range bufs { - s = append(s, r) - } - } else { - s = append(s, b) - } - } - } - offset += 1 + l - return string(s), offset, nil -} - // Helpers for dealing with escaped bytes func isDigit(b byte) bool { return b >= '0' && b <= '9' } func dddToByte(s []byte) byte { + _ = s[2] // bounds check hint to compiler; see golang.org/issue/14808 + return byte((s[0]-'0')*100 + (s[1]-'0')*10 + (s[2] - '0')) +} + +func dddStringToByte(s string) byte { + _ = s[2] // bounds check hint to compiler; see golang.org/issue/14808 return byte((s[0]-'0')*100 + (s[1]-'0')*10 + (s[2] - '0')) } @@ -574,19 +607,38 @@ func intToBytes(i *big.Int, length int) []byte { // PackRR packs a resource record rr into msg[off:]. // See PackDomainName for documentation about the compression. func PackRR(rr RR, msg []byte, off int, compression map[string]int, compress bool) (off1 int, err error) { + headerEnd, off1, err := packRR(rr, msg, off, compressionMap{ext: compression}, compress) + if err == nil { + // packRR no longer sets the Rdlength field on the rr, but + // callers might be expecting it so we set it here. + rr.Header().Rdlength = uint16(off1 - headerEnd) + } + return off1, err +} + +func packRR(rr RR, msg []byte, off int, compression compressionMap, compress bool) (headerEnd int, off1 int, err error) { if rr == nil { - return len(msg), &Error{err: "nil rr"} + return len(msg), len(msg), &Error{err: "nil rr"} } - off1, err = rr.pack(msg, off, compression, compress) + headerEnd, err = rr.Header().packHeader(msg, off, compression, compress) if err != nil { - return len(msg), err + return headerEnd, len(msg), err } - // TODO(miek): Not sure if this is needed? If removed we can remove rawmsg.go as well. - if rawSetRdlength(msg, off, off1) { - return off1, nil + + off1, err = rr.pack(msg, headerEnd, compression, compress) + if err != nil { + return headerEnd, len(msg), err } - return off, ErrRdata + + rdlength := off1 - headerEnd + if int(uint16(rdlength)) != rdlength { // overflow + return headerEnd, len(msg), ErrRdata + } + + // The RDLENGTH field is the last field in the header and we set it here. + binary.BigEndian.PutUint16(msg[headerEnd-2:], uint16(rdlength)) + return headerEnd, off1, nil } // UnpackRR unpacks msg[off:] into an RR. @@ -602,17 +654,28 @@ func UnpackRR(msg []byte, off int) (rr RR, off1 int, err error) { // UnpackRRWithHeader unpacks the record type specific payload given an existing // RR_Header. func UnpackRRWithHeader(h RR_Header, msg []byte, off int) (rr RR, off1 int, err error) { + if newFn, ok := TypeToRR[h.Rrtype]; ok { + rr = newFn() + *rr.Header() = h + } else { + rr = &RFC3597{Hdr: h} + } + + if noRdata(h) { + return rr, off, nil + } + end := off + int(h.Rdlength) - if fn, known := typeToUnpack[h.Rrtype]; !known { - rr, off, err = unpackRFC3597(h, msg, off) - } else { - rr, off, err = fn(h, msg, off) + off, err = rr.unpack(msg, off) + if err != nil { + return nil, end, err } if off != end { return &h, end, &Error{err: "bad rdlength"} } - return rr, off, err + + return rr, off, nil } // unpackRRslice unpacks msg[off:] into an []RR. @@ -693,32 +756,33 @@ func (dns *Msg) Pack() (msg []byte, err error) { // PackBuffer packs a Msg, using the given buffer buf. If buf is too small a new buffer is allocated. func (dns *Msg) PackBuffer(buf []byte) (msg []byte, err error) { - var compression map[string]int - if dns.Compress { - compression = make(map[string]int) // Compression pointer mappings. + // If this message can't be compressed, avoid filling the + // compression map and creating garbage. + if dns.Compress && dns.isCompressible() { + compression := make(map[string]uint16) // Compression pointer mappings. + return dns.packBufferWithCompressionMap(buf, compressionMap{int: compression}, true) } - return dns.packBufferWithCompressionMap(buf, compression) + + return dns.packBufferWithCompressionMap(buf, compressionMap{}, false) } // packBufferWithCompressionMap packs a Msg, using the given buffer buf. -func (dns *Msg) packBufferWithCompressionMap(buf []byte, compression map[string]int) (msg []byte, err error) { - // We use a similar function in tsig.go's stripTsig. - - var dh Header - +func (dns *Msg) packBufferWithCompressionMap(buf []byte, compression compressionMap, compress bool) (msg []byte, err error) { if dns.Rcode < 0 || dns.Rcode > 0xFFF { return nil, ErrRcode } - if dns.Rcode > 0xF { - // Regular RCODE field is 4 bits - opt := dns.IsEdns0() - if opt == nil { - return nil, ErrExtendedRcode - } - opt.SetExtendedRcode(uint8(dns.Rcode >> 4)) + + // Set extended rcode unconditionally if we have an opt, this will allow + // reseting the extended rcode bits if they need to. + if opt := dns.IsEdns0(); opt != nil { + opt.SetExtendedRcode(uint16(dns.Rcode)) + } else if dns.Rcode > 0xF { + // If Rcode is an extended one and opt is nil, error out. + return nil, ErrExtendedRcode } // Convert convenient Msg into wire-like Header. + var dh Header dh.Id = dns.Id dh.Bits = uint16(dns.Opcode)<<11 | uint16(dns.Rcode&0xF) if dns.Response { @@ -746,50 +810,44 @@ func (dns *Msg) packBufferWithCompressionMap(buf []byte, compression map[string] dh.Bits |= _CD } - // Prepare variable sized arrays. - question := dns.Question - answer := dns.Answer - ns := dns.Ns - extra := dns.Extra - - dh.Qdcount = uint16(len(question)) - dh.Ancount = uint16(len(answer)) - dh.Nscount = uint16(len(ns)) - dh.Arcount = uint16(len(extra)) + dh.Qdcount = uint16(len(dns.Question)) + dh.Ancount = uint16(len(dns.Answer)) + dh.Nscount = uint16(len(dns.Ns)) + dh.Arcount = uint16(len(dns.Extra)) // We need the uncompressed length here, because we first pack it and then compress it. msg = buf - uncompressedLen := compressedLen(dns, false) + uncompressedLen := msgLenWithCompressionMap(dns, nil) if packLen := uncompressedLen + 1; len(msg) < packLen { msg = make([]byte, packLen) } // Pack it in: header and then the pieces. off := 0 - off, err = dh.pack(msg, off, compression, dns.Compress) + off, err = dh.pack(msg, off, compression, compress) if err != nil { return nil, err } - for i := 0; i < len(question); i++ { - off, err = question[i].pack(msg, off, compression, dns.Compress) + for _, r := range dns.Question { + off, err = r.pack(msg, off, compression, compress) if err != nil { return nil, err } } - for i := 0; i < len(answer); i++ { - off, err = PackRR(answer[i], msg, off, compression, dns.Compress) + for _, r := range dns.Answer { + _, off, err = packRR(r, msg, off, compression, compress) if err != nil { return nil, err } } - for i := 0; i < len(ns); i++ { - off, err = PackRR(ns[i], msg, off, compression, dns.Compress) + for _, r := range dns.Ns { + _, off, err = packRR(r, msg, off, compression, compress) if err != nil { return nil, err } } - for i := 0; i < len(extra); i++ { - off, err = PackRR(extra[i], msg, off, compression, dns.Compress) + for _, r := range dns.Extra { + _, off, err = packRR(r, msg, off, compression, compress) if err != nil { return nil, err } @@ -797,28 +855,7 @@ func (dns *Msg) packBufferWithCompressionMap(buf []byte, compression map[string] return msg[:off], nil } -// Unpack unpacks a binary message to a Msg structure. -func (dns *Msg) Unpack(msg []byte) (err error) { - var ( - dh Header - off int - ) - if dh, off, err = unpackMsgHdr(msg, off); err != nil { - return err - } - - dns.Id = dh.Id - dns.Response = (dh.Bits & _QR) != 0 - dns.Opcode = int(dh.Bits>>11) & 0xF - dns.Authoritative = (dh.Bits & _AA) != 0 - dns.Truncated = (dh.Bits & _TC) != 0 - dns.RecursionDesired = (dh.Bits & _RD) != 0 - dns.RecursionAvailable = (dh.Bits & _RA) != 0 - dns.Zero = (dh.Bits & _Z) != 0 - dns.AuthenticatedData = (dh.Bits & _AD) != 0 - dns.CheckingDisabled = (dh.Bits & _CD) != 0 - dns.Rcode = int(dh.Bits & 0xF) - +func (dns *Msg) unpack(dh Header, msg []byte, off int) (err error) { // If we are at the end of the message we should return *just* the // header. This can still be useful to the caller. 9.9.9.9 sends these // when responding with REFUSED for instance. @@ -837,8 +874,6 @@ func (dns *Msg) Unpack(msg []byte) (err error) { var q Question q, off, err = unpackQuestion(msg, off) if err != nil { - // Even if Truncated is set, we only will set ErrTruncated if we - // actually got the questions return err } if off1 == off { // Offset does not increase anymore, dh.Qdcount is a lie! @@ -862,16 +897,29 @@ func (dns *Msg) Unpack(msg []byte) (err error) { // The header counts might have been wrong so we need to update it dh.Arcount = uint16(len(dns.Extra)) + // Set extended Rcode + if opt := dns.IsEdns0(); opt != nil { + dns.Rcode |= opt.ExtendedRcode() + } + if off != len(msg) { // TODO(miek) make this an error? // use PackOpt to let people tell how detailed the error reporting should be? // println("dns: extra bytes in dns packet", off, "<", len(msg)) - } else if dns.Truncated { - // Whether we ran into a an error or not, we want to return that it - // was truncated - err = ErrTruncated } return err + +} + +// Unpack unpacks a binary message to a Msg structure. +func (dns *Msg) Unpack(msg []byte) (err error) { + dh, off, err := unpackMsgHdr(msg, 0) + if err != nil { + return err + } + + dns.setHdr(dh) + return dns.unpack(dh, msg, off) } // Convert a complete message to a string with dig-like output. @@ -917,151 +965,117 @@ func (dns *Msg) String() string { return s } +// isCompressible returns whether the msg may be compressible. +func (dns *Msg) isCompressible() bool { + // If we only have one question, there is nothing we can ever compress. + return len(dns.Question) > 1 || len(dns.Answer) > 0 || + len(dns.Ns) > 0 || len(dns.Extra) > 0 +} + // Len returns the message length when in (un)compressed wire format. // If dns.Compress is true compression it is taken into account. Len() // is provided to be a faster way to get the size of the resulting packet, // than packing it, measuring the size and discarding the buffer. -func (dns *Msg) Len() int { return compressedLen(dns, dns.Compress) } - -func compressedLenWithCompressionMap(dns *Msg, compression map[string]int) int { - l := 12 // Message header is always 12 bytes - for _, r := range dns.Question { - compressionLenHelper(compression, r.Name, l) - l += r.len() +func (dns *Msg) Len() int { + // If this message can't be compressed, avoid filling the + // compression map and creating garbage. + if dns.Compress && dns.isCompressible() { + compression := make(map[string]struct{}) + return msgLenWithCompressionMap(dns, compression) } - l += compressionLenSlice(l, compression, dns.Answer) - l += compressionLenSlice(l, compression, dns.Ns) - l += compressionLenSlice(l, compression, dns.Extra) - return l + + return msgLenWithCompressionMap(dns, nil) } -// compressedLen returns the message length when in compressed wire format -// when compress is true, otherwise the uncompressed length is returned. -func compressedLen(dns *Msg, compress bool) int { - // We always return one more than needed. - if compress { - compression := map[string]int{} - return compressedLenWithCompressionMap(dns, compression) - } - l := 12 // Message header is always 12 bytes +func msgLenWithCompressionMap(dns *Msg, compression map[string]struct{}) int { + l := headerSize for _, r := range dns.Question { - l += r.len() + l += r.len(l, compression) } for _, r := range dns.Answer { if r != nil { - l += r.len() + l += r.len(l, compression) } } for _, r := range dns.Ns { if r != nil { - l += r.len() + l += r.len(l, compression) } } for _, r := range dns.Extra { if r != nil { - l += r.len() + l += r.len(l, compression) } } return l } -func compressionLenSlice(lenp int, c map[string]int, rs []RR) int { - initLen := lenp - for _, r := range rs { - if r == nil { +func domainNameLen(s string, off int, compression map[string]struct{}, compress bool) int { + if s == "" || s == "." { + return 1 + } + + escaped := strings.Contains(s, "\\") + + if compression != nil && (compress || off < maxCompressionOffset) { + // compressionLenSearch will insert the entry into the compression + // map if it doesn't contain it. + if l, ok := compressionLenSearch(compression, s, off); ok && compress { + if escaped { + return escapedNameLen(s[:l]) + 2 + } + + return l + 2 + } + } + + if escaped { + return escapedNameLen(s) + 1 + } + + return len(s) + 1 +} + +func escapedNameLen(s string) int { + nameLen := len(s) + for i := 0; i < len(s); i++ { + if s[i] != '\\' { continue } - // TmpLen is to track len of record at 14bits boudaries - tmpLen := lenp - x := r.len() - // track this length, and the global length in len, while taking compression into account for both. - k, ok, _ := compressionLenSearch(c, r.Header().Name) - if ok { - // Size of x is reduced by k, but we add 1 since k includes the '.' and label descriptor take 2 bytes - // so, basically x:= x - k - 1 + 2 - x += 1 - k - } - - tmpLen += compressionLenHelper(c, r.Header().Name, tmpLen) - k, ok, _ = compressionLenSearchType(c, r) - if ok { - x += 1 - k - } - lenp += x - tmpLen = lenp - tmpLen += compressionLenHelperType(c, r, tmpLen) - - } - return lenp - initLen -} - -// Put the parts of the name in the compression map, return the size in bytes added in payload -func compressionLenHelper(c map[string]int, s string, currentLen int) int { - if currentLen > maxCompressionOffset { - // We won't be able to add any label that could be re-used later anyway - return 0 - } - if _, ok := c[s]; ok { - return 0 - } - initLen := currentLen - pref := "" - prev := s - lbs := Split(s) - for j := 0; j < len(lbs); j++ { - pref = s[lbs[j]:] - currentLen += len(prev) - len(pref) - prev = pref - if _, ok := c[pref]; !ok { - // If first byte label is within the first 14bits, it might be re-used later - if currentLen < maxCompressionOffset { - c[pref] = currentLen - } + if i+3 < len(s) && isDigit(s[i+1]) && isDigit(s[i+2]) && isDigit(s[i+3]) { + nameLen -= 3 + i += 3 } else { - added := currentLen - initLen - if j > 0 { - // We added a new PTR - added += 2 - } - return added + nameLen-- + i++ } } - return currentLen - initLen + + return nameLen } -// Look for each part in the compression map and returns its length, -// keep on searching so we get the longest match. -// Will return the size of compression found, whether a match has been -// found and the size of record if added in payload -func compressionLenSearch(c map[string]int, s string) (int, bool, int) { - off := 0 - end := false - if s == "" { // don't bork on bogus data - return 0, false, 0 - } - fullSize := 0 - for { +func compressionLenSearch(c map[string]struct{}, s string, msgOff int) (int, bool) { + for off, end := 0, false; !end; off, end = NextLabel(s, off) { if _, ok := c[s[off:]]; ok { - return len(s[off:]), true, fullSize + off + return off, true } - if end { - break + + if msgOff+off < maxCompressionOffset { + c[s[off:]] = struct{}{} } - // Each label descriptor takes 2 bytes, add it - fullSize += 2 - off, end = NextLabel(s, off) } - return 0, false, fullSize + len(s) + + return 0, false } // Copy returns a new RR which is a deep-copy of r. -func Copy(r RR) RR { r1 := r.copy(); return r1 } +func Copy(r RR) RR { return r.copy() } // Len returns the length (in octets) of the uncompressed RR in wire format. -func Len(r RR) int { return r.len() } +func Len(r RR) int { return r.len(0, nil) } // Copy returns a new *Msg which is a deep-copy of dns. func (dns *Msg) Copy() *Msg { return dns.CopyTo(new(Msg)) } @@ -1109,8 +1123,8 @@ func (dns *Msg) CopyTo(r1 *Msg) *Msg { return r1 } -func (q *Question) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := PackDomainName(q.Name, msg, off, compression, compress) +func (q *Question) pack(msg []byte, off int, compression compressionMap, compress bool) (int, error) { + off, err := packDomainName(q.Name, msg, off, compression, compress) if err != nil { return off, err } @@ -1151,7 +1165,7 @@ func unpackQuestion(msg []byte, off int) (Question, int, error) { return q, off, err } -func (dh *Header) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { +func (dh *Header) pack(msg []byte, off int, compression compressionMap, compress bool) (int, error) { off, err := packUint16(dh.Id, msg, off) if err != nil { return off, err @@ -1173,7 +1187,10 @@ func (dh *Header) pack(msg []byte, off int, compression map[string]int, compress return off, err } off, err = packUint16(dh.Arcount, msg, off) - return off, err + if err != nil { + return off, err + } + return off, nil } func unpackMsgHdr(msg []byte, off int) (Header, int, error) { @@ -1202,5 +1219,23 @@ func unpackMsgHdr(msg []byte, off int) (Header, int, error) { return dh, off, err } dh.Arcount, off, err = unpackUint16(msg, off) - return dh, off, err + if err != nil { + return dh, off, err + } + return dh, off, nil +} + +// setHdr set the header in the dns using the binary data in dh. +func (dns *Msg) setHdr(dh Header) { + dns.Id = dh.Id + dns.Response = dh.Bits&_QR != 0 + dns.Opcode = int(dh.Bits>>11) & 0xF + dns.Authoritative = dh.Bits&_AA != 0 + dns.Truncated = dh.Bits&_TC != 0 + dns.RecursionDesired = dh.Bits&_RD != 0 + dns.RecursionAvailable = dh.Bits&_RA != 0 + dns.Zero = dh.Bits&_Z != 0 // _Z covers the zero bit, which should be zero; not sure why we set it to the opposite. + dns.AuthenticatedData = dh.Bits&_AD != 0 + dns.CheckingDisabled = dh.Bits&_CD != 0 + dns.Rcode = int(dh.Bits & 0xF) } diff --git a/vendor/github.com/miekg/dns/msg_generate.go b/vendor/github.com/miekg/dns/msg_generate.go index 8ba609f..721a0fc 100644 --- a/vendor/github.com/miekg/dns/msg_generate.go +++ b/vendor/github.com/miekg/dns/msg_generate.go @@ -80,13 +80,7 @@ func main() { o := scope.Lookup(name) st, _ := getTypeStruct(o.Type(), scope) - fmt.Fprintf(b, "func (rr *%s) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) {\n", name) - fmt.Fprint(b, `off, err := rr.Hdr.pack(msg, off, compression, compress) -if err != nil { - return off, err -} -headerEnd := off -`) + fmt.Fprintf(b, "func (rr *%s) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) {\n", name) for i := 1; i < st.NumFields(); i++ { o := func(s string) { fmt.Fprintf(b, s, st.Field(i).Name()) @@ -106,7 +100,7 @@ return off, err case `dns:"nsec"`: o("off, err = packDataNsec(rr.%s, msg, off)\n") case `dns:"domain-name"`: - o("off, err = packDataDomainNames(rr.%s, msg, off, compression, compress)\n") + o("off, err = packDataDomainNames(rr.%s, msg, off, compression, false)\n") default: log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) } @@ -116,9 +110,9 @@ return off, err switch { case st.Tag(i) == `dns:"-"`: // ignored case st.Tag(i) == `dns:"cdomain-name"`: - o("off, err = PackDomainName(rr.%s, msg, off, compression, compress)\n") + o("off, err = packDomainName(rr.%s, msg, off, compression, compress)\n") case st.Tag(i) == `dns:"domain-name"`: - o("off, err = PackDomainName(rr.%s, msg, off, compression, false)\n") + o("off, err = packDomainName(rr.%s, msg, off, compression, false)\n") case st.Tag(i) == `dns:"a"`: o("off, err = packDataA(rr.%s, msg, off)\n") case st.Tag(i) == `dns:"aaaa"`: @@ -154,7 +148,8 @@ if rr.%s != "-" { fallthrough case st.Tag(i) == `dns:"hex"`: o("off, err = packStringHex(rr.%s, msg, off)\n") - + case st.Tag(i) == `dns:"any"`: + o("off, err = packStringAny(rr.%s, msg, off)\n") case st.Tag(i) == `dns:"octet"`: o("off, err = packStringOctet(rr.%s, msg, off)\n") case st.Tag(i) == "": @@ -176,8 +171,6 @@ if rr.%s != "-" { log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) } } - // We have packed everything, only now we know the rdlength of this RR - fmt.Fprintln(b, "rr.Header().Rdlength = uint16(off-headerEnd)") fmt.Fprintln(b, "return off, nil }\n") } @@ -186,14 +179,8 @@ if rr.%s != "-" { o := scope.Lookup(name) st, _ := getTypeStruct(o.Type(), scope) - fmt.Fprintf(b, "func unpack%s(h RR_Header, msg []byte, off int) (RR, int, error) {\n", name) - fmt.Fprintf(b, "rr := new(%s)\n", name) - fmt.Fprint(b, "rr.Hdr = h\n") - fmt.Fprint(b, `if noRdata(h) { -return rr, off, nil - } -var err error -rdStart := off + fmt.Fprintf(b, "func (rr *%s) unpack(msg []byte, off int) (off1 int, err error) {\n", name) + fmt.Fprint(b, `rdStart := off _ = rdStart `) @@ -201,7 +188,7 @@ _ = rdStart o := func(s string) { fmt.Fprintf(b, s, st.Field(i).Name()) fmt.Fprint(b, `if err != nil { -return rr, off, err +return off, err } `) } @@ -221,7 +208,7 @@ return rr, off, err log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) } fmt.Fprint(b, `if err != nil { -return rr, off, err +return off, err } `) continue @@ -264,6 +251,8 @@ return rr, off, err o("rr.%s, off, err = unpackStringBase64(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") case `dns:"hex"`: o("rr.%s, off, err = unpackStringHex(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"any"`: + o("rr.%s, off, err = unpackStringAny(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") case `dns:"octet"`: o("rr.%s, off, err = unpackStringOctet(msg, off)\n") case "": @@ -287,22 +276,13 @@ return rr, off, err // If we've hit len(msg) we return without error. if i < st.NumFields()-1 { fmt.Fprintf(b, `if off == len(msg) { -return rr, off, nil +return off, nil } `) } } - fmt.Fprintf(b, "return rr, off, err }\n\n") + fmt.Fprintf(b, "return off, nil }\n\n") } - // Generate typeToUnpack map - fmt.Fprintln(b, "var typeToUnpack = map[uint16]func(RR_Header, []byte, int) (RR, int, error){") - for _, name := range namedTypes { - if name == "RFC3597" { - continue - } - fmt.Fprintf(b, "Type%s: unpack%s,\n", name, name) - } - fmt.Fprintln(b, "}\n") // gofmt res, err := format.Source(b.Bytes()) diff --git a/vendor/github.com/miekg/dns/msg_helpers.go b/vendor/github.com/miekg/dns/msg_helpers.go index 4a6e878..527621a 100644 --- a/vendor/github.com/miekg/dns/msg_helpers.go +++ b/vendor/github.com/miekg/dns/msg_helpers.go @@ -6,7 +6,7 @@ import ( "encoding/binary" "encoding/hex" "net" - "strconv" + "strings" ) // helper functions called from the generated zmsg.go @@ -99,14 +99,14 @@ func unpackHeader(msg []byte, off int) (rr RR_Header, off1 int, truncmsg []byte, return hdr, off, msg, err } -// pack packs an RR header, returning the offset to the end of the header. +// packHeader packs an RR header, returning the offset to the end of the header. // See PackDomainName for documentation about the compression. -func (hdr RR_Header) pack(msg []byte, off int, compression map[string]int, compress bool) (off1 int, err error) { +func (hdr RR_Header) packHeader(msg []byte, off int, compression compressionMap, compress bool) (int, error) { if off == len(msg) { return off, nil } - off, err = PackDomainName(hdr.Name, msg, off, compression, compress) + off, err := packDomainName(hdr.Name, msg, off, compression, compress) if err != nil { return len(msg), err } @@ -122,7 +122,7 @@ func (hdr RR_Header) pack(msg []byte, off int, compression map[string]int, compr if err != nil { return len(msg), err } - off, err = packUint16(hdr.Rdlength, msg, off) + off, err = packUint16(0, msg, off) // The RDLENGTH field will be set later in packRR. if err != nil { return len(msg), err } @@ -177,14 +177,14 @@ func unpackUint8(msg []byte, off int) (i uint8, off1 int, err error) { if off+1 > len(msg) { return 0, len(msg), &Error{err: "overflow unpacking uint8"} } - return uint8(msg[off]), off + 1, nil + return msg[off], off + 1, nil } func packUint8(i uint8, msg []byte, off int) (off1 int, err error) { if off+1 > len(msg) { return len(msg), &Error{err: "overflow packing uint8"} } - msg[off] = byte(i) + msg[off] = i return off + 1, nil } @@ -223,8 +223,8 @@ func unpackUint48(msg []byte, off int) (i uint64, off1 int, err error) { return 0, len(msg), &Error{err: "overflow unpacking uint64 as uint48"} } // Used in TSIG where the last 48 bits are occupied, so for now, assume a uint48 (6 bytes) - i = (uint64(uint64(msg[off])<<40 | uint64(msg[off+1])<<32 | uint64(msg[off+2])<<24 | uint64(msg[off+3])<<16 | - uint64(msg[off+4])<<8 | uint64(msg[off+5]))) + i = uint64(msg[off])<<40 | uint64(msg[off+1])<<32 | uint64(msg[off+2])<<24 | uint64(msg[off+3])<<16 | + uint64(msg[off+4])<<8 | uint64(msg[off+5]) off += 6 return i, off, nil } @@ -267,29 +267,21 @@ func unpackString(msg []byte, off int) (string, int, error) { if off+l+1 > len(msg) { return "", off, &Error{err: "overflow unpacking txt"} } - s := make([]byte, 0, l) + var s strings.Builder + s.Grow(l) for _, b := range msg[off+1 : off+1+l] { - switch b { - case '"', '\\': - s = append(s, '\\', b) + switch { + case b == '"' || b == '\\': + s.WriteByte('\\') + s.WriteByte(b) + case b < ' ' || b > '~': // unprintable + s.WriteString(escapeByte(b)) default: - if b < 32 || b > 127 { // unprintable - var buf [3]byte - bufs := strconv.AppendInt(buf[:0], int64(b), 10) - s = append(s, '\\') - for i := 0; i < 3-len(bufs); i++ { - s = append(s, '0') - } - for _, r := range bufs { - s = append(s, r) - } - } else { - s = append(s, b) - } + s.WriteByte(b) } } off += 1 + l - return string(s), off, nil + return s.String(), off, nil } func packString(s string, msg []byte, off int) (int, error) { @@ -363,7 +355,7 @@ func packStringHex(s string, msg []byte, off int) (int, error) { if err != nil { return len(msg), err } - if off+(len(h)) > len(msg) { + if off+len(h) > len(msg) { return len(msg), &Error{err: "overflow packing hex"} } copy(msg[off:off+len(h)], h) @@ -371,6 +363,22 @@ func packStringHex(s string, msg []byte, off int) (int, error) { return off, nil } +func unpackStringAny(msg []byte, off, end int) (string, int, error) { + if end > len(msg) { + return "", len(msg), &Error{err: "overflow unpacking anything"} + } + return string(msg[off:end]), end, nil +} + +func packStringAny(s string, msg []byte, off int) (int, error) { + if off+len(s) > len(msg) { + return len(msg), &Error{err: "overflow packing anything"} + } + copy(msg[off:off+len(s)], s) + off += len(s) + return off, nil +} + func unpackStringTxt(msg []byte, off int) ([]string, int, error) { txt, off, err := unpackTxt(msg, off) if err != nil { @@ -391,7 +399,7 @@ func packStringTxt(s []string, msg []byte, off int) (int, error) { func unpackDataOpt(msg []byte, off int) ([]EDNS0, int, error) { var edns []EDNS0 Option: - code := uint16(0) + var code uint16 if off+4 > len(msg) { return nil, len(msg), &Error{err: "overflow unpacking opt"} } @@ -603,7 +611,7 @@ func packDataNsec(bitmap []uint16, msg []byte, off int) (int, error) { // Setting the octets length msg[off+1] = byte(length) // Setting the bit value for the type in the right octet - msg[off+1+int(length)] |= byte(1 << (7 - (t % 8))) + msg[off+1+int(length)] |= byte(1 << (7 - t%8)) lastwindow, lastlength = window, length } off += int(lastlength) + 2 @@ -629,10 +637,10 @@ func unpackDataDomainNames(msg []byte, off, end int) ([]string, int, error) { return servers, off, nil } -func packDataDomainNames(names []string, msg []byte, off int, compression map[string]int, compress bool) (int, error) { +func packDataDomainNames(names []string, msg []byte, off int, compression compressionMap, compress bool) (int, error) { var err error for j := 0; j < len(names); j++ { - off, err = PackDomainName(names[j], msg, off, compression, false && compress) + off, err = packDomainName(names[j], msg, off, compression, compress) if err != nil { return len(msg), err } diff --git a/vendor/github.com/miekg/dns/msg_helpers_test.go b/vendor/github.com/miekg/dns/msg_helpers_test.go new file mode 100644 index 0000000..ebeac84 --- /dev/null +++ b/vendor/github.com/miekg/dns/msg_helpers_test.go @@ -0,0 +1,138 @@ +package dns + +import "testing" + +// TestPacketDataNsec tests generated using fuzz.go and with a message pack +// containing the following bytes: 0000\x00\x00000000\x00\x002000000\x0060000\x00\x130000000000000000000" +// That bytes sequence created the overflow error and further permutations of that sequence were able to trigger +// the other code paths. +func TestPackDataNsec(t *testing.T) { + type args struct { + bitmap []uint16 + msg []byte + off int + } + tests := []struct { + name string + args args + want int + wantErr bool + wantErrMsg string + }{ + { + name: "overflow", + args: args{ + bitmap: []uint16{ + 8962, 8963, 8970, 8971, 8978, 8979, + 8986, 8987, 8994, 8995, 9002, 9003, + 9010, 9011, 9018, 9019, 9026, 9027, + 9034, 9035, 9042, 9043, 9050, 9051, + 9058, 9059, 9066, + }, + msg: []byte{ + 48, 48, 48, 48, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 50, + 48, 48, 48, 48, 48, 48, + 0, 54, 48, 48, 48, 48, + 0, 19, 48, 48, + }, + off: 48, + }, + wantErr: true, + wantErrMsg: "dns: overflow packing nsec", + want: 31, + }, + { + name: "disordered nsec bits", + args: args{ + bitmap: []uint16{ + 8962, + 0, + }, + msg: []byte{ + 48, 48, 48, 48, 0, 0, 0, 1, 0, 0, 0, 0, + 0, 0, 50, 48, 48, 48, 48, 48, 48, 0, 54, 48, + 48, 48, 48, 0, 19, 48, 48, 48, 48, 48, 48, 0, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 50, 48, 48, + 48, 48, 48, 48, 0, 54, 48, 48, 48, 48, 0, 19, + 48, 48, 48, 48, 48, 48, 0, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 50, 48, 48, 48, 48, 48, 48, 0, + 54, 48, 48, 48, 48, 0, 19, 48, 48, 48, 48, 48, + 48, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 50, + 48, 48, 48, 48, 48, 48, 0, 54, 48, 48, 48, 48, + 0, 19, 48, 48, 48, 48, 48, 48, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 50, 48, 48, 48, 48, 48, + 48, 0, 54, 48, 48, 48, 48, 0, 19, 48, 48, + }, + off: 0, + }, + wantErr: true, + wantErrMsg: "dns: nsec bits out of order", + want: 155, + }, + { + name: "simple message with only one window", + args: args{ + bitmap: []uint16{ + 0, + }, + msg: []byte{ + 48, 48, 48, 48, 0, 0, + 0, 1, 0, 0, 0, 0, + 0, 0, 50, 48, 48, 48, + 48, 48, 48, 0, 54, 48, + 48, 48, 48, 0, 19, 48, 48, + }, + off: 0, + }, + wantErr: false, + want: 3, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := packDataNsec(tt.args.bitmap, tt.args.msg, tt.args.off) + if (err != nil) != tt.wantErr { + t.Errorf("packDataNsec() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err != nil && tt.wantErrMsg != err.Error() { + t.Errorf("packDataNsec() error msg = %v, wantErrMsg %v", err.Error(), tt.wantErrMsg) + return + } + if got != tt.want { + t.Errorf("packDataNsec() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestUnpackString(t *testing.T) { + msg := []byte("\x00abcdef\x0f\\\"ghi\x04mmm\x7f") + msg[0] = byte(len(msg) - 1) + + got, _, err := unpackString(msg, 0) + if err != nil { + t.Fatal(err) + } + + if want := `abcdef\015\\\"ghi\004mmm\127`; want != got { + t.Errorf("expected %q, got %q", want, got) + } +} + +func BenchmarkUnpackString(b *testing.B) { + msg := []byte("\x00abcdef\x0f\\\"ghi\x04mmm") + msg[0] = byte(len(msg) - 1) + + for n := 0; n < b.N; n++ { + got, _, err := unpackString(msg, 0) + if err != nil { + b.Fatal(err) + } + + if want := `abcdef\015\\\"ghi\004mmm`; want != got { + b.Errorf("expected %q, got %q", want, got) + } + } +} diff --git a/vendor/github.com/miekg/dns/msg_test.go b/vendor/github.com/miekg/dns/msg_test.go index afbe6d8..616e44d 100644 --- a/vendor/github.com/miekg/dns/msg_test.go +++ b/vendor/github.com/miekg/dns/msg_test.go @@ -8,14 +8,12 @@ import ( "testing" ) -const ( - maxPrintableLabel = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789x" - tooLongLabel = maxPrintableLabel + "x" -) +const maxPrintableLabel = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789x" var ( longDomain = maxPrintableLabel[:53] + strings.TrimSuffix( strings.Join([]string{".", ".", ".", ".", "."}, maxPrintableLabel[:49]), ".") + reChar = regexp.MustCompile(`.`) i = -1 maxUnprintableLabel = reChar.ReplaceAllStringFunc(maxPrintableLabel, func(ch string) string { @@ -24,6 +22,10 @@ var ( } return fmt.Sprintf("\\%03d", i) }) + + // These are the longest possible domain names in presentation format. + longestDomain = maxPrintableLabel[:61] + strings.Join([]string{".", ".", ".", "."}, maxPrintableLabel) + longestUnprintableDomain = maxUnprintableLabel[:61*4] + strings.Join([]string{".", ".", ".", "."}, maxUnprintableLabel) ) func TestPackNoSideEffect(t *testing.T) { @@ -48,6 +50,77 @@ func TestPackNoSideEffect(t *testing.T) { } } +func TestPackExtendedBadCookie(t *testing.T) { + m := new(Msg) + m.SetQuestion(Fqdn("example.com."), TypeNS) + + a := new(Msg) + a.SetReply(m) + o := &OPT{ + Hdr: RR_Header{ + Name: ".", + Rrtype: TypeOPT, + }, + } + o.SetUDPSize(DefaultMsgSize) + a.Extra = append(a.Extra, o) + + a.SetRcode(m, RcodeBadCookie) + + edns0 := a.IsEdns0() + if edns0 == nil { + t.Fatal("Expected OPT RR") + } + // SetExtendedRcode is only called as part of `Pack()`, hence at this stage, + // the OPT RR is not set yet. + if edns0.ExtendedRcode() == RcodeBadCookie&0xFFFFFFF0 { + t.Errorf("ExtendedRcode is expected to not be BADCOOKIE before Pack") + } + + a.Pack() + + edns0 = a.IsEdns0() + if edns0 == nil { + t.Fatal("Expected OPT RR") + } + + if edns0.ExtendedRcode() != RcodeBadCookie&0xFFFFFFF0 { + t.Errorf("ExtendedRcode is expected to be BADCOOKIE after Pack") + } +} + +func TestUnPackExtendedRcode(t *testing.T) { + m := new(Msg) + m.SetQuestion(Fqdn("example.com."), TypeNS) + + a := new(Msg) + a.SetReply(m) + o := &OPT{ + Hdr: RR_Header{ + Name: ".", + Rrtype: TypeOPT, + }, + } + o.SetUDPSize(DefaultMsgSize) + a.Extra = append(a.Extra, o) + + a.SetRcode(m, RcodeBadCookie) + + packed, err := a.Pack() + if err != nil { + t.Fatalf("Could not unpack %v", a) + } + + unpacked := new(Msg) + if err := unpacked.Unpack(packed); err != nil { + t.Fatalf("Failed to unpack message") + } + + if unpacked.Rcode != RcodeBadCookie { + t.Fatalf("Rcode should be matching RcodeBadCookie (%d), got (%d)", RcodeBadCookie, unpacked.Rcode) + } +} + func TestUnpackDomainName(t *testing.T) { var cases = []struct { label string @@ -80,10 +153,9 @@ func TestUnpackDomainName(t *testing.T) { "\x03foo" + "\x05\x03com\x00" + "\x07example" + "\xC0\x05", "foo.\\003com\\000.example.com.", ""}, - {"too long domain", string(54) + "x" + strings.Replace(longDomain, ".", string(49), -1) + "\x00", - "x" + longDomain + ".", + "", ErrLongDomain.Error()}, {"too long by pointer", // a matryoshka doll name to get over 255 octets after expansion via internal pointers @@ -125,10 +197,11 @@ func TestUnpackDomainName(t *testing.T) { ""}, {"truncated name", "\x07example\x03", "", "dns: buffer size too small"}, {"non-absolute name", "\x07example\x03com", "", "dns: buffer size too small"}, - {"compression pointer cycle", + {"compression pointer cycle (too many)", "\xC0\x00", "", "dns: too many compression pointers"}, + {"compression pointer cycle (too long)", "\x03foo" + "\x03bar" + "\x07example" + "\xC0\x04", "", - "dns: too many compression pointers"}, + ErrLongDomain.Error()}, {"reserved compression pointer 0b10", "\x07example\x80", "", "dns: bad rdata"}, {"reserved compression pointer 0b01", "\x07example\x40", "", "dns: bad rdata"}, } @@ -144,3 +217,92 @@ func TestUnpackDomainName(t *testing.T) { } } } + +func TestPackDomainNameCompressionMap(t *testing.T) { + expected := map[string]struct{}{ + `www\.this.is.\131an.example.org.`: struct{}{}, + `is.\131an.example.org.`: struct{}{}, + `\131an.example.org.`: struct{}{}, + `example.org.`: struct{}{}, + `org.`: struct{}{}, + } + + msg := make([]byte, 256) + for _, compress := range []bool{true, false} { + compression := make(map[string]int) + + _, err := PackDomainName(`www\.this.is.\131an.example.org.`, msg, 0, compression, compress) + if err != nil { + t.Fatalf("PackDomainName failed: %v", err) + } + + if !compressionMapsEqual(expected, compression) { + t.Errorf("expected compression maps to be equal\n%s", compressionMapsDifference(expected, compression)) + } + } +} + +func TestPackDomainNameNSECTypeBitmap(t *testing.T) { + ownername := "some-very-long-ownername.com." + msg := &Msg{ + Compress: true, + Answer: []RR{ + &NS{ + Hdr: RR_Header{ + Name: ownername, + Rrtype: TypeNS, + Class: ClassINET, + }, + Ns: "ns1.server.com.", + }, + &NSEC{ + Hdr: RR_Header{ + Name: ownername, + Rrtype: TypeNSEC, + Class: ClassINET, + }, + NextDomain: "a.com.", + TypeBitMap: []uint16{TypeNS, TypeNSEC}, + }, + }, + } + + // Pack msg and then unpack into msg2 + buf, err := msg.Pack() + if err != nil { + t.Fatalf("msg.Pack failed: %v", err) + } + + var msg2 Msg + if err := msg2.Unpack(buf); err != nil { + t.Fatalf("msg2.Unpack failed: %v", err) + } + + if !IsDuplicate(msg.Answer[1], msg2.Answer[1]) { + t.Error("message differs after packing and unpacking") + + // Print NSEC RR for both cases + t.Logf("expected: %v", msg.Answer[1]) + t.Logf("got: %v", msg2.Answer[1]) + } +} + +func TestPackUnpackManyCompressionPointers(t *testing.T) { + m := new(Msg) + m.Compress = true + m.SetQuestion("example.org.", TypeNS) + + for domain := "a."; len(domain) < maxDomainNameWireOctets; domain += "a." { + m.Answer = append(m.Answer, &NS{Hdr: RR_Header{Name: domain, Rrtype: TypeNS, Class: ClassINET}, Ns: "example.org."}) + + b, err := m.Pack() + if err != nil { + t.Fatalf("Pack failed for %q and %d records with: %v", domain, len(m.Answer), err) + } + + var m2 Msg + if err := m2.Unpack(b); err != nil { + t.Fatalf("Unpack failed for %q and %d records with: %v", domain, len(m.Answer), err) + } + } +} diff --git a/vendor/github.com/miekg/dns/nsecx.go b/vendor/github.com/miekg/dns/nsecx.go index 9b908c4..8f071a4 100644 --- a/vendor/github.com/miekg/dns/nsecx.go +++ b/vendor/github.com/miekg/dns/nsecx.go @@ -2,49 +2,44 @@ package dns import ( "crypto/sha1" - "hash" + "encoding/hex" "strings" ) -type saltWireFmt struct { - Salt string `dns:"size-hex"` -} - // HashName hashes a string (label) according to RFC 5155. It returns the hashed string in uppercase. func HashName(label string, ha uint8, iter uint16, salt string) string { - saltwire := new(saltWireFmt) - saltwire.Salt = salt - wire := make([]byte, DefaultMsgSize) - n, err := packSaltWire(saltwire, wire) + if ha != SHA1 { + return "" + } + + wireSalt := make([]byte, hex.DecodedLen(len(salt))) + n, err := packStringHex(salt, wireSalt, 0) if err != nil { return "" } - wire = wire[:n] + wireSalt = wireSalt[:n] + name := make([]byte, 255) off, err := PackDomainName(strings.ToLower(label), name, 0, nil, false) if err != nil { return "" } name = name[:off] - var s hash.Hash - switch ha { - case SHA1: - s = sha1.New() - default: - return "" - } + s := sha1.New() // k = 0 s.Write(name) - s.Write(wire) + s.Write(wireSalt) nsec3 := s.Sum(nil) + // k > 0 for k := uint16(0); k < iter; k++ { s.Reset() s.Write(nsec3) - s.Write(wire) + s.Write(wireSalt) nsec3 = s.Sum(nsec3[:0]) } + return toBase32(nsec3) } @@ -63,8 +58,10 @@ func (rr *NSEC3) Cover(name string) bool { } nextHash := rr.NextDomain - if ownerHash == nextHash { // empty interval - return false + + // if empty interval found, try cover wildcard hashes so nameHash shouldn't match with ownerHash + if ownerHash == nextHash && nameHash != ownerHash { // empty interval + return true } if ownerHash > nextHash { // end of zone if nameHash > ownerHash { // covered since there is nothing after ownerHash @@ -96,11 +93,3 @@ func (rr *NSEC3) Match(name string) bool { } return false } - -func packSaltWire(sw *saltWireFmt, msg []byte) (int, error) { - off, err := packStringHex(sw.Salt, msg, 0) - if err != nil { - return off, err - } - return off, nil -} diff --git a/vendor/github.com/miekg/dns/nsecx_test.go b/vendor/github.com/miekg/dns/nsecx_test.go index 74ebda3..ee92653 100644 --- a/vendor/github.com/miekg/dns/nsecx_test.go +++ b/vendor/github.com/miekg/dns/nsecx_test.go @@ -1,6 +1,9 @@ package dns -import "testing" +import ( + "strconv" + "testing" +) func TestPackNsec3(t *testing.T) { nsec3 := HashName("dnsex.nl.", SHA1, 0, "DEAD") @@ -112,6 +115,18 @@ func TestNsec3(t *testing.T) { name: "asd.com.", covers: false, }, + { // empty interval wildcard + rr: &NSEC3{ + Hdr: RR_Header{Name: "2n1tb3vairuobl6rkdvii42n9tfmialp.com."}, + Hash: 1, + Flags: 1, + Iterations: 5, + Salt: "F10E9F7EA83FC8F3", + NextDomain: "2N1TB3VAIRUOBL6RKDVII42N9TFMIALP", + }, + name: "*.asd.com.", + covers: true, + }, { // name hash is before owner hash, not covered rr: &NSEC3{ Hdr: RR_Header{Name: "3V62ULR0NRE83V0RJA2VJGTLIF9V6RAB.com."}, @@ -139,3 +154,17 @@ func TestNsec3EmptySalt(t *testing.T) { t.Fatalf("expected record to match com. label") } } + +func BenchmarkHashName(b *testing.B) { + for _, iter := range []uint16{ + 150, 2500, 5000, 10000, ^uint16(0), + } { + b.Run(strconv.Itoa(int(iter)), func(b *testing.B) { + for n := 0; n < b.N; n++ { + if HashName("some.example.org.", SHA1, iter, "deadbeef") == "" { + b.Fatalf("HashName failed") + } + } + }) + } +} diff --git a/vendor/github.com/miekg/dns/parse_test.go b/vendor/github.com/miekg/dns/parse_test.go index efdaf59..9adb648 100644 --- a/vendor/github.com/miekg/dns/parse_test.go +++ b/vendor/github.com/miekg/dns/parse_test.go @@ -341,7 +341,7 @@ func TestParseDirectiveMisc(t *testing.T) { func TestNSEC(t *testing.T) { nsectests := map[string]string{ - "nl. IN NSEC3PARAM 1 0 5 30923C44C6CBBB8F": "nl.\t3600\tIN\tNSEC3PARAM\t1 0 5 30923C44C6CBBB8F", + "nl. IN NSEC3PARAM 1 0 5 30923C44C6CBBB8F": "nl.\t3600\tIN\tNSEC3PARAM\t1 0 5 30923C44C6CBBB8F", "p2209hipbpnm681knjnu0m1febshlv4e.nl. IN NSEC3 1 1 5 30923C44C6CBBB8F P90DG1KE8QEAN0B01613LHQDG0SOJ0TA NS SOA TXT RRSIG DNSKEY NSEC3PARAM": "p2209hipbpnm681knjnu0m1febshlv4e.nl.\t3600\tIN\tNSEC3\t1 1 5 30923C44C6CBBB8F P90DG1KE8QEAN0B01613LHQDG0SOJ0TA NS SOA TXT RRSIG DNSKEY NSEC3PARAM", "localhost.dnssex.nl. IN NSEC www.dnssex.nl. A RRSIG NSEC": "localhost.dnssex.nl.\t3600\tIN\tNSEC\twww.dnssex.nl. A RRSIG NSEC", "localhost.dnssex.nl. IN NSEC www.dnssex.nl. A RRSIG NSEC TYPE65534": "localhost.dnssex.nl.\t3600\tIN\tNSEC\twww.dnssex.nl. A RRSIG NSEC TYPE65534", @@ -398,16 +398,16 @@ func TestQuotes(t *testing.T) { `t.example.com. IN TXT "a bc"`: "t.example.com.\t3600\tIN\tTXT\t\"a bc\"", `t.example.com. IN TXT "a bc"`: "t.example.com.\t3600\tIN\tTXT\t\"a\\010 bc\"", - `t.example.com. IN TXT ""`: "t.example.com.\t3600\tIN\tTXT\t\"\"", - `t.example.com. IN TXT "a"`: "t.example.com.\t3600\tIN\tTXT\t\"a\"", - `t.example.com. IN TXT "aa"`: "t.example.com.\t3600\tIN\tTXT\t\"aa\"", - `t.example.com. IN TXT "aaa" ;`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\"", - `t.example.com. IN TXT "abc" "DEF"`: "t.example.com.\t3600\tIN\tTXT\t\"abc\" \"DEF\"", - `t.example.com. IN TXT "abc" ( "DEF" )`: "t.example.com.\t3600\tIN\tTXT\t\"abc\" \"DEF\"", - `t.example.com. IN TXT aaa ;`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\"", - `t.example.com. IN TXT aaa aaa;`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\" \"aaa\"", - `t.example.com. IN TXT aaa aaa`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\" \"aaa\"", - `t.example.com. IN TXT aaa`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\"", + `t.example.com. IN TXT ""`: "t.example.com.\t3600\tIN\tTXT\t\"\"", + `t.example.com. IN TXT "a"`: "t.example.com.\t3600\tIN\tTXT\t\"a\"", + `t.example.com. IN TXT "aa"`: "t.example.com.\t3600\tIN\tTXT\t\"aa\"", + `t.example.com. IN TXT "aaa" ;`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\"", + `t.example.com. IN TXT "abc" "DEF"`: "t.example.com.\t3600\tIN\tTXT\t\"abc\" \"DEF\"", + `t.example.com. IN TXT "abc" ( "DEF" )`: "t.example.com.\t3600\tIN\tTXT\t\"abc\" \"DEF\"", + `t.example.com. IN TXT aaa ;`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\"", + `t.example.com. IN TXT aaa aaa;`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\" \"aaa\"", + `t.example.com. IN TXT aaa aaa`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\" \"aaa\"", + `t.example.com. IN TXT aaa`: "t.example.com.\t3600\tIN\tTXT\t\"aaa\"", "cid.urn.arpa. NAPTR 100 50 \"s\" \"z3950+I2L+I2C\" \"\" _z3950._tcp.gatech.edu.": "cid.urn.arpa.\t3600\tIN\tNAPTR\t100 50 \"s\" \"z3950+I2L+I2C\" \"\" _z3950._tcp.gatech.edu.", "cid.urn.arpa. NAPTR 100 50 \"s\" \"rcds+I2C\" \"\" _rcds._udp.gatech.edu.": "cid.urn.arpa.\t3600\tIN\tNAPTR\t100 50 \"s\" \"rcds+I2C\" \"\" _rcds._udp.gatech.edu.", "cid.urn.arpa. NAPTR 100 50 \"s\" \"http+I2L+I2C+I2R\" \"\" _http._tcp.gatech.edu.": "cid.urn.arpa.\t3600\tIN\tNAPTR\t100 50 \"s\" \"http+I2L+I2C+I2R\" \"\" _http._tcp.gatech.edu.", @@ -543,6 +543,10 @@ example.com. DNAME 10 ; TTL=314 after second $TTL continue } expected := reCaseFromComment.FindStringSubmatch(record.Comment) + if len(expected) != 3 { + t.Errorf("regexp didn't match for record %d", i) + continue + } expectedTTL, _ := strconv.ParseUint(expected[1], 10, 32) ttl := record.RR.Header().Ttl if ttl != uint32(expectedTTL) { @@ -847,20 +851,25 @@ func TestPX(t *testing.T) { func TestComment(t *testing.T) { // Comments we must see - comments := map[string]bool{"; this is comment 1": true, - "; this is comment 4": true, "; this is comment 6": true, - "; this is comment 7": true, "; this is comment 8": true} + comments := map[string]bool{ + "; this is comment 1": true, + "; this is comment 2": true, + "; this is comment 4": true, + "; this is comment 6": true, + "; this is comment 7": true, + "; this is comment 8": true, + } zone := ` foo. IN A 10.0.0.1 ; this is comment 1 foo. IN A ( - 10.0.0.2 ; this is comment2 + 10.0.0.2 ; this is comment 2 ) -; this is comment3 +; this is comment 3 foo. IN A 10.0.0.3 foo. IN A ( 10.0.0.4 ); this is comment 4 foo. IN A 10.0.0.5 -; this is comment5 +; this is comment 5 foo. IN A 10.0.0.6 @@ -872,13 +881,112 @@ foo. IN TXT "THIS IS TEXT MAN"; this is comment 8 if x.Error == nil { if x.Comment != "" { if _, ok := comments[x.Comment]; !ok { - t.Errorf("wrong comment %s", x.Comment) + t.Errorf("wrong comment %q", x.Comment) } } } } } +func TestParseZoneComments(t *testing.T) { + for i, test := range []struct { + zone string + comments []string + }{ + { + `name. IN SOA a6.nstld.com. hostmaster.nic.name. ( + 203362132 ; serial + 5m ; refresh (5 minutes) + 5m ; retry (5 minutes) + 2w ; expire (2 weeks) + 300 ; minimum (5 minutes) + ) ; y +. 3600000 IN NS ONE.MY-ROOTS.NET. ; x`, + []string{"; serial ; refresh (5 minutes) ; retry (5 minutes) ; expire (2 weeks) ; minimum (5 minutes) ; y", "; x"}, + }, + { + `name. IN SOA a6.nstld.com. hostmaster.nic.name. ( + 203362132 ; serial + 5m ; refresh (5 minutes) + 5m ; retry (5 minutes) + 2w ; expire (2 weeks) + 300 ; minimum (5 minutes) + ) ; y +. 3600000 IN NS ONE.MY-ROOTS.NET.`, + []string{"; serial ; refresh (5 minutes) ; retry (5 minutes) ; expire (2 weeks) ; minimum (5 minutes) ; y", ""}, + }, + { + `name. IN SOA a6.nstld.com. hostmaster.nic.name. ( + 203362132 ; serial + 5m ; refresh (5 minutes) + 5m ; retry (5 minutes) + 2w ; expire (2 weeks) + 300 ; minimum (5 minutes) + ) +. 3600000 IN NS ONE.MY-ROOTS.NET.`, + []string{"; serial ; refresh (5 minutes) ; retry (5 minutes) ; expire (2 weeks) ; minimum (5 minutes)", ""}, + }, + { + `name. IN SOA a6.nstld.com. hostmaster.nic.name. ( + 203362132 ; serial + 5m ; refresh (5 minutes) + 5m ; retry (5 minutes) + 2w ; expire (2 weeks) + 300 ; minimum (5 minutes) + ) +. 3600000 IN NS ONE.MY-ROOTS.NET. ; x`, + []string{"; serial ; refresh (5 minutes) ; retry (5 minutes) ; expire (2 weeks) ; minimum (5 minutes)", "; x"}, + }, + { + `name. IN SOA a6.nstld.com. hostmaster.nic.name. ( + 203362132 ; serial + 5m ; refresh (5 minutes) + 5m ; retry (5 minutes) + 2w ; expire (2 weeks) + 300 ; minimum (5 minutes) + )`, + []string{"; serial ; refresh (5 minutes) ; retry (5 minutes) ; expire (2 weeks) ; minimum (5 minutes)"}, + }, + { + `. 3600000 IN NS ONE.MY-ROOTS.NET. ; x`, + []string{"; x"}, + }, + { + `. 3600000 IN NS ONE.MY-ROOTS.NET.`, + []string{""}, + }, + { + `. 3600000 IN NS ONE.MY-ROOTS.NET. ;;x`, + []string{";;x"}, + }, + } { + r := strings.NewReader(test.zone) + + var j int + for r := range ParseZone(r, "", "") { + if r.Error != nil { + t.Fatal(r.Error) + } + + if j >= len(test.comments) { + t.Fatalf("too many records for zone %d at %d record, expected %d", i, j+1, len(test.comments)) + } + + if r.Comment != test.comments[j] { + t.Errorf("invalid comment for record %d:%d %v / %v", i, j, r.RR, r.Error) + t.Logf("expected %q", test.comments[j]) + t.Logf("got %q", r.Comment) + } + + j++ + } + + if j != len(test.comments) { + t.Errorf("too few records for zone %d, got %d, expected %d", i, j, len(test.comments)) + } + } +} + func TestEUIxx(t *testing.T) { tests := map[string]string{ "host.example. IN EUI48 00-00-5e-90-01-2a": "host.example.\t3600\tIN\tEUI48\t00-00-5e-90-01-2a", @@ -928,8 +1036,8 @@ func TestTXT(t *testing.T) { if rr.String() != `_raop._tcp.local. 60 IN TXT "single value"` { t.Error("bad representation of TXT record:", rr.String()) } - if rr.len() != 28+1+12 { - t.Error("bad size of serialized record:", rr.len()) + if Len(rr) != 28+1+12 { + t.Error("bad size of serialized record:", Len(rr)) } } @@ -948,8 +1056,8 @@ func TestTXT(t *testing.T) { if rr.String() != `_raop._tcp.local. 60 IN TXT "a=1" "b=2" "c=3" "d=4"` { t.Error("bad representation of TXT multi value record:", rr.String()) } - if rr.len() != 28+1+3+1+3+1+3+1+3 { - t.Error("bad size of serialized multi value record:", rr.len()) + if Len(rr) != 28+1+3+1+3+1+3+1+3 { + t.Error("bad size of serialized multi value record:", Len(rr)) } } @@ -968,8 +1076,8 @@ func TestTXT(t *testing.T) { if rr.String() != `_raop._tcp.local. 60 IN TXT ""` { t.Error("bad representation of empty-string TXT record:", rr.String()) } - if rr.len() != 28+1 { - t.Error("bad size of serialized record:", rr.len()) + if Len(rr) != 28+1 { + t.Error("bad size of serialized record:", Len(rr)) } } @@ -1463,3 +1571,17 @@ func TestBad(t *testing.T) { } } } + +func TestNULLRecord(t *testing.T) { + // packet captured from iodine + packet := `8116840000010001000000000569627a6c700474657374046d69656b026e6c00000a0001c00c000a0001000000000005497f000001` + data, _ := hex.DecodeString(packet) + msg := new(Msg) + err := msg.Unpack(data) + if err != nil { + t.Fatalf("Failed to unpack NULL record") + } + if _, ok := msg.Answer[0].(*NULL); !ok { + t.Fatalf("Expected packet to contain NULL record") + } +} diff --git a/vendor/github.com/miekg/dns/privaterr.go b/vendor/github.com/miekg/dns/privaterr.go index 41989e7..d9c0d26 100644 --- a/vendor/github.com/miekg/dns/privaterr.go +++ b/vendor/github.com/miekg/dns/privaterr.go @@ -39,11 +39,12 @@ func mkPrivateRR(rrtype uint16) *PrivateRR { } anyrr := rrfunc() - switch rr := anyrr.(type) { - case *PrivateRR: - return rr + rr, ok := anyrr.(*PrivateRR) + if !ok { + panic(fmt.Sprintf("dns: RR is not a PrivateRR, TypeToRR[%d] generator returned %T", rrtype, anyrr)) } - panic(fmt.Sprintf("dns: RR is not a PrivateRR, TypeToRR[%d] generator returned %T", rrtype, anyrr)) + + return rr } // Header return the RR header of r. @@ -52,7 +53,12 @@ func (r *PrivateRR) Header() *RR_Header { return &r.Hdr } func (r *PrivateRR) String() string { return r.Hdr.String() + r.Data.String() } // Private len and copy parts to satisfy RR interface. -func (r *PrivateRR) len() int { return r.Hdr.len() + r.Data.Len() } +func (r *PrivateRR) len(off int, compression map[string]struct{}) int { + l := r.Hdr.len(off, compression) + l += r.Data.Len() + return l +} + func (r *PrivateRR) copy() RR { // make new RR like this: rr := mkPrivateRR(r.Hdr.Rrtype) @@ -64,21 +70,47 @@ func (r *PrivateRR) copy() RR { } return rr } -func (r *PrivateRR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := r.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off + +func (r *PrivateRR) pack(msg []byte, off int, compression compressionMap, compress bool) (int, error) { n, err := r.Data.Pack(msg[off:]) if err != nil { return len(msg), err } off += n - r.Header().Rdlength = uint16(off - headerEnd) return off, nil } +func (r *PrivateRR) unpack(msg []byte, off int) (int, error) { + off1, err := r.Data.Unpack(msg[off:]) + off += off1 + return off, err +} + +func (r *PrivateRR) parse(c *zlexer, origin, file string) *ParseError { + var l lex + text := make([]string, 0, 2) // could be 0..N elements, median is probably 1 +Fetch: + for { + // TODO(miek): we could also be returning _QUOTE, this might or might not + // be an issue (basically parsing TXT becomes hard) + switch l, _ = c.Next(); l.value { + case zNewline, zEOF: + break Fetch + case zString: + text = append(text, l.token) + } + } + + err := r.Data.Parse(text) + if err != nil { + return &ParseError{file, err.Error(), l} + } + + return nil +} + +func (r1 *PrivateRR) isDuplicate(r2 RR) bool { return false } + // PrivateHandle registers a private resource record type. It requires // string and numeric representation of private RR type and generator function as argument. func PrivateHandle(rtypestr string, rtype uint16, generator func() PrivateRdata) { @@ -87,62 +119,14 @@ func PrivateHandle(rtypestr string, rtype uint16, generator func() PrivateRdata) TypeToRR[rtype] = func() RR { return &PrivateRR{RR_Header{}, generator()} } TypeToString[rtype] = rtypestr StringToType[rtypestr] = rtype - - typeToUnpack[rtype] = func(h RR_Header, msg []byte, off int) (RR, int, error) { - if noRdata(h) { - return &h, off, nil - } - var err error - - rr := mkPrivateRR(h.Rrtype) - rr.Hdr = h - - off1, err := rr.Data.Unpack(msg[off:]) - off += off1 - if err != nil { - return rr, off, err - } - return rr, off, err - } - - setPrivateRR := func(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := mkPrivateRR(h.Rrtype) - rr.Hdr = h - - var l lex - text := make([]string, 0, 2) // could be 0..N elements, median is probably 1 - Fetch: - for { - // TODO(miek): we could also be returning _QUOTE, this might or might not - // be an issue (basically parsing TXT becomes hard) - switch l = <-c; l.value { - case zNewline, zEOF: - break Fetch - case zString: - text = append(text, l.token) - } - } - - err := rr.Data.Parse(text) - if err != nil { - return nil, &ParseError{f, err.Error(), l}, "" - } - - return rr, nil, "" - } - - typeToparserFunc[rtype] = parserFunc{setPrivateRR, true} } -// PrivateHandleRemove removes defenitions required to support private RR type. +// PrivateHandleRemove removes definitions required to support private RR type. func PrivateHandleRemove(rtype uint16) { rtypestr, ok := TypeToString[rtype] if ok { delete(TypeToRR, rtype) delete(TypeToString, rtype) - delete(typeToparserFunc, rtype) delete(StringToType, rtypestr) - delete(typeToUnpack, rtype) } - return } diff --git a/vendor/github.com/miekg/dns/rawmsg.go b/vendor/github.com/miekg/dns/rawmsg.go deleted file mode 100644 index 6e21fba..0000000 --- a/vendor/github.com/miekg/dns/rawmsg.go +++ /dev/null @@ -1,49 +0,0 @@ -package dns - -import "encoding/binary" - -// rawSetRdlength sets the rdlength in the header of -// the RR. The offset 'off' must be positioned at the -// start of the header of the RR, 'end' must be the -// end of the RR. -func rawSetRdlength(msg []byte, off, end int) bool { - l := len(msg) -Loop: - for { - if off+1 > l { - return false - } - c := int(msg[off]) - off++ - switch c & 0xC0 { - case 0x00: - if c == 0x00 { - // End of the domainname - break Loop - } - if off+c > l { - return false - } - off += c - - case 0xC0: - // pointer, next byte included, ends domainname - off++ - break Loop - } - } - // The domainname has been seen, we at the start of the fixed part in the header. - // Type is 2 bytes, class is 2 bytes, ttl 4 and then 2 bytes for the length. - off += 2 + 2 + 4 - if off+2 > l { - return false - } - //off+1 is the end of the header, 'end' is the end of the rr - //so 'end' - 'off+2' is the length of the rdata - rdatalen := end - (off + 2) - if rdatalen > 0xFFFF { - return false - } - binary.BigEndian.PutUint16(msg[off:], uint16(rdatalen)) - return true -} diff --git a/vendor/github.com/miekg/dns/reverse.go b/vendor/github.com/miekg/dns/reverse.go index f6e7a47..28151af 100644 --- a/vendor/github.com/miekg/dns/reverse.go +++ b/vendor/github.com/miekg/dns/reverse.go @@ -12,6 +12,20 @@ var StringToOpcode = reverseInt(OpcodeToString) // StringToRcode is a map of rcodes to strings. var StringToRcode = reverseInt(RcodeToString) +func init() { + // Preserve previous NOTIMP typo, see github.com/miekg/dns/issues/733. + StringToRcode["NOTIMPL"] = RcodeNotImplemented +} + +// StringToAlgorithm is the reverse of AlgorithmToString. +var StringToAlgorithm = reverseInt8(AlgorithmToString) + +// StringToHash is a map of names to hash IDs. +var StringToHash = reverseInt8(HashToString) + +// StringToCertType is the reverseof CertTypeToString. +var StringToCertType = reverseInt16(CertTypeToString) + // Reverse a map func reverseInt8(m map[uint8]string) map[string]uint8 { n := make(map[string]uint8, len(m)) diff --git a/vendor/github.com/miekg/dns/rr_test.go b/vendor/github.com/miekg/dns/rr_test.go index 77a153b..2f99401 100644 --- a/vendor/github.com/miekg/dns/rr_test.go +++ b/vendor/github.com/miekg/dns/rr_test.go @@ -1,7 +1,11 @@ package dns -// testRR returns the RR from string s. The error is thrown away. +// testRR is a helper that wraps a call to NewRR and panics if the error is non-nil. func testRR(s string) RR { - r, _ := NewRR(s) + r, err := NewRR(s) + if err != nil { + panic(err) + } + return r } diff --git a/vendor/github.com/miekg/dns/sanitize.go b/vendor/github.com/miekg/dns/sanitize.go index c415bdd..a638e86 100644 --- a/vendor/github.com/miekg/dns/sanitize.go +++ b/vendor/github.com/miekg/dns/sanitize.go @@ -5,6 +5,7 @@ package dns // rrs. // m is used to store the RRs temporary. If it is nil a new map will be allocated. func Dedup(rrs []RR, m map[string]RR) []RR { + if m == nil { m = make(map[string]RR) } @@ -14,10 +15,11 @@ func Dedup(rrs []RR, m map[string]RR) []RR { for _, r := range rrs { key := normalizedString(r) keys = append(keys, &key) - if _, ok := m[key]; ok { + if mr, ok := m[key]; ok { // Shortest TTL wins. - if m[key].Header().Ttl > r.Header().Ttl { - m[key].Header().Ttl = r.Header().Ttl + rh, mrh := r.Header(), mr.Header() + if mrh.Ttl > rh.Ttl { + mrh.Ttl = rh.Ttl } continue } diff --git a/vendor/github.com/miekg/dns/scan.go b/vendor/github.com/miekg/dns/scan.go index f9cd474..a8691bc 100644 --- a/vendor/github.com/miekg/dns/scan.go +++ b/vendor/github.com/miekg/dns/scan.go @@ -1,6 +1,7 @@ package dns import ( + "bufio" "fmt" "io" "os" @@ -10,7 +11,10 @@ import ( ) const maxTok = 2048 // Largest token we can return. -const maxUint16 = 1<<16 - 1 + +// The maximum depth of $INCLUDE directives supported by the +// ZoneParser API. +const maxIncludeDepth = 7 // Tokinize a RFC 1035 zone file. The tokenizer will normalize it: // * Add ownernames if they are left blank; @@ -75,15 +79,12 @@ func (e *ParseError) Error() (s string) { } type lex struct { - token string // text of the token - tokenUpper string // uppercase text of the token - length int // length of the token - err bool // when true, token text has lexer error - value uint8 // value: zString, _BLANK, etc. - line int // line in the file - column int // column in the file - torc uint16 // type or class as parsed in the lexer, we only need to look this up in the grammar - comment string // any comment text seen + token string // text of the token + err bool // when true, token text has lexer error + value uint8 // value: zString, _BLANK, etc. + torc uint16 // type or class as parsed in the lexer, we only need to look this up in the grammar + line int // line in the file + column int // column in the file } // Token holds the token that are returned when a zone file is parsed. @@ -103,10 +104,14 @@ type ttlState struct { } // NewRR reads the RR contained in the string s. Only the first RR is -// returned. If s contains no RR, return nil with no error. The class -// defaults to IN and TTL defaults to 3600. The full zone file syntax -// like $TTL, $ORIGIN, etc. is supported. All fields of the returned -// RR are set, except RR.Header().Rdlength which is set to 0. +// returned. If s contains no records, NewRR will return nil with no +// error. +// +// The class defaults to IN and TTL defaults to 3600. The full zone +// file syntax like $TTL, $ORIGIN, etc. is supported. +// +// All fields of the returned RR are set, except RR.Header().Rdlength +// which is set to 0. func NewRR(s string) (RR, error) { if len(s) > 0 && s[len(s)-1] != '\n' { // We need a closing newline return ReadRR(strings.NewReader(s+"\n"), "") @@ -114,28 +119,31 @@ func NewRR(s string) (RR, error) { return ReadRR(strings.NewReader(s), "") } -// ReadRR reads the RR contained in q. +// ReadRR reads the RR contained in r. +// +// The string file is used in error reporting and to resolve relative +// $INCLUDE directives. +// // See NewRR for more documentation. -func ReadRR(q io.Reader, filename string) (RR, error) { - defttl := &ttlState{defaultTtl, false} - r := <-parseZoneHelper(q, ".", filename, defttl, 1) - if r == nil { - return nil, nil - } - - if r.Error != nil { - return nil, r.Error - } - return r.RR, nil +func ReadRR(r io.Reader, file string) (RR, error) { + zp := NewZoneParser(r, ".", file) + zp.SetDefaultTTL(defaultTtl) + zp.SetIncludeAllowed(true) + rr, _ := zp.Next() + return rr, zp.Err() } -// ParseZone reads a RFC 1035 style zonefile from r. It returns *Tokens on the -// returned channel, each consisting of either a parsed RR and optional comment -// or a nil RR and an error. The string file is only used -// in error reporting. The string origin is used as the initial origin, as -// if the file would start with an $ORIGIN directive. -// The directives $INCLUDE, $ORIGIN, $TTL and $GENERATE are supported. -// The channel t is closed by ParseZone when the end of r is reached. +// ParseZone reads a RFC 1035 style zonefile from r. It returns +// *Tokens on the returned channel, each consisting of either a +// parsed RR and optional comment or a nil RR and an error. The +// channel is closed by ParseZone when the end of r is reached. +// +// The string file is used in error reporting and to resolve relative +// $INCLUDE directives. The string origin is used as the initial +// origin, as if the file would start with an $ORIGIN directive. +// +// The directives $INCLUDE, $ORIGIN, $TTL and $GENERATE are all +// supported. // // Basic usage pattern when reading from a string (z) containing the // zone data: @@ -148,91 +156,249 @@ func ReadRR(q io.Reader, filename string) (RR, error) { // } // } // -// Comments specified after an RR (and on the same line!) are returned too: +// Comments specified after an RR (and on the same line!) are +// returned too: // // foo. IN A 10.0.0.1 ; this is a comment // -// The text "; this is comment" is returned in Token.Comment. Comments inside the -// RR are discarded. Comments on a line by themselves are discarded too. +// The text "; this is comment" is returned in Token.Comment. +// Comments inside the RR are returned concatenated along with the +// RR. Comments on a line by themselves are discarded. +// +// To prevent memory leaks it is important to always fully drain the +// returned channel. If an error occurs, it will always be the last +// Token sent on the channel. +// +// Deprecated: New users should prefer the ZoneParser API. func ParseZone(r io.Reader, origin, file string) chan *Token { - return parseZoneHelper(r, origin, file, nil, 10000) -} - -func parseZoneHelper(r io.Reader, origin, file string, defttl *ttlState, chansize int) chan *Token { - t := make(chan *Token, chansize) - go parseZone(r, origin, file, defttl, t, 0) + t := make(chan *Token, 10000) + go parseZone(r, origin, file, t) return t } -func parseZone(r io.Reader, origin, f string, defttl *ttlState, t chan *Token, include int) { - defer func() { - if include == 0 { - close(t) - } - }() - s, cancel := scanInit(r) - c := make(chan lex) - // Start the lexer - go zlexer(s, c) +func parseZone(r io.Reader, origin, file string, t chan *Token) { + defer close(t) - defer func() { - cancel() - // zlexer can send up to three tokens, the next one and possibly 2 remainders. - // Do a non-blocking read. - _, ok := <-c - _, ok = <-c - _, ok = <-c + zp := NewZoneParser(r, origin, file) + zp.SetIncludeAllowed(true) + + for rr, ok := zp.Next(); ok; rr, ok = zp.Next() { + t <- &Token{RR: rr, Comment: zp.Comment()} + } + + if err := zp.Err(); err != nil { + pe, ok := err.(*ParseError) if !ok { - // too bad + pe = &ParseError{file: file, err: err.Error()} } - }() - // 6 possible beginnings of a line, _ is a space - // 0. zRRTYPE -> all omitted until the rrtype - // 1. zOwner _ zRrtype -> class/ttl omitted - // 2. zOwner _ zString _ zRrtype -> class omitted - // 3. zOwner _ zString _ zClass _ zRrtype -> ttl/class - // 4. zOwner _ zClass _ zRrtype -> ttl omitted - // 5. zOwner _ zClass _ zString _ zRrtype -> class/ttl (reversed) - // After detecting these, we know the zRrtype so we can jump to functions - // handling the rdata for each of these types. + t <- &Token{Error: pe} + } +} + +// ZoneParser is a parser for an RFC 1035 style zonefile. +// +// Each parsed RR in the zone is returned sequentially from Next. An +// optional comment can be retrieved with Comment. +// +// The directives $INCLUDE, $ORIGIN, $TTL and $GENERATE are all +// supported. Although $INCLUDE is disabled by default. +// +// Basic usage pattern when reading from a string (z) containing the +// zone data: +// +// zp := NewZoneParser(strings.NewReader(z), "", "") +// +// for rr, ok := zp.Next(); ok; rr, ok = zp.Next() { +// // Do something with rr +// } +// +// if err := zp.Err(); err != nil { +// // log.Println(err) +// } +// +// Comments specified after an RR (and on the same line!) are +// returned too: +// +// foo. IN A 10.0.0.1 ; this is a comment +// +// The text "; this is comment" is returned from Comment. Comments inside +// the RR are returned concatenated along with the RR. Comments on a line +// by themselves are discarded. +type ZoneParser struct { + c *zlexer + + parseErr *ParseError + + origin string + file string + + defttl *ttlState + + h RR_Header + + // sub is used to parse $INCLUDE files and $GENERATE directives. + // Next, by calling subNext, forwards the resulting RRs from this + // sub parser to the calling code. + sub *ZoneParser + osFile *os.File + + includeDepth uint8 + + includeAllowed bool +} + +// NewZoneParser returns an RFC 1035 style zonefile parser that reads +// from r. +// +// The string file is used in error reporting and to resolve relative +// $INCLUDE directives. The string origin is used as the initial +// origin, as if the file would start with an $ORIGIN directive. +func NewZoneParser(r io.Reader, origin, file string) *ZoneParser { + var pe *ParseError if origin != "" { origin = Fqdn(origin) if _, ok := IsDomainName(origin); !ok { - t <- &Token{Error: &ParseError{f, "bad initial origin name", lex{}}} - return + pe = &ParseError{file, "bad initial origin name", lex{}} } } - st := zExpectOwnerDir // initial state - var h RR_Header - var prevName string - for l := range c { - // Lexer spotted an error already - if l.err == true { - t <- &Token{Error: &ParseError{f, l.token, l}} - return + return &ZoneParser{ + c: newZLexer(r), + parseErr: pe, + + origin: origin, + file: file, + } +} + +// SetDefaultTTL sets the parsers default TTL to ttl. +func (zp *ZoneParser) SetDefaultTTL(ttl uint32) { + zp.defttl = &ttlState{ttl, false} +} + +// SetIncludeAllowed controls whether $INCLUDE directives are +// allowed. $INCLUDE directives are not supported by default. +// +// The $INCLUDE directive will open and read from a user controlled +// file on the system. Even if the file is not a valid zonefile, the +// contents of the file may be revealed in error messages, such as: +// +// /etc/passwd: dns: not a TTL: "root:x:0:0:root:/root:/bin/bash" at line: 1:31 +// /etc/shadow: dns: not a TTL: "root:$6$::0:99999:7:::" at line: 1:125 +func (zp *ZoneParser) SetIncludeAllowed(v bool) { + zp.includeAllowed = v +} + +// Err returns the first non-EOF error that was encountered by the +// ZoneParser. +func (zp *ZoneParser) Err() error { + if zp.parseErr != nil { + return zp.parseErr + } + + if zp.sub != nil { + if err := zp.sub.Err(); err != nil { + return err } + } + + return zp.c.Err() +} + +func (zp *ZoneParser) setParseError(err string, l lex) (RR, bool) { + zp.parseErr = &ParseError{zp.file, err, l} + return nil, false +} + +// Comment returns an optional text comment that occurred alongside +// the RR. +func (zp *ZoneParser) Comment() string { + if zp.parseErr != nil { + return "" + } + + if zp.sub != nil { + return zp.sub.Comment() + } + + return zp.c.Comment() +} + +func (zp *ZoneParser) subNext() (RR, bool) { + if rr, ok := zp.sub.Next(); ok { + return rr, true + } + + if zp.sub.osFile != nil { + zp.sub.osFile.Close() + zp.sub.osFile = nil + } + + if zp.sub.Err() != nil { + // We have errors to surface. + return nil, false + } + + zp.sub = nil + return zp.Next() +} + +// Next advances the parser to the next RR in the zonefile and +// returns the (RR, true). It will return (nil, false) when the +// parsing stops, either by reaching the end of the input or an +// error. After Next returns (nil, false), the Err method will return +// any error that occurred during parsing. +func (zp *ZoneParser) Next() (RR, bool) { + if zp.parseErr != nil { + return nil, false + } + if zp.sub != nil { + return zp.subNext() + } + + // 6 possible beginnings of a line (_ is a space): + // + // 0. zRRTYPE -> all omitted until the rrtype + // 1. zOwner _ zRrtype -> class/ttl omitted + // 2. zOwner _ zString _ zRrtype -> class omitted + // 3. zOwner _ zString _ zClass _ zRrtype -> ttl/class + // 4. zOwner _ zClass _ zRrtype -> ttl omitted + // 5. zOwner _ zClass _ zString _ zRrtype -> class/ttl (reversed) + // + // After detecting these, we know the zRrtype so we can jump to functions + // handling the rdata for each of these types. + + st := zExpectOwnerDir // initial state + h := &zp.h + + for l, ok := zp.c.Next(); ok; l, ok = zp.c.Next() { + // zlexer spotted an error already + if l.err { + return zp.setParseError(l.token, l) + } + switch st { case zExpectOwnerDir: // We can also expect a directive, like $TTL or $ORIGIN - if defttl != nil { - h.Ttl = defttl.ttl + if zp.defttl != nil { + h.Ttl = zp.defttl.ttl } + h.Class = ClassINET + switch l.value { case zNewline: st = zExpectOwnerDir case zOwner: - h.Name = l.token - name, ok := toAbsoluteName(l.token, origin) + name, ok := toAbsoluteName(l.token, zp.origin) if !ok { - t <- &Token{Error: &ParseError{f, "bad owner name", l}} - return + return zp.setParseError("bad owner name", l) } + h.Name = name - prevName = h.Name + st = zExpectOwnerBl case zDirTTL: st = zExpectDirTTLBl @@ -243,12 +409,12 @@ func parseZone(r io.Reader, origin, f string, defttl *ttlState, t chan *Token, i case zDirGenerate: st = zExpectDirGenerateBl case zRrtpe: - h.Name = prevName h.Rrtype = l.torc + st = zExpectRdata case zClass: - h.Name = prevName h.Class = l.torc + st = zExpectAnyNoClassBl case zBlank: // Discard, can happen when there is nothing on the @@ -256,297 +422,401 @@ func parseZone(r io.Reader, origin, f string, defttl *ttlState, t chan *Token, i case zString: ttl, ok := stringToTTL(l.token) if !ok { - t <- &Token{Error: &ParseError{f, "not a TTL", l}} - return + return zp.setParseError("not a TTL", l) } - h.Ttl = ttl - if defttl == nil || !defttl.isByDirective { - defttl = &ttlState{ttl, false} - } - st = zExpectAnyNoTTLBl + h.Ttl = ttl + + if zp.defttl == nil || !zp.defttl.isByDirective { + zp.defttl = &ttlState{ttl, false} + } + + st = zExpectAnyNoTTLBl default: - t <- &Token{Error: &ParseError{f, "syntax error at beginning", l}} - return + return zp.setParseError("syntax error at beginning", l) } case zExpectDirIncludeBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank after $INCLUDE-directive", l}} - return + return zp.setParseError("no blank after $INCLUDE-directive", l) } + st = zExpectDirInclude case zExpectDirInclude: if l.value != zString { - t <- &Token{Error: &ParseError{f, "expecting $INCLUDE value, not this...", l}} - return + return zp.setParseError("expecting $INCLUDE value, not this...", l) } - neworigin := origin // There may be optionally a new origin set after the filename, if not use current one - switch l := <-c; l.value { + + neworigin := zp.origin // There may be optionally a new origin set after the filename, if not use current one + switch l, _ := zp.c.Next(); l.value { case zBlank: - l := <-c + l, _ := zp.c.Next() if l.value == zString { - name, ok := toAbsoluteName(l.token, origin) + name, ok := toAbsoluteName(l.token, zp.origin) if !ok { - t <- &Token{Error: &ParseError{f, "bad origin name", l}} - return + return zp.setParseError("bad origin name", l) } + neworigin = name } case zNewline, zEOF: // Ok default: - t <- &Token{Error: &ParseError{f, "garbage after $INCLUDE", l}} - return + return zp.setParseError("garbage after $INCLUDE", l) } + + if !zp.includeAllowed { + return zp.setParseError("$INCLUDE directive not allowed", l) + } + if zp.includeDepth >= maxIncludeDepth { + return zp.setParseError("too deeply nested $INCLUDE", l) + } + // Start with the new file includePath := l.token if !filepath.IsAbs(includePath) { - includePath = filepath.Join(filepath.Dir(f), includePath) + includePath = filepath.Join(filepath.Dir(zp.file), includePath) } + r1, e1 := os.Open(includePath) if e1 != nil { - msg := fmt.Sprintf("failed to open `%s'", l.token) + var as string if !filepath.IsAbs(l.token) { - msg += fmt.Sprintf(" as `%s'", includePath) + as = fmt.Sprintf(" as `%s'", includePath) } - t <- &Token{Error: &ParseError{f, msg, l}} - return + + msg := fmt.Sprintf("failed to open `%s'%s: %v", l.token, as, e1) + return zp.setParseError(msg, l) } - if include+1 > 7 { - t <- &Token{Error: &ParseError{f, "too deeply nested $INCLUDE", l}} - return - } - parseZone(r1, neworigin, includePath, defttl, t, include+1) - st = zExpectOwnerDir + + zp.sub = NewZoneParser(r1, neworigin, includePath) + zp.sub.defttl, zp.sub.includeDepth, zp.sub.osFile = zp.defttl, zp.includeDepth+1, r1 + zp.sub.SetIncludeAllowed(true) + return zp.subNext() case zExpectDirTTLBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank after $TTL-directive", l}} - return + return zp.setParseError("no blank after $TTL-directive", l) } + st = zExpectDirTTL case zExpectDirTTL: if l.value != zString { - t <- &Token{Error: &ParseError{f, "expecting $TTL value, not this...", l}} - return + return zp.setParseError("expecting $TTL value, not this...", l) } - if e, _ := slurpRemainder(c, f); e != nil { - t <- &Token{Error: e} - return + + if e := slurpRemainder(zp.c, zp.file); e != nil { + zp.parseErr = e + return nil, false } + ttl, ok := stringToTTL(l.token) if !ok { - t <- &Token{Error: &ParseError{f, "expecting $TTL value, not this...", l}} - return + return zp.setParseError("expecting $TTL value, not this...", l) } - defttl = &ttlState{ttl, true} + + zp.defttl = &ttlState{ttl, true} + st = zExpectOwnerDir case zExpectDirOriginBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank after $ORIGIN-directive", l}} - return + return zp.setParseError("no blank after $ORIGIN-directive", l) } + st = zExpectDirOrigin case zExpectDirOrigin: if l.value != zString { - t <- &Token{Error: &ParseError{f, "expecting $ORIGIN value, not this...", l}} - return + return zp.setParseError("expecting $ORIGIN value, not this...", l) } - if e, _ := slurpRemainder(c, f); e != nil { - t <- &Token{Error: e} + + if e := slurpRemainder(zp.c, zp.file); e != nil { + zp.parseErr = e + return nil, false } - name, ok := toAbsoluteName(l.token, origin) + + name, ok := toAbsoluteName(l.token, zp.origin) if !ok { - t <- &Token{Error: &ParseError{f, "bad origin name", l}} - return + return zp.setParseError("bad origin name", l) } - origin = name + + zp.origin = name + st = zExpectOwnerDir case zExpectDirGenerateBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank after $GENERATE-directive", l}} - return + return zp.setParseError("no blank after $GENERATE-directive", l) } + st = zExpectDirGenerate case zExpectDirGenerate: if l.value != zString { - t <- &Token{Error: &ParseError{f, "expecting $GENERATE value, not this...", l}} - return + return zp.setParseError("expecting $GENERATE value, not this...", l) } - if errMsg := generate(l, c, t, origin); errMsg != "" { - t <- &Token{Error: &ParseError{f, errMsg, l}} - return - } - st = zExpectOwnerDir + + return zp.generate(l) case zExpectOwnerBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank after owner", l}} - return + return zp.setParseError("no blank after owner", l) } + st = zExpectAny case zExpectAny: switch l.value { case zRrtpe: - if defttl == nil { - t <- &Token{Error: &ParseError{f, "missing TTL with no previous value", l}} - return + if zp.defttl == nil { + return zp.setParseError("missing TTL with no previous value", l) } + h.Rrtype = l.torc + st = zExpectRdata case zClass: h.Class = l.torc + st = zExpectAnyNoClassBl case zString: ttl, ok := stringToTTL(l.token) if !ok { - t <- &Token{Error: &ParseError{f, "not a TTL", l}} - return + return zp.setParseError("not a TTL", l) } + h.Ttl = ttl - if defttl == nil || !defttl.isByDirective { - defttl = &ttlState{ttl, false} + + if zp.defttl == nil || !zp.defttl.isByDirective { + zp.defttl = &ttlState{ttl, false} } + st = zExpectAnyNoTTLBl default: - t <- &Token{Error: &ParseError{f, "expecting RR type, TTL or class, not this...", l}} - return + return zp.setParseError("expecting RR type, TTL or class, not this...", l) } case zExpectAnyNoClassBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank before class", l}} - return + return zp.setParseError("no blank before class", l) } + st = zExpectAnyNoClass case zExpectAnyNoTTLBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank before TTL", l}} - return + return zp.setParseError("no blank before TTL", l) } + st = zExpectAnyNoTTL case zExpectAnyNoTTL: switch l.value { case zClass: h.Class = l.torc + st = zExpectRrtypeBl case zRrtpe: h.Rrtype = l.torc + st = zExpectRdata default: - t <- &Token{Error: &ParseError{f, "expecting RR type or class, not this...", l}} - return + return zp.setParseError("expecting RR type or class, not this...", l) } case zExpectAnyNoClass: switch l.value { case zString: ttl, ok := stringToTTL(l.token) if !ok { - t <- &Token{Error: &ParseError{f, "not a TTL", l}} - return + return zp.setParseError("not a TTL", l) } + h.Ttl = ttl - if defttl == nil || !defttl.isByDirective { - defttl = &ttlState{ttl, false} + + if zp.defttl == nil || !zp.defttl.isByDirective { + zp.defttl = &ttlState{ttl, false} } + st = zExpectRrtypeBl case zRrtpe: h.Rrtype = l.torc + st = zExpectRdata default: - t <- &Token{Error: &ParseError{f, "expecting RR type or TTL, not this...", l}} - return + return zp.setParseError("expecting RR type or TTL, not this...", l) } case zExpectRrtypeBl: if l.value != zBlank { - t <- &Token{Error: &ParseError{f, "no blank before RR type", l}} - return + return zp.setParseError("no blank before RR type", l) } + st = zExpectRrtype case zExpectRrtype: if l.value != zRrtpe { - t <- &Token{Error: &ParseError{f, "unknown RR type", l}} - return + return zp.setParseError("unknown RR type", l) } + h.Rrtype = l.torc + st = zExpectRdata case zExpectRdata: - r, e, c1 := setRR(h, c, origin, f) + r, e := setRR(*h, zp.c, zp.origin, zp.file) if e != nil { // If e.lex is nil than we have encounter a unknown RR type // in that case we substitute our current lex token if e.lex.token == "" && e.lex.value == 0 { e.lex = l // Uh, dirty } - t <- &Token{Error: e} - return + + zp.parseErr = e + return nil, false } - t <- &Token{RR: r, Comment: c1} - st = zExpectOwnerDir + + return r, true } } + // If we get here, we and the h.Rrtype is still zero, we haven't parsed anything, this // is not an error, because an empty zone file is still a zone file. + return nil, false } -// zlexer scans the sourcefile and returns tokens on the channel c. -func zlexer(s *scan, c chan lex) { - var l lex - str := make([]byte, maxTok) // Should be enough for any token - stri := 0 // Offset in str (0 means empty) - com := make([]byte, maxTok) // Hold comment text - comi := 0 - quote := false - escape := false - space := false - commt := false - rrtype := false - owner := true - brace := 0 - x, err := s.tokenText() - defer close(c) - for err == nil { - l.column = s.position.Column - l.line = s.position.Line - if stri >= maxTok { +type zlexer struct { + br io.ByteReader + + readErr error + + line int + column int + + comBuf string + comment string + + l lex + + brace int + quote bool + space bool + commt bool + rrtype bool + owner bool + + nextL bool + + eol bool // end-of-line +} + +func newZLexer(r io.Reader) *zlexer { + br, ok := r.(io.ByteReader) + if !ok { + br = bufio.NewReaderSize(r, 1024) + } + + return &zlexer{ + br: br, + + line: 1, + + owner: true, + } +} + +func (zl *zlexer) Err() error { + if zl.readErr == io.EOF { + return nil + } + + return zl.readErr +} + +// readByte returns the next byte from the input +func (zl *zlexer) readByte() (byte, bool) { + if zl.readErr != nil { + return 0, false + } + + c, err := zl.br.ReadByte() + if err != nil { + zl.readErr = err + return 0, false + } + + // delay the newline handling until the next token is delivered, + // fixes off-by-one errors when reporting a parse error. + if zl.eol { + zl.line++ + zl.column = 0 + zl.eol = false + } + + if c == '\n' { + zl.eol = true + } else { + zl.column++ + } + + return c, true +} + +func (zl *zlexer) Next() (lex, bool) { + l := &zl.l + if zl.nextL { + zl.nextL = false + return *l, true + } + if l.err { + // Parsing errors should be sticky. + return lex{value: zEOF}, false + } + + var ( + str [maxTok]byte // Hold string text + com [maxTok]byte // Hold comment text + + stri int // Offset in str (0 means empty) + comi int // Offset in com (0 means empty) + + escape bool + ) + + if zl.comBuf != "" { + comi = copy(com[:], zl.comBuf) + zl.comBuf = "" + } + + zl.comment = "" + + for x, ok := zl.readByte(); ok; x, ok = zl.readByte() { + l.line, l.column = zl.line, zl.column + + if stri >= len(str) { l.token = "token length insufficient for parsing" l.err = true - c <- l - return + return *l, true } - if comi >= maxTok { + if comi >= len(com) { l.token = "comment length insufficient for parsing" l.err = true - c <- l - return + return *l, true } switch x { case ' ', '\t': - if escape { + if escape || zl.quote { + // Inside quotes or escaped this is legal. + str[stri] = x + stri++ + escape = false - str[stri] = x - stri++ break } - if quote { - // Inside quotes this is legal - str[stri] = x - stri++ - break - } - if commt { + + if zl.commt { com[comi] = x comi++ break } + + var retL lex if stri == 0 { // Space directly in the beginning, handled in the grammar - } else if owner { + } else if zl.owner { // If we have a string and its the first, make it an owner l.value = zOwner l.token = string(str[:stri]) - l.tokenUpper = strings.ToUpper(l.token) - l.length = stri + // escape $... start with a \ not a $, so this will work - switch l.tokenUpper { + switch strings.ToUpper(l.token) { case "$TTL": l.value = zDirTTL case "$ORIGIN": @@ -556,259 +826,323 @@ func zlexer(s *scan, c chan lex) { case "$GENERATE": l.value = zDirGenerate } - c <- l + + retL = *l } else { l.value = zString l.token = string(str[:stri]) - l.tokenUpper = strings.ToUpper(l.token) - l.length = stri - if !rrtype { - if t, ok := StringToType[l.tokenUpper]; ok { + + if !zl.rrtype { + tokenUpper := strings.ToUpper(l.token) + if t, ok := StringToType[tokenUpper]; ok { l.value = zRrtpe l.torc = t - rrtype = true - } else { - if strings.HasPrefix(l.tokenUpper, "TYPE") { - t, ok := typeToInt(l.token) - if !ok { - l.token = "unknown RR type" - l.err = true - c <- l - return - } - l.value = zRrtpe - rrtype = true - l.torc = t + + zl.rrtype = true + } else if strings.HasPrefix(tokenUpper, "TYPE") { + t, ok := typeToInt(l.token) + if !ok { + l.token = "unknown RR type" + l.err = true + return *l, true } + + l.value = zRrtpe + l.torc = t + + zl.rrtype = true } - if t, ok := StringToClass[l.tokenUpper]; ok { + + if t, ok := StringToClass[tokenUpper]; ok { l.value = zClass l.torc = t - } else { - if strings.HasPrefix(l.tokenUpper, "CLASS") { - t, ok := classToInt(l.token) - if !ok { - l.token = "unknown class" - l.err = true - c <- l - return - } - l.value = zClass - l.torc = t + } else if strings.HasPrefix(tokenUpper, "CLASS") { + t, ok := classToInt(l.token) + if !ok { + l.token = "unknown class" + l.err = true + return *l, true } + + l.value = zClass + l.torc = t } } - c <- l - } - stri = 0 - if !space && !commt { + retL = *l + } + + zl.owner = false + + if !zl.space { + zl.space = true + l.value = zBlank l.token = " " - l.length = 1 - c <- l + + if retL == (lex{}) { + return *l, true + } + + zl.nextL = true + } + + if retL != (lex{}) { + return retL, true } - owner = false - space = true case ';': - if escape { + if escape || zl.quote { + // Inside quotes or escaped this is legal. + str[stri] = x + stri++ + escape = false - str[stri] = x - stri++ break } - if quote { - // Inside quotes this is legal - str[stri] = x - stri++ - break + + zl.commt = true + zl.comBuf = "" + + if comi > 1 { + // A newline was previously seen inside a comment that + // was inside braces and we delayed adding it until now. + com[comi] = ' ' // convert newline to space + comi++ } - if stri > 0 { - l.value = zString - l.token = string(str[:stri]) - l.tokenUpper = strings.ToUpper(l.token) - l.length = stri - c <- l - stri = 0 - } - commt = true + com[comi] = ';' comi++ + + if stri > 0 { + zl.comBuf = string(com[:comi]) + + l.value = zString + l.token = string(str[:stri]) + return *l, true + } case '\r': escape = false - if quote { + + if zl.quote { str[stri] = x stri++ - break } + // discard if outside of quotes case '\n': escape = false + // Escaped newline - if quote { + if zl.quote { str[stri] = x stri++ break } - // inside quotes this is legal - if commt { + + if zl.commt { // Reset a comment - commt = false - rrtype = false - stri = 0 + zl.commt = false + zl.rrtype = false + // If not in a brace this ends the comment AND the RR - if brace == 0 { - owner = true - owner = true + if zl.brace == 0 { + zl.owner = true + l.value = zNewline l.token = "\n" - l.tokenUpper = l.token - l.length = 1 - l.comment = string(com[:comi]) - c <- l - l.comment = "" - comi = 0 - break + zl.comment = string(com[:comi]) + return *l, true } - com[comi] = ' ' // convert newline to space - comi++ + + zl.comBuf = string(com[:comi]) break } - if brace == 0 { + if zl.brace == 0 { // If there is previous text, we should output it here + var retL lex if stri != 0 { l.value = zString l.token = string(str[:stri]) - l.tokenUpper = strings.ToUpper(l.token) - l.length = stri - if !rrtype { - if t, ok := StringToType[l.tokenUpper]; ok { + if !zl.rrtype { + tokenUpper := strings.ToUpper(l.token) + if t, ok := StringToType[tokenUpper]; ok { + zl.rrtype = true + l.value = zRrtpe l.torc = t - rrtype = true } } - c <- l + + retL = *l } + l.value = zNewline l.token = "\n" - l.tokenUpper = l.token - l.length = 1 - c <- l - stri = 0 - commt = false - rrtype = false - owner = true - comi = 0 + + zl.comment = zl.comBuf + zl.comBuf = "" + zl.rrtype = false + zl.owner = true + + if retL != (lex{}) { + zl.nextL = true + return retL, true + } + + return *l, true } case '\\': // comments do not get escaped chars, everything is copied - if commt { + if zl.commt { com[comi] = x comi++ break } + // something already escaped must be in string if escape { str[stri] = x stri++ + escape = false break } + // something escaped outside of string gets added to string str[stri] = x stri++ + escape = true case '"': - if commt { + if zl.commt { com[comi] = x comi++ break } + if escape { str[stri] = x stri++ + escape = false break } - space = false + + zl.space = false + // send previous gathered text and the quote + var retL lex if stri != 0 { l.value = zString l.token = string(str[:stri]) - l.tokenUpper = strings.ToUpper(l.token) - l.length = stri - c <- l - stri = 0 + retL = *l } // send quote itself as separate token l.value = zQuote l.token = "\"" - l.tokenUpper = l.token - l.length = 1 - c <- l - quote = !quote + + zl.quote = !zl.quote + + if retL != (lex{}) { + zl.nextL = true + return retL, true + } + + return *l, true case '(', ')': - if commt { + if zl.commt { com[comi] = x comi++ break } - if escape { + + if escape || zl.quote { + // Inside quotes or escaped this is legal. str[stri] = x stri++ + escape = false break } - if quote { - str[stri] = x - stri++ - break - } + switch x { case ')': - brace-- - if brace < 0 { + zl.brace-- + + if zl.brace < 0 { l.token = "extra closing brace" - l.tokenUpper = l.token l.err = true - c <- l - return + return *l, true } case '(': - brace++ + zl.brace++ } default: escape = false - if commt { + + if zl.commt { com[comi] = x comi++ break } + str[stri] = x stri++ - space = false + + zl.space = false } - x, err = s.tokenText() } + + if zl.readErr != nil && zl.readErr != io.EOF { + // Don't return any tokens after a read error occurs. + return lex{value: zEOF}, false + } + + var retL lex if stri > 0 { - // Send remainder - l.token = string(str[:stri]) - l.tokenUpper = strings.ToUpper(l.token) - l.length = stri + // Send remainder of str l.value = zString - c <- l + l.token = string(str[:stri]) + retL = *l + + if comi <= 0 { + return retL, true + } } - if brace != 0 { + + if comi > 0 { + // Send remainder of com + l.value = zNewline + l.token = "\n" + zl.comment = string(com[:comi]) + + if retL != (lex{}) { + zl.nextL = true + return retL, true + } + + return *l, true + } + + if zl.brace != 0 { l.token = "unbalanced brace" - l.tokenUpper = l.token l.err = true - c <- l + return *l, true } + + return lex{value: zEOF}, false +} + +func (zl *zlexer) Comment() string { + if zl.l.err { + return "" + } + + return zl.comment } // Extract the class number from CLASSxx @@ -839,8 +1173,7 @@ func typeToInt(token string) (uint16, bool) { // stringToTTL parses things like 2w, 2m, etc, and returns the time in seconds. func stringToTTL(token string) (uint32, bool) { - s := uint32(0) - i := uint32(0) + var s, i uint32 for _, c := range token { switch c { case 's', 'S': @@ -928,7 +1261,7 @@ func toAbsoluteName(name, origin string) (absolute string, ok bool) { } // check if name is already absolute - if name[len(name)-1] == '.' { + if IsFqdn(name) { return name, true } @@ -968,24 +1301,21 @@ func locCheckEast(token string, longitude uint32) (uint32, bool) { return longitude, false } -// "Eat" the rest of the "line". Return potential comments -func slurpRemainder(c chan lex, f string) (*ParseError, string) { - l := <-c - com := "" +// "Eat" the rest of the "line" +func slurpRemainder(c *zlexer, f string) *ParseError { + l, _ := c.Next() switch l.value { case zBlank: - l = <-c - com = l.comment + l, _ = c.Next() if l.value != zNewline && l.value != zEOF { - return &ParseError{f, "garbage after rdata", l}, "" + return &ParseError{f, "garbage after rdata", l} } case zNewline: - com = l.comment case zEOF: default: - return &ParseError{f, "garbage after rdata", l}, "" + return &ParseError{f, "garbage after rdata", l} } - return nil, com + return nil } // Parse a 64 bit-like ipv6 address: "0014:4fff:ff20:ee64" diff --git a/vendor/github.com/miekg/dns/scan_rr.go b/vendor/github.com/miekg/dns/scan_rr.go index fb6f95d..f48ff78 100644 --- a/vendor/github.com/miekg/dns/scan_rr.go +++ b/vendor/github.com/miekg/dns/scan_rr.go @@ -7,70 +7,69 @@ import ( "strings" ) -type parserFunc struct { - // Func defines the function that parses the tokens and returns the RR - // or an error. The last string contains any comments in the line as - // they returned by the lexer as well. - Func func(h RR_Header, c chan lex, origin string, file string) (RR, *ParseError, string) - // Signals if the RR ending is of variable length, like TXT or records - // that have Hexadecimal or Base64 as their last element in the Rdata. Records - // that have a fixed ending or for instance A, AAAA, SOA and etc. - Variable bool -} - // Parse the rdata of each rrtype. // All data from the channel c is either zString or zBlank. // After the rdata there may come a zBlank and then a zNewline // or immediately a zNewline. If this is not the case we flag // an *ParseError: garbage after rdata. -func setRR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - parserfunc, ok := typeToparserFunc[h.Rrtype] - if ok { - r, e, cm := parserfunc.Func(h, c, o, f) - if parserfunc.Variable { - return r, e, cm - } - if e != nil { - return nil, e, "" - } - e, cm = slurpRemainder(c, f) - if e != nil { - return nil, e, "" - } - return r, nil, cm +func setRR(h RR_Header, c *zlexer, o, f string) (RR, *ParseError) { + var rr RR + if newFn, ok := TypeToRR[h.Rrtype]; ok && canParseAsRR(h.Rrtype) { + rr = newFn() + *rr.Header() = h + } else { + rr = &RFC3597{Hdr: h} + } + + err := rr.parse(c, o, f) + if err != nil { + return nil, err + } + + return rr, nil +} + +// canParseAsRR returns true if the record type can be parsed as a +// concrete RR. It blacklists certain record types that must be parsed +// according to RFC 3597 because they lack a presentation format. +func canParseAsRR(rrtype uint16) bool { + switch rrtype { + case TypeANY, TypeNULL, TypeOPT, TypeTSIG: + return false + default: + return true } - // RFC3957 RR (Unknown RR handling) - return setRFC3597(h, c, o, f) } // A remainder of the rdata with embedded spaces, return the parsed string (sans the spaces) // or an error -func endingToString(c chan lex, errstr, f string) (string, *ParseError, string) { - s := "" - l := <-c // zString +func endingToString(c *zlexer, errstr, f string) (string, *ParseError) { + var s string + l, _ := c.Next() // zString for l.value != zNewline && l.value != zEOF { if l.err { - return s, &ParseError{f, errstr, l}, "" + return s, &ParseError{f, errstr, l} } switch l.value { case zString: s += l.token case zBlank: // Ok default: - return "", &ParseError{f, errstr, l}, "" + return "", &ParseError{f, errstr, l} } - l = <-c + l, _ = c.Next() } - return s, nil, l.comment + + return s, nil } // A remainder of the rdata with embedded spaces, split on unquoted whitespace // and return the parsed string slice or an error -func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, string) { +func endingToTxtSlice(c *zlexer, errstr, f string) ([]string, *ParseError) { // Get the remaining data until we see a zNewline - l := <-c + l, _ := c.Next() if l.err { - return nil, &ParseError{f, errstr, l}, "" + return nil, &ParseError{f, errstr, l} } // Build the slice @@ -79,7 +78,7 @@ func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, stri empty := false for l.value != zNewline && l.value != zEOF { if l.err { - return nil, &ParseError{f, errstr, l}, "" + return nil, &ParseError{f, errstr, l} } switch l.value { case zString: @@ -106,7 +105,7 @@ func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, stri case zBlank: if quote { // zBlank can only be seen in between txt parts. - return nil, &ParseError{f, errstr, l}, "" + return nil, &ParseError{f, errstr, l} } case zQuote: if empty && quote { @@ -115,196 +114,168 @@ func endingToTxtSlice(c chan lex, errstr, f string) ([]string, *ParseError, stri quote = !quote empty = true default: - return nil, &ParseError{f, errstr, l}, "" + return nil, &ParseError{f, errstr, l} } - l = <-c + l, _ = c.Next() } + if quote { - return nil, &ParseError{f, errstr, l}, "" + return nil, &ParseError{f, errstr, l} } - return s, nil, l.comment + + return s, nil } -func setA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(A) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *A) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } rr.A = net.ParseIP(l.token) if rr.A == nil || l.err { - return nil, &ParseError{f, "bad A A", l}, "" + return &ParseError{f, "bad A A", l} } - return rr, nil, "" + return slurpRemainder(c, f) } -func setAAAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(AAAA) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *AAAA) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } rr.AAAA = net.ParseIP(l.token) if rr.AAAA == nil || l.err { - return nil, &ParseError{f, "bad AAAA AAAA", l}, "" + return &ParseError{f, "bad AAAA AAAA", l} } - return rr, nil, "" + return slurpRemainder(c, f) } -func setNS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NS) - rr.Hdr = h - - l := <-c +func (rr *NS) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Ns = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad NS Ns", l}, "" + return &ParseError{f, "bad NS Ns", l} } rr.Ns = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(PTR) - rr.Hdr = h - - l := <-c +func (rr *PTR) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Ptr = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad PTR Ptr", l}, "" + return &ParseError{f, "bad PTR Ptr", l} } rr.Ptr = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setNSAPPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NSAPPTR) - rr.Hdr = h - - l := <-c +func (rr *NSAPPTR) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Ptr = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad NSAP-PTR Ptr", l}, "" + return &ParseError{f, "bad NSAP-PTR Ptr", l} } rr.Ptr = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setRP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(RP) - rr.Hdr = h - - l := <-c +func (rr *RP) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Mbox = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } mbox, mboxOk := toAbsoluteName(l.token, o) if l.err || !mboxOk { - return nil, &ParseError{f, "bad RP Mbox", l}, "" + return &ParseError{f, "bad RP Mbox", l} } rr.Mbox = mbox - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() rr.Txt = l.token txt, txtOk := toAbsoluteName(l.token, o) if l.err || !txtOk { - return nil, &ParseError{f, "bad RP Txt", l}, "" + return &ParseError{f, "bad RP Txt", l} } rr.Txt = txt - return rr, nil, "" + return slurpRemainder(c, f) } -func setMR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MR) - rr.Hdr = h - - l := <-c +func (rr *MR) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Mr = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad MR Mr", l}, "" + return &ParseError{f, "bad MR Mr", l} } rr.Mr = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setMB(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MB) - rr.Hdr = h - - l := <-c +func (rr *MB) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Mb = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad MB Mb", l}, "" + return &ParseError{f, "bad MB Mb", l} } rr.Mb = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setMG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MG) - rr.Hdr = h - - l := <-c +func (rr *MG) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Mg = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad MG Mg", l}, "" + return &ParseError{f, "bad MG Mg", l} } rr.Mg = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setHINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(HINFO) - rr.Hdr = h - - chunks, e, c1 := endingToTxtSlice(c, "bad HINFO Fields", f) +func (rr *HINFO) parse(c *zlexer, o, f string) *ParseError { + chunks, e := endingToTxtSlice(c, "bad HINFO Fields", f) if e != nil { - return nil, e, c1 + return e } if ln := len(chunks); ln == 0 { - return rr, nil, "" + return nil } else if ln == 1 { // Can we split it? if out := strings.Fields(chunks[0]); len(out) > 1 { @@ -317,281 +288,248 @@ func setHINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { rr.Cpu = chunks[0] rr.Os = strings.Join(chunks[1:], " ") - return rr, nil, "" + return nil } -func setMINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MINFO) - rr.Hdr = h - - l := <-c +func (rr *MINFO) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Rmail = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } rmail, rmailOk := toAbsoluteName(l.token, o) if l.err || !rmailOk { - return nil, &ParseError{f, "bad MINFO Rmail", l}, "" + return &ParseError{f, "bad MINFO Rmail", l} } rr.Rmail = rmail - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() rr.Email = l.token email, emailOk := toAbsoluteName(l.token, o) if l.err || !emailOk { - return nil, &ParseError{f, "bad MINFO Email", l}, "" + return &ParseError{f, "bad MINFO Email", l} } rr.Email = email - return rr, nil, "" + return slurpRemainder(c, f) } -func setMF(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MF) - rr.Hdr = h - - l := <-c +func (rr *MF) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Mf = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad MF Mf", l}, "" + return &ParseError{f, "bad MF Mf", l} } rr.Mf = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setMD(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MD) - rr.Hdr = h - - l := <-c +func (rr *MD) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Md = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad MD Md", l}, "" + return &ParseError{f, "bad MD Md", l} } rr.Md = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setMX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(MX) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *MX) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad MX Pref", l}, "" + return &ParseError{f, "bad MX Pref", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Mx = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad MX Mx", l}, "" + return &ParseError{f, "bad MX Mx", l} } rr.Mx = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setRT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(RT) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *RT) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil { - return nil, &ParseError{f, "bad RT Preference", l}, "" + return &ParseError{f, "bad RT Preference", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Host = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad RT Host", l}, "" + return &ParseError{f, "bad RT Host", l} } rr.Host = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setAFSDB(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(AFSDB) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *AFSDB) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad AFSDB Subtype", l}, "" + return &ParseError{f, "bad AFSDB Subtype", l} } rr.Subtype = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Hostname = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad AFSDB Hostname", l}, "" + return &ParseError{f, "bad AFSDB Hostname", l} } rr.Hostname = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setX25(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(X25) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *X25) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } if l.err { - return nil, &ParseError{f, "bad X25 PSDNAddress", l}, "" + return &ParseError{f, "bad X25 PSDNAddress", l} } rr.PSDNAddress = l.token - return rr, nil, "" + return slurpRemainder(c, f) } -func setKX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(KX) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *KX) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad KX Pref", l}, "" + return &ParseError{f, "bad KX Pref", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Exchanger = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad KX Exchanger", l}, "" + return &ParseError{f, "bad KX Exchanger", l} } rr.Exchanger = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setCNAME(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(CNAME) - rr.Hdr = h - - l := <-c +func (rr *CNAME) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Target = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad CNAME Target", l}, "" + return &ParseError{f, "bad CNAME Target", l} } rr.Target = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setDNAME(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(DNAME) - rr.Hdr = h - - l := <-c +func (rr *DNAME) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Target = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad DNAME Target", l}, "" + return &ParseError{f, "bad DNAME Target", l} } rr.Target = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setSOA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(SOA) - rr.Hdr = h - - l := <-c +func (rr *SOA) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.Ns = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } ns, nsOk := toAbsoluteName(l.token, o) if l.err || !nsOk { - return nil, &ParseError{f, "bad SOA Ns", l}, "" + return &ParseError{f, "bad SOA Ns", l} } rr.Ns = ns - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() rr.Mbox = l.token mbox, mboxOk := toAbsoluteName(l.token, o) if l.err || !mboxOk { - return nil, &ParseError{f, "bad SOA Mbox", l}, "" + return &ParseError{f, "bad SOA Mbox", l} } rr.Mbox = mbox - <-c // zBlank + c.Next() // zBlank var ( v uint32 ok bool ) for i := 0; i < 5; i++ { - l = <-c + l, _ = c.Next() if l.err { - return nil, &ParseError{f, "bad SOA zone parameter", l}, "" + return &ParseError{f, "bad SOA zone parameter", l} } if j, e := strconv.ParseUint(l.token, 10, 32); e != nil { if i == 0 { // Serial must be a number - return nil, &ParseError{f, "bad SOA zone parameter", l}, "" + return &ParseError{f, "bad SOA zone parameter", l} } // We allow other fields to be unitful duration strings if v, ok = stringToTTL(l.token); !ok { - return nil, &ParseError{f, "bad SOA zone parameter", l}, "" + return &ParseError{f, "bad SOA zone parameter", l} } } else { @@ -600,191 +538,180 @@ func setSOA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { switch i { case 0: rr.Serial = v - <-c // zBlank + c.Next() // zBlank case 1: rr.Refresh = v - <-c // zBlank + c.Next() // zBlank case 2: rr.Retry = v - <-c // zBlank + c.Next() // zBlank case 3: rr.Expire = v - <-c // zBlank + c.Next() // zBlank case 4: rr.Minttl = v } } - return rr, nil, "" + return slurpRemainder(c, f) } -func setSRV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(SRV) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *SRV) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad SRV Priority", l}, "" + return &ParseError{f, "bad SRV Priority", l} } rr.Priority = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad SRV Weight", l}, "" + return &ParseError{f, "bad SRV Weight", l} } rr.Weight = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad SRV Port", l}, "" + return &ParseError{f, "bad SRV Port", l} } rr.Port = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Target = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad SRV Target", l}, "" + return &ParseError{f, "bad SRV Target", l} } rr.Target = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setNAPTR(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NAPTR) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *NAPTR) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad NAPTR Order", l}, "" + return &ParseError{f, "bad NAPTR Order", l} } rr.Order = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad NAPTR Preference", l}, "" + return &ParseError{f, "bad NAPTR Preference", l} } rr.Preference = uint16(i) // Flags - <-c // zBlank - l = <-c // _QUOTE + c.Next() // zBlank + l, _ = c.Next() // _QUOTE if l.value != zQuote { - return nil, &ParseError{f, "bad NAPTR Flags", l}, "" + return &ParseError{f, "bad NAPTR Flags", l} } - l = <-c // Either String or Quote + l, _ = c.Next() // Either String or Quote if l.value == zString { rr.Flags = l.token - l = <-c // _QUOTE + l, _ = c.Next() // _QUOTE if l.value != zQuote { - return nil, &ParseError{f, "bad NAPTR Flags", l}, "" + return &ParseError{f, "bad NAPTR Flags", l} } } else if l.value == zQuote { rr.Flags = "" } else { - return nil, &ParseError{f, "bad NAPTR Flags", l}, "" + return &ParseError{f, "bad NAPTR Flags", l} } // Service - <-c // zBlank - l = <-c // _QUOTE + c.Next() // zBlank + l, _ = c.Next() // _QUOTE if l.value != zQuote { - return nil, &ParseError{f, "bad NAPTR Service", l}, "" + return &ParseError{f, "bad NAPTR Service", l} } - l = <-c // Either String or Quote + l, _ = c.Next() // Either String or Quote if l.value == zString { rr.Service = l.token - l = <-c // _QUOTE + l, _ = c.Next() // _QUOTE if l.value != zQuote { - return nil, &ParseError{f, "bad NAPTR Service", l}, "" + return &ParseError{f, "bad NAPTR Service", l} } } else if l.value == zQuote { rr.Service = "" } else { - return nil, &ParseError{f, "bad NAPTR Service", l}, "" + return &ParseError{f, "bad NAPTR Service", l} } // Regexp - <-c // zBlank - l = <-c // _QUOTE + c.Next() // zBlank + l, _ = c.Next() // _QUOTE if l.value != zQuote { - return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" + return &ParseError{f, "bad NAPTR Regexp", l} } - l = <-c // Either String or Quote + l, _ = c.Next() // Either String or Quote if l.value == zString { rr.Regexp = l.token - l = <-c // _QUOTE + l, _ = c.Next() // _QUOTE if l.value != zQuote { - return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" + return &ParseError{f, "bad NAPTR Regexp", l} } } else if l.value == zQuote { rr.Regexp = "" } else { - return nil, &ParseError{f, "bad NAPTR Regexp", l}, "" + return &ParseError{f, "bad NAPTR Regexp", l} } // After quote no space?? - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Replacement = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad NAPTR Replacement", l}, "" + return &ParseError{f, "bad NAPTR Replacement", l} } rr.Replacement = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setTALINK(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(TALINK) - rr.Hdr = h - - l := <-c +func (rr *TALINK) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.PreviousName = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } previousName, previousNameOk := toAbsoluteName(l.token, o) if l.err || !previousNameOk { - return nil, &ParseError{f, "bad TALINK PreviousName", l}, "" + return &ParseError{f, "bad TALINK PreviousName", l} } rr.PreviousName = previousName - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() rr.NextName = l.token nextName, nextNameOk := toAbsoluteName(l.token, o) if l.err || !nextNameOk { - return nil, &ParseError{f, "bad TALINK NextName", l}, "" + return &ParseError{f, "bad TALINK NextName", l} } rr.NextName = nextName - return rr, nil, "" + return slurpRemainder(c, f) } -func setLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(LOC) - rr.Hdr = h +func (rr *LOC) parse(c *zlexer, o, f string) *ParseError { // Non zero defaults for LOC record, see RFC 1876, Section 3. rr.HorizPre = 165 // 10000 rr.VertPre = 162 // 10 @@ -792,97 +719,97 @@ func setLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { ok := false // North - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 32) if e != nil || l.err { - return nil, &ParseError{f, "bad LOC Latitude", l}, "" + return &ParseError{f, "bad LOC Latitude", l} } rr.Latitude = 1000 * 60 * 60 * uint32(i) - <-c // zBlank + c.Next() // zBlank // Either number, 'N' or 'S' - l = <-c + l, _ = c.Next() if rr.Latitude, ok = locCheckNorth(l.token, rr.Latitude); ok { goto East } i, e = strconv.ParseUint(l.token, 10, 32) if e != nil || l.err { - return nil, &ParseError{f, "bad LOC Latitude minutes", l}, "" + return &ParseError{f, "bad LOC Latitude minutes", l} } rr.Latitude += 1000 * 60 * uint32(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, e := strconv.ParseFloat(l.token, 32); e != nil || l.err { - return nil, &ParseError{f, "bad LOC Latitude seconds", l}, "" + return &ParseError{f, "bad LOC Latitude seconds", l} } else { rr.Latitude += uint32(1000 * i) } - <-c // zBlank + c.Next() // zBlank // Either number, 'N' or 'S' - l = <-c + l, _ = c.Next() if rr.Latitude, ok = locCheckNorth(l.token, rr.Latitude); ok { goto East } // If still alive, flag an error - return nil, &ParseError{f, "bad LOC Latitude North/South", l}, "" + return &ParseError{f, "bad LOC Latitude North/South", l} East: // East - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, e := strconv.ParseUint(l.token, 10, 32); e != nil || l.err { - return nil, &ParseError{f, "bad LOC Longitude", l}, "" + return &ParseError{f, "bad LOC Longitude", l} } else { rr.Longitude = 1000 * 60 * 60 * uint32(i) } - <-c // zBlank + c.Next() // zBlank // Either number, 'E' or 'W' - l = <-c + l, _ = c.Next() if rr.Longitude, ok = locCheckEast(l.token, rr.Longitude); ok { goto Altitude } if i, e := strconv.ParseUint(l.token, 10, 32); e != nil || l.err { - return nil, &ParseError{f, "bad LOC Longitude minutes", l}, "" + return &ParseError{f, "bad LOC Longitude minutes", l} } else { rr.Longitude += 1000 * 60 * uint32(i) } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, e := strconv.ParseFloat(l.token, 32); e != nil || l.err { - return nil, &ParseError{f, "bad LOC Longitude seconds", l}, "" + return &ParseError{f, "bad LOC Longitude seconds", l} } else { rr.Longitude += uint32(1000 * i) } - <-c // zBlank + c.Next() // zBlank // Either number, 'E' or 'W' - l = <-c + l, _ = c.Next() if rr.Longitude, ok = locCheckEast(l.token, rr.Longitude); ok { goto Altitude } // If still alive, flag an error - return nil, &ParseError{f, "bad LOC Longitude East/West", l}, "" + return &ParseError{f, "bad LOC Longitude East/West", l} Altitude: - <-c // zBlank - l = <-c - if l.length == 0 || l.err { - return nil, &ParseError{f, "bad LOC Altitude", l}, "" + c.Next() // zBlank + l, _ = c.Next() + if len(l.token) == 0 || l.err { + return &ParseError{f, "bad LOC Altitude", l} } if l.token[len(l.token)-1] == 'M' || l.token[len(l.token)-1] == 'm' { l.token = l.token[0 : len(l.token)-1] } if i, e := strconv.ParseFloat(l.token, 32); e != nil { - return nil, &ParseError{f, "bad LOC Altitude", l}, "" + return &ParseError{f, "bad LOC Altitude", l} } else { rr.Altitude = uint32(i*100.0 + 10000000.0 + 0.5) } // And now optionally the other values - l = <-c + l, _ = c.Next() count := 0 for l.value != zNewline && l.value != zEOF { switch l.value { @@ -891,161 +818,150 @@ Altitude: case 0: // Size e, m, ok := stringToCm(l.token) if !ok { - return nil, &ParseError{f, "bad LOC Size", l}, "" + return &ParseError{f, "bad LOC Size", l} } - rr.Size = (e & 0x0f) | (m << 4 & 0xf0) + rr.Size = e&0x0f | m<<4&0xf0 case 1: // HorizPre e, m, ok := stringToCm(l.token) if !ok { - return nil, &ParseError{f, "bad LOC HorizPre", l}, "" + return &ParseError{f, "bad LOC HorizPre", l} } - rr.HorizPre = (e & 0x0f) | (m << 4 & 0xf0) + rr.HorizPre = e&0x0f | m<<4&0xf0 case 2: // VertPre e, m, ok := stringToCm(l.token) if !ok { - return nil, &ParseError{f, "bad LOC VertPre", l}, "" + return &ParseError{f, "bad LOC VertPre", l} } - rr.VertPre = (e & 0x0f) | (m << 4 & 0xf0) + rr.VertPre = e&0x0f | m<<4&0xf0 } count++ case zBlank: // Ok default: - return nil, &ParseError{f, "bad LOC Size, HorizPre or VertPre", l}, "" + return &ParseError{f, "bad LOC Size, HorizPre or VertPre", l} } - l = <-c + l, _ = c.Next() } - return rr, nil, "" + return nil } -func setHIP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(HIP) - rr.Hdr = h - +func (rr *HIP) parse(c *zlexer, o, f string) *ParseError { // HitLength is not represented - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad HIP PublicKeyAlgorithm", l}, "" + return &ParseError{f, "bad HIP PublicKeyAlgorithm", l} } rr.PublicKeyAlgorithm = uint8(i) - <-c // zBlank - l = <-c // zString - if l.length == 0 || l.err { - return nil, &ParseError{f, "bad HIP Hit", l}, "" + c.Next() // zBlank + l, _ = c.Next() // zString + if len(l.token) == 0 || l.err { + return &ParseError{f, "bad HIP Hit", l} } rr.Hit = l.token // This can not contain spaces, see RFC 5205 Section 6. rr.HitLength = uint8(len(rr.Hit)) / 2 - <-c // zBlank - l = <-c // zString - if l.length == 0 || l.err { - return nil, &ParseError{f, "bad HIP PublicKey", l}, "" + c.Next() // zBlank + l, _ = c.Next() // zString + if len(l.token) == 0 || l.err { + return &ParseError{f, "bad HIP PublicKey", l} } rr.PublicKey = l.token // This cannot contain spaces rr.PublicKeyLength = uint16(base64.StdEncoding.DecodedLen(len(rr.PublicKey))) // RendezvousServers (if any) - l = <-c + l, _ = c.Next() var xs []string for l.value != zNewline && l.value != zEOF { switch l.value { case zString: name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad HIP RendezvousServers", l}, "" + return &ParseError{f, "bad HIP RendezvousServers", l} } xs = append(xs, name) case zBlank: // Ok default: - return nil, &ParseError{f, "bad HIP RendezvousServers", l}, "" + return &ParseError{f, "bad HIP RendezvousServers", l} } - l = <-c + l, _ = c.Next() } + rr.RendezvousServers = xs - return rr, nil, l.comment + return nil } -func setCERT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(CERT) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *CERT) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } if v, ok := StringToCertType[l.token]; ok { rr.Type = v } else if i, e := strconv.ParseUint(l.token, 10, 16); e != nil { - return nil, &ParseError{f, "bad CERT Type", l}, "" + return &ParseError{f, "bad CERT Type", l} } else { rr.Type = uint16(i) } - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad CERT KeyTag", l}, "" + return &ParseError{f, "bad CERT KeyTag", l} } rr.KeyTag = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString if v, ok := StringToAlgorithm[l.token]; ok { rr.Algorithm = v } else if i, e := strconv.ParseUint(l.token, 10, 8); e != nil { - return nil, &ParseError{f, "bad CERT Algorithm", l}, "" + return &ParseError{f, "bad CERT Algorithm", l} } else { rr.Algorithm = uint8(i) } - s, e1, c1 := endingToString(c, "bad CERT Certificate", f) + s, e1 := endingToString(c, "bad CERT Certificate", f) if e1 != nil { - return nil, e1, c1 + return e1 } rr.Certificate = s - return rr, nil, c1 + return nil } -func setOPENPGPKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(OPENPGPKEY) - rr.Hdr = h - - s, e, c1 := endingToString(c, "bad OPENPGPKEY PublicKey", f) +func (rr *OPENPGPKEY) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToString(c, "bad OPENPGPKEY PublicKey", f) if e != nil { - return nil, e, c1 + return e } rr.PublicKey = s - return rr, nil, c1 + return nil } -func setCSYNC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(CSYNC) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *CSYNC) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } j, e := strconv.ParseUint(l.token, 10, 32) if e != nil { // Serial must be a number - return nil, &ParseError{f, "bad CSYNC serial", l}, "" + return &ParseError{f, "bad CSYNC serial", l} } rr.Serial = uint32(j) - <-c // zBlank + c.Next() // zBlank - l = <-c + l, _ = c.Next() j, e = strconv.ParseUint(l.token, 10, 16) if e != nil { // Serial must be a number - return nil, &ParseError{f, "bad CSYNC flags", l}, "" + return &ParseError{f, "bad CSYNC flags", l} } rr.Flags = uint16(j) @@ -1054,146 +970,138 @@ func setCSYNC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { k uint16 ok bool ) - l = <-c + l, _ = c.Next() for l.value != zNewline && l.value != zEOF { switch l.value { case zBlank: // Ok case zString: - if k, ok = StringToType[l.tokenUpper]; !ok { - if k, ok = typeToInt(l.tokenUpper); !ok { - return nil, &ParseError{f, "bad CSYNC TypeBitMap", l}, "" + tokenUpper := strings.ToUpper(l.token) + if k, ok = StringToType[tokenUpper]; !ok { + if k, ok = typeToInt(l.token); !ok { + return &ParseError{f, "bad CSYNC TypeBitMap", l} } } rr.TypeBitMap = append(rr.TypeBitMap, k) default: - return nil, &ParseError{f, "bad CSYNC TypeBitMap", l}, "" + return &ParseError{f, "bad CSYNC TypeBitMap", l} } - l = <-c + l, _ = c.Next() } - return rr, nil, l.comment + return nil } -func setSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setRRSIG(h, c, o, f) - if r != nil { - return &SIG{*r.(*RRSIG)}, e, s - } - return nil, e, s +func (rr *SIG) parse(c *zlexer, o, f string) *ParseError { + return rr.RRSIG.parse(c, o, f) } -func setRRSIG(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(RRSIG) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *RRSIG) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } - if t, ok := StringToType[l.tokenUpper]; !ok { - if strings.HasPrefix(l.tokenUpper, "TYPE") { - t, ok = typeToInt(l.tokenUpper) + tokenUpper := strings.ToUpper(l.token) + if t, ok := StringToType[tokenUpper]; !ok { + if strings.HasPrefix(tokenUpper, "TYPE") { + t, ok = typeToInt(l.token) if !ok { - return nil, &ParseError{f, "bad RRSIG Typecovered", l}, "" + return &ParseError{f, "bad RRSIG Typecovered", l} } rr.TypeCovered = t } else { - return nil, &ParseError{f, "bad RRSIG Typecovered", l}, "" + return &ParseError{f, "bad RRSIG Typecovered", l} } } else { rr.TypeCovered = t } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, err := strconv.ParseUint(l.token, 10, 8) if err != nil || l.err { - return nil, &ParseError{f, "bad RRSIG Algorithm", l}, "" + return &ParseError{f, "bad RRSIG Algorithm", l} } rr.Algorithm = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, err = strconv.ParseUint(l.token, 10, 8) if err != nil || l.err { - return nil, &ParseError{f, "bad RRSIG Labels", l}, "" + return &ParseError{f, "bad RRSIG Labels", l} } rr.Labels = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, err = strconv.ParseUint(l.token, 10, 32) if err != nil || l.err { - return nil, &ParseError{f, "bad RRSIG OrigTtl", l}, "" + return &ParseError{f, "bad RRSIG OrigTtl", l} } rr.OrigTtl = uint32(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, err := StringToTime(l.token); err != nil { // Try to see if all numeric and use it as epoch if i, err := strconv.ParseInt(l.token, 10, 64); err == nil { // TODO(miek): error out on > MAX_UINT32, same below rr.Expiration = uint32(i) } else { - return nil, &ParseError{f, "bad RRSIG Expiration", l}, "" + return &ParseError{f, "bad RRSIG Expiration", l} } } else { rr.Expiration = i } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, err := StringToTime(l.token); err != nil { if i, err := strconv.ParseInt(l.token, 10, 64); err == nil { rr.Inception = uint32(i) } else { - return nil, &ParseError{f, "bad RRSIG Inception", l}, "" + return &ParseError{f, "bad RRSIG Inception", l} } } else { rr.Inception = i } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, err = strconv.ParseUint(l.token, 10, 16) if err != nil || l.err { - return nil, &ParseError{f, "bad RRSIG KeyTag", l}, "" + return &ParseError{f, "bad RRSIG KeyTag", l} } rr.KeyTag = uint16(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() rr.SignerName = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad RRSIG SignerName", l}, "" + return &ParseError{f, "bad RRSIG SignerName", l} } rr.SignerName = name - s, e, c1 := endingToString(c, "bad RRSIG Signature", f) + s, e := endingToString(c, "bad RRSIG Signature", f) if e != nil { - return nil, e, c1 + return e } rr.Signature = s - return rr, nil, c1 + return nil } -func setNSEC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NSEC) - rr.Hdr = h - - l := <-c +func (rr *NSEC) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() rr.NextDomain = l.token - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment + if len(l.token) == 0 { // dynamic update rr. + return nil } name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad NSEC NextDomain", l}, "" + return &ParseError{f, "bad NSEC NextDomain", l} } rr.NextDomain = name @@ -1202,68 +1110,66 @@ func setNSEC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { k uint16 ok bool ) - l = <-c + l, _ = c.Next() for l.value != zNewline && l.value != zEOF { switch l.value { case zBlank: // Ok case zString: - if k, ok = StringToType[l.tokenUpper]; !ok { - if k, ok = typeToInt(l.tokenUpper); !ok { - return nil, &ParseError{f, "bad NSEC TypeBitMap", l}, "" + tokenUpper := strings.ToUpper(l.token) + if k, ok = StringToType[tokenUpper]; !ok { + if k, ok = typeToInt(l.token); !ok { + return &ParseError{f, "bad NSEC TypeBitMap", l} } } rr.TypeBitMap = append(rr.TypeBitMap, k) default: - return nil, &ParseError{f, "bad NSEC TypeBitMap", l}, "" + return &ParseError{f, "bad NSEC TypeBitMap", l} } - l = <-c + l, _ = c.Next() } - return rr, nil, l.comment + return nil } -func setNSEC3(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NSEC3) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *NSEC3) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad NSEC3 Hash", l}, "" + return &ParseError{f, "bad NSEC3 Hash", l} } rr.Hash = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad NSEC3 Flags", l}, "" + return &ParseError{f, "bad NSEC3 Flags", l} } rr.Flags = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad NSEC3 Iterations", l}, "" + return &ParseError{f, "bad NSEC3 Iterations", l} } rr.Iterations = uint16(i) - <-c - l = <-c + c.Next() + l, _ = c.Next() if len(l.token) == 0 || l.err { - return nil, &ParseError{f, "bad NSEC3 Salt", l}, "" + return &ParseError{f, "bad NSEC3 Salt", l} } if l.token != "-" { rr.SaltLength = uint8(len(l.token)) / 2 rr.Salt = l.token } - <-c - l = <-c + c.Next() + l, _ = c.Next() if len(l.token) == 0 || l.err { - return nil, &ParseError{f, "bad NSEC3 NextDomain", l}, "" + return &ParseError{f, "bad NSEC3 NextDomain", l} } rr.HashLength = 20 // Fix for NSEC3 (sha1 160 bits) rr.NextDomain = l.token @@ -1273,74 +1179,69 @@ func setNSEC3(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { k uint16 ok bool ) - l = <-c + l, _ = c.Next() for l.value != zNewline && l.value != zEOF { switch l.value { case zBlank: // Ok case zString: - if k, ok = StringToType[l.tokenUpper]; !ok { - if k, ok = typeToInt(l.tokenUpper); !ok { - return nil, &ParseError{f, "bad NSEC3 TypeBitMap", l}, "" + tokenUpper := strings.ToUpper(l.token) + if k, ok = StringToType[tokenUpper]; !ok { + if k, ok = typeToInt(l.token); !ok { + return &ParseError{f, "bad NSEC3 TypeBitMap", l} } } rr.TypeBitMap = append(rr.TypeBitMap, k) default: - return nil, &ParseError{f, "bad NSEC3 TypeBitMap", l}, "" + return &ParseError{f, "bad NSEC3 TypeBitMap", l} } - l = <-c + l, _ = c.Next() } - return rr, nil, l.comment + return nil } -func setNSEC3PARAM(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NSEC3PARAM) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *NSEC3PARAM) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad NSEC3PARAM Hash", l}, "" + return &ParseError{f, "bad NSEC3PARAM Hash", l} } rr.Hash = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad NSEC3PARAM Flags", l}, "" + return &ParseError{f, "bad NSEC3PARAM Flags", l} } rr.Flags = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad NSEC3PARAM Iterations", l}, "" + return &ParseError{f, "bad NSEC3PARAM Iterations", l} } rr.Iterations = uint16(i) - <-c - l = <-c + c.Next() + l, _ = c.Next() if l.token != "-" { rr.SaltLength = uint8(len(l.token)) rr.Salt = l.token } - return rr, nil, "" + return slurpRemainder(c, f) } -func setEUI48(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(EUI48) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *EUI48) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } - if l.length != 17 || l.err { - return nil, &ParseError{f, "bad EUI48 Address", l}, "" + if len(l.token) != 17 || l.err { + return &ParseError{f, "bad EUI48 Address", l} } addr := make([]byte, 12) dash := 0 @@ -1349,7 +1250,7 @@ func setEUI48(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { addr[i+1] = l.token[i+1+dash] dash++ if l.token[i+1+dash] != '-' { - return nil, &ParseError{f, "bad EUI48 Address", l}, "" + return &ParseError{f, "bad EUI48 Address", l} } } addr[10] = l.token[15] @@ -1357,23 +1258,20 @@ func setEUI48(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { i, e := strconv.ParseUint(string(addr), 16, 48) if e != nil { - return nil, &ParseError{f, "bad EUI48 Address", l}, "" + return &ParseError{f, "bad EUI48 Address", l} } rr.Address = i - return rr, nil, "" + return slurpRemainder(c, f) } -func setEUI64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(EUI64) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *EUI64) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } - if l.length != 23 || l.err { - return nil, &ParseError{f, "bad EUI64 Address", l}, "" + if len(l.token) != 23 || l.err { + return &ParseError{f, "bad EUI64 Address", l} } addr := make([]byte, 16) dash := 0 @@ -1382,7 +1280,7 @@ func setEUI64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { addr[i+1] = l.token[i+1+dash] dash++ if l.token[i+1+dash] != '-' { - return nil, &ParseError{f, "bad EUI64 Address", l}, "" + return &ParseError{f, "bad EUI64 Address", l} } } addr[14] = l.token[21] @@ -1390,814 +1288,650 @@ func setEUI64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { i, e := strconv.ParseUint(string(addr), 16, 64) if e != nil { - return nil, &ParseError{f, "bad EUI68 Address", l}, "" + return &ParseError{f, "bad EUI68 Address", l} } - rr.Address = uint64(i) - return rr, nil, "" + rr.Address = i + return slurpRemainder(c, f) } -func setSSHFP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(SSHFP) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *SSHFP) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad SSHFP Algorithm", l}, "" + return &ParseError{f, "bad SSHFP Algorithm", l} } rr.Algorithm = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad SSHFP Type", l}, "" + return &ParseError{f, "bad SSHFP Type", l} } rr.Type = uint8(i) - <-c // zBlank - s, e1, c1 := endingToString(c, "bad SSHFP Fingerprint", f) + c.Next() // zBlank + s, e1 := endingToString(c, "bad SSHFP Fingerprint", f) if e1 != nil { - return nil, e1, c1 + return e1 } rr.FingerPrint = s - return rr, nil, "" + return nil } -func setDNSKEYs(h RR_Header, c chan lex, o, f, typ string) (RR, *ParseError, string) { - rr := new(DNSKEY) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *DNSKEY) parseDNSKEY(c *zlexer, o, f, typ string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad " + typ + " Flags", l}, "" + return &ParseError{f, "bad " + typ + " Flags", l} } rr.Flags = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad " + typ + " Protocol", l}, "" + return &ParseError{f, "bad " + typ + " Protocol", l} } rr.Protocol = uint8(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad " + typ + " Algorithm", l}, "" + return &ParseError{f, "bad " + typ + " Algorithm", l} } rr.Algorithm = uint8(i) - s, e1, c1 := endingToString(c, "bad "+typ+" PublicKey", f) + s, e1 := endingToString(c, "bad "+typ+" PublicKey", f) if e1 != nil { - return nil, e1, c1 + return e1 } rr.PublicKey = s - return rr, nil, c1 + return nil } -func setKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setDNSKEYs(h, c, o, f, "KEY") - if r != nil { - return &KEY{*r.(*DNSKEY)}, e, s - } - return nil, e, s +func (rr *DNSKEY) parse(c *zlexer, o, f string) *ParseError { + return rr.parseDNSKEY(c, o, f, "DNSKEY") } -func setDNSKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setDNSKEYs(h, c, o, f, "DNSKEY") - return r, e, s +func (rr *KEY) parse(c *zlexer, o, f string) *ParseError { + return rr.parseDNSKEY(c, o, f, "KEY") } -func setCDNSKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setDNSKEYs(h, c, o, f, "CDNSKEY") - if r != nil { - return &CDNSKEY{*r.(*DNSKEY)}, e, s - } - return nil, e, s +func (rr *CDNSKEY) parse(c *zlexer, o, f string) *ParseError { + return rr.parseDNSKEY(c, o, f, "CDNSKEY") } -func setRKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(RKEY) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *RKEY) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad RKEY Flags", l}, "" + return &ParseError{f, "bad RKEY Flags", l} } rr.Flags = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad RKEY Protocol", l}, "" + return &ParseError{f, "bad RKEY Protocol", l} } rr.Protocol = uint8(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad RKEY Algorithm", l}, "" + return &ParseError{f, "bad RKEY Algorithm", l} } rr.Algorithm = uint8(i) - s, e1, c1 := endingToString(c, "bad RKEY PublicKey", f) + s, e1 := endingToString(c, "bad RKEY PublicKey", f) if e1 != nil { - return nil, e1, c1 + return e1 } rr.PublicKey = s - return rr, nil, c1 + return nil } -func setEID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(EID) - rr.Hdr = h - s, e, c1 := endingToString(c, "bad EID Endpoint", f) +func (rr *EID) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToString(c, "bad EID Endpoint", f) if e != nil { - return nil, e, c1 + return e } rr.Endpoint = s - return rr, nil, c1 + return nil } -func setNIMLOC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NIMLOC) - rr.Hdr = h - s, e, c1 := endingToString(c, "bad NIMLOC Locator", f) +func (rr *NIMLOC) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToString(c, "bad NIMLOC Locator", f) if e != nil { - return nil, e, c1 + return e } rr.Locator = s - return rr, nil, c1 + return nil } -func setGPOS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(GPOS) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *GPOS) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } _, e := strconv.ParseFloat(l.token, 64) if e != nil || l.err { - return nil, &ParseError{f, "bad GPOS Longitude", l}, "" + return &ParseError{f, "bad GPOS Longitude", l} } rr.Longitude = l.token - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() _, e = strconv.ParseFloat(l.token, 64) if e != nil || l.err { - return nil, &ParseError{f, "bad GPOS Latitude", l}, "" + return &ParseError{f, "bad GPOS Latitude", l} } rr.Latitude = l.token - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() _, e = strconv.ParseFloat(l.token, 64) if e != nil || l.err { - return nil, &ParseError{f, "bad GPOS Altitude", l}, "" + return &ParseError{f, "bad GPOS Altitude", l} } rr.Altitude = l.token - return rr, nil, "" + return slurpRemainder(c, f) } -func setDSs(h RR_Header, c chan lex, o, f, typ string) (RR, *ParseError, string) { - rr := new(DS) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *DS) parseDS(c *zlexer, o, f, typ string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad " + typ + " KeyTag", l}, "" + return &ParseError{f, "bad " + typ + " KeyTag", l} } rr.KeyTag = uint16(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, e = strconv.ParseUint(l.token, 10, 8); e != nil { - i, ok := StringToAlgorithm[l.tokenUpper] + tokenUpper := strings.ToUpper(l.token) + i, ok := StringToAlgorithm[tokenUpper] if !ok || l.err { - return nil, &ParseError{f, "bad " + typ + " Algorithm", l}, "" + return &ParseError{f, "bad " + typ + " Algorithm", l} } rr.Algorithm = i } else { rr.Algorithm = uint8(i) } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad " + typ + " DigestType", l}, "" + return &ParseError{f, "bad " + typ + " DigestType", l} } rr.DigestType = uint8(i) - s, e1, c1 := endingToString(c, "bad "+typ+" Digest", f) + s, e1 := endingToString(c, "bad "+typ+" Digest", f) if e1 != nil { - return nil, e1, c1 + return e1 } rr.Digest = s - return rr, nil, c1 + return nil } -func setDS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setDSs(h, c, o, f, "DS") - return r, e, s +func (rr *DS) parse(c *zlexer, o, f string) *ParseError { + return rr.parseDS(c, o, f, "DS") } -func setDLV(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setDSs(h, c, o, f, "DLV") - if r != nil { - return &DLV{*r.(*DS)}, e, s - } - return nil, e, s +func (rr *DLV) parse(c *zlexer, o, f string) *ParseError { + return rr.parseDS(c, o, f, "DLV") } -func setCDS(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - r, e, s := setDSs(h, c, o, f, "CDS") - if r != nil { - return &CDS{*r.(*DS)}, e, s - } - return nil, e, s +func (rr *CDS) parse(c *zlexer, o, f string) *ParseError { + return rr.parseDS(c, o, f, "CDS") } -func setTA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(TA) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *TA) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad TA KeyTag", l}, "" + return &ParseError{f, "bad TA KeyTag", l} } rr.KeyTag = uint16(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if i, e := strconv.ParseUint(l.token, 10, 8); e != nil { - i, ok := StringToAlgorithm[l.tokenUpper] + tokenUpper := strings.ToUpper(l.token) + i, ok := StringToAlgorithm[tokenUpper] if !ok || l.err { - return nil, &ParseError{f, "bad TA Algorithm", l}, "" + return &ParseError{f, "bad TA Algorithm", l} } rr.Algorithm = i } else { rr.Algorithm = uint8(i) } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad TA DigestType", l}, "" + return &ParseError{f, "bad TA DigestType", l} } rr.DigestType = uint8(i) - s, e, c1 := endingToString(c, "bad TA Digest", f) - if e != nil { - return nil, e.(*ParseError), c1 + s, err := endingToString(c, "bad TA Digest", f) + if err != nil { + return err } rr.Digest = s - return rr, nil, c1 + return nil } -func setTLSA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(TLSA) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *TLSA) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad TLSA Usage", l}, "" + return &ParseError{f, "bad TLSA Usage", l} } rr.Usage = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad TLSA Selector", l}, "" + return &ParseError{f, "bad TLSA Selector", l} } rr.Selector = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad TLSA MatchingType", l}, "" + return &ParseError{f, "bad TLSA MatchingType", l} } rr.MatchingType = uint8(i) // So this needs be e2 (i.e. different than e), because...??t - s, e2, c1 := endingToString(c, "bad TLSA Certificate", f) + s, e2 := endingToString(c, "bad TLSA Certificate", f) if e2 != nil { - return nil, e2, c1 + return e2 } rr.Certificate = s - return rr, nil, c1 + return nil } -func setSMIMEA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(SMIMEA) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *SMIMEA) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad SMIMEA Usage", l}, "" + return &ParseError{f, "bad SMIMEA Usage", l} } rr.Usage = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad SMIMEA Selector", l}, "" + return &ParseError{f, "bad SMIMEA Selector", l} } rr.Selector = uint8(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 8) if e != nil || l.err { - return nil, &ParseError{f, "bad SMIMEA MatchingType", l}, "" + return &ParseError{f, "bad SMIMEA MatchingType", l} } rr.MatchingType = uint8(i) // So this needs be e2 (i.e. different than e), because...??t - s, e2, c1 := endingToString(c, "bad SMIMEA Certificate", f) + s, e2 := endingToString(c, "bad SMIMEA Certificate", f) if e2 != nil { - return nil, e2, c1 + return e2 } rr.Certificate = s - return rr, nil, c1 + return nil } -func setRFC3597(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(RFC3597) - rr.Hdr = h - - l := <-c +func (rr *RFC3597) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() if l.token != "\\#" { - return nil, &ParseError{f, "bad RFC3597 Rdata", l}, "" + return &ParseError{f, "bad RFC3597 Rdata", l} } - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() rdlength, e := strconv.Atoi(l.token) if e != nil || l.err { - return nil, &ParseError{f, "bad RFC3597 Rdata ", l}, "" + return &ParseError{f, "bad RFC3597 Rdata ", l} } - s, e1, c1 := endingToString(c, "bad RFC3597 Rdata", f) + s, e1 := endingToString(c, "bad RFC3597 Rdata", f) if e1 != nil { - return nil, e1, c1 + return e1 } if rdlength*2 != len(s) { - return nil, &ParseError{f, "bad RFC3597 Rdata", l}, "" + return &ParseError{f, "bad RFC3597 Rdata", l} } rr.Rdata = s - return rr, nil, c1 + return nil } -func setSPF(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(SPF) - rr.Hdr = h - - s, e, c1 := endingToTxtSlice(c, "bad SPF Txt", f) +func (rr *SPF) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToTxtSlice(c, "bad SPF Txt", f) if e != nil { - return nil, e, "" + return e } rr.Txt = s - return rr, nil, c1 + return nil } -func setAVC(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(AVC) - rr.Hdr = h - - s, e, c1 := endingToTxtSlice(c, "bad AVC Txt", f) +func (rr *AVC) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToTxtSlice(c, "bad AVC Txt", f) if e != nil { - return nil, e, "" + return e } rr.Txt = s - return rr, nil, c1 + return nil } -func setTXT(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(TXT) - rr.Hdr = h - +func (rr *TXT) parse(c *zlexer, o, f string) *ParseError { // no zBlank reading here, because all this rdata is TXT - s, e, c1 := endingToTxtSlice(c, "bad TXT Txt", f) + s, e := endingToTxtSlice(c, "bad TXT Txt", f) if e != nil { - return nil, e, "" + return e } rr.Txt = s - return rr, nil, c1 + return nil } // identical to setTXT -func setNINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NINFO) - rr.Hdr = h - - s, e, c1 := endingToTxtSlice(c, "bad NINFO ZSData", f) +func (rr *NINFO) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToTxtSlice(c, "bad NINFO ZSData", f) if e != nil { - return nil, e, "" + return e } rr.ZSData = s - return rr, nil, c1 + return nil } -func setURI(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(URI) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *URI) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad URI Priority", l}, "" + return &ParseError{f, "bad URI Priority", l} } rr.Priority = uint16(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() i, e = strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad URI Weight", l}, "" + return &ParseError{f, "bad URI Weight", l} } rr.Weight = uint16(i) - <-c // zBlank - s, err, c1 := endingToTxtSlice(c, "bad URI Target", f) + c.Next() // zBlank + s, err := endingToTxtSlice(c, "bad URI Target", f) if err != nil { - return nil, err, "" + return err } if len(s) != 1 { - return nil, &ParseError{f, "bad URI Target", l}, "" + return &ParseError{f, "bad URI Target", l} } rr.Target = s[0] - return rr, nil, c1 + return nil } -func setDHCID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { +func (rr *DHCID) parse(c *zlexer, o, f string) *ParseError { // awesome record to parse! - rr := new(DHCID) - rr.Hdr = h - - s, e, c1 := endingToString(c, "bad DHCID Digest", f) + s, e := endingToString(c, "bad DHCID Digest", f) if e != nil { - return nil, e, c1 + return e } rr.Digest = s - return rr, nil, c1 + return nil } -func setNID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(NID) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *NID) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad NID Preference", l}, "" + return &ParseError{f, "bad NID Preference", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString u, err := stringToNodeID(l) if err != nil || l.err { - return nil, err, "" + return err } rr.NodeID = u - return rr, nil, "" + return slurpRemainder(c, f) } -func setL32(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(L32) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *L32) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad L32 Preference", l}, "" + return &ParseError{f, "bad L32 Preference", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Locator32 = net.ParseIP(l.token) if rr.Locator32 == nil || l.err { - return nil, &ParseError{f, "bad L32 Locator", l}, "" + return &ParseError{f, "bad L32 Locator", l} } - return rr, nil, "" + return slurpRemainder(c, f) } -func setLP(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(LP) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *LP) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad LP Preference", l}, "" + return &ParseError{f, "bad LP Preference", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Fqdn = l.token name, nameOk := toAbsoluteName(l.token, o) if l.err || !nameOk { - return nil, &ParseError{f, "bad LP Fqdn", l}, "" + return &ParseError{f, "bad LP Fqdn", l} } rr.Fqdn = name - return rr, nil, "" + return slurpRemainder(c, f) } -func setL64(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(L64) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *L64) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad L64 Preference", l}, "" + return &ParseError{f, "bad L64 Preference", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString u, err := stringToNodeID(l) if err != nil || l.err { - return nil, err, "" + return err } rr.Locator64 = u - return rr, nil, "" + return slurpRemainder(c, f) } -func setUID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(UID) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *UID) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 32) if e != nil || l.err { - return nil, &ParseError{f, "bad UID Uid", l}, "" + return &ParseError{f, "bad UID Uid", l} } rr.Uid = uint32(i) - return rr, nil, "" + return slurpRemainder(c, f) } -func setGID(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(GID) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *GID) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 32) if e != nil || l.err { - return nil, &ParseError{f, "bad GID Gid", l}, "" + return &ParseError{f, "bad GID Gid", l} } rr.Gid = uint32(i) - return rr, nil, "" + return slurpRemainder(c, f) } -func setUINFO(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(UINFO) - rr.Hdr = h - - s, e, c1 := endingToTxtSlice(c, "bad UINFO Uinfo", f) +func (rr *UINFO) parse(c *zlexer, o, f string) *ParseError { + s, e := endingToTxtSlice(c, "bad UINFO Uinfo", f) if e != nil { - return nil, e, c1 + return e } if ln := len(s); ln == 0 { - return rr, nil, c1 + return nil } rr.Uinfo = s[0] // silently discard anything after the first character-string - return rr, nil, c1 + return nil } -func setPX(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(PX) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, "" +func (rr *PX) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return slurpRemainder(c, f) } i, e := strconv.ParseUint(l.token, 10, 16) if e != nil || l.err { - return nil, &ParseError{f, "bad PX Preference", l}, "" + return &ParseError{f, "bad PX Preference", l} } rr.Preference = uint16(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Map822 = l.token map822, map822Ok := toAbsoluteName(l.token, o) if l.err || !map822Ok { - return nil, &ParseError{f, "bad PX Map822", l}, "" + return &ParseError{f, "bad PX Map822", l} } rr.Map822 = map822 - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString rr.Mapx400 = l.token mapx400, mapx400Ok := toAbsoluteName(l.token, o) if l.err || !mapx400Ok { - return nil, &ParseError{f, "bad PX Mapx400", l}, "" + return &ParseError{f, "bad PX Mapx400", l} } rr.Mapx400 = mapx400 - return rr, nil, "" + return slurpRemainder(c, f) } -func setCAA(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(CAA) - rr.Hdr = h - - l := <-c - if l.length == 0 { // dynamic update rr. - return rr, nil, l.comment +func (rr *CAA) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() + if len(l.token) == 0 { // dynamic update rr. + return nil } i, err := strconv.ParseUint(l.token, 10, 8) if err != nil || l.err { - return nil, &ParseError{f, "bad CAA Flag", l}, "" + return &ParseError{f, "bad CAA Flag", l} } rr.Flag = uint8(i) - <-c // zBlank - l = <-c // zString + c.Next() // zBlank + l, _ = c.Next() // zString if l.value != zString { - return nil, &ParseError{f, "bad CAA Tag", l}, "" + return &ParseError{f, "bad CAA Tag", l} } rr.Tag = l.token - <-c // zBlank - s, e, c1 := endingToTxtSlice(c, "bad CAA Value", f) + c.Next() // zBlank + s, e := endingToTxtSlice(c, "bad CAA Value", f) if e != nil { - return nil, e, "" + return e } if len(s) != 1 { - return nil, &ParseError{f, "bad CAA Value", l}, "" + return &ParseError{f, "bad CAA Value", l} } rr.Value = s[0] - return rr, nil, c1 + return nil } -func setTKEY(h RR_Header, c chan lex, o, f string) (RR, *ParseError, string) { - rr := new(TKEY) - rr.Hdr = h - - l := <-c +func (rr *TKEY) parse(c *zlexer, o, f string) *ParseError { + l, _ := c.Next() // Algorithm if l.value != zString { - return nil, &ParseError{f, "bad TKEY algorithm", l}, "" + return &ParseError{f, "bad TKEY algorithm", l} } rr.Algorithm = l.token - <-c // zBlank + c.Next() // zBlank // Get the key length and key values - l = <-c + l, _ = c.Next() i, err := strconv.ParseUint(l.token, 10, 8) if err != nil || l.err { - return nil, &ParseError{f, "bad TKEY key length", l}, "" + return &ParseError{f, "bad TKEY key length", l} } rr.KeySize = uint16(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if l.value != zString { - return nil, &ParseError{f, "bad TKEY key", l}, "" + return &ParseError{f, "bad TKEY key", l} } rr.Key = l.token - <-c // zBlank + c.Next() // zBlank // Get the otherdata length and string data - l = <-c + l, _ = c.Next() i, err = strconv.ParseUint(l.token, 10, 8) if err != nil || l.err { - return nil, &ParseError{f, "bad TKEY otherdata length", l}, "" + return &ParseError{f, "bad TKEY otherdata length", l} } rr.OtherLen = uint16(i) - <-c // zBlank - l = <-c + c.Next() // zBlank + l, _ = c.Next() if l.value != zString { - return nil, &ParseError{f, "bad TKEY otherday", l}, "" + return &ParseError{f, "bad TKEY otherday", l} } rr.OtherData = l.token - return rr, nil, "" -} - -var typeToparserFunc = map[uint16]parserFunc{ - TypeAAAA: {setAAAA, false}, - TypeAFSDB: {setAFSDB, false}, - TypeA: {setA, false}, - TypeCAA: {setCAA, true}, - TypeCDS: {setCDS, true}, - TypeCDNSKEY: {setCDNSKEY, true}, - TypeCERT: {setCERT, true}, - TypeCNAME: {setCNAME, false}, - TypeCSYNC: {setCSYNC, true}, - TypeDHCID: {setDHCID, true}, - TypeDLV: {setDLV, true}, - TypeDNAME: {setDNAME, false}, - TypeKEY: {setKEY, true}, - TypeDNSKEY: {setDNSKEY, true}, - TypeDS: {setDS, true}, - TypeEID: {setEID, true}, - TypeEUI48: {setEUI48, false}, - TypeEUI64: {setEUI64, false}, - TypeGID: {setGID, false}, - TypeGPOS: {setGPOS, false}, - TypeHINFO: {setHINFO, true}, - TypeHIP: {setHIP, true}, - TypeKX: {setKX, false}, - TypeL32: {setL32, false}, - TypeL64: {setL64, false}, - TypeLOC: {setLOC, true}, - TypeLP: {setLP, false}, - TypeMB: {setMB, false}, - TypeMD: {setMD, false}, - TypeMF: {setMF, false}, - TypeMG: {setMG, false}, - TypeMINFO: {setMINFO, false}, - TypeMR: {setMR, false}, - TypeMX: {setMX, false}, - TypeNAPTR: {setNAPTR, false}, - TypeNID: {setNID, false}, - TypeNIMLOC: {setNIMLOC, true}, - TypeNINFO: {setNINFO, true}, - TypeNSAPPTR: {setNSAPPTR, false}, - TypeNSEC3PARAM: {setNSEC3PARAM, false}, - TypeNSEC3: {setNSEC3, true}, - TypeNSEC: {setNSEC, true}, - TypeNS: {setNS, false}, - TypeOPENPGPKEY: {setOPENPGPKEY, true}, - TypePTR: {setPTR, false}, - TypePX: {setPX, false}, - TypeSIG: {setSIG, true}, - TypeRKEY: {setRKEY, true}, - TypeRP: {setRP, false}, - TypeRRSIG: {setRRSIG, true}, - TypeRT: {setRT, false}, - TypeSMIMEA: {setSMIMEA, true}, - TypeSOA: {setSOA, false}, - TypeSPF: {setSPF, true}, - TypeAVC: {setAVC, true}, - TypeSRV: {setSRV, false}, - TypeSSHFP: {setSSHFP, true}, - TypeTALINK: {setTALINK, false}, - TypeTA: {setTA, true}, - TypeTLSA: {setTLSA, true}, - TypeTXT: {setTXT, true}, - TypeUID: {setUID, false}, - TypeUINFO: {setUINFO, true}, - TypeURI: {setURI, true}, - TypeX25: {setX25, false}, - TypeTKEY: {setTKEY, true}, + return nil } diff --git a/vendor/github.com/miekg/dns/scan_test.go b/vendor/github.com/miekg/dns/scan_test.go index e43ad44..a939d21 100644 --- a/vendor/github.com/miekg/dns/scan_test.go +++ b/vendor/github.com/miekg/dns/scan_test.go @@ -1,18 +1,57 @@ package dns import ( + "io" "io/ioutil" + "net" "os" "strings" "testing" ) +func TestParseZoneGenerate(t *testing.T) { + zone := "$ORIGIN example.org.\n$GENERATE 10-12 foo${2,3,d} IN A 127.0.0.$" + + wantRRs := []RR{ + &A{Hdr: RR_Header{Name: "foo012.example.org."}, A: net.ParseIP("127.0.0.10")}, + &A{Hdr: RR_Header{Name: "foo013.example.org."}, A: net.ParseIP("127.0.0.11")}, + &A{Hdr: RR_Header{Name: "foo014.example.org."}, A: net.ParseIP("127.0.0.12")}, + } + wantIdx := 0 + + tok := ParseZone(strings.NewReader(zone), "", "") + for x := range tok { + if wantIdx >= len(wantRRs) { + t.Fatalf("expected %d RRs, but got more", len(wantRRs)) + } + if x.Error != nil { + t.Fatalf("expected no error, but got %s", x.Error) + } + if got, want := x.RR.Header().Name, wantRRs[wantIdx].Header().Name; got != want { + t.Fatalf("expected name %s, but got %s", want, got) + } + a, ok := x.RR.(*A) + if !ok { + t.Fatalf("expected *A RR, but got %T", x.RR) + } + if got, want := a.A, wantRRs[wantIdx].(*A).A; !got.Equal(want) { + t.Fatalf("expected A with IP %v, but got %v", got, want) + } + wantIdx++ + } + + if wantIdx != len(wantRRs) { + t.Errorf("too few records, expected %d, got %d", len(wantRRs), wantIdx) + } +} + func TestParseZoneInclude(t *testing.T) { tmpfile, err := ioutil.TempFile("", "dns") if err != nil { t.Fatalf("could not create tmpfile for test: %s", err) } + defer os.Remove(tmpfile.Name()) if _, err := tmpfile.WriteString("foo\tIN\tA\t127.0.0.1"); err != nil { t.Fatalf("unable to write content to tmpfile %q: %s", tmpfile.Name(), err) @@ -21,16 +60,24 @@ func TestParseZoneInclude(t *testing.T) { t.Fatalf("could not close tmpfile %q: %s", tmpfile.Name(), err) } - zone := "$ORIGIN example.org.\n$INCLUDE " + tmpfile.Name() + zone := "$ORIGIN example.org.\n$INCLUDE " + tmpfile.Name() + "\nbar\tIN\tA\t127.0.0.2" + var got int tok := ParseZone(strings.NewReader(zone), "", "") for x := range tok { if x.Error != nil { t.Fatalf("expected no error, but got %s", x.Error) } - if x.RR.Header().Name != "foo.example.org." { - t.Fatalf("expected %s, but got %s", "foo.example.org.", x.RR.Header().Name) + switch x.RR.Header().Name { + case "foo.example.org.", "bar.example.org.": + default: + t.Fatalf("expected foo.example.org. or bar.example.org., but got %s", x.RR.Header().Name) } + got++ + } + + if expected := 2; got != expected { + t.Errorf("failed to parse zone after include, expected %d records, got %d", expected, got) } os.Remove(tmpfile.Name()) @@ -41,8 +88,131 @@ func TestParseZoneInclude(t *testing.T) { t.Fatalf("expected first token to contain an error but it didn't") } if !strings.Contains(x.Error.Error(), "failed to open") || - !strings.Contains(x.Error.Error(), tmpfile.Name()) { - t.Fatalf(`expected error to contain: "failed to open" and %q but got: %s`, tmpfile.Name(), x.Error) + !strings.Contains(x.Error.Error(), tmpfile.Name()) || + !strings.Contains(x.Error.Error(), "no such file or directory") { + t.Fatalf(`expected error to contain: "failed to open", %q and "no such file or directory" but got: %s`, + tmpfile.Name(), x.Error) + } + } +} + +func TestZoneParserIncludeDisallowed(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dns") + if err != nil { + t.Fatalf("could not create tmpfile for test: %s", err) + } + defer os.Remove(tmpfile.Name()) + + if _, err := tmpfile.WriteString("foo\tIN\tA\t127.0.0.1"); err != nil { + t.Fatalf("unable to write content to tmpfile %q: %s", tmpfile.Name(), err) + } + if err := tmpfile.Close(); err != nil { + t.Fatalf("could not close tmpfile %q: %s", tmpfile.Name(), err) + } + + zp := NewZoneParser(strings.NewReader("$INCLUDE "+tmpfile.Name()), "example.org.", "") + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + const expect = "$INCLUDE directive not allowed" + if err := zp.Err(); err == nil || !strings.Contains(err.Error(), expect) { + t.Errorf("expected error to contain %q, got %v", expect, err) + } +} + +func TestParseTA(t *testing.T) { + rr, err := NewRR(` Ta 0 0 0`) + if err != nil { + t.Fatalf("expected no error, but got %s", err) + } + if rr == nil { + t.Fatal(`expected a normal RR, but got nil`) + } +} + +var errTestReadError = &Error{"test error"} + +type errReader struct{} + +func (errReader) Read(p []byte) (int, error) { return 0, errTestReadError } + +func TestParseZoneReadError(t *testing.T) { + rr, err := ReadRR(errReader{}, "") + if err == nil || !strings.Contains(err.Error(), errTestReadError.Error()) { + t.Errorf("expected error to contain %q, but got %v", errTestReadError, err) + } + if rr != nil { + t.Errorf("expected a nil RR, but got %v", rr) + } +} + +func BenchmarkNewRR(b *testing.B) { + const name1 = "12345678901234567890123456789012345.12345678.123." + const s = name1 + " 3600 IN MX 10 " + name1 + + for n := 0; n < b.N; n++ { + _, err := NewRR(s) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkReadRR(b *testing.B) { + const name1 = "12345678901234567890123456789012345.12345678.123." + const s = name1 + " 3600 IN MX 10 " + name1 + "\n" + + for n := 0; n < b.N; n++ { + r := struct{ io.Reader }{strings.NewReader(s)} + // r is now only an io.Reader and won't benefit from the + // io.ByteReader special-case in zlexer.Next. + + _, err := ReadRR(r, "") + if err != nil { + b.Fatal(err) + } + } +} + +const benchZone = ` +foo. IN A 10.0.0.1 ; this is comment 1 +foo. IN A ( + 10.0.0.2 ; this is comment 2 +) +; this is comment 3 +foo. IN A 10.0.0.3 +foo. IN A ( 10.0.0.4 ); this is comment 4 + +foo. IN A 10.0.0.5 +; this is comment 5 + +foo. IN A 10.0.0.6 + +foo. IN DNSKEY 256 3 5 AwEAAb+8l ; this is comment 6 +foo. IN NSEC miek.nl. TXT RRSIG NSEC; this is comment 7 +foo. IN TXT "THIS IS TEXT MAN"; this is comment 8 +` + +func BenchmarkParseZone(b *testing.B) { + for n := 0; n < b.N; n++ { + for tok := range ParseZone(strings.NewReader(benchZone), "example.org.", "") { + if tok.Error != nil { + b.Fatal(tok.Error) + } + } + } +} + +func BenchmarkZoneParser(b *testing.B) { + for n := 0; n < b.N; n++ { + zp := NewZoneParser(strings.NewReader(benchZone), "example.org.", "") + + for _, ok := zp.Next(); ok; _, ok = zp.Next() { + } + + if err := zp.Err(); err != nil { + b.Fatal(err) } } } diff --git a/vendor/github.com/miekg/dns/scanner.go b/vendor/github.com/miekg/dns/scanner.go deleted file mode 100644 index 424e5af..0000000 --- a/vendor/github.com/miekg/dns/scanner.go +++ /dev/null @@ -1,56 +0,0 @@ -package dns - -// Implement a simple scanner, return a byte stream from an io reader. - -import ( - "bufio" - "context" - "io" - "text/scanner" -) - -type scan struct { - src *bufio.Reader - position scanner.Position - eof bool // Have we just seen a eof - ctx context.Context -} - -func scanInit(r io.Reader) (*scan, context.CancelFunc) { - s := new(scan) - s.src = bufio.NewReader(r) - s.position.Line = 1 - - ctx, cancel := context.WithCancel(context.Background()) - s.ctx = ctx - - return s, cancel -} - -// tokenText returns the next byte from the input -func (s *scan) tokenText() (byte, error) { - c, err := s.src.ReadByte() - if err != nil { - return c, err - } - select { - case <-s.ctx.Done(): - return c, context.Canceled - default: - break - } - - // delay the newline handling until the next token is delivered, - // fixes off-by-one errors when reporting a parse error. - if s.eof == true { - s.position.Line++ - s.position.Column = 0 - s.eof = false - } - if c == '\n' { - s.eof = true - return c, nil - } - s.position.Column++ - return c, nil -} diff --git a/vendor/github.com/miekg/dns/serve_mux.go b/vendor/github.com/miekg/dns/serve_mux.go new file mode 100644 index 0000000..ae304db --- /dev/null +++ b/vendor/github.com/miekg/dns/serve_mux.go @@ -0,0 +1,147 @@ +package dns + +import ( + "strings" + "sync" +) + +// ServeMux is an DNS request multiplexer. It matches the zone name of +// each incoming request against a list of registered patterns add calls +// the handler for the pattern that most closely matches the zone name. +// +// ServeMux is DNSSEC aware, meaning that queries for the DS record are +// redirected to the parent zone (if that is also registered), otherwise +// the child gets the query. +// +// ServeMux is also safe for concurrent access from multiple goroutines. +// +// The zero ServeMux is empty and ready for use. +type ServeMux struct { + z map[string]Handler + m sync.RWMutex +} + +// NewServeMux allocates and returns a new ServeMux. +func NewServeMux() *ServeMux { + return new(ServeMux) +} + +// DefaultServeMux is the default ServeMux used by Serve. +var DefaultServeMux = NewServeMux() + +func (mux *ServeMux) match(q string, t uint16) Handler { + mux.m.RLock() + defer mux.m.RUnlock() + if mux.z == nil { + return nil + } + + var handler Handler + + // TODO(tmthrgd): Once https://go-review.googlesource.com/c/go/+/137575 + // lands in a go release, replace the following with strings.ToLower. + var sb strings.Builder + for i := 0; i < len(q); i++ { + c := q[i] + if !(c >= 'A' && c <= 'Z') { + continue + } + + sb.Grow(len(q)) + sb.WriteString(q[:i]) + + for ; i < len(q); i++ { + c := q[i] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + + sb.WriteByte(c) + } + + q = sb.String() + break + } + + for off, end := 0, false; !end; off, end = NextLabel(q, off) { + if h, ok := mux.z[q[off:]]; ok { + if t != TypeDS { + return h + } + // Continue for DS to see if we have a parent too, if so delegate to the parent + handler = h + } + } + + // Wildcard match, if we have found nothing try the root zone as a last resort. + if h, ok := mux.z["."]; ok { + return h + } + + return handler +} + +// Handle adds a handler to the ServeMux for pattern. +func (mux *ServeMux) Handle(pattern string, handler Handler) { + if pattern == "" { + panic("dns: invalid pattern " + pattern) + } + mux.m.Lock() + if mux.z == nil { + mux.z = make(map[string]Handler) + } + mux.z[Fqdn(pattern)] = handler + mux.m.Unlock() +} + +// HandleFunc adds a handler function to the ServeMux for pattern. +func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Msg)) { + mux.Handle(pattern, HandlerFunc(handler)) +} + +// HandleRemove deregisters the handler specific for pattern from the ServeMux. +func (mux *ServeMux) HandleRemove(pattern string) { + if pattern == "" { + panic("dns: invalid pattern " + pattern) + } + mux.m.Lock() + delete(mux.z, Fqdn(pattern)) + mux.m.Unlock() +} + +// ServeDNS dispatches the request to the handler whose pattern most +// closely matches the request message. +// +// ServeDNS is DNSSEC aware, meaning that queries for the DS record +// are redirected to the parent zone (if that is also registered), +// otherwise the child gets the query. +// +// If no handler is found, or there is no question, a standard SERVFAIL +// message is returned +func (mux *ServeMux) ServeDNS(w ResponseWriter, req *Msg) { + var h Handler + if len(req.Question) >= 1 { // allow more than one question + h = mux.match(req.Question[0].Name, req.Question[0].Qtype) + } + + if h != nil { + h.ServeDNS(w, req) + } else { + HandleFailed(w, req) + } +} + +// Handle registers the handler with the given pattern +// in the DefaultServeMux. The documentation for +// ServeMux explains how patterns are matched. +func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } + +// HandleRemove deregisters the handle with the given pattern +// in the DefaultServeMux. +func HandleRemove(pattern string) { DefaultServeMux.HandleRemove(pattern) } + +// HandleFunc registers the handler function with the given pattern +// in the DefaultServeMux. +func HandleFunc(pattern string, handler func(ResponseWriter, *Msg)) { + DefaultServeMux.HandleFunc(pattern, handler) +} diff --git a/vendor/github.com/miekg/dns/serve_mux_test.go b/vendor/github.com/miekg/dns/serve_mux_test.go new file mode 100644 index 0000000..3d990ce --- /dev/null +++ b/vendor/github.com/miekg/dns/serve_mux_test.go @@ -0,0 +1,72 @@ +package dns + +import "testing" + +func TestDotAsCatchAllWildcard(t *testing.T) { + mux := NewServeMux() + mux.Handle(".", HandlerFunc(HelloServer)) + mux.Handle("example.com.", HandlerFunc(AnotherHelloServer)) + + handler := mux.match("www.miek.nl.", TypeTXT) + if handler == nil { + t.Error("wildcard match failed") + } + + handler = mux.match("www.example.com.", TypeTXT) + if handler == nil { + t.Error("example.com match failed") + } + + handler = mux.match("a.www.example.com.", TypeTXT) + if handler == nil { + t.Error("a.www.example.com match failed") + } + + handler = mux.match("boe.", TypeTXT) + if handler == nil { + t.Error("boe. match failed") + } +} + +func TestCaseFolding(t *testing.T) { + mux := NewServeMux() + mux.Handle("_udp.example.com.", HandlerFunc(HelloServer)) + + handler := mux.match("_dns._udp.example.com.", TypeSRV) + if handler == nil { + t.Error("case sensitive characters folded") + } + + handler = mux.match("_DNS._UDP.EXAMPLE.COM.", TypeSRV) + if handler == nil { + t.Error("case insensitive characters not folded") + } +} + +func TestRootServer(t *testing.T) { + mux := NewServeMux() + mux.Handle(".", HandlerFunc(HelloServer)) + + handler := mux.match(".", TypeNS) + if handler == nil { + t.Error("root match failed") + } +} + +func BenchmarkMuxMatch(b *testing.B) { + mux := NewServeMux() + mux.Handle("_udp.example.com.", HandlerFunc(HelloServer)) + + bench := func(q string) func(*testing.B) { + return func(b *testing.B) { + for n := 0; n < b.N; n++ { + handler := mux.match(q, TypeSRV) + if handler == nil { + b.Fatal("couldn't find match") + } + } + } + } + b.Run("lowercase", bench("_dns._udp.example.com.")) + b.Run("uppercase", bench("_DNS._UDP.EXAMPLE.COM.")) +} diff --git a/vendor/github.com/miekg/dns/server.go b/vendor/github.com/miekg/dns/server.go index 2d98f14..8824037 100644 --- a/vendor/github.com/miekg/dns/server.go +++ b/vendor/github.com/miekg/dns/server.go @@ -4,10 +4,13 @@ package dns import ( "bytes" + "context" "crypto/tls" "encoding/binary" + "errors" "io" "net" + "strings" "sync" "sync/atomic" "time" @@ -16,17 +19,39 @@ import ( // Default maximum number of TCP queries before we close the socket. const maxTCPQueries = 128 -// Interval for stop worker if no load +// The maximum number of idle workers. +// +// This controls the maximum number of workers that are allowed to stay +// idle waiting for incoming requests before being torn down. +// +// If this limit is reached, the server will just keep spawning new +// workers (goroutines) for each incoming request. In this case, each +// worker will only be used for a single request. +const maxIdleWorkersCount = 10000 + +// The maximum length of time a worker may idle for before being destroyed. const idleWorkerTimeout = 10 * time.Second -// Maximum number of workers -const maxWorkersCount = 10000 +// aLongTimeAgo is a non-zero time, far in the past, used for +// immediate cancelation of network operations. +var aLongTimeAgo = time.Unix(1, 0) // Handler is implemented by any value that implements ServeDNS. type Handler interface { ServeDNS(w ResponseWriter, r *Msg) } +// The HandlerFunc type is an adapter to allow the use of +// ordinary functions as DNS handlers. If f is a function +// with the appropriate signature, HandlerFunc(f) is a +// Handler object that calls f. +type HandlerFunc func(ResponseWriter, *Msg) + +// ServeDNS calls f(w, r). +func (f HandlerFunc) ServeDNS(w ResponseWriter, r *Msg) { + f(w, r) +} + // A ResponseWriter interface is used by an DNS handler to // construct an DNS response. type ResponseWriter interface { @@ -49,46 +74,25 @@ type ResponseWriter interface { Hijack() } +// A ConnectionStater interface is used by a DNS Handler to access TLS connection state +// when available. +type ConnectionStater interface { + ConnectionState() *tls.ConnectionState +} + type response struct { msg []byte + closed bool // connection has been closed hijacked bool // connection has been hijacked by handler - tsigStatus error tsigTimersOnly bool + tsigStatus error tsigRequestMAC string tsigSecret map[string]string // the tsig secrets udp *net.UDPConn // i/o connection if UDP was used tcp net.Conn // i/o connection if TCP was used udpSession *SessionUDP // oob data to get egress interface right writer Writer // writer to output the raw DNS bits -} - -// ServeMux is an DNS request multiplexer. It matches the -// zone name of each incoming request against a list of -// registered patterns add calls the handler for the pattern -// that most closely matches the zone name. ServeMux is DNSSEC aware, meaning -// that queries for the DS record are redirected to the parent zone (if that -// is also registered), otherwise the child gets the query. -// ServeMux is also safe for concurrent access from multiple goroutines. -type ServeMux struct { - z map[string]Handler - m *sync.RWMutex -} - -// NewServeMux allocates and returns a new ServeMux. -func NewServeMux() *ServeMux { return &ServeMux{z: make(map[string]Handler), m: new(sync.RWMutex)} } - -// DefaultServeMux is the default ServeMux used by Serve. -var DefaultServeMux = NewServeMux() - -// The HandlerFunc type is an adapter to allow the use of -// ordinary functions as DNS handlers. If f is a function -// with the appropriate signature, HandlerFunc(f) is a -// Handler object that calls f. -type HandlerFunc func(ResponseWriter, *Msg) - -// ServeDNS calls f(w, r). -func (f HandlerFunc) ServeDNS(w ResponseWriter, r *Msg) { - f(w, r) + wg *sync.WaitGroup // for gracefull shutdown } // HandleFailed returns a HandlerFunc that returns SERVFAIL for every request it gets. @@ -99,8 +103,6 @@ func HandleFailed(w ResponseWriter, r *Msg) { w.WriteMsg(m) } -func failedHandler() Handler { return HandlerFunc(HandleFailed) } - // ListenAndServe Starts a server on address and network specified Invoke handler // for incoming queries. func ListenAndServe(addr string, network string, handler Handler) error { @@ -139,99 +141,6 @@ func ActivateAndServe(l net.Listener, p net.PacketConn, handler Handler) error { return server.ActivateAndServe() } -func (mux *ServeMux) match(q string, t uint16) Handler { - mux.m.RLock() - defer mux.m.RUnlock() - var handler Handler - b := make([]byte, len(q)) // worst case, one label of length q - off := 0 - end := false - for { - l := len(q[off:]) - for i := 0; i < l; i++ { - b[i] = q[off+i] - if b[i] >= 'A' && b[i] <= 'Z' { - b[i] |= ('a' - 'A') - } - } - if h, ok := mux.z[string(b[:l])]; ok { // causes garbage, might want to change the map key - if t != TypeDS { - return h - } - // Continue for DS to see if we have a parent too, if so delegeate to the parent - handler = h - } - off, end = NextLabel(q, off) - if end { - break - } - } - // Wildcard match, if we have found nothing try the root zone as a last resort. - if h, ok := mux.z["."]; ok { - return h - } - return handler -} - -// Handle adds a handler to the ServeMux for pattern. -func (mux *ServeMux) Handle(pattern string, handler Handler) { - if pattern == "" { - panic("dns: invalid pattern " + pattern) - } - mux.m.Lock() - mux.z[Fqdn(pattern)] = handler - mux.m.Unlock() -} - -// HandleFunc adds a handler function to the ServeMux for pattern. -func (mux *ServeMux) HandleFunc(pattern string, handler func(ResponseWriter, *Msg)) { - mux.Handle(pattern, HandlerFunc(handler)) -} - -// HandleRemove deregistrars the handler specific for pattern from the ServeMux. -func (mux *ServeMux) HandleRemove(pattern string) { - if pattern == "" { - panic("dns: invalid pattern " + pattern) - } - mux.m.Lock() - delete(mux.z, Fqdn(pattern)) - mux.m.Unlock() -} - -// ServeDNS dispatches the request to the handler whose -// pattern most closely matches the request message. If DefaultServeMux -// is used the correct thing for DS queries is done: a possible parent -// is sought. -// If no handler is found a standard SERVFAIL message is returned -// If the request message does not have exactly one question in the -// question section a SERVFAIL is returned, unlesss Unsafe is true. -func (mux *ServeMux) ServeDNS(w ResponseWriter, request *Msg) { - var h Handler - if len(request.Question) < 1 { // allow more than one question - h = failedHandler() - } else { - if h = mux.match(request.Question[0].Name, request.Question[0].Qtype); h == nil { - h = failedHandler() - } - } - h.ServeDNS(w, request) -} - -// Handle registers the handler with the given pattern -// in the DefaultServeMux. The documentation for -// ServeMux explains how patterns are matched. -func Handle(pattern string, handler Handler) { DefaultServeMux.Handle(pattern, handler) } - -// HandleRemove deregisters the handle with the given pattern -// in the DefaultServeMux. -func HandleRemove(pattern string) { DefaultServeMux.HandleRemove(pattern) } - -// HandleFunc registers the handler function with the given pattern -// in the DefaultServeMux. -func HandleFunc(pattern string, handler func(ResponseWriter, *Msg)) { - DefaultServeMux.HandleFunc(pattern, handler) -} - // Writer writes raw DNS messages; each call to Write should send an entire message. type Writer interface { io.Writer @@ -253,11 +162,11 @@ type defaultReader struct { *Server } -func (dr *defaultReader) ReadTCP(conn net.Conn, timeout time.Duration) ([]byte, error) { +func (dr defaultReader) ReadTCP(conn net.Conn, timeout time.Duration) ([]byte, error) { return dr.readTCP(conn, timeout) } -func (dr *defaultReader) ReadUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *SessionUDP, error) { +func (dr defaultReader) ReadUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *SessionUDP, error) { return dr.readUDP(conn, timeout) } @@ -294,9 +203,6 @@ type Server struct { IdleTimeout func() time.Duration // Secret(s) for Tsig map[]. The zonename must be in canonical form (lowercase, fqdn, see RFC 4034 Section 6.2). TsigSecret map[string]string - // Unsafe instructs the server to disregard any sanity checks and directly hand the message to - // the handler. It will specifically not check if the query has the QR bit not set. - Unsafe bool // If NotifyStartedFunc is set it is called once the server has started listening. NotifyStartedFunc func() // DecorateReader is optional, allows customization of the process that reads raw DNS messages. @@ -305,14 +211,33 @@ type Server struct { DecorateWriter DecorateWriter // Maximum number of TCP queries before we close the socket. Default is maxTCPQueries (unlimited if -1). MaxTCPQueries int + // Whether to set the SO_REUSEPORT socket option, allowing multiple listeners to be bound to a single address. + // It is only supported on go1.11+ and when using ListenAndServe. + ReusePort bool + // AcceptMsgFunc will check the incoming message and will reject it early in the process. + // By default DefaultMsgAcceptFunc will be used. + MsgAcceptFunc MsgAcceptFunc // UDP packet or TCP connection queue queue chan *response // Workers count workersCount int32 + // Shutdown handling - lock sync.RWMutex - started bool + lock sync.RWMutex + started bool + shutdown chan struct{} + conns map[net.Conn]struct{} + + // A pool for UDP message buffers. + udpPool sync.Pool +} + +func (srv *Server) isStarted() bool { + srv.lock.RLock() + started := srv.started + srv.lock.RUnlock() + return started } func (srv *Server) worker(w *response) { @@ -320,7 +245,7 @@ func (srv *Server) worker(w *response) { for { count := atomic.LoadInt32(&srv.workersCount) - if count > maxWorkersCount { + if count > maxIdleWorkersCount { return } if atomic.CompareAndSwapInt32(&srv.workersCount, count, count+1) { @@ -360,10 +285,39 @@ func (srv *Server) spawnWorker(w *response) { } } +func makeUDPBuffer(size int) func() interface{} { + return func() interface{} { + return make([]byte, size) + } +} + +func (srv *Server) init() { + srv.queue = make(chan *response) + + srv.shutdown = make(chan struct{}) + srv.conns = make(map[net.Conn]struct{}) + + if srv.UDPSize == 0 { + srv.UDPSize = MinMsgSize + } + if srv.MsgAcceptFunc == nil { + srv.MsgAcceptFunc = defaultMsgAcceptFunc + } + + srv.udpPool.New = makeUDPBuffer(srv.UDPSize) +} + +func unlockOnce(l sync.Locker) func() { + var once sync.Once + return func() { once.Do(l.Unlock) } +} + // ListenAndServe starts a nameserver on the configured address in *Server. func (srv *Server) ListenAndServe() error { + unlock := unlockOnce(&srv.lock) srv.lock.Lock() - defer srv.lock.Unlock() + defer unlock() + if srv.started { return &Error{err: "server already started"} } @@ -372,63 +326,47 @@ func (srv *Server) ListenAndServe() error { if addr == "" { addr = ":domain" } - if srv.UDPSize == 0 { - srv.UDPSize = MinMsgSize - } - srv.queue = make(chan *response) + + srv.init() defer close(srv.queue) + switch srv.Net { case "tcp", "tcp4", "tcp6": - a, err := net.ResolveTCPAddr(srv.Net, addr) - if err != nil { - return err - } - l, err := net.ListenTCP(srv.Net, a) + l, err := listenTCP(srv.Net, addr, srv.ReusePort) if err != nil { return err } srv.Listener = l srv.started = true - srv.lock.Unlock() - err = srv.serveTCP(l) - srv.lock.Lock() // to satisfy the defer at the top - return err + unlock() + return srv.serveTCP(l) case "tcp-tls", "tcp4-tls", "tcp6-tls": - network := "tcp" - if srv.Net == "tcp4-tls" { - network = "tcp4" - } else if srv.Net == "tcp6-tls" { - network = "tcp6" + if srv.TLSConfig == nil || (len(srv.TLSConfig.Certificates) == 0 && srv.TLSConfig.GetCertificate == nil) { + return errors.New("dns: neither Certificates nor GetCertificate set in Config") } - - l, err := tls.Listen(network, addr, srv.TLSConfig) + network := strings.TrimSuffix(srv.Net, "-tls") + l, err := listenTCP(network, addr, srv.ReusePort) if err != nil { return err } + l = tls.NewListener(l, srv.TLSConfig) srv.Listener = l srv.started = true - srv.lock.Unlock() - err = srv.serveTCP(l) - srv.lock.Lock() // to satisfy the defer at the top - return err + unlock() + return srv.serveTCP(l) case "udp", "udp4", "udp6": - a, err := net.ResolveUDPAddr(srv.Net, addr) + l, err := listenUDP(srv.Net, addr, srv.ReusePort) if err != nil { return err } - l, err := net.ListenUDP(srv.Net, a) - if err != nil { - return err - } - if e := setUDPSocketOptions(l); e != nil { + u := l.(*net.UDPConn) + if e := setUDPSocketOptions(u); e != nil { return e } srv.PacketConn = l srv.started = true - srv.lock.Unlock() - err = srv.serveUDP(l) - srv.lock.Lock() // to satisfy the defer at the top - return err + unlock() + return srv.serveUDP(u) } return &Error{err: "bad network"} } @@ -436,20 +374,20 @@ func (srv *Server) ListenAndServe() error { // ActivateAndServe starts a nameserver with the PacketConn or Listener // configured in *Server. Its main use is to start a server from systemd. func (srv *Server) ActivateAndServe() error { + unlock := unlockOnce(&srv.lock) srv.lock.Lock() - defer srv.lock.Unlock() + defer unlock() + if srv.started { return &Error{err: "server already started"} } + srv.init() + defer close(srv.queue) + pConn := srv.PacketConn l := srv.Listener - srv.queue = make(chan *response) - defer close(srv.queue) if pConn != nil { - if srv.UDPSize == 0 { - srv.UDPSize = MinMsgSize - } // Check PacketConn interface's type is valid and value // is not nil if t, ok := pConn.(*net.UDPConn); ok && t != nil { @@ -457,18 +395,14 @@ func (srv *Server) ActivateAndServe() error { return e } srv.started = true - srv.lock.Unlock() - e := srv.serveUDP(t) - srv.lock.Lock() // to satisfy the defer at the top - return e + unlock() + return srv.serveUDP(t) } } if l != nil { srv.started = true - srv.lock.Unlock() - e := srv.serveTCP(l) - srv.lock.Lock() // to satisfy the defer at the top - return e + unlock() + return srv.serveTCP(l) } return &Error{err: "bad listeners"} } @@ -476,30 +410,63 @@ func (srv *Server) ActivateAndServe() error { // Shutdown shuts down a server. After a call to Shutdown, ListenAndServe and // ActivateAndServe will return. func (srv *Server) Shutdown() error { + return srv.ShutdownContext(context.Background()) +} + +// ShutdownContext shuts down a server. After a call to ShutdownContext, +// ListenAndServe and ActivateAndServe will return. +// +// A context.Context may be passed to limit how long to wait for connections +// to terminate. +func (srv *Server) ShutdownContext(ctx context.Context) error { srv.lock.Lock() if !srv.started { srv.lock.Unlock() return &Error{err: "server not started"} } + srv.started = false + + if srv.PacketConn != nil { + srv.PacketConn.SetReadDeadline(aLongTimeAgo) // Unblock reads + } + + if srv.Listener != nil { + srv.Listener.Close() + } + + for rw := range srv.conns { + rw.SetReadDeadline(aLongTimeAgo) // Unblock reads + } + srv.lock.Unlock() + if testShutdownNotify != nil { + testShutdownNotify.Broadcast() + } + + var ctxErr error + select { + case <-srv.shutdown: + case <-ctx.Done(): + ctxErr = ctx.Err() + } + if srv.PacketConn != nil { srv.PacketConn.Close() } - if srv.Listener != nil { - srv.Listener.Close() - } - return nil + + return ctxErr } +var testShutdownNotify *sync.Cond + // getReadTimeout is a helper func to use system timeout if server did not intend to change it. func (srv *Server) getReadTimeout() time.Duration { - rtimeout := dnsTimeout if srv.ReadTimeout != 0 { - rtimeout = srv.ReadTimeout + return srv.ReadTimeout } - return rtimeout + return dnsTimeout } // serveTCP starts a TCP listener for the server. @@ -510,22 +477,36 @@ func (srv *Server) serveTCP(l net.Listener) error { srv.NotifyStartedFunc() } - for { + var wg sync.WaitGroup + defer func() { + wg.Wait() + close(srv.shutdown) + }() + + for srv.isStarted() { rw, err := l.Accept() - srv.lock.RLock() - if !srv.started { - srv.lock.RUnlock() - return nil - } - srv.lock.RUnlock() if err != nil { + if !srv.isStarted() { + return nil + } if neterr, ok := err.(net.Error); ok && neterr.Temporary() { continue } return err } - srv.spawnWorker(&response{tsigSecret: srv.TsigSecret, tcp: rw}) + srv.lock.Lock() + // Track the connection to allow unblocking reads on shutdown. + srv.conns[rw] = struct{}{} + srv.lock.Unlock() + wg.Add(1) + srv.spawnWorker(&response{ + tsigSecret: srv.TsigSecret, + tcp: rw, + wg: &wg, + }) } + + return nil } // serveUDP starts a UDP listener for the server. @@ -536,32 +517,47 @@ func (srv *Server) serveUDP(l *net.UDPConn) error { srv.NotifyStartedFunc() } - reader := Reader(&defaultReader{srv}) + reader := Reader(defaultReader{srv}) if srv.DecorateReader != nil { reader = srv.DecorateReader(reader) } + var wg sync.WaitGroup + defer func() { + wg.Wait() + close(srv.shutdown) + }() + rtimeout := srv.getReadTimeout() // deadline is not used here - for { + for srv.isStarted() { m, s, err := reader.ReadUDP(l, rtimeout) - srv.lock.RLock() - if !srv.started { - srv.lock.RUnlock() - return nil - } - srv.lock.RUnlock() if err != nil { + if !srv.isStarted() { + return nil + } if netErr, ok := err.(net.Error); ok && netErr.Temporary() { continue } return err } if len(m) < headerSize { + if cap(m) == srv.UDPSize { + srv.udpPool.Put(m[:srv.UDPSize]) + } continue } - srv.spawnWorker(&response{msg: m, tsigSecret: srv.TsigSecret, udp: l, udpSession: s}) + wg.Add(1) + srv.spawnWorker(&response{ + msg: m, + tsigSecret: srv.TsigSecret, + udp: l, + udpSession: s, + wg: &wg, + }) } + + return nil } func (srv *Server) serve(w *response) { @@ -574,20 +570,28 @@ func (srv *Server) serve(w *response) { if w.udp != nil { // serve UDP srv.serveDNS(w) - return - } - reader := Reader(&defaultReader{srv}) - if srv.DecorateReader != nil { - reader = srv.DecorateReader(reader) + w.wg.Done() + return } defer func() { if !w.hijacked { w.Close() } + + srv.lock.Lock() + delete(srv.conns, w.tcp) + srv.lock.Unlock() + + w.wg.Done() }() + reader := Reader(defaultReader{srv}) + if srv.DecorateReader != nil { + reader = srv.DecorateReader(reader) + } + idleTimeout := tcpIdleTimeout if srv.IdleTimeout != nil { idleTimeout = srv.IdleTimeout() @@ -600,7 +604,7 @@ func (srv *Server) serve(w *response) { limit = maxTCPQueries } - for q := 0; q < limit || limit == -1; q++ { + for q := 0; (q < limit || limit == -1) && srv.isStarted(); q++ { var err error w.msg, err = reader.ReadTCP(w.tcp, timeout) if err != nil { @@ -620,16 +624,43 @@ func (srv *Server) serve(w *response) { } } +func (srv *Server) disposeBuffer(w *response) { + if w.udp != nil && cap(w.msg) == srv.UDPSize { + srv.udpPool.Put(w.msg[:srv.UDPSize]) + } + w.msg = nil +} + func (srv *Server) serveDNS(w *response) { - req := new(Msg) - err := req.Unpack(w.msg) - if err != nil { // Send a FormatError back - x := new(Msg) - x.SetRcodeFormatError(req) - w.WriteMsg(x) + dh, off, err := unpackMsgHdr(w.msg, 0) + if err != nil { + // Let client hang, they are sending crap; any reply can be used to amplify. return } - if !srv.Unsafe && req.Response { + + req := new(Msg) + req.setHdr(dh) + + switch srv.MsgAcceptFunc(dh) { + case MsgAccept: + case MsgIgnore: + return + case MsgReject: + req.SetRcodeFormatError(req) + // Are we allowed to delete any OPT records here? + req.Ns, req.Answer, req.Extra = nil, nil, nil + + w.WriteMsg(req) + srv.disposeBuffer(w) + return + } + + if err := req.unpack(dh, w.msg, off); err != nil { + req.SetRcodeFormatError(req) + req.Ns, req.Answer, req.Extra = nil, nil, nil + + w.WriteMsg(req) + srv.disposeBuffer(w) return } @@ -646,6 +677,8 @@ func (srv *Server) serveDNS(w *response) { } } + srv.disposeBuffer(w) + handler := srv.Handler if handler == nil { handler = DefaultServeMux @@ -655,7 +688,16 @@ func (srv *Server) serveDNS(w *response) { } func (srv *Server) readTCP(conn net.Conn, timeout time.Duration) ([]byte, error) { - conn.SetReadDeadline(time.Now().Add(timeout)) + // If we race with ShutdownContext, the read deadline may + // have been set in the distant past to unblock the read + // below. We must not override it, otherwise we may block + // ShutdownContext. + srv.lock.RLock() + if srv.started { + conn.SetReadDeadline(time.Now().Add(timeout)) + } + srv.lock.RUnlock() + l := make([]byte, 2) n, err := conn.Read(l) if err != nil || n != 2 { @@ -690,10 +732,17 @@ func (srv *Server) readTCP(conn net.Conn, timeout time.Duration) ([]byte, error) } func (srv *Server) readUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *SessionUDP, error) { - conn.SetReadDeadline(time.Now().Add(timeout)) - m := make([]byte, srv.UDPSize) + srv.lock.RLock() + if srv.started { + // See the comment in readTCP above. + conn.SetReadDeadline(time.Now().Add(timeout)) + } + srv.lock.RUnlock() + + m := srv.udpPool.Get().([]byte) n, s, err := ReadFromSessionUDP(conn, m) if err != nil { + srv.udpPool.Put(m) return nil, nil, err } m = m[:n] @@ -702,6 +751,10 @@ func (srv *Server) readUDP(conn *net.UDPConn, timeout time.Duration) ([]byte, *S // WriteMsg implements the ResponseWriter.WriteMsg method. func (w *response) WriteMsg(m *Msg) (err error) { + if w.closed { + return &Error{err: "WriteMsg called after Close"} + } + var data []byte if w.tsigSecret != nil { // if no secrets, dont check for the tsig (which is a longer check) if t := m.IsTsig(); t != nil { @@ -723,10 +776,13 @@ func (w *response) WriteMsg(m *Msg) (err error) { // Write implements the ResponseWriter.Write method. func (w *response) Write(m []byte) (int, error) { + if w.closed { + return 0, &Error{err: "Write called after Close"} + } + switch { case w.udp != nil: - n, err := WriteToSessionUDP(w.udp, m, w.udpSession) - return n, err + return WriteToSessionUDP(w.udp, m, w.udpSession) case w.tcp != nil: lm := len(m) if lm < 2 { @@ -741,24 +797,33 @@ func (w *response) Write(m []byte) (int, error) { n, err := io.Copy(w.tcp, bytes.NewReader(m)) return int(n), err + default: + panic("dns: internal error: udp and tcp both nil") } - panic("not reached") } // LocalAddr implements the ResponseWriter.LocalAddr method. func (w *response) LocalAddr() net.Addr { - if w.tcp != nil { + switch { + case w.udp != nil: + return w.udp.LocalAddr() + case w.tcp != nil: return w.tcp.LocalAddr() + default: + panic("dns: internal error: udp and tcp both nil") } - return w.udp.LocalAddr() } // RemoteAddr implements the ResponseWriter.RemoteAddr method. func (w *response) RemoteAddr() net.Addr { - if w.tcp != nil { + switch { + case w.udpSession != nil: + return w.udpSession.RemoteAddr() + case w.tcp != nil: return w.tcp.RemoteAddr() + default: + panic("dns: internal error: udpSession and tcp both nil") } - return w.udpSession.RemoteAddr() } // TsigStatus implements the ResponseWriter.TsigStatus method. @@ -772,11 +837,30 @@ func (w *response) Hijack() { w.hijacked = true } // Close implements the ResponseWriter.Close method func (w *response) Close() error { - // Can't close the udp conn, as that is actually the listener. - if w.tcp != nil { - e := w.tcp.Close() - w.tcp = nil - return e + if w.closed { + return &Error{err: "connection already closed"} + } + w.closed = true + + switch { + case w.udp != nil: + // Can't close the udp conn, as that is actually the listener. + return nil + case w.tcp != nil: + return w.tcp.Close() + default: + panic("dns: internal error: udp and tcp both nil") + } +} + +// ConnectionState() implements the ConnectionStater.ConnectionState() interface. +func (w *response) ConnectionState() *tls.ConnectionState { + type tlsConnectionStater interface { + ConnectionState() tls.ConnectionState + } + if v, ok := w.tcp.(tlsConnectionStater); ok { + t := v.ConnectionState() + return &t } return nil } diff --git a/vendor/github.com/miekg/dns/server_test.go b/vendor/github.com/miekg/dns/server_test.go index 6fc9e88..d850b8e 100644 --- a/vendor/github.com/miekg/dns/server_test.go +++ b/vendor/github.com/miekg/dns/server_test.go @@ -1,14 +1,19 @@ package dns import ( + "context" "crypto/tls" "fmt" "io" "net" "runtime" + "strings" "sync" + "sync/atomic" "testing" "time" + + "golang.org/x/sync/errgroup" ) func HelloServer(w ResponseWriter, req *Msg) { @@ -55,7 +60,7 @@ func RunLocalUDPServer(laddr string) (*Server, string, error) { return server, l, err } -func RunLocalUDPServerWithFinChan(laddr string) (*Server, string, chan error, error) { +func RunLocalUDPServerWithFinChan(laddr string, opts ...func(*Server)) (*Server, string, chan error, error) { pc, err := net.ListenPacket("udp", laddr) if err != nil { return nil, "", nil, err @@ -71,6 +76,10 @@ func RunLocalUDPServerWithFinChan(laddr string) (*Server, string, chan error, er // in RunLocalUDPServer and can happen in TestShutdownUDP. fin := make(chan error, 1) + for _, opt := range opts { + opt(server) + } + go func() { fin <- server.ActivateAndServe() pc.Close() @@ -80,27 +89,6 @@ func RunLocalUDPServerWithFinChan(laddr string) (*Server, string, chan error, er return server, pc.LocalAddr().String(), fin, nil } -func RunLocalUDPServerUnsafe(laddr string) (*Server, string, error) { - pc, err := net.ListenPacket("udp", laddr) - if err != nil { - return nil, "", err - } - server := &Server{PacketConn: pc, Unsafe: true, - ReadTimeout: time.Hour, WriteTimeout: time.Hour} - - waitLock := sync.Mutex{} - waitLock.Lock() - server.NotifyStartedFunc = waitLock.Unlock - - go func() { - server.ActivateAndServe() - pc.Close() - }() - - waitLock.Lock() - return server, pc.LocalAddr().String(), nil -} - func RunLocalTCPServer(laddr string) (*Server, string, error) { server, l, _, err := RunLocalTCPServerWithFinChan(laddr) @@ -259,6 +247,98 @@ func TestServingTLS(t *testing.T) { } } +// TestServingTLSConnectionState tests that we only can access +// tls.ConnectionState under a DNS query handled by a TLS DNS server. +// This test will sequentially create a TLS, UDP and TCP server, attach a custom +// handler which will set a testing error if tls.ConnectionState is available +// when it is not expected, or the other way around. +func TestServingTLSConnectionState(t *testing.T) { + handlerResponse := "Hello example" + // tlsHandlerTLS is a HandlerFunc that can be set to expect or not TLS + // connection state. + tlsHandlerTLS := func(tlsExpected bool) func(ResponseWriter, *Msg) { + return func(w ResponseWriter, req *Msg) { + m := new(Msg) + m.SetReply(req) + tlsFound := true + if connState := w.(ConnectionStater).ConnectionState(); connState == nil { + tlsFound = false + } + if tlsFound != tlsExpected { + t.Errorf("TLS connection state available: %t, expected: %t", tlsFound, tlsExpected) + } + m.Extra = make([]RR, 1) + m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{handlerResponse}} + w.WriteMsg(m) + } + } + + // Question used in tests + m := new(Msg) + m.SetQuestion("tlsstate.example.net.", TypeTXT) + + // TLS DNS server + HandleFunc(".", tlsHandlerTLS(true)) + cert, err := tls.X509KeyPair(CertPEMBlock, KeyPEMBlock) + if err != nil { + t.Fatalf("unable to build certificate: %v", err) + } + + config := tls.Config{ + Certificates: []tls.Certificate{cert}, + } + + s, addrstr, err := RunLocalTLSServer(":0", &config) + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + defer s.Shutdown() + + // TLS DNS query + c := &Client{ + Net: "tcp-tls", + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + _, _, err = c.Exchange(m, addrstr) + if err != nil { + t.Error("failed to exchange tlsstate.example.net", err) + } + + HandleRemove(".") + // UDP DNS Server + HandleFunc(".", tlsHandlerTLS(false)) + defer HandleRemove(".") + s, addrstr, err = RunLocalUDPServer(":0") + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + defer s.Shutdown() + + // UDP DNS query + c = new(Client) + _, _, err = c.Exchange(m, addrstr) + if err != nil { + t.Error("failed to exchange tlsstate.example.net", err) + } + + // TCP DNS Server + s, addrstr, err = RunLocalTCPServer(":0") + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + defer s.Shutdown() + + // TCP DNS query + c = &Client{Net: "tcp"} + _, _, err = c.Exchange(m, addrstr) + if err != nil { + t.Error("failed to exchange tlsstate.example.net", err) + } +} + func TestServingListenAndServe(t *testing.T) { HandleFunc("example.com.", AnotherHelloServer) defer HandleRemove("example.com.") @@ -337,33 +417,42 @@ func BenchmarkServe(b *testing.B) { c := new(Client) m := new(Msg) - m.SetQuestion("miek.nl", TypeSOA) + m.SetQuestion("miek.nl.", TypeSOA) b.StartTimer() for i := 0; i < b.N; i++ { - c.Exchange(m, addrstr) + _, _, err := c.Exchange(m, addrstr) + if err != nil { + b.Fatalf("Exchange failed: %v", err) + } } runtime.GOMAXPROCS(a) } -func benchmarkServe6(b *testing.B) { +func BenchmarkServe6(b *testing.B) { b.StopTimer() HandleFunc("miek.nl.", HelloServer) defer HandleRemove("miek.nl.") a := runtime.GOMAXPROCS(4) s, addrstr, err := RunLocalUDPServer("[::1]:0") if err != nil { + if strings.Contains(err.Error(), "bind: cannot assign requested address") { + b.Skip("missing IPv6 support") + } b.Fatalf("unable to run test server: %v", err) } defer s.Shutdown() c := new(Client) m := new(Msg) - m.SetQuestion("miek.nl", TypeSOA) + m.SetQuestion("miek.nl.", TypeSOA) b.StartTimer() for i := 0; i < b.N; i++ { - c.Exchange(m, addrstr) + _, _, err := c.Exchange(m, addrstr) + if err != nil { + b.Fatalf("Exchange failed: %v", err) + } } runtime.GOMAXPROCS(a) } @@ -390,65 +479,17 @@ func BenchmarkServeCompress(b *testing.B) { c := new(Client) m := new(Msg) - m.SetQuestion("miek.nl", TypeSOA) + m.SetQuestion("miek.nl.", TypeSOA) b.StartTimer() for i := 0; i < b.N; i++ { - c.Exchange(m, addrstr) + _, _, err := c.Exchange(m, addrstr) + if err != nil { + b.Fatalf("Exchange failed: %v", err) + } } runtime.GOMAXPROCS(a) } -func TestDotAsCatchAllWildcard(t *testing.T) { - mux := NewServeMux() - mux.Handle(".", HandlerFunc(HelloServer)) - mux.Handle("example.com.", HandlerFunc(AnotherHelloServer)) - - handler := mux.match("www.miek.nl.", TypeTXT) - if handler == nil { - t.Error("wildcard match failed") - } - - handler = mux.match("www.example.com.", TypeTXT) - if handler == nil { - t.Error("example.com match failed") - } - - handler = mux.match("a.www.example.com.", TypeTXT) - if handler == nil { - t.Error("a.www.example.com match failed") - } - - handler = mux.match("boe.", TypeTXT) - if handler == nil { - t.Error("boe. match failed") - } -} - -func TestCaseFolding(t *testing.T) { - mux := NewServeMux() - mux.Handle("_udp.example.com.", HandlerFunc(HelloServer)) - - handler := mux.match("_dns._udp.example.com.", TypeSRV) - if handler == nil { - t.Error("case sensitive characters folded") - } - - handler = mux.match("_DNS._UDP.EXAMPLE.COM.", TypeSRV) - if handler == nil { - t.Error("case insensitive characters not folded") - } -} - -func TestRootServer(t *testing.T) { - mux := NewServeMux() - mux.Handle(".", HandlerFunc(HelloServer)) - - handler := mux.match(".", TypeNS) - if handler == nil { - t.Error("root match failed") - } -} - type maxRec struct { max int sync.RWMutex @@ -527,6 +568,7 @@ func TestServingResponse(t *testing.T) { if err != nil { t.Fatalf("unable to run test server: %v", err) } + defer s.Shutdown() c := new(Client) m := new(Msg) @@ -541,19 +583,6 @@ func TestServingResponse(t *testing.T) { if err == nil { t.Fatal("exchanged response message") } - - s.Shutdown() - s, addrstr, err = RunLocalUDPServerUnsafe(":0") - if err != nil { - t.Fatalf("unable to run test server: %v", err) - } - defer s.Shutdown() - - m.Response = true - _, _, err = c.Exchange(m, addrstr) - if err != nil { - t.Fatal("could exchanged response message in Unsafe mode") - } } func TestShutdownTCP(t *testing.T) { @@ -575,6 +604,128 @@ func TestShutdownTCP(t *testing.T) { } } +func init() { + testShutdownNotify = &sync.Cond{ + L: new(sync.Mutex), + } +} + +func checkInProgressQueriesAtShutdownServer(t *testing.T, srv *Server, addr string, client *Client) { + const requests = 100 + + var errOnce sync.Once + // t.Fail will panic if it's called after the test function has + // finished. Burning the sync.Once with a defer will prevent the + // handler from calling t.Errorf after we've returned. + defer errOnce.Do(func() {}) + + toHandle := int32(requests) + HandleFunc("example.com.", func(w ResponseWriter, req *Msg) { + defer atomic.AddInt32(&toHandle, -1) + + // Wait until ShutdownContext is called before replying. + testShutdownNotify.L.Lock() + testShutdownNotify.Wait() + testShutdownNotify.L.Unlock() + + m := new(Msg) + m.SetReply(req) + m.Extra = make([]RR, 1) + m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello world"}} + + if err := w.WriteMsg(m); err != nil { + errOnce.Do(func() { + t.Errorf("ResponseWriter.WriteMsg error: %s", err) + }) + } + }) + defer HandleRemove("example.com.") + + client.Timeout = 10 * time.Second + + conns := make([]*Conn, requests) + eg := new(errgroup.Group) + + for i := range conns { + conn := &conns[i] + eg.Go(func() error { + var err error + *conn, err = client.Dial(addr) + return err + }) + } + + if eg.Wait() != nil { + t.Fatalf("client.Dial error: %v", eg.Wait()) + } + + m := new(Msg) + m.SetQuestion("example.com.", TypeTXT) + eg = new(errgroup.Group) + + for _, conn := range conns { + conn := conn + eg.Go(func() error { + conn.SetWriteDeadline(time.Now().Add(client.Timeout)) + + return conn.WriteMsg(m) + }) + } + + if eg.Wait() != nil { + t.Fatalf("conn.WriteMsg error: %v", eg.Wait()) + } + + // This sleep is needed to allow time for the requests to + // pass from the client through the kernel and back into + // the server. Without it, some requests may still be in + // the kernel's buffer when ShutdownContext is called. + time.Sleep(100 * time.Millisecond) + + eg = new(errgroup.Group) + + for _, conn := range conns { + conn := conn + eg.Go(func() error { + conn.SetReadDeadline(time.Now().Add(client.Timeout)) + + _, err := conn.ReadMsg() + return err + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), client.Timeout) + defer cancel() + + if err := srv.ShutdownContext(ctx); err != nil { + t.Errorf("could not shutdown test server: %v", err) + } + + if left := atomic.LoadInt32(&toHandle); left != 0 { + t.Errorf("ShutdownContext returned before %d replies", left) + } + + if eg.Wait() != nil { + t.Errorf("conn.ReadMsg error: %v", eg.Wait()) + } + + srv.lock.RLock() + defer srv.lock.RUnlock() + if len(srv.conns) != 0 { + t.Errorf("TCP connection tracking map not empty after ShutdownContext; map still contains %d connections", len(srv.conns)) + } +} + +func TestInProgressQueriesAtShutdownTCP(t *testing.T) { + s, addr, _, err := RunLocalTCPServerWithFinChan(":0") + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + + c := &Client{Net: "tcp"} + checkInProgressQueriesAtShutdownServer(t, s, addr, c) +} + func TestShutdownTLS(t *testing.T) { cert, err := tls.X509KeyPair(CertPEMBlock, KeyPEMBlock) if err != nil { @@ -595,20 +746,28 @@ func TestShutdownTLS(t *testing.T) { } } -type trigger struct { - done bool - sync.RWMutex -} +func TestInProgressQueriesAtShutdownTLS(t *testing.T) { + cert, err := tls.X509KeyPair(CertPEMBlock, KeyPEMBlock) + if err != nil { + t.Fatalf("unable to build certificate: %v", err) + } -func (t *trigger) Set() { - t.Lock() - defer t.Unlock() - t.done = true -} -func (t *trigger) Get() bool { - t.RLock() - defer t.RUnlock() - return t.done + config := tls.Config{ + Certificates: []tls.Certificate{cert}, + } + + s, addr, err := RunLocalTLSServer(":0", &config) + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + + c := &Client{ + Net: "tcp-tls", + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + checkInProgressQueriesAtShutdownServer(t, s, addr, c) } func TestHandlerCloseTCP(t *testing.T) { @@ -622,9 +781,9 @@ func TestHandlerCloseTCP(t *testing.T) { server := &Server{Addr: addr, Net: "tcp", Listener: ln} hname := "testhandlerclosetcp." - triggered := &trigger{} + triggered := make(chan struct{}) HandleFunc(hname, func(w ResponseWriter, r *Msg) { - triggered.Set() + close(triggered) w.Close() }) defer HandleRemove(hname) @@ -637,7 +796,7 @@ func TestHandlerCloseTCP(t *testing.T) { exchange: _, _, err := c.Exchange(m, addr) if err != nil && err != io.EOF { - t.Errorf("exchange failed: %s\n", err) + t.Errorf("exchange failed: %v", err) if tries == 3 { return } @@ -646,8 +805,12 @@ func TestHandlerCloseTCP(t *testing.T) { goto exchange } }() - server.ActivateAndServe() - if !triggered.Get() { + if err := server.ActivateAndServe(); err != nil { + t.Fatalf("ActivateAndServe failed: %v", err) + } + select { + case <-triggered: + default: t.Fatalf("handler never called") } } @@ -671,20 +834,159 @@ func TestShutdownUDP(t *testing.T) { } } +func TestInProgressQueriesAtShutdownUDP(t *testing.T) { + s, addr, _, err := RunLocalUDPServerWithFinChan(":0") + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + + c := &Client{Net: "udp"} + checkInProgressQueriesAtShutdownServer(t, s, addr, c) +} + func TestServerStartStopRace(t *testing.T) { + var wg sync.WaitGroup for i := 0; i < 10; i++ { - var err error - s := &Server{} - s, _, _, err = RunLocalUDPServerWithFinChan(":0") + wg.Add(1) + s, _, _, err := RunLocalUDPServerWithFinChan(":0") if err != nil { t.Fatalf("could not start server: %s", err) } go func() { + defer wg.Done() if err := s.Shutdown(); err != nil { - t.Fatalf("could not stop server: %s", err) + t.Errorf("could not stop server: %s", err) } }() } + wg.Wait() +} + +func TestServerReuseport(t *testing.T) { + if !supportsReusePort { + t.Skip("reuseport is not supported") + } + + startServer := func(addr string) (*Server, chan error) { + wait := make(chan struct{}) + srv := &Server{ + Net: "udp", + Addr: addr, + NotifyStartedFunc: func() { close(wait) }, + ReusePort: true, + } + + fin := make(chan error, 1) + go func() { + fin <- srv.ListenAndServe() + }() + + select { + case <-wait: + case err := <-fin: + t.Fatalf("failed to start server: %v", err) + } + + return srv, fin + } + + srv1, fin1 := startServer(":0") // :0 is resolved to a random free port by the kernel + srv2, fin2 := startServer(srv1.PacketConn.LocalAddr().String()) + + if err := srv1.Shutdown(); err != nil { + t.Fatalf("failed to shutdown first server: %v", err) + } + if err := srv2.Shutdown(); err != nil { + t.Fatalf("failed to shutdown second server: %v", err) + } + + if err := <-fin1; err != nil { + t.Fatalf("first ListenAndServe returned error after Shutdown: %v", err) + } + if err := <-fin2; err != nil { + t.Fatalf("second ListenAndServe returned error after Shutdown: %v", err) + } +} + +func TestServerRoundtripTsig(t *testing.T) { + secret := map[string]string{"test.": "so6ZGir4GPAqINNh9U5c3A=="} + + s, addrstr, _, err := RunLocalUDPServerWithFinChan(":0", func(srv *Server) { + srv.TsigSecret = secret + }) + if err != nil { + t.Fatalf("unable to run test server: %v", err) + } + defer s.Shutdown() + + HandleFunc("example.com.", func(w ResponseWriter, r *Msg) { + m := new(Msg) + m.SetReply(r) + if r.IsTsig() != nil { + status := w.TsigStatus() + if status == nil { + // *Msg r has an TSIG record and it was validated + m.SetTsig("test.", HmacMD5, 300, time.Now().Unix()) + } else { + // *Msg r has an TSIG records and it was not valided + t.Errorf("invalid TSIG: %v", status) + } + } else { + t.Error("missing TSIG") + } + w.WriteMsg(m) + }) + + c := new(Client) + m := new(Msg) + m.Opcode = OpcodeUpdate + m.SetQuestion("example.com.", TypeSOA) + m.Ns = []RR{&CNAME{ + Hdr: RR_Header{ + Name: "foo.example.com.", + Rrtype: TypeCNAME, + Class: ClassINET, + Ttl: 300, + }, + Target: "bar.example.com.", + }} + c.TsigSecret = secret + m.SetTsig("test.", HmacMD5, 300, time.Now().Unix()) + _, _, err = c.Exchange(m, addrstr) + if err != nil { + t.Fatal("failed to exchange", err) + } +} + +func TestResponseAfterClose(t *testing.T) { + testError := func(name string, err error) { + t.Helper() + + expect := fmt.Sprintf("dns: %s called after Close", name) + if err == nil { + t.Errorf("expected error from %s after Close", name) + } else if err.Error() != expect { + t.Errorf("expected explicit error from %s after Close, expected %q, got %q", name, expect, err) + } + } + + rw := &response{ + closed: true, + } + + _, err := rw.Write(make([]byte, 2)) + testError("Write", err) + + testError("WriteMsg", rw.WriteMsg(new(Msg))) +} + +func TestResponseDoubleClose(t *testing.T) { + rw := &response{ + closed: true, + } + if err, expect := rw.Close(), "dns: connection already closed"; err == nil || err.Error() != expect { + t.Errorf("Close did not return expected: error %q, got: %v", expect, err) + } } type ExampleFrameLengthWriter struct { diff --git a/vendor/github.com/miekg/dns/sig0.go b/vendor/github.com/miekg/dns/sig0.go index f31e9e6..ec65dd7 100644 --- a/vendor/github.com/miekg/dns/sig0.go +++ b/vendor/github.com/miekg/dns/sig0.go @@ -21,15 +21,11 @@ func (rr *SIG) Sign(k crypto.Signer, m *Msg) ([]byte, error) { if rr.KeyTag == 0 || len(rr.SignerName) == 0 || rr.Algorithm == 0 { return nil, ErrKey } - rr.Header().Rrtype = TypeSIG - rr.Header().Class = ClassANY - rr.Header().Ttl = 0 - rr.Header().Name = "." - rr.OrigTtl = 0 - rr.TypeCovered = 0 - rr.Labels = 0 - buf := make([]byte, m.Len()+rr.len()) + rr.Hdr = RR_Header{Name: ".", Rrtype: TypeSIG, Class: ClassANY, Ttl: 0} + rr.OrigTtl, rr.TypeCovered, rr.Labels = 0, 0, 0 + + buf := make([]byte, m.Len()+Len(rr)) mbuf, err := m.PackBuffer(buf) if err != nil { return nil, err @@ -107,7 +103,7 @@ func (rr *SIG) Verify(k *KEY, buf []byte) error { anc := binary.BigEndian.Uint16(buf[6:]) auc := binary.BigEndian.Uint16(buf[8:]) adc := binary.BigEndian.Uint16(buf[10:]) - offset := 12 + offset := headerSize var err error for i := uint16(0); i < qdc && offset < buflen; i++ { _, offset, err = UnpackDomainName(buf, offset) @@ -127,8 +123,7 @@ func (rr *SIG) Verify(k *KEY, buf []byte) error { if offset+1 >= buflen { continue } - var rdlen uint16 - rdlen = binary.BigEndian.Uint16(buf[offset:]) + rdlen := binary.BigEndian.Uint16(buf[offset:]) offset += 2 offset += int(rdlen) } @@ -168,7 +163,7 @@ func (rr *SIG) Verify(k *KEY, buf []byte) error { } // If key has come from the DNS name compression might // have mangled the case of the name - if strings.ToLower(signername) != strings.ToLower(k.Header().Name) { + if !strings.EqualFold(signername, k.Header().Name) { return &Error{err: "signer name doesn't match key name"} } sigend := offset diff --git a/vendor/github.com/miekg/dns/singleinflight.go b/vendor/github.com/miekg/dns/singleinflight.go index 9573c7d..febcc30 100644 --- a/vendor/github.com/miekg/dns/singleinflight.go +++ b/vendor/github.com/miekg/dns/singleinflight.go @@ -23,6 +23,8 @@ type call struct { type singleflight struct { sync.Mutex // protects m m map[string]*call // lazily initialized + + dontDeleteForTesting bool // this is only to be used by TestConcurrentExchanges } // Do executes and returns the results of the given function, making @@ -49,9 +51,11 @@ func (g *singleflight) Do(key string, fn func() (*Msg, time.Duration, error)) (v c.val, c.rtt, c.err = fn() c.wg.Done() - g.Lock() - delete(g.m, key) - g.Unlock() + if !g.dontDeleteForTesting { + g.Lock() + delete(g.m, key) + g.Unlock() + } return c.val, c.rtt, c.err, c.dups > 0 } diff --git a/vendor/github.com/miekg/dns/smimea.go b/vendor/github.com/miekg/dns/smimea.go index 4e7ded4..89f09f0 100644 --- a/vendor/github.com/miekg/dns/smimea.go +++ b/vendor/github.com/miekg/dns/smimea.go @@ -14,10 +14,7 @@ func (r *SMIMEA) Sign(usage, selector, matchingType int, cert *x509.Certificate) r.MatchingType = uint8(matchingType) r.Certificate, err = CertificateToDANE(r.Selector, r.MatchingType, cert) - if err != nil { - return err - } - return nil + return err } // Verify verifies a SMIMEA record against an SSL certificate. If it is OK diff --git a/vendor/github.com/miekg/dns/tlsa.go b/vendor/github.com/miekg/dns/tlsa.go index 431e2fb..4e07983 100644 --- a/vendor/github.com/miekg/dns/tlsa.go +++ b/vendor/github.com/miekg/dns/tlsa.go @@ -14,10 +14,7 @@ func (r *TLSA) Sign(usage, selector, matchingType int, cert *x509.Certificate) ( r.MatchingType = uint8(matchingType) r.Certificate, err = CertificateToDANE(r.Selector, r.MatchingType, cert) - if err != nil { - return err - } - return nil + return err } // Verify verifies a TLSA record against an SSL certificate. If it is OK diff --git a/vendor/github.com/miekg/dns/tsig.go b/vendor/github.com/miekg/dns/tsig.go index 4837b4a..afa462f 100644 --- a/vendor/github.com/miekg/dns/tsig.go +++ b/vendor/github.com/miekg/dns/tsig.go @@ -54,6 +54,10 @@ func (rr *TSIG) String() string { return s } +func (rr *TSIG) parse(c *zlexer, origin, file string) *ParseError { + panic("dns: internal error: parse should never be called on TSIG") +} + // The following values must be put in wireformat, so that the MAC can be calculated. // RFC 2845, section 3.4.2. TSIG Variables. type tsigWireFmt struct { @@ -113,13 +117,13 @@ func TsigGenerate(m *Msg, secret, requestMAC string, timersOnly bool) ([]byte, s var h hash.Hash switch strings.ToLower(rr.Algorithm) { case HmacMD5: - h = hmac.New(md5.New, []byte(rawsecret)) + h = hmac.New(md5.New, rawsecret) case HmacSHA1: - h = hmac.New(sha1.New, []byte(rawsecret)) + h = hmac.New(sha1.New, rawsecret) case HmacSHA256: - h = hmac.New(sha256.New, []byte(rawsecret)) + h = hmac.New(sha256.New, rawsecret) case HmacSHA512: - h = hmac.New(sha512.New, []byte(rawsecret)) + h = hmac.New(sha512.New, rawsecret) default: return nil, "", ErrKeyAlg } @@ -133,13 +137,12 @@ func TsigGenerate(m *Msg, secret, requestMAC string, timersOnly bool) ([]byte, s t.Algorithm = rr.Algorithm t.OrigId = m.Id - tbuf := make([]byte, t.len()) - if off, err := PackRR(t, tbuf, 0, nil, false); err == nil { - tbuf = tbuf[:off] // reset to actual size used - } else { + tbuf := make([]byte, Len(t)) + off, err := PackRR(t, tbuf, 0, nil, false) + if err != nil { return nil, "", err } - mbuf = append(mbuf, tbuf...) + mbuf = append(mbuf, tbuf[:off]...) // Update the ArCount directly in the buffer. binary.BigEndian.PutUint16(mbuf[10:], uint16(len(m.Extra)+1)) diff --git a/vendor/github.com/miekg/dns/tsig_test.go b/vendor/github.com/miekg/dns/tsig_test.go index 4bc5273..3f1f3c8 100644 --- a/vendor/github.com/miekg/dns/tsig_test.go +++ b/vendor/github.com/miekg/dns/tsig_test.go @@ -32,7 +32,7 @@ func TestTsig(t *testing.T) { t.Fatal(err) } - binary.BigEndian.PutUint16(buf[0:2], uint16(42)) + binary.BigEndian.PutUint16(buf[0:2], 42) err = TsigVerify(buf, "pRZgBrBvI4NAHZYhxmhs/Q==", "", false) if err != nil { t.Fatal(err) diff --git a/vendor/github.com/miekg/dns/types.go b/vendor/github.com/miekg/dns/types.go index a779ca8..efa3424 100644 --- a/vendor/github.com/miekg/dns/types.go +++ b/vendor/github.com/miekg/dns/types.go @@ -205,9 +205,6 @@ var CertTypeToString = map[uint16]string{ CertOID: "OID", } -// StringToCertType is the reverseof CertTypeToString. -var StringToCertType = reverseInt16(CertTypeToString) - //go:generate go run types_generate.go // Question holds a DNS question. There can be multiple questions in the @@ -218,8 +215,10 @@ type Question struct { Qclass uint16 } -func (q *Question) len() int { - return len(q.Name) + 1 + 2 + 2 +func (q *Question) len(off int, compression map[string]struct{}) int { + l := domainNameLen(q.Name, off, compression, true) + l += 2 + 2 + return l } func (q *Question) String() (s string) { @@ -239,6 +238,25 @@ type ANY struct { func (rr *ANY) String() string { return rr.Hdr.String() } +func (rr *ANY) parse(c *zlexer, origin, file string) *ParseError { + panic("dns: internal error: parse should never be called on ANY") +} + +// NULL RR. See RFC 1035. +type NULL struct { + Hdr RR_Header + Data string `dns:"any"` +} + +func (rr *NULL) String() string { + // There is no presentation format; prefix string with a comment. + return ";" + rr.Hdr.String() + rr.Data +} + +func (rr *NULL) parse(c *zlexer, origin, file string) *ParseError { + panic("dns: internal error: parse should never be called on NULL") +} + // CNAME RR. See RFC 1034. type CNAME struct { Hdr RR_Header @@ -330,7 +348,7 @@ func (rr *MX) String() string { type AFSDB struct { Hdr RR_Header Subtype uint16 - Hostname string `dns:"cdomain-name"` + Hostname string `dns:"domain-name"` } func (rr *AFSDB) String() string { @@ -351,7 +369,7 @@ func (rr *X25) String() string { type RT struct { Hdr RR_Header Preference uint16 - Host string `dns:"cdomain-name"` + Host string `dns:"domain-name"` // RFC 3597 prohibits compressing records not defined in RFC 1035. } func (rr *RT) String() string { @@ -419,128 +437,154 @@ type TXT struct { func (rr *TXT) String() string { return rr.Hdr.String() + sprintTxt(rr.Txt) } func sprintName(s string) string { - src := []byte(s) - dst := make([]byte, 0, len(src)) - for i := 0; i < len(src); { - if i+1 < len(src) && src[i] == '\\' && src[i+1] == '.' { - dst = append(dst, src[i:i+2]...) + var dst strings.Builder + dst.Grow(len(s)) + for i := 0; i < len(s); { + if i+1 < len(s) && s[i] == '\\' && s[i+1] == '.' { + dst.WriteString(s[i : i+2]) i += 2 - } else { - b, n := nextByte(src, i) - if n == 0 { - i++ // dangling back slash - } else if b == '.' { - dst = append(dst, b) - } else { - dst = appendDomainNameByte(dst, b) - } - i += n + continue } + + b, n := nextByte(s, i) + switch { + case n == 0: + i++ // dangling back slash + case b == '.': + dst.WriteByte('.') + default: + writeDomainNameByte(&dst, b) + } + i += n } - return string(dst) + return dst.String() } func sprintTxtOctet(s string) string { - src := []byte(s) - dst := make([]byte, 0, len(src)) - dst = append(dst, '"') - for i := 0; i < len(src); { - if i+1 < len(src) && src[i] == '\\' && src[i+1] == '.' { - dst = append(dst, src[i:i+2]...) + var dst strings.Builder + dst.Grow(2 + len(s)) + dst.WriteByte('"') + for i := 0; i < len(s); { + if i+1 < len(s) && s[i] == '\\' && s[i+1] == '.' { + dst.WriteString(s[i : i+2]) i += 2 - } else { - b, n := nextByte(src, i) - if n == 0 { - i++ // dangling back slash - } else if b == '.' { - dst = append(dst, b) - } else { - if b < ' ' || b > '~' { - dst = appendByte(dst, b) - } else { - dst = append(dst, b) - } - } - i += n + continue } + + b, n := nextByte(s, i) + switch { + case n == 0: + i++ // dangling back slash + case b == '.': + dst.WriteByte('.') + case b < ' ' || b > '~': + dst.WriteString(escapeByte(b)) + default: + dst.WriteByte(b) + } + i += n } - dst = append(dst, '"') - return string(dst) + dst.WriteByte('"') + return dst.String() } func sprintTxt(txt []string) string { - var out []byte + var out strings.Builder for i, s := range txt { + out.Grow(3 + len(s)) if i > 0 { - out = append(out, ` "`...) + out.WriteString(` "`) } else { - out = append(out, '"') + out.WriteByte('"') } - bs := []byte(s) - for j := 0; j < len(bs); { - b, n := nextByte(bs, j) + for j := 0; j < len(s); { + b, n := nextByte(s, j) if n == 0 { break } - out = appendTXTStringByte(out, b) + writeTXTStringByte(&out, b) j += n } - out = append(out, '"') + out.WriteByte('"') } - return string(out) + return out.String() } -func appendDomainNameByte(s []byte, b byte) []byte { +func writeDomainNameByte(s *strings.Builder, b byte) { switch b { case '.', ' ', '\'', '@', ';', '(', ')': // additional chars to escape - return append(s, '\\', b) + s.WriteByte('\\') + s.WriteByte(b) + default: + writeTXTStringByte(s, b) } - return appendTXTStringByte(s, b) } -func appendTXTStringByte(s []byte, b byte) []byte { - switch b { - case '"', '\\': - return append(s, '\\', b) +func writeTXTStringByte(s *strings.Builder, b byte) { + switch { + case b == '"' || b == '\\': + s.WriteByte('\\') + s.WriteByte(b) + case b < ' ' || b > '~': + s.WriteString(escapeByte(b)) + default: + s.WriteByte(b) } - if b < ' ' || b > '~' { - return appendByte(s, b) - } - return append(s, b) } -func appendByte(s []byte, b byte) []byte { - var buf [3]byte - bufs := strconv.AppendInt(buf[:0], int64(b), 10) - s = append(s, '\\') - for i := 0; i < 3-len(bufs); i++ { - s = append(s, '0') +const ( + escapedByteSmall = "" + + `\000\001\002\003\004\005\006\007\008\009` + + `\010\011\012\013\014\015\016\017\018\019` + + `\020\021\022\023\024\025\026\027\028\029` + + `\030\031` + escapedByteLarge = `\127\128\129` + + `\130\131\132\133\134\135\136\137\138\139` + + `\140\141\142\143\144\145\146\147\148\149` + + `\150\151\152\153\154\155\156\157\158\159` + + `\160\161\162\163\164\165\166\167\168\169` + + `\170\171\172\173\174\175\176\177\178\179` + + `\180\181\182\183\184\185\186\187\188\189` + + `\190\191\192\193\194\195\196\197\198\199` + + `\200\201\202\203\204\205\206\207\208\209` + + `\210\211\212\213\214\215\216\217\218\219` + + `\220\221\222\223\224\225\226\227\228\229` + + `\230\231\232\233\234\235\236\237\238\239` + + `\240\241\242\243\244\245\246\247\248\249` + + `\250\251\252\253\254\255` +) + +// escapeByte returns the \DDD escaping of b which must +// satisfy b < ' ' || b > '~'. +func escapeByte(b byte) string { + if b < ' ' { + return escapedByteSmall[b*4 : b*4+4] } - for _, r := range bufs { - s = append(s, r) - } - return s + + b -= '~' + 1 + // The cast here is needed as b*4 may overflow byte. + return escapedByteLarge[int(b)*4 : int(b)*4+4] } -func nextByte(b []byte, offset int) (byte, int) { - if offset >= len(b) { +func nextByte(s string, offset int) (byte, int) { + if offset >= len(s) { return 0, 0 } - if b[offset] != '\\' { + if s[offset] != '\\' { // not an escape sequence - return b[offset], 1 + return s[offset], 1 } - switch len(b) - offset { + switch len(s) - offset { case 1: // dangling escape return 0, 0 case 2, 3: // too short to be \ddd default: // maybe \ddd - if isDigit(b[offset+1]) && isDigit(b[offset+2]) && isDigit(b[offset+3]) { - return dddToByte(b[offset+1:]), 4 + if isDigit(s[offset+1]) && isDigit(s[offset+2]) && isDigit(s[offset+3]) { + return dddStringToByte(s[offset+1:]), 4 } } // not \ddd, just an RFC 1035 "quoted" character - return b[offset+1], 2 + return s[offset+1], 2 } // SPF RR. See RFC 4408, Section 3.1.1. @@ -728,7 +772,7 @@ func (rr *LOC) String() string { lat = lat % LOC_DEGREES m := lat / LOC_HOURS lat = lat % LOC_HOURS - s += fmt.Sprintf("%02d %02d %0.3f %s ", h, m, (float64(lat) / 1000), ns) + s += fmt.Sprintf("%02d %02d %0.3f %s ", h, m, float64(lat)/1000, ns) lon := rr.Longitude ew := "E" @@ -742,7 +786,7 @@ func (rr *LOC) String() string { lon = lon % LOC_DEGREES m = lon / LOC_HOURS lon = lon % LOC_HOURS - s += fmt.Sprintf("%02d %02d %0.3f %s ", h, m, (float64(lon) / 1000), ew) + s += fmt.Sprintf("%02d %02d %0.3f %s ", h, m, float64(lon)/1000, ew) var alt = float64(rr.Altitude) / 100 alt -= LOC_ALTITUDEBASE @@ -752,9 +796,9 @@ func (rr *LOC) String() string { s += fmt.Sprintf("%.0fm ", alt) } - s += cmToM((rr.Size&0xf0)>>4, rr.Size&0x0f) + "m " - s += cmToM((rr.HorizPre&0xf0)>>4, rr.HorizPre&0x0f) + "m " - s += cmToM((rr.VertPre&0xf0)>>4, rr.VertPre&0x0f) + "m" + s += cmToM(rr.Size&0xf0>>4, rr.Size&0x0f) + "m " + s += cmToM(rr.HorizPre&0xf0>>4, rr.HorizPre&0x0f) + "m " + s += cmToM(rr.VertPre&0xf0>>4, rr.VertPre&0x0f) + "m" return s } @@ -807,8 +851,9 @@ func (rr *NSEC) String() string { return s } -func (rr *NSEC) len() int { - l := rr.Hdr.len() + len(rr.NextDomain) + 1 +func (rr *NSEC) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.NextDomain, off+l, compression, false) lastwindow := uint32(2 ^ 32 + 1) for _, t := range rr.TypeBitMap { window := t / 256 @@ -972,8 +1017,9 @@ func (rr *NSEC3) String() string { return s } -func (rr *NSEC3) len() int { - l := rr.Hdr.len() + 6 + len(rr.Salt)/2 + 1 + len(rr.NextDomain) + 1 +func (rr *NSEC3) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += 6 + len(rr.Salt)/2 + 1 + len(rr.NextDomain) + 1 lastwindow := uint32(2 ^ 32 + 1) for _, t := range rr.TypeBitMap { window := t / 256 @@ -1020,10 +1066,16 @@ type TKEY struct { // TKEY has no official presentation format, but this will suffice. func (rr *TKEY) String() string { - s := "\n;; TKEY PSEUDOSECTION:\n" - s += rr.Hdr.String() + " " + rr.Algorithm + " " + - strconv.Itoa(int(rr.KeySize)) + " " + rr.Key + " " + - strconv.Itoa(int(rr.OtherLen)) + " " + rr.OtherData + s := ";" + rr.Hdr.String() + + " " + rr.Algorithm + + " " + TimeToString(rr.Inception) + + " " + TimeToString(rr.Expiration) + + " " + strconv.Itoa(int(rr.Mode)) + + " " + strconv.Itoa(int(rr.Error)) + + " " + strconv.Itoa(int(rr.KeySize)) + + " " + rr.Key + + " " + strconv.Itoa(int(rr.OtherLen)) + + " " + rr.OtherData return s } @@ -1289,8 +1341,9 @@ func (rr *CSYNC) String() string { return s } -func (rr *CSYNC) len() int { - l := rr.Hdr.len() + 4 + 2 +func (rr *CSYNC) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += 4 + 2 lastwindow := uint32(2 ^ 32 + 1) for _, t := range rr.TypeBitMap { window := t / 256 @@ -1306,11 +1359,11 @@ func (rr *CSYNC) len() int { // string representation used when printing the record. // It takes serial arithmetic (RFC 1982) into account. func TimeToString(t uint32) string { - mod := ((int64(t) - time.Now().Unix()) / year68) - 1 + mod := (int64(t)-time.Now().Unix())/year68 - 1 if mod < 0 { mod = 0 } - ti := time.Unix(int64(t)-(mod*year68), 0).UTC() + ti := time.Unix(int64(t)-mod*year68, 0).UTC() return ti.Format("20060102150405") } @@ -1322,11 +1375,11 @@ func StringToTime(s string) (uint32, error) { if err != nil { return 0, err } - mod := (t.Unix() / year68) - 1 + mod := t.Unix()/year68 - 1 if mod < 0 { mod = 0 } - return uint32(t.Unix() - (mod * year68)), nil + return uint32(t.Unix() - mod*year68), nil } // saltToString converts a NSECX salt to uppercase and returns "-" when it is empty. diff --git a/vendor/github.com/miekg/dns/types_generate.go b/vendor/github.com/miekg/dns/types_generate.go index b8db4f3..aa05a08 100644 --- a/vendor/github.com/miekg/dns/types_generate.go +++ b/vendor/github.com/miekg/dns/types_generate.go @@ -153,8 +153,8 @@ func main() { if isEmbedded { continue } - fmt.Fprintf(b, "func (rr *%s) len() int {\n", name) - fmt.Fprintf(b, "l := rr.Hdr.len()\n") + fmt.Fprintf(b, "func (rr *%s) len(off int, compression map[string]struct{}) int {\n", name) + fmt.Fprintf(b, "l := rr.Hdr.len(off, compression)\n") for i := 1; i < st.NumFields(); i++ { o := func(s string) { fmt.Fprintf(b, s, st.Field(i).Name()) } @@ -162,7 +162,11 @@ func main() { switch st.Tag(i) { case `dns:"-"`: // ignored - case `dns:"cdomain-name"`, `dns:"domain-name"`, `dns:"txt"`: + case `dns:"cdomain-name"`: + o("for _, x := range rr.%s { l += domainNameLen(x, off+l, compression, true) }\n") + case `dns:"domain-name"`: + o("for _, x := range rr.%s { l += domainNameLen(x, off+l, compression, false) }\n") + case `dns:"txt"`: o("for _, x := range rr.%s { l += len(x) + 1 }\n") default: log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) @@ -173,8 +177,10 @@ func main() { switch { case st.Tag(i) == `dns:"-"`: // ignored - case st.Tag(i) == `dns:"cdomain-name"`, st.Tag(i) == `dns:"domain-name"`: - o("l += len(rr.%s) + 1\n") + case st.Tag(i) == `dns:"cdomain-name"`: + o("l += domainNameLen(rr.%s, off+l, compression, true)\n") + case st.Tag(i) == `dns:"domain-name"`: + o("l += domainNameLen(rr.%s, off+l, compression, false)\n") case st.Tag(i) == `dns:"octet"`: o("l += len(rr.%s)\n") case strings.HasPrefix(st.Tag(i), `dns:"size-base64`): @@ -187,6 +193,8 @@ func main() { fallthrough case st.Tag(i) == `dns:"hex"`: o("l += len(rr.%s)/2 + 1\n") + case st.Tag(i) == `dns:"any"`: + o("l += len(rr.%s)\n") case st.Tag(i) == `dns:"a"`: o("l += net.IPv4len // %s\n") case st.Tag(i) == `dns:"aaaa"`: @@ -236,6 +244,13 @@ func main() { splits := strings.Split(t, ".") t = splits[len(splits)-1] } + // For the EDNS0 interface (used in the OPT RR), we need to call the copy method on each element. + if t == "EDNS0" { + fmt.Fprintf(b, "%s := make([]%s, len(rr.%s));\nfor i,e := range rr.%s {\n %s[i] = e.copy()\n}\n", + f, t, f, f, f) + fields = append(fields, f) + continue + } fmt.Fprintf(b, "%s := make([]%s, len(rr.%s)); copy(%s, rr.%s)\n", f, t, f, f, f) fields = append(fields, f) diff --git a/vendor/github.com/miekg/dns/types_test.go b/vendor/github.com/miekg/dns/types_test.go index 3dbddee..81a67fd 100644 --- a/vendor/github.com/miekg/dns/types_test.go +++ b/vendor/github.com/miekg/dns/types_test.go @@ -72,3 +72,65 @@ func TestSplitN(t *testing.T) { t.Errorf("failure to split 510 char long string: %d", len(xs)) } } + +func TestSprintName(t *testing.T) { + got := sprintName("abc\\.def\007\"\127@\255\x05\xef\\") + + if want := "abc\\.def\\007\\\"W\\@\\173\\005\\239"; got != want { + t.Errorf("expected %q, got %q", got, want) + } +} + +func TestSprintTxtOctet(t *testing.T) { + got := sprintTxtOctet("abc\\.def\007\"\127@\255\x05\xef\\") + + if want := "\"abc\\.def\\007\"W@\\173\\005\\239\""; got != want { + t.Errorf("expected %q, got %q", got, want) + } +} + +func TestSprintTxt(t *testing.T) { + got := sprintTxt([]string{ + "abc\\.def\007\"\127@\255\x05\xef\\", + "example.com", + }) + + if want := "\"abc.def\\007\\\"W@\\173\\005\\239\" \"example.com\""; got != want { + t.Errorf("expected %q, got %q", got, want) + } +} + +func BenchmarkSprintName(b *testing.B) { + for n := 0; n < b.N; n++ { + got := sprintName("abc\\.def\007\"\127@\255\x05\xef\\") + + if want := "abc\\.def\\007\\\"W\\@\\173\\005\\239"; got != want { + b.Fatalf("expected %q, got %q", got, want) + } + } +} + +func BenchmarkSprintTxtOctet(b *testing.B) { + for n := 0; n < b.N; n++ { + got := sprintTxtOctet("abc\\.def\007\"\127@\255\x05\xef\\") + + if want := "\"abc\\.def\\007\"W@\\173\\005\\239\""; got != want { + b.Fatalf("expected %q, got %q", got, want) + } + } +} + +func BenchmarkSprintTxt(b *testing.B) { + txt := []string{ + "abc\\.def\007\"\127@\255\x05\xef\\", + "example.com", + } + + for n := 0; n < b.N; n++ { + got := sprintTxt(txt) + + if want := "\"abc.def\\007\\\"W@\\173\\005\\239\" \"example.com\""; got != want { + b.Fatalf("expected %q, got %q", got, want) + } + } +} diff --git a/vendor/github.com/miekg/dns/udp_test.go b/vendor/github.com/miekg/dns/udp_test.go index f27d79d..18be622 100644 --- a/vendor/github.com/miekg/dns/udp_test.go +++ b/vendor/github.com/miekg/dns/udp_test.go @@ -35,7 +35,8 @@ func TestSetUDPSocketOptions(t *testing.T) { b := make([]byte, 1) _, sess, err := ReadFromSessionUDP(c, b) if err != nil { - t.Fatalf("failed to read from conn: %v", err) + t.Errorf("failed to read from conn: %v", err) + // fallthrough to chan send below } ch <- sess }() @@ -48,6 +49,10 @@ func TestSetUDPSocketOptions(t *testing.T) { t.Fatalf("failed to write to conn: %v", err) } sess := <-ch + if sess == nil { + // t.Error was already called in the goroutine above. + t.FailNow() + } if len(sess.context) == 0 { t.Fatalf("empty session context: %v", sess) } diff --git a/vendor/github.com/miekg/dns/udp_windows.go b/vendor/github.com/miekg/dns/udp_windows.go index 6778c3c..e7dd8ca 100644 --- a/vendor/github.com/miekg/dns/udp_windows.go +++ b/vendor/github.com/miekg/dns/udp_windows.go @@ -20,15 +20,13 @@ func ReadFromSessionUDP(conn *net.UDPConn, b []byte) (int, *SessionUDP, error) { if err != nil { return n, nil, err } - session := &SessionUDP{raddr.(*net.UDPAddr)} - return n, session, err + return n, &SessionUDP{raddr.(*net.UDPAddr)}, err } // WriteToSessionUDP acts just like net.UDPConn.WriteTo(), but uses a *SessionUDP instead of a net.Addr. // TODO(fastest963): Once go1.10 is released, use WriteMsgUDP. func WriteToSessionUDP(conn *net.UDPConn, b []byte, session *SessionUDP) (int, error) { - n, err := conn.WriteTo(b, session.raddr) - return n, err + return conn.WriteTo(b, session.raddr) } // TODO(fastest963): Once go1.10 is released and we can use *MsgUDP methods diff --git a/vendor/github.com/miekg/dns/update.go b/vendor/github.com/miekg/dns/update.go index e90c5c9..69dd386 100644 --- a/vendor/github.com/miekg/dns/update.go +++ b/vendor/github.com/miekg/dns/update.go @@ -44,7 +44,8 @@ func (u *Msg) RRsetUsed(rr []RR) { u.Answer = make([]RR, 0, len(rr)) } for _, r := range rr { - u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: r.Header().Rrtype, Class: ClassANY}}) + h := r.Header() + u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: h.Name, Ttl: 0, Rrtype: h.Rrtype, Class: ClassANY}}) } } @@ -55,7 +56,8 @@ func (u *Msg) RRsetNotUsed(rr []RR) { u.Answer = make([]RR, 0, len(rr)) } for _, r := range rr { - u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: r.Header().Rrtype, Class: ClassNONE}}) + h := r.Header() + u.Answer = append(u.Answer, &ANY{Hdr: RR_Header{Name: h.Name, Ttl: 0, Rrtype: h.Rrtype, Class: ClassNONE}}) } } @@ -79,7 +81,8 @@ func (u *Msg) RemoveRRset(rr []RR) { u.Ns = make([]RR, 0, len(rr)) } for _, r := range rr { - u.Ns = append(u.Ns, &ANY{Hdr: RR_Header{Name: r.Header().Name, Ttl: 0, Rrtype: r.Header().Rrtype, Class: ClassANY}}) + h := r.Header() + u.Ns = append(u.Ns, &ANY{Hdr: RR_Header{Name: h.Name, Ttl: 0, Rrtype: h.Rrtype, Class: ClassANY}}) } } @@ -99,8 +102,9 @@ func (u *Msg) Remove(rr []RR) { u.Ns = make([]RR, 0, len(rr)) } for _, r := range rr { - r.Header().Class = ClassNONE - r.Header().Ttl = 0 + h := r.Header() + h.Class = ClassNONE + h.Ttl = 0 u.Ns = append(u.Ns, r) } } diff --git a/vendor/github.com/miekg/dns/version.go b/vendor/github.com/miekg/dns/version.go index dcc84e4..46d644c 100644 --- a/vendor/github.com/miekg/dns/version.go +++ b/vendor/github.com/miekg/dns/version.go @@ -3,7 +3,7 @@ package dns import "fmt" // Version is current version of this library. -var Version = V{1, 0, 8} +var Version = V{1, 1, 4} // V holds the version of this library. type V struct { diff --git a/vendor/github.com/miekg/dns/xfr.go b/vendor/github.com/miekg/dns/xfr.go index 5d0ff5c..82afc52 100644 --- a/vendor/github.com/miekg/dns/xfr.go +++ b/vendor/github.com/miekg/dns/xfr.go @@ -35,30 +35,36 @@ type Transfer struct { // channel, err := transfer.In(message, master) // func (t *Transfer) In(q *Msg, a string) (env chan *Envelope, err error) { + switch q.Question[0].Qtype { + case TypeAXFR, TypeIXFR: + default: + return nil, &Error{"unsupported question type"} + } + timeout := dnsTimeout if t.DialTimeout != 0 { timeout = t.DialTimeout } + if t.Conn == nil { t.Conn, err = DialTimeout("tcp", a, timeout) if err != nil { return nil, err } } + if err := t.WriteMsg(q); err != nil { return nil, err } + env = make(chan *Envelope) - go func() { - if q.Question[0].Qtype == TypeAXFR { - go t.inAxfr(q, env) - return - } - if q.Question[0].Qtype == TypeIXFR { - go t.inIxfr(q, env) - return - } - }() + switch q.Question[0].Qtype { + case TypeAXFR: + go t.inAxfr(q, env) + case TypeIXFR: + go t.inIxfr(q, env) + } + return env, nil } @@ -111,7 +117,7 @@ func (t *Transfer) inAxfr(q *Msg, c chan *Envelope) { } func (t *Transfer) inIxfr(q *Msg, c chan *Envelope) { - serial := uint32(0) // The first serial seen is the current server serial + var serial uint32 // The first serial seen is the current server serial axfr := true n := 0 qser := q.Ns[0].(*SOA).Serial @@ -237,24 +243,18 @@ func (t *Transfer) WriteMsg(m *Msg) (err error) { if err != nil { return err } - if _, err = t.Write(out); err != nil { - return err - } - return nil + _, err = t.Write(out) + return err } func isSOAFirst(in *Msg) bool { - if len(in.Answer) > 0 { - return in.Answer[0].Header().Rrtype == TypeSOA - } - return false + return len(in.Answer) > 0 && + in.Answer[0].Header().Rrtype == TypeSOA } func isSOALast(in *Msg) bool { - if len(in.Answer) > 0 { - return in.Answer[len(in.Answer)-1].Header().Rrtype == TypeSOA - } - return false + return len(in.Answer) > 0 && + in.Answer[len(in.Answer)-1].Header().Rrtype == TypeSOA } const errXFR = "bad xfr rcode: %d" diff --git a/vendor/github.com/miekg/dns/zcompress.go b/vendor/github.com/miekg/dns/zcompress.go deleted file mode 100644 index a2c09dd..0000000 --- a/vendor/github.com/miekg/dns/zcompress.go +++ /dev/null @@ -1,155 +0,0 @@ -// Code generated by "go run compress_generate.go"; DO NOT EDIT. - -package dns - -func compressionLenHelperType(c map[string]int, r RR, initLen int) int { - currentLen := initLen - switch x := r.(type) { - case *AFSDB: - currentLen -= len(x.Hostname) + 1 - currentLen += compressionLenHelper(c, x.Hostname, currentLen) - case *CNAME: - currentLen -= len(x.Target) + 1 - currentLen += compressionLenHelper(c, x.Target, currentLen) - case *DNAME: - currentLen -= len(x.Target) + 1 - currentLen += compressionLenHelper(c, x.Target, currentLen) - case *HIP: - for i := range x.RendezvousServers { - currentLen -= len(x.RendezvousServers[i]) + 1 - } - for i := range x.RendezvousServers { - currentLen += compressionLenHelper(c, x.RendezvousServers[i], currentLen) - } - case *KX: - currentLen -= len(x.Exchanger) + 1 - currentLen += compressionLenHelper(c, x.Exchanger, currentLen) - case *LP: - currentLen -= len(x.Fqdn) + 1 - currentLen += compressionLenHelper(c, x.Fqdn, currentLen) - case *MB: - currentLen -= len(x.Mb) + 1 - currentLen += compressionLenHelper(c, x.Mb, currentLen) - case *MD: - currentLen -= len(x.Md) + 1 - currentLen += compressionLenHelper(c, x.Md, currentLen) - case *MF: - currentLen -= len(x.Mf) + 1 - currentLen += compressionLenHelper(c, x.Mf, currentLen) - case *MG: - currentLen -= len(x.Mg) + 1 - currentLen += compressionLenHelper(c, x.Mg, currentLen) - case *MINFO: - currentLen -= len(x.Rmail) + 1 - currentLen += compressionLenHelper(c, x.Rmail, currentLen) - currentLen -= len(x.Email) + 1 - currentLen += compressionLenHelper(c, x.Email, currentLen) - case *MR: - currentLen -= len(x.Mr) + 1 - currentLen += compressionLenHelper(c, x.Mr, currentLen) - case *MX: - currentLen -= len(x.Mx) + 1 - currentLen += compressionLenHelper(c, x.Mx, currentLen) - case *NAPTR: - currentLen -= len(x.Replacement) + 1 - currentLen += compressionLenHelper(c, x.Replacement, currentLen) - case *NS: - currentLen -= len(x.Ns) + 1 - currentLen += compressionLenHelper(c, x.Ns, currentLen) - case *NSAPPTR: - currentLen -= len(x.Ptr) + 1 - currentLen += compressionLenHelper(c, x.Ptr, currentLen) - case *NSEC: - currentLen -= len(x.NextDomain) + 1 - currentLen += compressionLenHelper(c, x.NextDomain, currentLen) - case *PTR: - currentLen -= len(x.Ptr) + 1 - currentLen += compressionLenHelper(c, x.Ptr, currentLen) - case *PX: - currentLen -= len(x.Map822) + 1 - currentLen += compressionLenHelper(c, x.Map822, currentLen) - currentLen -= len(x.Mapx400) + 1 - currentLen += compressionLenHelper(c, x.Mapx400, currentLen) - case *RP: - currentLen -= len(x.Mbox) + 1 - currentLen += compressionLenHelper(c, x.Mbox, currentLen) - currentLen -= len(x.Txt) + 1 - currentLen += compressionLenHelper(c, x.Txt, currentLen) - case *RRSIG: - currentLen -= len(x.SignerName) + 1 - currentLen += compressionLenHelper(c, x.SignerName, currentLen) - case *RT: - currentLen -= len(x.Host) + 1 - currentLen += compressionLenHelper(c, x.Host, currentLen) - case *SIG: - currentLen -= len(x.SignerName) + 1 - currentLen += compressionLenHelper(c, x.SignerName, currentLen) - case *SOA: - currentLen -= len(x.Ns) + 1 - currentLen += compressionLenHelper(c, x.Ns, currentLen) - currentLen -= len(x.Mbox) + 1 - currentLen += compressionLenHelper(c, x.Mbox, currentLen) - case *SRV: - currentLen -= len(x.Target) + 1 - currentLen += compressionLenHelper(c, x.Target, currentLen) - case *TALINK: - currentLen -= len(x.PreviousName) + 1 - currentLen += compressionLenHelper(c, x.PreviousName, currentLen) - currentLen -= len(x.NextName) + 1 - currentLen += compressionLenHelper(c, x.NextName, currentLen) - case *TKEY: - currentLen -= len(x.Algorithm) + 1 - currentLen += compressionLenHelper(c, x.Algorithm, currentLen) - case *TSIG: - currentLen -= len(x.Algorithm) + 1 - currentLen += compressionLenHelper(c, x.Algorithm, currentLen) - } - return currentLen - initLen -} - -func compressionLenSearchType(c map[string]int, r RR) (int, bool, int) { - switch x := r.(type) { - case *AFSDB: - k1, ok1, sz1 := compressionLenSearch(c, x.Hostname) - return k1, ok1, sz1 - case *CNAME: - k1, ok1, sz1 := compressionLenSearch(c, x.Target) - return k1, ok1, sz1 - case *MB: - k1, ok1, sz1 := compressionLenSearch(c, x.Mb) - return k1, ok1, sz1 - case *MD: - k1, ok1, sz1 := compressionLenSearch(c, x.Md) - return k1, ok1, sz1 - case *MF: - k1, ok1, sz1 := compressionLenSearch(c, x.Mf) - return k1, ok1, sz1 - case *MG: - k1, ok1, sz1 := compressionLenSearch(c, x.Mg) - return k1, ok1, sz1 - case *MINFO: - k1, ok1, sz1 := compressionLenSearch(c, x.Rmail) - k2, ok2, sz2 := compressionLenSearch(c, x.Email) - return k1 + k2, ok1 && ok2, sz1 + sz2 - case *MR: - k1, ok1, sz1 := compressionLenSearch(c, x.Mr) - return k1, ok1, sz1 - case *MX: - k1, ok1, sz1 := compressionLenSearch(c, x.Mx) - return k1, ok1, sz1 - case *NS: - k1, ok1, sz1 := compressionLenSearch(c, x.Ns) - return k1, ok1, sz1 - case *PTR: - k1, ok1, sz1 := compressionLenSearch(c, x.Ptr) - return k1, ok1, sz1 - case *RT: - k1, ok1, sz1 := compressionLenSearch(c, x.Host) - return k1, ok1, sz1 - case *SOA: - k1, ok1, sz1 := compressionLenSearch(c, x.Ns) - k2, ok2, sz2 := compressionLenSearch(c, x.Mbox) - return k1 + k2, ok1 && ok2, sz1 + sz2 - } - return 0, false, 0 -} diff --git a/vendor/github.com/miekg/dns/zduplicate.go b/vendor/github.com/miekg/dns/zduplicate.go new file mode 100644 index 0000000..81e99e0 --- /dev/null +++ b/vendor/github.com/miekg/dns/zduplicate.go @@ -0,0 +1,1140 @@ +// Code generated by "go run duplicate_generate.go"; DO NOT EDIT. + +package dns + +// isDuplicate() functions + +func (r1 *A) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*A) + if !ok { + return false + } + _ = r2 + if !r1.A.Equal(r2.A) { + return false + } + return true +} + +func (r1 *AAAA) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*AAAA) + if !ok { + return false + } + _ = r2 + if !r1.AAAA.Equal(r2.AAAA) { + return false + } + return true +} + +func (r1 *AFSDB) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*AFSDB) + if !ok { + return false + } + _ = r2 + if r1.Subtype != r2.Subtype { + return false + } + if !isDulicateName(r1.Hostname, r2.Hostname) { + return false + } + return true +} + +func (r1 *ANY) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*ANY) + if !ok { + return false + } + _ = r2 + return true +} + +func (r1 *AVC) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*AVC) + if !ok { + return false + } + _ = r2 + if len(r1.Txt) != len(r2.Txt) { + return false + } + for i := 0; i < len(r1.Txt); i++ { + if r1.Txt[i] != r2.Txt[i] { + return false + } + } + return true +} + +func (r1 *CAA) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*CAA) + if !ok { + return false + } + _ = r2 + if r1.Flag != r2.Flag { + return false + } + if r1.Tag != r2.Tag { + return false + } + if r1.Value != r2.Value { + return false + } + return true +} + +func (r1 *CERT) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*CERT) + if !ok { + return false + } + _ = r2 + if r1.Type != r2.Type { + return false + } + if r1.KeyTag != r2.KeyTag { + return false + } + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.Certificate != r2.Certificate { + return false + } + return true +} + +func (r1 *CNAME) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*CNAME) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Target, r2.Target) { + return false + } + return true +} + +func (r1 *CSYNC) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*CSYNC) + if !ok { + return false + } + _ = r2 + if r1.Serial != r2.Serial { + return false + } + if r1.Flags != r2.Flags { + return false + } + if len(r1.TypeBitMap) != len(r2.TypeBitMap) { + return false + } + for i := 0; i < len(r1.TypeBitMap); i++ { + if r1.TypeBitMap[i] != r2.TypeBitMap[i] { + return false + } + } + return true +} + +func (r1 *DHCID) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*DHCID) + if !ok { + return false + } + _ = r2 + if r1.Digest != r2.Digest { + return false + } + return true +} + +func (r1 *DNAME) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*DNAME) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Target, r2.Target) { + return false + } + return true +} + +func (r1 *DNSKEY) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*DNSKEY) + if !ok { + return false + } + _ = r2 + if r1.Flags != r2.Flags { + return false + } + if r1.Protocol != r2.Protocol { + return false + } + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.PublicKey != r2.PublicKey { + return false + } + return true +} + +func (r1 *DS) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*DS) + if !ok { + return false + } + _ = r2 + if r1.KeyTag != r2.KeyTag { + return false + } + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.DigestType != r2.DigestType { + return false + } + if r1.Digest != r2.Digest { + return false + } + return true +} + +func (r1 *EID) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*EID) + if !ok { + return false + } + _ = r2 + if r1.Endpoint != r2.Endpoint { + return false + } + return true +} + +func (r1 *EUI48) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*EUI48) + if !ok { + return false + } + _ = r2 + if r1.Address != r2.Address { + return false + } + return true +} + +func (r1 *EUI64) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*EUI64) + if !ok { + return false + } + _ = r2 + if r1.Address != r2.Address { + return false + } + return true +} + +func (r1 *GID) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*GID) + if !ok { + return false + } + _ = r2 + if r1.Gid != r2.Gid { + return false + } + return true +} + +func (r1 *GPOS) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*GPOS) + if !ok { + return false + } + _ = r2 + if r1.Longitude != r2.Longitude { + return false + } + if r1.Latitude != r2.Latitude { + return false + } + if r1.Altitude != r2.Altitude { + return false + } + return true +} + +func (r1 *HINFO) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*HINFO) + if !ok { + return false + } + _ = r2 + if r1.Cpu != r2.Cpu { + return false + } + if r1.Os != r2.Os { + return false + } + return true +} + +func (r1 *HIP) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*HIP) + if !ok { + return false + } + _ = r2 + if r1.HitLength != r2.HitLength { + return false + } + if r1.PublicKeyAlgorithm != r2.PublicKeyAlgorithm { + return false + } + if r1.PublicKeyLength != r2.PublicKeyLength { + return false + } + if r1.Hit != r2.Hit { + return false + } + if r1.PublicKey != r2.PublicKey { + return false + } + if len(r1.RendezvousServers) != len(r2.RendezvousServers) { + return false + } + for i := 0; i < len(r1.RendezvousServers); i++ { + if !isDulicateName(r1.RendezvousServers[i], r2.RendezvousServers[i]) { + return false + } + } + return true +} + +func (r1 *KX) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*KX) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if !isDulicateName(r1.Exchanger, r2.Exchanger) { + return false + } + return true +} + +func (r1 *L32) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*L32) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if !r1.Locator32.Equal(r2.Locator32) { + return false + } + return true +} + +func (r1 *L64) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*L64) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if r1.Locator64 != r2.Locator64 { + return false + } + return true +} + +func (r1 *LOC) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*LOC) + if !ok { + return false + } + _ = r2 + if r1.Version != r2.Version { + return false + } + if r1.Size != r2.Size { + return false + } + if r1.HorizPre != r2.HorizPre { + return false + } + if r1.VertPre != r2.VertPre { + return false + } + if r1.Latitude != r2.Latitude { + return false + } + if r1.Longitude != r2.Longitude { + return false + } + if r1.Altitude != r2.Altitude { + return false + } + return true +} + +func (r1 *LP) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*LP) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if !isDulicateName(r1.Fqdn, r2.Fqdn) { + return false + } + return true +} + +func (r1 *MB) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MB) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Mb, r2.Mb) { + return false + } + return true +} + +func (r1 *MD) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MD) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Md, r2.Md) { + return false + } + return true +} + +func (r1 *MF) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MF) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Mf, r2.Mf) { + return false + } + return true +} + +func (r1 *MG) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MG) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Mg, r2.Mg) { + return false + } + return true +} + +func (r1 *MINFO) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MINFO) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Rmail, r2.Rmail) { + return false + } + if !isDulicateName(r1.Email, r2.Email) { + return false + } + return true +} + +func (r1 *MR) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MR) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Mr, r2.Mr) { + return false + } + return true +} + +func (r1 *MX) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*MX) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if !isDulicateName(r1.Mx, r2.Mx) { + return false + } + return true +} + +func (r1 *NAPTR) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NAPTR) + if !ok { + return false + } + _ = r2 + if r1.Order != r2.Order { + return false + } + if r1.Preference != r2.Preference { + return false + } + if r1.Flags != r2.Flags { + return false + } + if r1.Service != r2.Service { + return false + } + if r1.Regexp != r2.Regexp { + return false + } + if !isDulicateName(r1.Replacement, r2.Replacement) { + return false + } + return true +} + +func (r1 *NID) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NID) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if r1.NodeID != r2.NodeID { + return false + } + return true +} + +func (r1 *NIMLOC) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NIMLOC) + if !ok { + return false + } + _ = r2 + if r1.Locator != r2.Locator { + return false + } + return true +} + +func (r1 *NINFO) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NINFO) + if !ok { + return false + } + _ = r2 + if len(r1.ZSData) != len(r2.ZSData) { + return false + } + for i := 0; i < len(r1.ZSData); i++ { + if r1.ZSData[i] != r2.ZSData[i] { + return false + } + } + return true +} + +func (r1 *NS) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NS) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Ns, r2.Ns) { + return false + } + return true +} + +func (r1 *NSAPPTR) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NSAPPTR) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Ptr, r2.Ptr) { + return false + } + return true +} + +func (r1 *NSEC) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NSEC) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.NextDomain, r2.NextDomain) { + return false + } + if len(r1.TypeBitMap) != len(r2.TypeBitMap) { + return false + } + for i := 0; i < len(r1.TypeBitMap); i++ { + if r1.TypeBitMap[i] != r2.TypeBitMap[i] { + return false + } + } + return true +} + +func (r1 *NSEC3) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NSEC3) + if !ok { + return false + } + _ = r2 + if r1.Hash != r2.Hash { + return false + } + if r1.Flags != r2.Flags { + return false + } + if r1.Iterations != r2.Iterations { + return false + } + if r1.SaltLength != r2.SaltLength { + return false + } + if r1.Salt != r2.Salt { + return false + } + if r1.HashLength != r2.HashLength { + return false + } + if r1.NextDomain != r2.NextDomain { + return false + } + if len(r1.TypeBitMap) != len(r2.TypeBitMap) { + return false + } + for i := 0; i < len(r1.TypeBitMap); i++ { + if r1.TypeBitMap[i] != r2.TypeBitMap[i] { + return false + } + } + return true +} + +func (r1 *NSEC3PARAM) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NSEC3PARAM) + if !ok { + return false + } + _ = r2 + if r1.Hash != r2.Hash { + return false + } + if r1.Flags != r2.Flags { + return false + } + if r1.Iterations != r2.Iterations { + return false + } + if r1.SaltLength != r2.SaltLength { + return false + } + if r1.Salt != r2.Salt { + return false + } + return true +} + +func (r1 *NULL) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*NULL) + if !ok { + return false + } + _ = r2 + if r1.Data != r2.Data { + return false + } + return true +} + +func (r1 *OPENPGPKEY) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*OPENPGPKEY) + if !ok { + return false + } + _ = r2 + if r1.PublicKey != r2.PublicKey { + return false + } + return true +} + +func (r1 *PTR) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*PTR) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Ptr, r2.Ptr) { + return false + } + return true +} + +func (r1 *PX) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*PX) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if !isDulicateName(r1.Map822, r2.Map822) { + return false + } + if !isDulicateName(r1.Mapx400, r2.Mapx400) { + return false + } + return true +} + +func (r1 *RFC3597) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*RFC3597) + if !ok { + return false + } + _ = r2 + if r1.Rdata != r2.Rdata { + return false + } + return true +} + +func (r1 *RKEY) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*RKEY) + if !ok { + return false + } + _ = r2 + if r1.Flags != r2.Flags { + return false + } + if r1.Protocol != r2.Protocol { + return false + } + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.PublicKey != r2.PublicKey { + return false + } + return true +} + +func (r1 *RP) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*RP) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Mbox, r2.Mbox) { + return false + } + if !isDulicateName(r1.Txt, r2.Txt) { + return false + } + return true +} + +func (r1 *RRSIG) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*RRSIG) + if !ok { + return false + } + _ = r2 + if r1.TypeCovered != r2.TypeCovered { + return false + } + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.Labels != r2.Labels { + return false + } + if r1.OrigTtl != r2.OrigTtl { + return false + } + if r1.Expiration != r2.Expiration { + return false + } + if r1.Inception != r2.Inception { + return false + } + if r1.KeyTag != r2.KeyTag { + return false + } + if !isDulicateName(r1.SignerName, r2.SignerName) { + return false + } + if r1.Signature != r2.Signature { + return false + } + return true +} + +func (r1 *RT) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*RT) + if !ok { + return false + } + _ = r2 + if r1.Preference != r2.Preference { + return false + } + if !isDulicateName(r1.Host, r2.Host) { + return false + } + return true +} + +func (r1 *SMIMEA) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*SMIMEA) + if !ok { + return false + } + _ = r2 + if r1.Usage != r2.Usage { + return false + } + if r1.Selector != r2.Selector { + return false + } + if r1.MatchingType != r2.MatchingType { + return false + } + if r1.Certificate != r2.Certificate { + return false + } + return true +} + +func (r1 *SOA) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*SOA) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Ns, r2.Ns) { + return false + } + if !isDulicateName(r1.Mbox, r2.Mbox) { + return false + } + if r1.Serial != r2.Serial { + return false + } + if r1.Refresh != r2.Refresh { + return false + } + if r1.Retry != r2.Retry { + return false + } + if r1.Expire != r2.Expire { + return false + } + if r1.Minttl != r2.Minttl { + return false + } + return true +} + +func (r1 *SPF) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*SPF) + if !ok { + return false + } + _ = r2 + if len(r1.Txt) != len(r2.Txt) { + return false + } + for i := 0; i < len(r1.Txt); i++ { + if r1.Txt[i] != r2.Txt[i] { + return false + } + } + return true +} + +func (r1 *SRV) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*SRV) + if !ok { + return false + } + _ = r2 + if r1.Priority != r2.Priority { + return false + } + if r1.Weight != r2.Weight { + return false + } + if r1.Port != r2.Port { + return false + } + if !isDulicateName(r1.Target, r2.Target) { + return false + } + return true +} + +func (r1 *SSHFP) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*SSHFP) + if !ok { + return false + } + _ = r2 + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.Type != r2.Type { + return false + } + if r1.FingerPrint != r2.FingerPrint { + return false + } + return true +} + +func (r1 *TA) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*TA) + if !ok { + return false + } + _ = r2 + if r1.KeyTag != r2.KeyTag { + return false + } + if r1.Algorithm != r2.Algorithm { + return false + } + if r1.DigestType != r2.DigestType { + return false + } + if r1.Digest != r2.Digest { + return false + } + return true +} + +func (r1 *TALINK) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*TALINK) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.PreviousName, r2.PreviousName) { + return false + } + if !isDulicateName(r1.NextName, r2.NextName) { + return false + } + return true +} + +func (r1 *TKEY) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*TKEY) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Algorithm, r2.Algorithm) { + return false + } + if r1.Inception != r2.Inception { + return false + } + if r1.Expiration != r2.Expiration { + return false + } + if r1.Mode != r2.Mode { + return false + } + if r1.Error != r2.Error { + return false + } + if r1.KeySize != r2.KeySize { + return false + } + if r1.Key != r2.Key { + return false + } + if r1.OtherLen != r2.OtherLen { + return false + } + if r1.OtherData != r2.OtherData { + return false + } + return true +} + +func (r1 *TLSA) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*TLSA) + if !ok { + return false + } + _ = r2 + if r1.Usage != r2.Usage { + return false + } + if r1.Selector != r2.Selector { + return false + } + if r1.MatchingType != r2.MatchingType { + return false + } + if r1.Certificate != r2.Certificate { + return false + } + return true +} + +func (r1 *TSIG) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*TSIG) + if !ok { + return false + } + _ = r2 + if !isDulicateName(r1.Algorithm, r2.Algorithm) { + return false + } + if r1.TimeSigned != r2.TimeSigned { + return false + } + if r1.Fudge != r2.Fudge { + return false + } + if r1.MACSize != r2.MACSize { + return false + } + if r1.MAC != r2.MAC { + return false + } + if r1.OrigId != r2.OrigId { + return false + } + if r1.Error != r2.Error { + return false + } + if r1.OtherLen != r2.OtherLen { + return false + } + if r1.OtherData != r2.OtherData { + return false + } + return true +} + +func (r1 *TXT) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*TXT) + if !ok { + return false + } + _ = r2 + if len(r1.Txt) != len(r2.Txt) { + return false + } + for i := 0; i < len(r1.Txt); i++ { + if r1.Txt[i] != r2.Txt[i] { + return false + } + } + return true +} + +func (r1 *UID) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*UID) + if !ok { + return false + } + _ = r2 + if r1.Uid != r2.Uid { + return false + } + return true +} + +func (r1 *UINFO) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*UINFO) + if !ok { + return false + } + _ = r2 + if r1.Uinfo != r2.Uinfo { + return false + } + return true +} + +func (r1 *URI) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*URI) + if !ok { + return false + } + _ = r2 + if r1.Priority != r2.Priority { + return false + } + if r1.Weight != r2.Weight { + return false + } + if r1.Target != r2.Target { + return false + } + return true +} + +func (r1 *X25) isDuplicate(_r2 RR) bool { + r2, ok := _r2.(*X25) + if !ok { + return false + } + _ = r2 + if r1.PSDNAddress != r2.PSDNAddress { + return false + } + return true +} diff --git a/vendor/github.com/miekg/dns/zmsg.go b/vendor/github.com/miekg/dns/zmsg.go index 0d1f6f4..c4cf475 100644 --- a/vendor/github.com/miekg/dns/zmsg.go +++ b/vendor/github.com/miekg/dns/zmsg.go @@ -4,82 +4,47 @@ package dns // pack*() functions -func (rr *A) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *A) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packDataA(rr.A, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *AAAA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *AAAA) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packDataAAAA(rr.AAAA, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *AFSDB) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *AFSDB) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Subtype, msg, off) if err != nil { return off, err } - off, err = PackDomainName(rr.Hostname, msg, off, compression, compress) + off, err = packDomainName(rr.Hostname, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *ANY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off - rr.Header().Rdlength = uint16(off - headerEnd) +func (rr *ANY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { return off, nil } -func (rr *AVC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *AVC) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringTxt(rr.Txt, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *CAA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *CAA) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Flag, msg, off) if err != nil { return off, err @@ -92,16 +57,10 @@ func (rr *CAA) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *CDNSKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *CDNSKEY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Flags, msg, off) if err != nil { return off, err @@ -118,16 +77,10 @@ func (rr *CDNSKEY) pack(msg []byte, off int, compression map[string]int, compres if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *CDS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *CDS) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.KeyTag, msg, off) if err != nil { return off, err @@ -144,16 +97,10 @@ func (rr *CDS) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *CERT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *CERT) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Type, msg, off) if err != nil { return off, err @@ -170,30 +117,18 @@ func (rr *CERT) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *CNAME) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *CNAME) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Target, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Target, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *CSYNC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *CSYNC) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint32(rr.Serial, msg, off) if err != nil { return off, err @@ -206,30 +141,18 @@ func (rr *CSYNC) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *DHCID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *DHCID) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringBase64(rr.Digest, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *DLV) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *DLV) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.KeyTag, msg, off) if err != nil { return off, err @@ -246,30 +169,18 @@ func (rr *DLV) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *DNAME) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *DNAME) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Target, msg, off, compression, false) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Target, msg, off, compression, false) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *DNSKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *DNSKEY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Flags, msg, off) if err != nil { return off, err @@ -286,16 +197,10 @@ func (rr *DNSKEY) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *DS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *DS) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.KeyTag, msg, off) if err != nil { return off, err @@ -312,72 +217,42 @@ func (rr *DS) pack(msg []byte, off int, compression map[string]int, compress boo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *EID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *EID) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringHex(rr.Endpoint, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *EUI48) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *EUI48) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint48(rr.Address, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *EUI64) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *EUI64) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint64(rr.Address, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *GID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *GID) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint32(rr.Gid, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *GPOS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *GPOS) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packString(rr.Longitude, msg, off) if err != nil { return off, err @@ -390,16 +265,10 @@ func (rr *GPOS) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *HINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *HINFO) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packString(rr.Cpu, msg, off) if err != nil { return off, err @@ -408,16 +277,10 @@ func (rr *HINFO) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *HIP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *HIP) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.HitLength, msg, off) if err != nil { return off, err @@ -438,20 +301,14 @@ func (rr *HIP) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - off, err = packDataDomainNames(rr.RendezvousServers, msg, off, compression, compress) + off, err = packDataDomainNames(rr.RendezvousServers, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *KEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *KEY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Flags, msg, off) if err != nil { return off, err @@ -468,34 +325,22 @@ func (rr *KEY) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *KX) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *KX) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err } - off, err = PackDomainName(rr.Exchanger, msg, off, compression, false) + off, err = packDomainName(rr.Exchanger, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *L32) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *L32) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err @@ -504,16 +349,10 @@ func (rr *L32) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *L64) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *L64) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err @@ -522,16 +361,10 @@ func (rr *L64) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *LOC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *LOC) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Version, msg, off) if err != nil { return off, err @@ -560,140 +393,86 @@ func (rr *LOC) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *LP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *LP) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err } - off, err = PackDomainName(rr.Fqdn, msg, off, compression, false) + off, err = packDomainName(rr.Fqdn, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MB) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *MB) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Mb, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Mb, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MD) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *MD) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Md, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Md, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MF) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *MF) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Mf, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Mf, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *MG) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Mg, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Mg, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *MINFO) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Rmail, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Rmail, msg, off, compression, compress) + off, err = packDomainName(rr.Email, msg, off, compression, compress) if err != nil { return off, err } - off, err = PackDomainName(rr.Email, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *MR) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Mr, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Mr, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *MX) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *MX) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err } - off, err = PackDomainName(rr.Mx, msg, off, compression, compress) + off, err = packDomainName(rr.Mx, msg, off, compression, compress) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NAPTR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *NAPTR) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Order, msg, off) if err != nil { return off, err @@ -714,20 +493,14 @@ func (rr *NAPTR) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - off, err = PackDomainName(rr.Replacement, msg, off, compression, false) + off, err = packDomainName(rr.Replacement, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *NID) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err @@ -736,73 +509,43 @@ func (rr *NID) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NIMLOC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *NIMLOC) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringHex(rr.Locator, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *NINFO) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringTxt(rr.ZSData, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NS) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *NS) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Ns, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Ns, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NSAPPTR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *NSAPPTR) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Ptr, msg, off, compression, false) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Ptr, msg, off, compression, false) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NSEC) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off - off, err = PackDomainName(rr.NextDomain, msg, off, compression, false) +func (rr *NSEC) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.NextDomain, msg, off, compression, false) if err != nil { return off, err } @@ -810,16 +553,10 @@ func (rr *NSEC) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NSEC3) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *NSEC3) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Hash, msg, off) if err != nil { return off, err @@ -855,16 +592,10 @@ func (rr *NSEC3) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *NSEC3PARAM) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *NSEC3PARAM) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Hash, msg, off) if err != nil { return off, err @@ -888,94 +619,66 @@ func (rr *NSEC3PARAM) pack(msg []byte, off int, compression map[string]int, comp return off, err } } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *OPENPGPKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *NULL) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packStringAny(rr.Data, msg, off) if err != nil { return off, err } - headerEnd := off + return off, nil +} + +func (rr *OPENPGPKEY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringBase64(rr.PublicKey, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *OPT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *OPT) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packDataOpt(rr.Option, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *PTR) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *PTR) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Ptr, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Ptr, msg, off, compression, compress) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *PX) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *PX) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err } - off, err = PackDomainName(rr.Map822, msg, off, compression, false) + off, err = packDomainName(rr.Map822, msg, off, compression, false) if err != nil { return off, err } - off, err = PackDomainName(rr.Mapx400, msg, off, compression, false) + off, err = packDomainName(rr.Mapx400, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *RFC3597) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *RFC3597) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringHex(rr.Rdata, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *RKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *RKEY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Flags, msg, off) if err != nil { return off, err @@ -992,34 +695,22 @@ func (rr *RKEY) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *RP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *RP) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Mbox, msg, off, compression, false) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Mbox, msg, off, compression, false) + off, err = packDomainName(rr.Txt, msg, off, compression, false) if err != nil { return off, err } - off, err = PackDomainName(rr.Txt, msg, off, compression, false) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *RRSIG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *RRSIG) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.TypeCovered, msg, off) if err != nil { return off, err @@ -1048,7 +739,7 @@ func (rr *RRSIG) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - off, err = PackDomainName(rr.SignerName, msg, off, compression, false) + off, err = packDomainName(rr.SignerName, msg, off, compression, false) if err != nil { return off, err } @@ -1056,34 +747,22 @@ func (rr *RRSIG) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *RT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *RT) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Preference, msg, off) if err != nil { return off, err } - off, err = PackDomainName(rr.Host, msg, off, compression, compress) + off, err = packDomainName(rr.Host, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *SIG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *SIG) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.TypeCovered, msg, off) if err != nil { return off, err @@ -1112,7 +791,7 @@ func (rr *SIG) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - off, err = PackDomainName(rr.SignerName, msg, off, compression, false) + off, err = packDomainName(rr.SignerName, msg, off, compression, false) if err != nil { return off, err } @@ -1120,16 +799,10 @@ func (rr *SIG) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *SMIMEA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *SMIMEA) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Usage, msg, off) if err != nil { return off, err @@ -1146,21 +819,15 @@ func (rr *SMIMEA) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *SOA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *SOA) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Ns, msg, off, compression, compress) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.Ns, msg, off, compression, compress) - if err != nil { - return off, err - } - off, err = PackDomainName(rr.Mbox, msg, off, compression, compress) + off, err = packDomainName(rr.Mbox, msg, off, compression, compress) if err != nil { return off, err } @@ -1184,30 +851,18 @@ func (rr *SOA) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *SPF) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *SPF) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringTxt(rr.Txt, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *SRV) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *SRV) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Priority, msg, off) if err != nil { return off, err @@ -1220,20 +875,14 @@ func (rr *SRV) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - off, err = PackDomainName(rr.Target, msg, off, compression, false) + off, err = packDomainName(rr.Target, msg, off, compression, false) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *SSHFP) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *SSHFP) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Algorithm, msg, off) if err != nil { return off, err @@ -1246,16 +895,10 @@ func (rr *SSHFP) pack(msg []byte, off int, compression map[string]int, compress if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *TA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *TA) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.KeyTag, msg, off) if err != nil { return off, err @@ -1272,35 +915,23 @@ func (rr *TA) pack(msg []byte, off int, compression map[string]int, compress boo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *TALINK) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) +func (rr *TALINK) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.PreviousName, msg, off, compression, false) if err != nil { return off, err } - headerEnd := off - off, err = PackDomainName(rr.PreviousName, msg, off, compression, false) + off, err = packDomainName(rr.NextName, msg, off, compression, false) if err != nil { return off, err } - off, err = PackDomainName(rr.NextName, msg, off, compression, false) - if err != nil { - return off, err - } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *TKEY) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off - off, err = PackDomainName(rr.Algorithm, msg, off, compression, false) +func (rr *TKEY) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Algorithm, msg, off, compression, false) if err != nil { return off, err } @@ -1336,16 +967,10 @@ func (rr *TKEY) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *TLSA) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *TLSA) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint8(rr.Usage, msg, off) if err != nil { return off, err @@ -1362,17 +987,11 @@ func (rr *TLSA) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *TSIG) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off - off, err = PackDomainName(rr.Algorithm, msg, off, compression, false) +func (rr *TSIG) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { + off, err = packDomainName(rr.Algorithm, msg, off, compression, false) if err != nil { return off, err } @@ -1408,58 +1027,34 @@ func (rr *TSIG) pack(msg []byte, off int, compression map[string]int, compress b if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *TXT) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *TXT) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packStringTxt(rr.Txt, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *UID) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *UID) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint32(rr.Uid, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *UINFO) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *UINFO) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packString(rr.Uinfo, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *URI) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *URI) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packUint16(rr.Priority, msg, off) if err != nil { return off, err @@ -1472,2144 +1067,1656 @@ func (rr *URI) pack(msg []byte, off int, compression map[string]int, compress bo if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } -func (rr *X25) pack(msg []byte, off int, compression map[string]int, compress bool) (int, error) { - off, err := rr.Hdr.pack(msg, off, compression, compress) - if err != nil { - return off, err - } - headerEnd := off +func (rr *X25) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) { off, err = packString(rr.PSDNAddress, msg, off) if err != nil { return off, err } - rr.Header().Rdlength = uint16(off - headerEnd) return off, nil } // unpack*() functions -func unpackA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(A) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *A) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.A, off, err = unpackDataA(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackAAAA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(AAAA) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *AAAA) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.AAAA, off, err = unpackDataAAAA(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackAFSDB(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(AFSDB) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *AFSDB) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Subtype, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Hostname, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackANY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(ANY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *ANY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart - return rr, off, err + return off, nil } -func unpackAVC(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(AVC) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *AVC) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Txt, off, err = unpackStringTxt(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackCAA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(CAA) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *CAA) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Flag, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Tag, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Value, off, err = unpackStringOctet(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackCDNSKEY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(CDNSKEY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *CDNSKEY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Flags, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Protocol, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackCDS(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(CDS) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *CDS) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.DigestType, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackCERT(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(CERT) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *CERT) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Type, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Certificate, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackCNAME(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(CNAME) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *CNAME) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Target, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackCSYNC(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(CSYNC) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *CSYNC) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Serial, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Flags, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.TypeBitMap, off, err = unpackDataNsec(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackDHCID(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(DHCID) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *DHCID) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Digest, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackDLV(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(DLV) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *DLV) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.DigestType, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackDNAME(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(DNAME) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *DNAME) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Target, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackDNSKEY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(DNSKEY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *DNSKEY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Flags, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Protocol, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackDS(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(DS) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *DS) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.DigestType, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackEID(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(EID) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *EID) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Endpoint, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackEUI48(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(EUI48) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *EUI48) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Address, off, err = unpackUint48(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackEUI64(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(EUI64) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *EUI64) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Address, off, err = unpackUint64(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackGID(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(GID) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *GID) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Gid, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackGPOS(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(GPOS) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *GPOS) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Longitude, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Latitude, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Altitude, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackHINFO(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(HINFO) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *HINFO) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Cpu, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Os, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackHIP(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(HIP) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *HIP) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.HitLength, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.PublicKeyAlgorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.PublicKeyLength, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Hit, off, err = unpackStringHex(msg, off, off+int(rr.HitLength)) if err != nil { - return rr, off, err + return off, err } rr.PublicKey, off, err = unpackStringBase64(msg, off, off+int(rr.PublicKeyLength)) if err != nil { - return rr, off, err + return off, err } rr.RendezvousServers, off, err = unpackDataDomainNames(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackKEY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(KEY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *KEY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Flags, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Protocol, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackKX(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(KX) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *KX) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Exchanger, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackL32(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(L32) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *L32) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Locator32, off, err = unpackDataA(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackL64(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(L64) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *L64) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Locator64, off, err = unpackUint64(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackLOC(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(LOC) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *LOC) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Version, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Size, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.HorizPre, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.VertPre, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Latitude, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Longitude, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Altitude, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackLP(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(LP) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *LP) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Fqdn, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMB(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MB) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MB) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Mb, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMD(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MD) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MD) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Md, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMF(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MF) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MF) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Mf, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMG(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MG) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MG) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Mg, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMINFO(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MINFO) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MINFO) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Rmail, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Email, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMR(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MR) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MR) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Mr, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackMX(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(MX) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *MX) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Mx, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNAPTR(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NAPTR) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NAPTR) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Order, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Flags, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Service, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Regexp, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Replacement, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNID(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NID) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NID) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.NodeID, off, err = unpackUint64(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNIMLOC(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NIMLOC) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NIMLOC) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Locator, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNINFO(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NINFO) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NINFO) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.ZSData, off, err = unpackStringTxt(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNS(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NS) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NS) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Ns, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNSAPPTR(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NSAPPTR) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NSAPPTR) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Ptr, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNSEC(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NSEC) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NSEC) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.NextDomain, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.TypeBitMap, off, err = unpackDataNsec(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNSEC3(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NSEC3) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NSEC3) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Hash, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Flags, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Iterations, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.SaltLength, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Salt, off, err = unpackStringHex(msg, off, off+int(rr.SaltLength)) if err != nil { - return rr, off, err + return off, err } rr.HashLength, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.NextDomain, off, err = unpackStringBase32(msg, off, off+int(rr.HashLength)) if err != nil { - return rr, off, err + return off, err } rr.TypeBitMap, off, err = unpackDataNsec(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackNSEC3PARAM(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(NSEC3PARAM) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *NSEC3PARAM) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Hash, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Flags, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Iterations, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.SaltLength, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Salt, off, err = unpackStringHex(msg, off, off+int(rr.SaltLength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackOPENPGPKEY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(OPENPGPKEY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil +func (rr *NULL) unpack(msg []byte, off int) (off1 int, err error) { + rdStart := off + _ = rdStart + + rr.Data, off, err = unpackStringAny(msg, off, rdStart+int(rr.Hdr.Rdlength)) + if err != nil { + return off, err } - var err error + return off, nil +} + +func (rr *OPENPGPKEY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackOPT(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(OPT) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *OPT) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Option, off, err = unpackDataOpt(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackPTR(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(PTR) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *PTR) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Ptr, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackPX(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(PX) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *PX) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Map822, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Mapx400, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackRFC3597(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(RFC3597) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *RFC3597) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Rdata, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackRKEY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(RKEY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *RKEY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Flags, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Protocol, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.PublicKey, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackRP(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(RP) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *RP) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Mbox, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Txt, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackRRSIG(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(RRSIG) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *RRSIG) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.TypeCovered, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Labels, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.OrigTtl, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Expiration, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Inception, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.SignerName, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Signature, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackRT(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(RT) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *RT) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Preference, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Host, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackSIG(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(SIG) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *SIG) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.TypeCovered, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Labels, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.OrigTtl, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Expiration, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Inception, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.SignerName, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Signature, off, err = unpackStringBase64(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackSMIMEA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(SMIMEA) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *SMIMEA) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Usage, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Selector, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.MatchingType, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Certificate, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackSOA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(SOA) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *SOA) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Ns, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Mbox, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Serial, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Refresh, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Retry, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Expire, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Minttl, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackSPF(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(SPF) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *SPF) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Txt, off, err = unpackStringTxt(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackSRV(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(SRV) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *SRV) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Priority, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Weight, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Port, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Target, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackSSHFP(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(SSHFP) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *SSHFP) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Type, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.FingerPrint, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackTA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(TA) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *TA) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.KeyTag, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Algorithm, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.DigestType, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Digest, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackTALINK(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(TALINK) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *TALINK) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.PreviousName, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.NextName, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackTKEY(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(TKEY) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *TKEY) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Algorithm, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Inception, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Expiration, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Mode, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Error, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.KeySize, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Key, off, err = unpackStringHex(msg, off, off+int(rr.KeySize)) if err != nil { - return rr, off, err + return off, err } rr.OtherLen, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.OtherData, off, err = unpackStringHex(msg, off, off+int(rr.OtherLen)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackTLSA(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(TLSA) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *TLSA) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Usage, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Selector, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.MatchingType, off, err = unpackUint8(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Certificate, off, err = unpackStringHex(msg, off, rdStart+int(rr.Hdr.Rdlength)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackTSIG(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(TSIG) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *TSIG) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Algorithm, off, err = UnpackDomainName(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.TimeSigned, off, err = unpackUint48(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Fudge, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.MACSize, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.MAC, off, err = unpackStringHex(msg, off, off+int(rr.MACSize)) if err != nil { - return rr, off, err + return off, err } rr.OrigId, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Error, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.OtherLen, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.OtherData, off, err = unpackStringHex(msg, off, off+int(rr.OtherLen)) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackTXT(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(TXT) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *TXT) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Txt, off, err = unpackStringTxt(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackUID(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(UID) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *UID) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Uid, off, err = unpackUint32(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackUINFO(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(UINFO) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *UINFO) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Uinfo, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackURI(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(URI) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *URI) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.Priority, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Weight, off, err = unpackUint16(msg, off) if err != nil { - return rr, off, err + return off, err } if off == len(msg) { - return rr, off, nil + return off, nil } rr.Target, off, err = unpackStringOctet(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err + return off, nil } -func unpackX25(h RR_Header, msg []byte, off int) (RR, int, error) { - rr := new(X25) - rr.Hdr = h - if noRdata(h) { - return rr, off, nil - } - var err error +func (rr *X25) unpack(msg []byte, off int) (off1 int, err error) { rdStart := off _ = rdStart rr.PSDNAddress, off, err = unpackString(msg, off) if err != nil { - return rr, off, err + return off, err } - return rr, off, err -} - -var typeToUnpack = map[uint16]func(RR_Header, []byte, int) (RR, int, error){ - TypeA: unpackA, - TypeAAAA: unpackAAAA, - TypeAFSDB: unpackAFSDB, - TypeANY: unpackANY, - TypeAVC: unpackAVC, - TypeCAA: unpackCAA, - TypeCDNSKEY: unpackCDNSKEY, - TypeCDS: unpackCDS, - TypeCERT: unpackCERT, - TypeCNAME: unpackCNAME, - TypeCSYNC: unpackCSYNC, - TypeDHCID: unpackDHCID, - TypeDLV: unpackDLV, - TypeDNAME: unpackDNAME, - TypeDNSKEY: unpackDNSKEY, - TypeDS: unpackDS, - TypeEID: unpackEID, - TypeEUI48: unpackEUI48, - TypeEUI64: unpackEUI64, - TypeGID: unpackGID, - TypeGPOS: unpackGPOS, - TypeHINFO: unpackHINFO, - TypeHIP: unpackHIP, - TypeKEY: unpackKEY, - TypeKX: unpackKX, - TypeL32: unpackL32, - TypeL64: unpackL64, - TypeLOC: unpackLOC, - TypeLP: unpackLP, - TypeMB: unpackMB, - TypeMD: unpackMD, - TypeMF: unpackMF, - TypeMG: unpackMG, - TypeMINFO: unpackMINFO, - TypeMR: unpackMR, - TypeMX: unpackMX, - TypeNAPTR: unpackNAPTR, - TypeNID: unpackNID, - TypeNIMLOC: unpackNIMLOC, - TypeNINFO: unpackNINFO, - TypeNS: unpackNS, - TypeNSAPPTR: unpackNSAPPTR, - TypeNSEC: unpackNSEC, - TypeNSEC3: unpackNSEC3, - TypeNSEC3PARAM: unpackNSEC3PARAM, - TypeOPENPGPKEY: unpackOPENPGPKEY, - TypeOPT: unpackOPT, - TypePTR: unpackPTR, - TypePX: unpackPX, - TypeRKEY: unpackRKEY, - TypeRP: unpackRP, - TypeRRSIG: unpackRRSIG, - TypeRT: unpackRT, - TypeSIG: unpackSIG, - TypeSMIMEA: unpackSMIMEA, - TypeSOA: unpackSOA, - TypeSPF: unpackSPF, - TypeSRV: unpackSRV, - TypeSSHFP: unpackSSHFP, - TypeTA: unpackTA, - TypeTALINK: unpackTALINK, - TypeTKEY: unpackTKEY, - TypeTLSA: unpackTLSA, - TypeTSIG: unpackTSIG, - TypeTXT: unpackTXT, - TypeUID: unpackUID, - TypeUINFO: unpackUINFO, - TypeURI: unpackURI, - TypeX25: unpackX25, + return off, nil } diff --git a/vendor/github.com/miekg/dns/ztypes.go b/vendor/github.com/miekg/dns/ztypes.go index 965753b..19a542d 100644 --- a/vendor/github.com/miekg/dns/ztypes.go +++ b/vendor/github.com/miekg/dns/ztypes.go @@ -54,6 +54,7 @@ var TypeToRR = map[uint16]func() RR{ TypeNSEC: func() RR { return new(NSEC) }, TypeNSEC3: func() RR { return new(NSEC3) }, TypeNSEC3PARAM: func() RR { return new(NSEC3PARAM) }, + TypeNULL: func() RR { return new(NULL) }, TypeOPENPGPKEY: func() RR { return new(OPENPGPKEY) }, TypeOPT: func() RR { return new(OPT) }, TypePTR: func() RR { return new(PTR) }, @@ -209,6 +210,7 @@ func (rr *NSAPPTR) Header() *RR_Header { return &rr.Hdr } func (rr *NSEC) Header() *RR_Header { return &rr.Hdr } func (rr *NSEC3) Header() *RR_Header { return &rr.Hdr } func (rr *NSEC3PARAM) Header() *RR_Header { return &rr.Hdr } +func (rr *NULL) Header() *RR_Header { return &rr.Hdr } func (rr *OPENPGPKEY) Header() *RR_Header { return &rr.Hdr } func (rr *OPT) Header() *RR_Header { return &rr.Hdr } func (rr *PTR) Header() *RR_Header { return &rr.Hdr } @@ -236,144 +238,144 @@ func (rr *URI) Header() *RR_Header { return &rr.Hdr } func (rr *X25) Header() *RR_Header { return &rr.Hdr } // len() functions -func (rr *A) len() int { - l := rr.Hdr.len() +func (rr *A) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += net.IPv4len // A return l } -func (rr *AAAA) len() int { - l := rr.Hdr.len() +func (rr *AAAA) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += net.IPv6len // AAAA return l } -func (rr *AFSDB) len() int { - l := rr.Hdr.len() +func (rr *AFSDB) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Subtype - l += len(rr.Hostname) + 1 + l += domainNameLen(rr.Hostname, off+l, compression, false) return l } -func (rr *ANY) len() int { - l := rr.Hdr.len() +func (rr *ANY) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) return l } -func (rr *AVC) len() int { - l := rr.Hdr.len() +func (rr *AVC) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) for _, x := range rr.Txt { l += len(x) + 1 } return l } -func (rr *CAA) len() int { - l := rr.Hdr.len() +func (rr *CAA) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // Flag l += len(rr.Tag) + 1 l += len(rr.Value) return l } -func (rr *CERT) len() int { - l := rr.Hdr.len() +func (rr *CERT) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Type l += 2 // KeyTag l++ // Algorithm l += base64.StdEncoding.DecodedLen(len(rr.Certificate)) return l } -func (rr *CNAME) len() int { - l := rr.Hdr.len() - l += len(rr.Target) + 1 +func (rr *CNAME) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Target, off+l, compression, true) return l } -func (rr *DHCID) len() int { - l := rr.Hdr.len() +func (rr *DHCID) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += base64.StdEncoding.DecodedLen(len(rr.Digest)) return l } -func (rr *DNAME) len() int { - l := rr.Hdr.len() - l += len(rr.Target) + 1 +func (rr *DNAME) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Target, off+l, compression, false) return l } -func (rr *DNSKEY) len() int { - l := rr.Hdr.len() +func (rr *DNSKEY) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Flags l++ // Protocol l++ // Algorithm l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) return l } -func (rr *DS) len() int { - l := rr.Hdr.len() +func (rr *DS) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // KeyTag l++ // Algorithm l++ // DigestType l += len(rr.Digest)/2 + 1 return l } -func (rr *EID) len() int { - l := rr.Hdr.len() +func (rr *EID) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.Endpoint)/2 + 1 return l } -func (rr *EUI48) len() int { - l := rr.Hdr.len() +func (rr *EUI48) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 6 // Address return l } -func (rr *EUI64) len() int { - l := rr.Hdr.len() +func (rr *EUI64) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 8 // Address return l } -func (rr *GID) len() int { - l := rr.Hdr.len() +func (rr *GID) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 4 // Gid return l } -func (rr *GPOS) len() int { - l := rr.Hdr.len() +func (rr *GPOS) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.Longitude) + 1 l += len(rr.Latitude) + 1 l += len(rr.Altitude) + 1 return l } -func (rr *HINFO) len() int { - l := rr.Hdr.len() +func (rr *HINFO) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.Cpu) + 1 l += len(rr.Os) + 1 return l } -func (rr *HIP) len() int { - l := rr.Hdr.len() +func (rr *HIP) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // HitLength l++ // PublicKeyAlgorithm l += 2 // PublicKeyLength l += len(rr.Hit) / 2 l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) for _, x := range rr.RendezvousServers { - l += len(x) + 1 + l += domainNameLen(x, off+l, compression, false) } return l } -func (rr *KX) len() int { - l := rr.Hdr.len() +func (rr *KX) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference - l += len(rr.Exchanger) + 1 + l += domainNameLen(rr.Exchanger, off+l, compression, false) return l } -func (rr *L32) len() int { - l := rr.Hdr.len() +func (rr *L32) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference l += net.IPv4len // Locator32 return l } -func (rr *L64) len() int { - l := rr.Hdr.len() +func (rr *L64) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference l += 8 // Locator64 return l } -func (rr *LOC) len() int { - l := rr.Hdr.len() +func (rr *LOC) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // Version l++ // Size l++ // HorizPre @@ -383,89 +385,89 @@ func (rr *LOC) len() int { l += 4 // Altitude return l } -func (rr *LP) len() int { - l := rr.Hdr.len() +func (rr *LP) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference - l += len(rr.Fqdn) + 1 + l += domainNameLen(rr.Fqdn, off+l, compression, false) return l } -func (rr *MB) len() int { - l := rr.Hdr.len() - l += len(rr.Mb) + 1 +func (rr *MB) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Mb, off+l, compression, true) return l } -func (rr *MD) len() int { - l := rr.Hdr.len() - l += len(rr.Md) + 1 +func (rr *MD) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Md, off+l, compression, true) return l } -func (rr *MF) len() int { - l := rr.Hdr.len() - l += len(rr.Mf) + 1 +func (rr *MF) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Mf, off+l, compression, true) return l } -func (rr *MG) len() int { - l := rr.Hdr.len() - l += len(rr.Mg) + 1 +func (rr *MG) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Mg, off+l, compression, true) return l } -func (rr *MINFO) len() int { - l := rr.Hdr.len() - l += len(rr.Rmail) + 1 - l += len(rr.Email) + 1 +func (rr *MINFO) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Rmail, off+l, compression, true) + l += domainNameLen(rr.Email, off+l, compression, true) return l } -func (rr *MR) len() int { - l := rr.Hdr.len() - l += len(rr.Mr) + 1 +func (rr *MR) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Mr, off+l, compression, true) return l } -func (rr *MX) len() int { - l := rr.Hdr.len() +func (rr *MX) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference - l += len(rr.Mx) + 1 + l += domainNameLen(rr.Mx, off+l, compression, true) return l } -func (rr *NAPTR) len() int { - l := rr.Hdr.len() +func (rr *NAPTR) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Order l += 2 // Preference l += len(rr.Flags) + 1 l += len(rr.Service) + 1 l += len(rr.Regexp) + 1 - l += len(rr.Replacement) + 1 + l += domainNameLen(rr.Replacement, off+l, compression, false) return l } -func (rr *NID) len() int { - l := rr.Hdr.len() +func (rr *NID) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference l += 8 // NodeID return l } -func (rr *NIMLOC) len() int { - l := rr.Hdr.len() +func (rr *NIMLOC) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.Locator)/2 + 1 return l } -func (rr *NINFO) len() int { - l := rr.Hdr.len() +func (rr *NINFO) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) for _, x := range rr.ZSData { l += len(x) + 1 } return l } -func (rr *NS) len() int { - l := rr.Hdr.len() - l += len(rr.Ns) + 1 +func (rr *NS) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Ns, off+l, compression, true) return l } -func (rr *NSAPPTR) len() int { - l := rr.Hdr.len() - l += len(rr.Ptr) + 1 +func (rr *NSAPPTR) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Ptr, off+l, compression, false) return l } -func (rr *NSEC3PARAM) len() int { - l := rr.Hdr.len() +func (rr *NSEC3PARAM) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // Hash l++ // Flags l += 2 // Iterations @@ -473,44 +475,49 @@ func (rr *NSEC3PARAM) len() int { l += len(rr.Salt) / 2 return l } -func (rr *OPENPGPKEY) len() int { - l := rr.Hdr.len() +func (rr *NULL) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += len(rr.Data) + return l +} +func (rr *OPENPGPKEY) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) return l } -func (rr *PTR) len() int { - l := rr.Hdr.len() - l += len(rr.Ptr) + 1 +func (rr *PTR) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Ptr, off+l, compression, true) return l } -func (rr *PX) len() int { - l := rr.Hdr.len() +func (rr *PX) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference - l += len(rr.Map822) + 1 - l += len(rr.Mapx400) + 1 + l += domainNameLen(rr.Map822, off+l, compression, false) + l += domainNameLen(rr.Mapx400, off+l, compression, false) return l } -func (rr *RFC3597) len() int { - l := rr.Hdr.len() +func (rr *RFC3597) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.Rdata)/2 + 1 return l } -func (rr *RKEY) len() int { - l := rr.Hdr.len() +func (rr *RKEY) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Flags l++ // Protocol l++ // Algorithm l += base64.StdEncoding.DecodedLen(len(rr.PublicKey)) return l } -func (rr *RP) len() int { - l := rr.Hdr.len() - l += len(rr.Mbox) + 1 - l += len(rr.Txt) + 1 +func (rr *RP) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Mbox, off+l, compression, false) + l += domainNameLen(rr.Txt, off+l, compression, false) return l } -func (rr *RRSIG) len() int { - l := rr.Hdr.len() +func (rr *RRSIG) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // TypeCovered l++ // Algorithm l++ // Labels @@ -518,28 +525,28 @@ func (rr *RRSIG) len() int { l += 4 // Expiration l += 4 // Inception l += 2 // KeyTag - l += len(rr.SignerName) + 1 + l += domainNameLen(rr.SignerName, off+l, compression, false) l += base64.StdEncoding.DecodedLen(len(rr.Signature)) return l } -func (rr *RT) len() int { - l := rr.Hdr.len() +func (rr *RT) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Preference - l += len(rr.Host) + 1 + l += domainNameLen(rr.Host, off+l, compression, false) return l } -func (rr *SMIMEA) len() int { - l := rr.Hdr.len() +func (rr *SMIMEA) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // Usage l++ // Selector l++ // MatchingType l += len(rr.Certificate)/2 + 1 return l } -func (rr *SOA) len() int { - l := rr.Hdr.len() - l += len(rr.Ns) + 1 - l += len(rr.Mbox) + 1 +func (rr *SOA) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Ns, off+l, compression, true) + l += domainNameLen(rr.Mbox, off+l, compression, true) l += 4 // Serial l += 4 // Refresh l += 4 // Retry @@ -547,45 +554,45 @@ func (rr *SOA) len() int { l += 4 // Minttl return l } -func (rr *SPF) len() int { - l := rr.Hdr.len() +func (rr *SPF) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) for _, x := range rr.Txt { l += len(x) + 1 } return l } -func (rr *SRV) len() int { - l := rr.Hdr.len() +func (rr *SRV) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Priority l += 2 // Weight l += 2 // Port - l += len(rr.Target) + 1 + l += domainNameLen(rr.Target, off+l, compression, false) return l } -func (rr *SSHFP) len() int { - l := rr.Hdr.len() +func (rr *SSHFP) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // Algorithm l++ // Type l += len(rr.FingerPrint)/2 + 1 return l } -func (rr *TA) len() int { - l := rr.Hdr.len() +func (rr *TA) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // KeyTag l++ // Algorithm l++ // DigestType l += len(rr.Digest)/2 + 1 return l } -func (rr *TALINK) len() int { - l := rr.Hdr.len() - l += len(rr.PreviousName) + 1 - l += len(rr.NextName) + 1 +func (rr *TALINK) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.PreviousName, off+l, compression, false) + l += domainNameLen(rr.NextName, off+l, compression, false) return l } -func (rr *TKEY) len() int { - l := rr.Hdr.len() - l += len(rr.Algorithm) + 1 +func (rr *TKEY) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Algorithm, off+l, compression, false) l += 4 // Inception l += 4 // Expiration l += 2 // Mode @@ -596,17 +603,17 @@ func (rr *TKEY) len() int { l += len(rr.OtherData) / 2 return l } -func (rr *TLSA) len() int { - l := rr.Hdr.len() +func (rr *TLSA) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l++ // Usage l++ // Selector l++ // MatchingType l += len(rr.Certificate)/2 + 1 return l } -func (rr *TSIG) len() int { - l := rr.Hdr.len() - l += len(rr.Algorithm) + 1 +func (rr *TSIG) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) + l += domainNameLen(rr.Algorithm, off+l, compression, false) l += 6 // TimeSigned l += 2 // Fudge l += 2 // MACSize @@ -617,32 +624,32 @@ func (rr *TSIG) len() int { l += len(rr.OtherData) / 2 return l } -func (rr *TXT) len() int { - l := rr.Hdr.len() +func (rr *TXT) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) for _, x := range rr.Txt { l += len(x) + 1 } return l } -func (rr *UID) len() int { - l := rr.Hdr.len() +func (rr *UID) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 4 // Uid return l } -func (rr *UINFO) len() int { - l := rr.Hdr.len() +func (rr *UINFO) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.Uinfo) + 1 return l } -func (rr *URI) len() int { - l := rr.Hdr.len() +func (rr *URI) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += 2 // Priority l += 2 // Weight l += len(rr.Target) return l } -func (rr *X25) len() int { - l := rr.Hdr.len() +func (rr *X25) len(off int, compression map[string]struct{}) int { + l := rr.Hdr.len(off, compression) l += len(rr.PSDNAddress) + 1 return l } @@ -783,12 +790,17 @@ func (rr *NSEC3) copy() RR { func (rr *NSEC3PARAM) copy() RR { return &NSEC3PARAM{rr.Hdr, rr.Hash, rr.Flags, rr.Iterations, rr.SaltLength, rr.Salt} } +func (rr *NULL) copy() RR { + return &NULL{rr.Hdr, rr.Data} +} func (rr *OPENPGPKEY) copy() RR { return &OPENPGPKEY{rr.Hdr, rr.PublicKey} } func (rr *OPT) copy() RR { Option := make([]EDNS0, len(rr.Option)) - copy(Option, rr.Option) + for i, e := range rr.Option { + Option[i] = e.copy() + } return &OPT{rr.Hdr, Option} } func (rr *PTR) copy() RR { diff --git a/vendor/github.com/rs/cors/README.md b/vendor/github.com/rs/cors/README.md index 87413cc..ecc83b2 100644 --- a/vendor/github.com/rs/cors/README.md +++ b/vendor/github.com/rs/cors/README.md @@ -68,6 +68,7 @@ Please refer to [#55](https://github.com/rs/cors/issues/55) for more information * [Gorilla](http://www.gorillatoolkit.org/pkg/mux): [examples/gorilla/server.go](https://github.com/rs/cors/blob/master/examples/gorilla/server.go) * [Buffalo](https://gobuffalo.io): [examples/buffalo/server.go](https://github.com/rs/cors/blob/master/examples/buffalo/server.go) * [Gin](https://gin-gonic.github.io/gin): [examples/gin/server.go](https://github.com/rs/cors/blob/master/examples/gin/server.go) +* [Chi](https://github.com/go-chi/chi): [examples/chi/server.go](https://github.com/rs/cors/blob/master/examples/chi/server.go) ## Parameters @@ -86,7 +87,8 @@ handler = c.Handler(handler) ``` * **AllowedOrigins** `[]string`: A list of origins a cross-domain request can be executed from. If the special `*` value is present in the list, all origins will be allowed. An origin may contain a wildcard (`*`) to replace 0 or more characters (i.e.: `http://*.domain.com`). Usage of wildcards implies a small performance penality. Only one wildcard can be used per origin. The default value is `*`. -* **AllowOriginFunc** `func (origin string) bool`: A custom function to validate the origin. It take the origin as argument and returns true if allowed or false otherwise. If this option is set, the content of `AllowedOrigins` is ignored +* **AllowOriginFunc** `func (origin string) bool`: A custom function to validate the origin. It takes the origin as an argument and returns true if allowed, or false otherwise. If this option is set, the content of `AllowedOrigins` is ignored. +* **AllowOriginRequestFunc** `func (r *http.Request origin string) bool`: A custom function to validate the origin. It takes the HTTP Request object and the origin as argument and returns true if allowed or false otherwise. If this option is set, the content of `AllowedOrigins` and `AllowOriginFunc` is ignored * **AllowedMethods** `[]string`: A list of methods the client is allowed to use with cross-domain requests. Default value is simple methods (`GET` and `POST`). * **AllowedHeaders** `[]string`: A list of non simple headers the client is allowed to use with cross-domain requests. * **ExposedHeaders** `[]string`: Indicates which headers are safe to expose to the API of a CORS API specification diff --git a/vendor/github.com/rs/cors/cors.go b/vendor/github.com/rs/cors/cors.go index 1518108..d301ca7 100644 --- a/vendor/github.com/rs/cors/cors.go +++ b/vendor/github.com/rs/cors/cors.go @@ -41,6 +41,10 @@ type Options struct { // as argument and returns true if allowed or false otherwise. If this option is // set, the content of AllowedOrigins is ignored. AllowOriginFunc func(origin string) bool + // AllowOriginFunc is a custom function to validate the origin. It takes the HTTP Request object and the origin as + // argument and returns true if allowed or false otherwise. If this option is set, the content of `AllowedOrigins` + // and `AllowOriginFunc` is ignored. + AllowOriginRequestFunc func(r *http.Request, origin string) bool // AllowedMethods is a list of methods the client is allowed to use with // cross-domain requests. Default value is simple methods (HEAD, GET and POST). AllowedMethods []string @@ -75,6 +79,8 @@ type Cors struct { allowedWOrigins []wildcard // Optional origin validator function allowOriginFunc func(origin string) bool + // Optional origin validator (with request) function + allowOriginRequestFunc func(r *http.Request, origin string) bool // Normalized list of allowed headers allowedHeaders []string // Normalized list of allowed methods @@ -93,11 +99,12 @@ type Cors struct { // New creates a new Cors handler with the provided options. func New(options Options) *Cors { c := &Cors{ - exposedHeaders: convert(options.ExposedHeaders, http.CanonicalHeaderKey), - allowOriginFunc: options.AllowOriginFunc, - allowCredentials: options.AllowCredentials, - maxAge: options.MaxAge, - optionPassthrough: options.OptionsPassthrough, + exposedHeaders: convert(options.ExposedHeaders, http.CanonicalHeaderKey), + allowOriginFunc: options.AllowOriginFunc, + allowOriginRequestFunc: options.AllowOriginRequestFunc, + allowCredentials: options.AllowCredentials, + maxAge: options.MaxAge, + optionPassthrough: options.OptionsPassthrough, } if options.Debug { c.Log = log.New(os.Stdout, "[cors] ", log.LstdFlags) @@ -109,7 +116,7 @@ func New(options Options) *Cors { // Allowed Origins if len(options.AllowedOrigins) == 0 { - if options.AllowOriginFunc == nil { + if options.AllowOriginFunc == nil && options.AllowOriginRequestFunc == nil { // Default is all origins c.allowedOriginsAll = true } @@ -254,7 +261,7 @@ func (c *Cors) handlePreflight(w http.ResponseWriter, r *http.Request) { c.logf(" Preflight aborted: empty origin") return } - if !c.isOriginAllowed(origin) { + if !c.isOriginAllowed(r, origin) { c.logf(" Preflight aborted: origin '%s' not allowed", origin) return } @@ -307,7 +314,7 @@ func (c *Cors) handleActualRequest(w http.ResponseWriter, r *http.Request) { c.logf(" Actual request no headers added: missing origin") return } - if !c.isOriginAllowed(origin) { + if !c.isOriginAllowed(r, origin) { c.logf(" Actual request no headers added: origin '%s' not allowed", origin) return } @@ -344,7 +351,10 @@ func (c *Cors) logf(format string, a ...interface{}) { // isOriginAllowed checks if a given origin is allowed to perform cross-domain requests // on the endpoint -func (c *Cors) isOriginAllowed(origin string) bool { +func (c *Cors) isOriginAllowed(r *http.Request, origin string) bool { + if c.allowOriginRequestFunc != nil { + return c.allowOriginRequestFunc(r, origin) + } if c.allowOriginFunc != nil { return c.allowOriginFunc(origin) } diff --git a/vendor/github.com/rs/cors/cors_test.go b/vendor/github.com/rs/cors/cors_test.go index d3dbcba..68c12eb 100644 --- a/vendor/github.com/rs/cors/cors_test.go +++ b/vendor/github.com/rs/cors/cors_test.go @@ -49,7 +49,7 @@ func TestSpec(t *testing.T) { { "NoConfig", Options{ - // Intentionally left blank. + // Intentionally left blank. }, "GET", map[string]string{}, @@ -158,15 +158,33 @@ func TestSpec(t *testing.T) { }, }, { - "AllowedOriginFuncNotMatch", + "AllowOriginRequestFuncMatch", Options{ - AllowOriginFunc: func(o string) bool { - return regexp.MustCompile("^http://foo").MatchString(o) + AllowOriginRequestFunc: func(r *http.Request, o string) bool { + return regexp.MustCompile("^http://foo").MatchString(o) && r.Header.Get("Authorization") == "secret" }, }, "GET", map[string]string{ - "Origin": "http://barfoo.com", + "Origin": "http://foobar.com", + "Authorization": "secret", + }, + map[string]string{ + "Vary": "Origin", + "Access-Control-Allow-Origin": "http://foobar.com", + }, + }, + { + "AllowOriginRequestFuncNotMatch", + Options{ + AllowOriginRequestFunc: func(r *http.Request, o string) bool { + return regexp.MustCompile("^http://foo").MatchString(o) && r.Header.Get("Authorization") == "secret" + }, + }, + "GET", + map[string]string{ + "Origin": "http://foobar.com", + "Authorization": "not-secret", }, map[string]string{ "Vary": "Origin", @@ -447,7 +465,7 @@ func TestHandlePreflightInvalidOriginAbortion(t *testing.T) { func TestHandlePreflightNoOptionsAbortion(t *testing.T) { s := New(Options{ - // Intentionally left blank. + // Intentionally left blank. }) res := httptest.NewRecorder() req, _ := http.NewRequest("GET", "http://example.com/foo", nil) @@ -503,7 +521,7 @@ func TestHandleActualRequestInvalidMethodAbortion(t *testing.T) { func TestIsMethodAllowedReturnsFalseWithNoMethods(t *testing.T) { s := New(Options{ - // Intentionally left blank. + // Intentionally left blank. }) s.allowedMethods = []string{} if s.isMethodAllowed("") { @@ -513,7 +531,7 @@ func TestIsMethodAllowedReturnsFalseWithNoMethods(t *testing.T) { func TestIsMethodAllowedReturnsTrueWithOptions(t *testing.T) { s := New(Options{ - // Intentionally left blank. + // Intentionally left blank. }) if !s.isMethodAllowed("OPTIONS") { t.Error("IsMethodAllowed should return true when c.allowedMethods is nil.") diff --git a/vendor/github.com/rs/cors/examples/chi/server.go b/vendor/github.com/rs/cors/examples/chi/server.go new file mode 100644 index 0000000..8587da0 --- /dev/null +++ b/vendor/github.com/rs/cors/examples/chi/server.go @@ -0,0 +1,21 @@ +package main + +import ( + "net/http" + + "github.com/pressly/chi" + "github.com/rs/cors" +) + +func main() { + r := chi.NewRouter() + + // Use default options + r.Use(cors.Default().Handler) + + r.Get("/", func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("welcome")) + }) + + http.ListenAndServe(":8080", r) +} diff --git a/vendor/github.com/rs/cors/utils.go b/vendor/github.com/rs/cors/utils.go index c7a0aa0..53ad9e9 100644 --- a/vendor/github.com/rs/cors/utils.go +++ b/vendor/github.com/rs/cors/utils.go @@ -39,19 +39,20 @@ func parseHeaderList(headerList string) []string { headers := make([]string, 0, t) for i := 0; i < l; i++ { b := headerList[i] - if b >= 'a' && b <= 'z' { + switch { + case b >= 'a' && b <= 'z': if upper { h = append(h, b-toLower) } else { h = append(h, b) } - } else if b >= 'A' && b <= 'Z' { + case b >= 'A' && b <= 'Z': if !upper { h = append(h, b+toLower) } else { h = append(h, b) } - } else if b == '-' || b == '_' || (b >= '0' && b <= '9') { + case b == '-' || b == '_' || (b >= '0' && b <= '9'): h = append(h, b) } diff --git a/vendor/github.com/sirupsen/logrus/.gitignore b/vendor/github.com/sirupsen/logrus/.gitignore index 66be63a..6b7d7d1 100644 --- a/vendor/github.com/sirupsen/logrus/.gitignore +++ b/vendor/github.com/sirupsen/logrus/.gitignore @@ -1 +1,2 @@ logrus +vendor diff --git a/vendor/github.com/sirupsen/logrus/.travis.yml b/vendor/github.com/sirupsen/logrus/.travis.yml index 2f19b4a..a8f1545 100644 --- a/vendor/github.com/sirupsen/logrus/.travis.yml +++ b/vendor/github.com/sirupsen/logrus/.travis.yml @@ -1,13 +1,52 @@ language: go -go: - - 1.9.x - - 1.10.x +go_import_path: github.com/sirupsen/logrus env: - GOMAXPROCS=4 GORACE=halt_on_error=1 -install: - - go get github.com/stretchr/testify/assert - - go get gopkg.in/gemnasium/logrus-airbrake-hook.v2 - - go get golang.org/x/sys/unix - - go get golang.org/x/sys/windows -script: - - go test -race -v ./... +matrix: + include: + - go: 1.10.x + install: + - go get github.com/stretchr/testify/assert + - go get golang.org/x/crypto/ssh/terminal + - go get golang.org/x/sys/unix + - go get golang.org/x/sys/windows + script: + - go test -race -v ./... + - go: 1.11.x + env: GO111MODULE=on + install: + - go mod download + script: + - go test -race -v ./... + - go: 1.11.x + env: GO111MODULE=off + install: + - go get github.com/stretchr/testify/assert + - go get golang.org/x/crypto/ssh/terminal + - go get golang.org/x/sys/unix + - go get golang.org/x/sys/windows + script: + - go test -race -v ./... + - go: 1.10.x + install: + - go get github.com/stretchr/testify/assert + - go get golang.org/x/crypto/ssh/terminal + - go get golang.org/x/sys/unix + - go get golang.org/x/sys/windows + script: + - go test -race -v -tags appengine ./... + - go: 1.11.x + env: GO111MODULE=on + install: + - go mod download + script: + - go test -race -v -tags appengine ./... + - go: 1.11.x + env: GO111MODULE=off + install: + - go get github.com/stretchr/testify/assert + - go get golang.org/x/crypto/ssh/terminal + - go get golang.org/x/sys/unix + - go get golang.org/x/sys/windows + script: + - go test -race -v -tags appengine ./... diff --git a/vendor/github.com/sirupsen/logrus/CHANGELOG.md b/vendor/github.com/sirupsen/logrus/CHANGELOG.md index 1bd1deb..cb85d9f 100644 --- a/vendor/github.com/sirupsen/logrus/CHANGELOG.md +++ b/vendor/github.com/sirupsen/logrus/CHANGELOG.md @@ -1,3 +1,45 @@ +# 1.2.0 +This new release introduces: + * A new method `SetReportCaller` in the `Logger` to enable the file, line and calling function from which the trace has been issued + * A new trace level named `Trace` whose level is below `Debug` + * A configurable exit function to be called upon a Fatal trace + * The `Level` object now implements `encoding.TextUnmarshaler` interface + +# 1.1.1 +This is a bug fix release. + * fix the build break on Solaris + * don't drop a whole trace in JSONFormatter when a field param is a function pointer which can not be serialized + +# 1.1.0 +This new release introduces: + * several fixes: + * a fix for a race condition on entry formatting + * proper cleanup of previously used entries before putting them back in the pool + * the extra new line at the end of message in text formatter has been removed + * a new global public API to check if a level is activated: IsLevelEnabled + * the following methods have been added to the Logger object + * IsLevelEnabled + * SetFormatter + * SetOutput + * ReplaceHooks + * introduction of go module + * an indent configuration for the json formatter + * output colour support for windows + * the field sort function is now configurable for text formatter + * the CLICOLOR and CLICOLOR\_FORCE environment variable support in text formater + +# 1.0.6 + +This new release introduces: + * a new api WithTime which allows to easily force the time of the log entry + which is mostly useful for logger wrapper + * a fix reverting the immutability of the entry given as parameter to the hooks + a new configuration field of the json formatter in order to put all the fields + in a nested dictionnary + * a new SetOutput method in the Logger + * a new configuration of the textformatter to configure the name of the default keys + * a new configuration of the text formatter to disable the level truncation + # 1.0.5 * Fix hooks race (#707) diff --git a/vendor/github.com/sirupsen/logrus/README.md b/vendor/github.com/sirupsen/logrus/README.md index 072e99b..3987310 100644 --- a/vendor/github.com/sirupsen/logrus/README.md +++ b/vendor/github.com/sirupsen/logrus/README.md @@ -56,8 +56,39 @@ time="2015-03-26T01:27:38-04:00" level=warning msg="The group's number increased time="2015-03-26T01:27:38-04:00" level=debug msg="Temperature changes" temperature=-4 time="2015-03-26T01:27:38-04:00" level=panic msg="It's over 9000!" animal=orca size=9009 time="2015-03-26T01:27:38-04:00" level=fatal msg="The ice breaks!" err=&{0x2082280c0 map[animal:orca size:9009] 2015-03-26 01:27:38.441574009 -0400 EDT panic It's over 9000!} number=100 omg=true -exit status 1 ``` +To ensure this behaviour even if a TTY is attached, set your formatter as follows: + +```go + log.SetFormatter(&log.TextFormatter{ + DisableColors: true, + FullTimestamp: true, + }) +``` + +#### Logging Method Name + +If you wish to add the calling method as a field, instruct the logger via: +```go +log.SetReportCaller(true) +``` +This adds the caller as 'method' like so: + +```json +{"animal":"penguin","level":"fatal","method":"github.com/sirupsen/arcticcreatures.migrate","msg":"a penguin swims by", +"time":"2014-03-10 19:57:38.562543129 -0400 EDT"} +``` + +```text +time="2015-03-26T01:27:38-04:00" level=fatal method=github.com/sirupsen/arcticcreatures.migrate msg="a penguin swims by" animal=penguin +``` +Note that this does add measurable overhead - the cost will depend on the version of Go, but is +between 20 and 40% in recent tests with 1.6 and 1.7. You can validate this in your +environment via benchmarks: +``` +go test -bench=.*CallerTracing +``` + #### Case-sensitivity @@ -246,9 +277,10 @@ A list of currently known of service hook can be found in this wiki [page](https #### Level logging -Logrus has six logging levels: Debug, Info, Warning, Error, Fatal and Panic. +Logrus has seven logging levels: Trace, Debug, Info, Warning, Error, Fatal and Panic. ```go +log.Trace("Something very low level.") log.Debug("Useful debugging information.") log.Info("Something noteworthy happened!") log.Warn("You should probably take a look at this.") @@ -329,6 +361,7 @@ The built-in logging formatters are: Third party logging formatters: * [`FluentdFormatter`](https://github.com/joonix/log). Formats entries that can be parsed by Kubernetes and Google Container Engine. +* [`GELF`](https://github.com/fabienm/go-logrus-formatters). Formats entries so they comply to Graylog's [GELF 1.1 specification](http://docs.graylog.org/en/2.4/pages/gelf.html). * [`logstash`](https://github.com/bshuster-repo/logrus-logstash-hook). Logs fields as [Logstash](http://logstash.net) Events. * [`prefixed`](https://github.com/x-cray/logrus-prefixed-formatter). Displays log entry source along with alternative layout. * [`zalgo`](https://github.com/aybabtme/logzalgo). Invoking the P͉̫o̳̼̊w̖͈̰͎e̬͔̭͂r͚̼̹̲ ̫͓͉̳͈ō̠͕͖̚f̝͍̠ ͕̲̞͖͑Z̖̫̤̫ͪa͉̬͈̗l͖͎g̳̥o̰̥̅!̣͔̲̻͊̄ ̙̘̦̹̦. diff --git a/vendor/github.com/sirupsen/logrus/alt_exit_test.go b/vendor/github.com/sirupsen/logrus/alt_exit_test.go index a08b1a8..0a2ff56 100644 --- a/vendor/github.com/sirupsen/logrus/alt_exit_test.go +++ b/vendor/github.com/sirupsen/logrus/alt_exit_test.go @@ -6,6 +6,8 @@ import ( "os" "os/exec" "path/filepath" + "runtime" + "strings" "testing" "time" ) @@ -19,6 +21,9 @@ func TestRegister(t *testing.T) { } func TestHandler(t *testing.T) { + testprog := testprogleader + testprog = append(testprog, getPackage()...) + testprog = append(testprog, testprogtrailer...) tempDir, err := ioutil.TempDir("", "test_handler") if err != nil { log.Fatalf("can't create temp dir. %q", err) @@ -47,13 +52,24 @@ func TestHandler(t *testing.T) { } } -var testprog = []byte(` +// getPackage returns the name of the current package, which makes running this +// test in a fork simpler +func getPackage() []byte { + pc, _, _, _ := runtime.Caller(0) + fullFuncName := runtime.FuncForPC(pc).Name() + idx := strings.LastIndex(fullFuncName, ".") + return []byte(fullFuncName[:idx]) // trim off function details +} + +var testprogleader = []byte(` // Test program for atexit, gets output file and data as arguments and writes // data to output file in atexit handler. package main import ( - "github.com/sirupsen/logrus" + "`) +var testprogtrailer = []byte( + `" "flag" "fmt" "io/ioutil" diff --git a/vendor/github.com/sirupsen/logrus/entry.go b/vendor/github.com/sirupsen/logrus/entry.go index 473bd1a..df6d188 100644 --- a/vendor/github.com/sirupsen/logrus/entry.go +++ b/vendor/github.com/sirupsen/logrus/entry.go @@ -4,11 +4,30 @@ import ( "bytes" "fmt" "os" + "reflect" + "runtime" + "strings" "sync" "time" ) -var bufferPool *sync.Pool +var ( + bufferPool *sync.Pool + + // qualified package name, cached at first use + logrusPackage string + + // Positions in the call stack when tracing to report the calling method + minimumCallerDepth int + + // Used for caller information initialisation + callerInitOnce sync.Once +) + +const ( + maximumCallerDepth int = 25 + knownLogrusFrames int = 4 +) func init() { bufferPool = &sync.Pool{ @@ -16,15 +35,18 @@ func init() { return new(bytes.Buffer) }, } + + // start at the bottom of the stack before the package-name cache is primed + minimumCallerDepth = 1 } // Defines the key when adding errors using WithError. var ErrorKey = "error" // An entry is the final or intermediate Logrus logging entry. It contains all -// the fields passed with WithField{,s}. It's finally logged when Debug, Info, -// Warn, Error, Fatal or Panic is called on it. These objects can be reused and -// passed around as much as you wish to avoid field duplication. +// the fields passed with WithField{,s}. It's finally logged when Trace, Debug, +// Info, Warn, Error, Fatal or Panic is called on it. These objects can be +// reused and passed around as much as you wish to avoid field duplication. type Entry struct { Logger *Logger @@ -34,22 +56,28 @@ type Entry struct { // Time at which the log entry was created Time time.Time - // Level the log entry was logged at: Debug, Info, Warn, Error, Fatal or Panic + // Level the log entry was logged at: Trace, Debug, Info, Warn, Error, Fatal or Panic // This field will be set on entry firing and the value will be equal to the one in Logger struct field. Level Level - // Message passed to Debug, Info, Warn, Error, Fatal or Panic + // Calling method, with package name + Caller *runtime.Frame + + // Message passed to Trace, Debug, Info, Warn, Error, Fatal or Panic Message string - // When formatter is called in entry.log(), an Buffer may be set to entry + // When formatter is called in entry.log(), a Buffer may be set to entry Buffer *bytes.Buffer + + // err may contain a field formatting error + err string } func NewEntry(logger *Logger) *Entry { return &Entry{ Logger: logger, - // Default is five fields, give a little extra room - Data: make(Fields, 5), + // Default is three fields, plus one optional. Give a little extra room. + Data: make(Fields, 6), } } @@ -80,15 +108,85 @@ func (entry *Entry) WithFields(fields Fields) *Entry { for k, v := range entry.Data { data[k] = v } + fieldErr := entry.err for k, v := range fields { - data[k] = v + isErrField := false + if t := reflect.TypeOf(v); t != nil { + switch t.Kind() { + case reflect.Func: + isErrField = true + case reflect.Ptr: + isErrField = t.Elem().Kind() == reflect.Func + } + } + if isErrField { + tmp := fmt.Sprintf("can not add field %q", k) + if fieldErr != "" { + fieldErr = entry.err + ", " + tmp + } else { + fieldErr = tmp + } + } else { + data[k] = v + } } - return &Entry{Logger: entry.Logger, Data: data, Time: entry.Time} + return &Entry{Logger: entry.Logger, Data: data, Time: entry.Time, err: fieldErr} } // Overrides the time of the Entry. func (entry *Entry) WithTime(t time.Time) *Entry { - return &Entry{Logger: entry.Logger, Data: entry.Data, Time: t} + return &Entry{Logger: entry.Logger, Data: entry.Data, Time: t, err: entry.err} +} + +// getPackageName reduces a fully qualified function name to the package name +// There really ought to be to be a better way... +func getPackageName(f string) string { + for { + lastPeriod := strings.LastIndex(f, ".") + lastSlash := strings.LastIndex(f, "/") + if lastPeriod > lastSlash { + f = f[:lastPeriod] + } else { + break + } + } + + return f +} + +// getCaller retrieves the name of the first non-logrus calling function +func getCaller() *runtime.Frame { + // Restrict the lookback frames to avoid runaway lookups + pcs := make([]uintptr, maximumCallerDepth) + depth := runtime.Callers(minimumCallerDepth, pcs) + frames := runtime.CallersFrames(pcs[:depth]) + + // cache this package's fully-qualified name + callerInitOnce.Do(func() { + logrusPackage = getPackageName(runtime.FuncForPC(pcs[0]).Name()) + + // now that we have the cache, we can skip a minimum count of known-logrus functions + // XXX this is dubious, the number of frames may vary store an entry in a logger interface + minimumCallerDepth = knownLogrusFrames + }) + + for f, again := frames.Next(); again; f, again = frames.Next() { + pkg := getPackageName(f.Function) + + // If the caller isn't part of this package, we're done + if pkg != logrusPackage { + return &f + } + } + + // if we got here, we failed to find the caller's context + return nil +} + +func (entry Entry) HasCaller() (has bool) { + return entry.Logger != nil && + entry.Logger.ReportCaller && + entry.Caller != nil } // This function is not declared with a pointer value because otherwise @@ -107,6 +205,9 @@ func (entry Entry) log(level Level, msg string) { entry.Level = level entry.Message = msg + if entry.Logger.ReportCaller { + entry.Caller = getCaller() + } entry.fireHooks() @@ -137,9 +238,9 @@ func (entry *Entry) fireHooks() { } func (entry *Entry) write() { - serialized, err := entry.Logger.Formatter.Format(entry) entry.Logger.mu.Lock() defer entry.Logger.mu.Unlock() + serialized, err := entry.Logger.Formatter.Format(entry) if err != nil { fmt.Fprintf(os.Stderr, "Failed to obtain reader, %v\n", err) } else { @@ -150,26 +251,30 @@ func (entry *Entry) write() { } } -func (entry *Entry) Debug(args ...interface{}) { - if entry.Logger.level() >= DebugLevel { - entry.log(DebugLevel, fmt.Sprint(args...)) +func (entry *Entry) Log(level Level, args ...interface{}) { + if entry.Logger.IsLevelEnabled(level) { + entry.log(level, fmt.Sprint(args...)) } } +func (entry *Entry) Trace(args ...interface{}) { + entry.Log(TraceLevel, args...) +} + +func (entry *Entry) Debug(args ...interface{}) { + entry.Log(DebugLevel, args...) +} + func (entry *Entry) Print(args ...interface{}) { entry.Info(args...) } func (entry *Entry) Info(args ...interface{}) { - if entry.Logger.level() >= InfoLevel { - entry.log(InfoLevel, fmt.Sprint(args...)) - } + entry.Log(InfoLevel, args...) } func (entry *Entry) Warn(args ...interface{}) { - if entry.Logger.level() >= WarnLevel { - entry.log(WarnLevel, fmt.Sprint(args...)) - } + entry.Log(WarnLevel, args...) } func (entry *Entry) Warning(args ...interface{}) { @@ -177,37 +282,35 @@ func (entry *Entry) Warning(args ...interface{}) { } func (entry *Entry) Error(args ...interface{}) { - if entry.Logger.level() >= ErrorLevel { - entry.log(ErrorLevel, fmt.Sprint(args...)) - } + entry.Log(ErrorLevel, args...) } func (entry *Entry) Fatal(args ...interface{}) { - if entry.Logger.level() >= FatalLevel { - entry.log(FatalLevel, fmt.Sprint(args...)) - } - Exit(1) + entry.Log(FatalLevel, args...) + entry.Logger.Exit(1) } func (entry *Entry) Panic(args ...interface{}) { - if entry.Logger.level() >= PanicLevel { - entry.log(PanicLevel, fmt.Sprint(args...)) - } + entry.Log(PanicLevel, args...) panic(fmt.Sprint(args...)) } // Entry Printf family functions +func (entry *Entry) Logf(level Level, format string, args ...interface{}) { + entry.Log(level, fmt.Sprintf(format, args...)) +} + +func (entry *Entry) Tracef(format string, args ...interface{}) { + entry.Logf(TraceLevel, format, args...) +} + func (entry *Entry) Debugf(format string, args ...interface{}) { - if entry.Logger.level() >= DebugLevel { - entry.Debug(fmt.Sprintf(format, args...)) - } + entry.Logf(DebugLevel, format, args...) } func (entry *Entry) Infof(format string, args ...interface{}) { - if entry.Logger.level() >= InfoLevel { - entry.Info(fmt.Sprintf(format, args...)) - } + entry.Logf(InfoLevel, format, args...) } func (entry *Entry) Printf(format string, args ...interface{}) { @@ -215,9 +318,7 @@ func (entry *Entry) Printf(format string, args ...interface{}) { } func (entry *Entry) Warnf(format string, args ...interface{}) { - if entry.Logger.level() >= WarnLevel { - entry.Warn(fmt.Sprintf(format, args...)) - } + entry.Logf(WarnLevel, format, args...) } func (entry *Entry) Warningf(format string, args ...interface{}) { @@ -225,36 +326,36 @@ func (entry *Entry) Warningf(format string, args ...interface{}) { } func (entry *Entry) Errorf(format string, args ...interface{}) { - if entry.Logger.level() >= ErrorLevel { - entry.Error(fmt.Sprintf(format, args...)) - } + entry.Logf(ErrorLevel, format, args...) } func (entry *Entry) Fatalf(format string, args ...interface{}) { - if entry.Logger.level() >= FatalLevel { - entry.Fatal(fmt.Sprintf(format, args...)) - } - Exit(1) + entry.Logf(FatalLevel, format, args...) + entry.Logger.Exit(1) } func (entry *Entry) Panicf(format string, args ...interface{}) { - if entry.Logger.level() >= PanicLevel { - entry.Panic(fmt.Sprintf(format, args...)) - } + entry.Logf(PanicLevel, format, args...) } // Entry Println family functions -func (entry *Entry) Debugln(args ...interface{}) { - if entry.Logger.level() >= DebugLevel { - entry.Debug(entry.sprintlnn(args...)) +func (entry *Entry) Logln(level Level, args ...interface{}) { + if entry.Logger.IsLevelEnabled(level) { + entry.Log(level, entry.sprintlnn(args...)) } } +func (entry *Entry) Traceln(args ...interface{}) { + entry.Logln(TraceLevel, args...) +} + +func (entry *Entry) Debugln(args ...interface{}) { + entry.Logln(DebugLevel, args...) +} + func (entry *Entry) Infoln(args ...interface{}) { - if entry.Logger.level() >= InfoLevel { - entry.Info(entry.sprintlnn(args...)) - } + entry.Logln(InfoLevel, args...) } func (entry *Entry) Println(args ...interface{}) { @@ -262,9 +363,7 @@ func (entry *Entry) Println(args ...interface{}) { } func (entry *Entry) Warnln(args ...interface{}) { - if entry.Logger.level() >= WarnLevel { - entry.Warn(entry.sprintlnn(args...)) - } + entry.Logln(WarnLevel, args...) } func (entry *Entry) Warningln(args ...interface{}) { @@ -272,22 +371,16 @@ func (entry *Entry) Warningln(args ...interface{}) { } func (entry *Entry) Errorln(args ...interface{}) { - if entry.Logger.level() >= ErrorLevel { - entry.Error(entry.sprintlnn(args...)) - } + entry.Logln(ErrorLevel, args...) } func (entry *Entry) Fatalln(args ...interface{}) { - if entry.Logger.level() >= FatalLevel { - entry.Fatal(entry.sprintlnn(args...)) - } - Exit(1) + entry.Logln(FatalLevel, args...) + entry.Logger.Exit(1) } func (entry *Entry) Panicln(args ...interface{}) { - if entry.Logger.level() >= PanicLevel { - entry.Panic(entry.sprintlnn(args...)) - } + entry.Logln(PanicLevel, args...) } // Sprintlnn => Sprint no newline. This is to get the behavior of how diff --git a/vendor/github.com/sirupsen/logrus/entry_test.go b/vendor/github.com/sirupsen/logrus/entry_test.go index a81e2b3..5e66341 100644 --- a/vendor/github.com/sirupsen/logrus/entry_test.go +++ b/vendor/github.com/sirupsen/logrus/entry_test.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -113,3 +114,28 @@ func TestEntryHooksPanic(t *testing.T) { entry := NewEntry(logger) entry.Info(badMessage) } + +func TestEntryWithIncorrectField(t *testing.T) { + assert := assert.New(t) + + fn := func() {} + + e := Entry{} + eWithFunc := e.WithFields(Fields{"func": fn}) + eWithFuncPtr := e.WithFields(Fields{"funcPtr": &fn}) + + assert.Equal(eWithFunc.err, `can not add field "func"`) + assert.Equal(eWithFuncPtr.err, `can not add field "funcPtr"`) + + eWithFunc = eWithFunc.WithField("not_a_func", "it is a string") + eWithFuncPtr = eWithFuncPtr.WithField("not_a_func", "it is a string") + + assert.Equal(eWithFunc.err, `can not add field "func"`) + assert.Equal(eWithFuncPtr.err, `can not add field "funcPtr"`) + + eWithFunc = eWithFunc.WithTime(time.Now()) + eWithFuncPtr = eWithFuncPtr.WithTime(time.Now()) + + assert.Equal(eWithFunc.err, `can not add field "func"`) + assert.Equal(eWithFuncPtr.err, `can not add field "funcPtr"`) +} diff --git a/vendor/github.com/sirupsen/logrus/example_basic_test.go b/vendor/github.com/sirupsen/logrus/example_basic_test.go index a2acf55..9ff5655 100644 --- a/vendor/github.com/sirupsen/logrus/example_basic_test.go +++ b/vendor/github.com/sirupsen/logrus/example_basic_test.go @@ -1,16 +1,18 @@ package logrus_test import ( - "github.com/sirupsen/logrus" "os" + + "github.com/sirupsen/logrus" ) func Example_basic() { var log = logrus.New() log.Formatter = new(logrus.JSONFormatter) log.Formatter = new(logrus.TextFormatter) //default + log.Formatter.(*logrus.TextFormatter).DisableColors = true // remove colors log.Formatter.(*logrus.TextFormatter).DisableTimestamp = true // remove timestamp from test output - log.Level = logrus.DebugLevel + log.Level = logrus.TraceLevel log.Out = os.Stdout // file, err := os.OpenFile("logrus.log", os.O_CREATE|os.O_WRONLY, 0666) @@ -35,6 +37,11 @@ func Example_basic() { } }() + log.WithFields(logrus.Fields{ + "animal": "walrus", + "number": 0, + }).Trace("Went to the beach") + log.WithFields(logrus.Fields{ "animal": "walrus", "number": 8, @@ -60,6 +67,7 @@ func Example_basic() { }).Panic("It's over 9000!") // Output: + // level=trace msg="Went to the beach" animal=walrus number=0 // level=debug msg="Started observing beach" animal=walrus number=8 // level=info msg="A group of walrus emerges from the ocean" animal=walrus size=10 // level=warning msg="The group's number increased tremendously!" number=122 omg=true diff --git a/vendor/github.com/sirupsen/logrus/example_global_hook_test.go b/vendor/github.com/sirupsen/logrus/example_global_hook_test.go new file mode 100644 index 0000000..c81e448 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/example_global_hook_test.go @@ -0,0 +1,36 @@ +package logrus_test + +import ( + "github.com/sirupsen/logrus" + "os" +) + +var ( + mystring string +) + +type GlobalHook struct { +} + +func (h *GlobalHook) Levels() []logrus.Level { + return logrus.AllLevels +} + +func (h *GlobalHook) Fire(e *logrus.Entry) error { + e.Data["mystring"] = mystring + return nil +} + +func Example() { + l := logrus.New() + l.Out = os.Stdout + l.Formatter = &logrus.TextFormatter{DisableTimestamp: true, DisableColors: true} + l.AddHook(&GlobalHook{}) + mystring = "first value" + l.Info("first log") + mystring = "another value" + l.Info("second log") + // Output: + // level=info msg="first log" mystring="first value" + // level=info msg="second log" mystring="another value" +} diff --git a/vendor/github.com/sirupsen/logrus/example_hook_test.go b/vendor/github.com/sirupsen/logrus/example_hook_test.go index d4ddffc..dc0e69f 100644 --- a/vendor/github.com/sirupsen/logrus/example_hook_test.go +++ b/vendor/github.com/sirupsen/logrus/example_hook_test.go @@ -1,16 +1,24 @@ +// +build !windows + package logrus_test import ( - "github.com/sirupsen/logrus" - "gopkg.in/gemnasium/logrus-airbrake-hook.v2" + "log/syslog" "os" + + "github.com/sirupsen/logrus" + slhooks "github.com/sirupsen/logrus/hooks/syslog" ) +// An example on how to use a hook func Example_hook() { var log = logrus.New() log.Formatter = new(logrus.TextFormatter) // default + log.Formatter.(*logrus.TextFormatter).DisableColors = true // remove colors log.Formatter.(*logrus.TextFormatter).DisableTimestamp = true // remove timestamp from test output - log.Hooks.Add(airbrake.NewHook(123, "xyz", "development")) + if sl, err := slhooks.NewSyslogHook("udp", "localhost:514", syslog.LOG_INFO, ""); err == nil { + log.Hooks.Add(sl) + } log.Out = os.Stdout log.WithFields(logrus.Fields{ diff --git a/vendor/github.com/sirupsen/logrus/exported.go b/vendor/github.com/sirupsen/logrus/exported.go index eb612a6..7342613 100644 --- a/vendor/github.com/sirupsen/logrus/exported.go +++ b/vendor/github.com/sirupsen/logrus/exported.go @@ -21,30 +21,33 @@ func SetOutput(out io.Writer) { // SetFormatter sets the standard logger formatter. func SetFormatter(formatter Formatter) { - std.mu.Lock() - defer std.mu.Unlock() - std.Formatter = formatter + std.SetFormatter(formatter) +} + +// SetReportCaller sets whether the standard logger will include the calling +// method as a field. +func SetReportCaller(include bool) { + std.SetReportCaller(include) } // SetLevel sets the standard logger level. func SetLevel(level Level) { - std.mu.Lock() - defer std.mu.Unlock() std.SetLevel(level) } // GetLevel returns the standard logger level. func GetLevel() Level { - std.mu.Lock() - defer std.mu.Unlock() - return std.level() + return std.GetLevel() +} + +// IsLevelEnabled checks if the log level of the standard logger is greater than the level param +func IsLevelEnabled(level Level) bool { + return std.IsLevelEnabled(level) } // AddHook adds a hook to the standard logger hooks. func AddHook(hook Hook) { - std.mu.Lock() - defer std.mu.Unlock() - std.Hooks.Add(hook) + std.AddHook(hook) } // WithError creates an entry from the standard logger and adds an error to it, using the value defined in ErrorKey as key. @@ -80,6 +83,11 @@ func WithTime(t time.Time) *Entry { return std.WithTime(t) } +// Trace logs a message at level Trace on the standard logger. +func Trace(args ...interface{}) { + std.Trace(args...) +} + // Debug logs a message at level Debug on the standard logger. func Debug(args ...interface{}) { std.Debug(args...) @@ -120,6 +128,11 @@ func Fatal(args ...interface{}) { std.Fatal(args...) } +// Tracef logs a message at level Trace on the standard logger. +func Tracef(format string, args ...interface{}) { + std.Tracef(format, args...) +} + // Debugf logs a message at level Debug on the standard logger. func Debugf(format string, args ...interface{}) { std.Debugf(format, args...) @@ -160,6 +173,11 @@ func Fatalf(format string, args ...interface{}) { std.Fatalf(format, args...) } +// Traceln logs a message at level Trace on the standard logger. +func Traceln(args ...interface{}) { + std.Traceln(args...) +} + // Debugln logs a message at level Debug on the standard logger. func Debugln(args ...interface{}) { std.Debugln(args...) diff --git a/vendor/github.com/sirupsen/logrus/formatter.go b/vendor/github.com/sirupsen/logrus/formatter.go index 83c7494..4088837 100644 --- a/vendor/github.com/sirupsen/logrus/formatter.go +++ b/vendor/github.com/sirupsen/logrus/formatter.go @@ -2,7 +2,16 @@ package logrus import "time" -const defaultTimestampFormat = time.RFC3339 +// Default key names for the default fields +const ( + defaultTimestampFormat = time.RFC3339 + FieldKeyMsg = "msg" + FieldKeyLevel = "level" + FieldKeyTime = "time" + FieldKeyLogrusError = "logrus_error" + FieldKeyFunc = "func" + FieldKeyFile = "file" +) // The Formatter interface is used to implement a custom Formatter. It takes an // `Entry`. It exposes all the fields, including the default ones: @@ -18,7 +27,7 @@ type Formatter interface { Format(*Entry) ([]byte, error) } -// This is to not silently overwrite `time`, `msg` and `level` fields when +// This is to not silently overwrite `time`, `msg`, `func` and `level` fields when // dumping it. If this code wasn't there doing: // // logrus.WithField("level", 1).Info("hello") @@ -30,7 +39,7 @@ type Formatter interface { // // It's not exported because it's still using Data in an opinionated way. It's to // avoid code duplication between the two default formatters. -func prefixFieldClashes(data Fields, fieldMap FieldMap) { +func prefixFieldClashes(data Fields, fieldMap FieldMap, reportCaller bool) { timeKey := fieldMap.resolve(FieldKeyTime) if t, ok := data[timeKey]; ok { data["fields."+timeKey] = t @@ -48,4 +57,22 @@ func prefixFieldClashes(data Fields, fieldMap FieldMap) { data["fields."+levelKey] = l delete(data, levelKey) } + + logrusErrKey := fieldMap.resolve(FieldKeyLogrusError) + if l, ok := data[logrusErrKey]; ok { + data["fields."+logrusErrKey] = l + delete(data, logrusErrKey) + } + + // If reportCaller is not set, 'func' will not conflict. + if reportCaller { + funcKey := fieldMap.resolve(FieldKeyFunc) + if l, ok := data[funcKey]; ok { + data["fields."+funcKey] = l + } + fileKey := fieldMap.resolve(FieldKeyFile) + if l, ok := data[fileKey]; ok { + data["fields."+fileKey] = l + } + } } diff --git a/vendor/github.com/sirupsen/logrus/go.mod b/vendor/github.com/sirupsen/logrus/go.mod new file mode 100644 index 0000000..94574cc --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/go.mod @@ -0,0 +1,11 @@ +module github.com/sirupsen/logrus + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/konsorten/go-windows-terminal-sequences v1.0.1 + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.1.1 // indirect + github.com/stretchr/testify v1.2.2 + golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 + golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 +) diff --git a/vendor/github.com/sirupsen/logrus/go.sum b/vendor/github.com/sirupsen/logrus/go.sum new file mode 100644 index 0000000..133d34a --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/go.sum @@ -0,0 +1,15 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe h1:CHRGQ8V7OlCYtwaKPJi3iA7J+YdNKdo8j7nG5IgDhjs= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/vendor/github.com/sirupsen/logrus/hook_test.go b/vendor/github.com/sirupsen/logrus/hook_test.go index 4fea751..b967593 100644 --- a/vendor/github.com/sirupsen/logrus/hook_test.go +++ b/vendor/github.com/sirupsen/logrus/hook_test.go @@ -1,10 +1,16 @@ -package logrus +package logrus_test import ( + "bytes" + "encoding/json" "sync" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + . "github.com/sirupsen/logrus" + . "github.com/sirupsen/logrus/internal/testutils" ) type TestHook struct { @@ -18,6 +24,7 @@ func (hook *TestHook) Fire(entry *Entry) error { func (hook *TestHook) Levels() []Level { return []Level{ + TraceLevel, DebugLevel, InfoLevel, WarnLevel, @@ -50,6 +57,7 @@ func (hook *ModifyHook) Fire(entry *Entry) error { func (hook *ModifyHook) Levels() []Level { return []Level{ + TraceLevel, DebugLevel, InfoLevel, WarnLevel, @@ -85,6 +93,46 @@ func TestCanFireMultipleHooks(t *testing.T) { }) } +type SingleLevelModifyHook struct { + ModifyHook +} + +func (h *SingleLevelModifyHook) Levels() []Level { + return []Level{InfoLevel} +} + +func TestHookEntryIsPristine(t *testing.T) { + l := New() + b := &bytes.Buffer{} + l.Formatter = &JSONFormatter{} + l.Out = b + l.AddHook(&SingleLevelModifyHook{}) + + l.Error("error message") + data := map[string]string{} + err := json.Unmarshal(b.Bytes(), &data) + require.NoError(t, err) + _, ok := data["wow"] + require.False(t, ok) + b.Reset() + + l.Info("error message") + data = map[string]string{} + err = json.Unmarshal(b.Bytes(), &data) + require.NoError(t, err) + _, ok = data["wow"] + require.True(t, ok) + b.Reset() + + l.Error("error message") + data = map[string]string{} + err = json.Unmarshal(b.Bytes(), &data) + require.NoError(t, err) + _, ok = data["wow"] + require.False(t, ok) + b.Reset() +} + type ErrorHook struct { Fired bool } diff --git a/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog.go b/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog.go index 329ce0d..02b8df3 100644 --- a/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog.go +++ b/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog.go @@ -43,7 +43,7 @@ func (hook *SyslogHook) Fire(entry *logrus.Entry) error { return hook.Writer.Warning(line) case logrus.InfoLevel: return hook.Writer.Info(line) - case logrus.DebugLevel: + case logrus.DebugLevel, logrus.TraceLevel: return hook.Writer.Debug(line) default: return nil diff --git a/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog_test.go b/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog_test.go index 5ec3a44..bec6efd 100644 --- a/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog_test.go +++ b/vendor/github.com/sirupsen/logrus/hooks/syslog/syslog_test.go @@ -1,3 +1,5 @@ +// +build !windows,!nacl,!plan9 + package syslog import ( diff --git a/vendor/github.com/sirupsen/logrus/hooks/test/test_test.go b/vendor/github.com/sirupsen/logrus/hooks/test/test_test.go index d6f6d30..636bad5 100644 --- a/vendor/github.com/sirupsen/logrus/hooks/test/test_test.go +++ b/vendor/github.com/sirupsen/logrus/hooks/test/test_test.go @@ -71,3 +71,15 @@ func TestLoggingWithHooksRace(t *testing.T) { entries := hook.AllEntries() assert.Equal(100, len(entries)) } + +func TestFatalWithAlternateExit(t *testing.T) { + assert := assert.New(t) + + logger, hook := NewNullLogger() + logger.ExitFunc = func(code int) {} + + logger.Fatal("something went very wrong") + assert.Equal(logrus.FatalLevel, hook.LastEntry().Level) + assert.Equal("something went very wrong", hook.LastEntry().Message) + assert.Equal(1, len(hook.Entries)) +} diff --git a/vendor/github.com/sirupsen/logrus/internal/testutils/testutils.go b/vendor/github.com/sirupsen/logrus/internal/testutils/testutils.go new file mode 100644 index 0000000..20bc3c3 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/internal/testutils/testutils.go @@ -0,0 +1,58 @@ +package testutils + +import ( + "bytes" + "encoding/json" + "strconv" + "strings" + "testing" + + . "github.com/sirupsen/logrus" + + "github.com/stretchr/testify/require" +) + +func LogAndAssertJSON(t *testing.T, log func(*Logger), assertions func(fields Fields)) { + var buffer bytes.Buffer + var fields Fields + + logger := New() + logger.Out = &buffer + logger.Formatter = new(JSONFormatter) + + log(logger) + + err := json.Unmarshal(buffer.Bytes(), &fields) + require.Nil(t, err) + + assertions(fields) +} + +func LogAndAssertText(t *testing.T, log func(*Logger), assertions func(fields map[string]string)) { + var buffer bytes.Buffer + + logger := New() + logger.Out = &buffer + logger.Formatter = &TextFormatter{ + DisableColors: true, + } + + log(logger) + + fields := make(map[string]string) + for _, kv := range strings.Split(buffer.String(), " ") { + if !strings.Contains(kv, "=") { + continue + } + kvArr := strings.Split(kv, "=") + key := strings.TrimSpace(kvArr[0]) + val := kvArr[1] + if kvArr[1][0] == '"' { + var err error + val, err = strconv.Unquote(val) + require.NoError(t, err) + } + fields[key] = val + } + assertions(fields) +} diff --git a/vendor/github.com/sirupsen/logrus/json_formatter.go b/vendor/github.com/sirupsen/logrus/json_formatter.go index dab1761..2605753 100644 --- a/vendor/github.com/sirupsen/logrus/json_formatter.go +++ b/vendor/github.com/sirupsen/logrus/json_formatter.go @@ -1,6 +1,7 @@ package logrus import ( + "bytes" "encoding/json" "fmt" ) @@ -10,13 +11,6 @@ type fieldKey string // FieldMap allows customization of the key names for default fields. type FieldMap map[fieldKey]string -// Default key names for the default fields -const ( - FieldKeyMsg = "msg" - FieldKeyLevel = "level" - FieldKeyTime = "time" -) - func (f FieldMap) resolve(key fieldKey) string { if k, ok := f[key]; ok { return k @@ -40,17 +34,21 @@ type JSONFormatter struct { // As an example: // formatter := &JSONFormatter{ // FieldMap: FieldMap{ - // FieldKeyTime: "@timestamp", + // FieldKeyTime: "@timestamp", // FieldKeyLevel: "@level", - // FieldKeyMsg: "@message", + // FieldKeyMsg: "@message", + // FieldKeyFunc: "@caller", // }, // } FieldMap FieldMap + + // PrettyPrint will indent all json logs + PrettyPrint bool } // Format renders a single log entry func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { - data := make(Fields, len(entry.Data)+3) + data := make(Fields, len(entry.Data)+4) for k, v := range entry.Data { switch v := v.(type) { case error: @@ -68,22 +66,40 @@ func (f *JSONFormatter) Format(entry *Entry) ([]byte, error) { data = newData } - prefixFieldClashes(data, f.FieldMap) + prefixFieldClashes(data, f.FieldMap, entry.HasCaller()) timestampFormat := f.TimestampFormat if timestampFormat == "" { timestampFormat = defaultTimestampFormat } + if entry.err != "" { + data[f.FieldMap.resolve(FieldKeyLogrusError)] = entry.err + } if !f.DisableTimestamp { data[f.FieldMap.resolve(FieldKeyTime)] = entry.Time.Format(timestampFormat) } data[f.FieldMap.resolve(FieldKeyMsg)] = entry.Message data[f.FieldMap.resolve(FieldKeyLevel)] = entry.Level.String() + if entry.HasCaller() { + data[f.FieldMap.resolve(FieldKeyFunc)] = entry.Caller.Function + data[f.FieldMap.resolve(FieldKeyFile)] = fmt.Sprintf("%s:%d", entry.Caller.File, entry.Caller.Line) + } - serialized, err := json.Marshal(data) - if err != nil { + var b *bytes.Buffer + if entry.Buffer != nil { + b = entry.Buffer + } else { + b = &bytes.Buffer{} + } + + encoder := json.NewEncoder(b) + if f.PrettyPrint { + encoder.SetIndent("", " ") + } + if err := encoder.Encode(data); err != nil { return nil, fmt.Errorf("Failed to marshal fields to JSON, %v", err) } - return append(serialized, '\n'), nil + + return b.Bytes(), nil } diff --git a/vendor/github.com/sirupsen/logrus/json_formatter_test.go b/vendor/github.com/sirupsen/logrus/json_formatter_test.go index 0dde300..695c36e 100644 --- a/vendor/github.com/sirupsen/logrus/json_formatter_test.go +++ b/vendor/github.com/sirupsen/logrus/json_formatter_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "runtime" "strings" "testing" ) @@ -167,8 +168,8 @@ func TestFieldsInNestedDictionary(t *testing.T) { } logEntry := WithFields(Fields{ - "level": "level", - "test": "test", + "level": "level", + "test": "test", }) logEntry.Level = InfoLevel @@ -267,6 +268,55 @@ func TestJSONTimeKey(t *testing.T) { } } +func TestFieldDoesNotClashWithCaller(t *testing.T) { + SetReportCaller(false) + formatter := &JSONFormatter{} + + b, err := formatter.Format(WithField("func", "howdy pardner")) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["func"] != "howdy pardner" { + t.Fatal("func field replaced when ReportCaller=false") + } +} + +func TestFieldClashWithCaller(t *testing.T) { + SetReportCaller(true) + formatter := &JSONFormatter{} + e := WithField("func", "howdy pardner") + e.Caller = &runtime.Frame{Function: "somefunc"} + b, err := formatter.Format(e) + if err != nil { + t.Fatal("Unable to format entry: ", err) + } + + entry := make(map[string]interface{}) + err = json.Unmarshal(b, &entry) + if err != nil { + t.Fatal("Unable to unmarshal formatted entry: ", err) + } + + if entry["fields.func"] != "howdy pardner" { + t.Fatalf("fields.func not set to original func field when ReportCaller=true (got '%s')", + entry["fields.func"]) + } + + if entry["func"] != "somefunc" { + t.Fatalf("func not set as expected when ReportCaller=true (got '%s')", + entry["func"]) + } + + SetReportCaller(false) // return to default value +} + func TestJSONDisableTimestamp(t *testing.T) { formatter := &JSONFormatter{ DisableTimestamp: true, diff --git a/vendor/github.com/sirupsen/logrus/level_test.go b/vendor/github.com/sirupsen/logrus/level_test.go new file mode 100644 index 0000000..78915c4 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/level_test.go @@ -0,0 +1,62 @@ +package logrus_test + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestLevelJsonEncoding(t *testing.T) { + type X struct { + Level logrus.Level + } + + var x X + x.Level = logrus.WarnLevel + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + require.NoError(t, enc.Encode(x)) + dec := json.NewDecoder(&buf) + var y X + require.NoError(t, dec.Decode(&y)) +} + +func TestLevelUnmarshalText(t *testing.T) { + var u logrus.Level + for _, level := range logrus.AllLevels { + t.Run(level.String(), func(t *testing.T) { + require.NoError(t, u.UnmarshalText([]byte(level.String()))) + require.Equal(t, level, u) + }) + } + t.Run("invalid", func(t *testing.T) { + require.Error(t, u.UnmarshalText([]byte("invalid"))) + }) +} + +func TestLevelMarshalText(t *testing.T) { + levelStrings := []string{ + "panic", + "fatal", + "error", + "warning", + "info", + "debug", + "trace", + } + for idx, val := range logrus.AllLevels { + level := val + t.Run(level.String(), func(t *testing.T) { + var cmp logrus.Level + b, err := level.MarshalText() + require.NoError(t, err) + require.Equal(t, levelStrings[idx], string(b)) + err = cmp.UnmarshalText(b) + require.NoError(t, err) + require.Equal(t, level, cmp) + }) + } +} diff --git a/vendor/github.com/sirupsen/logrus/logger.go b/vendor/github.com/sirupsen/logrus/logger.go index 342f797..9bf64e2 100644 --- a/vendor/github.com/sirupsen/logrus/logger.go +++ b/vendor/github.com/sirupsen/logrus/logger.go @@ -11,7 +11,7 @@ import ( type Logger struct { // The logs are `io.Copy`'d to this in a mutex. It's common to set this to a // file, or leave it default which is `os.Stderr`. You can also set this to - // something more adventorous, such as logging to Kafka. + // something more adventurous, such as logging to Kafka. Out io.Writer // Hooks for the logger instance. These allow firing events based on logging // levels and log entries. For example, to send errors to an error tracking @@ -24,6 +24,10 @@ type Logger struct { // own that implements the `Formatter` interface, see the `README` or included // formatters for examples. Formatter Formatter + + // Flag for whether to log caller info (off by default) + ReportCaller bool + // The logging level the logger should log at. This is typically (and defaults // to) `logrus.Info`, which allows Info(), Warn(), Error() and Fatal() to be // logged. @@ -32,8 +36,12 @@ type Logger struct { mu MutexWrap // Reusable empty entry entryPool sync.Pool + // Function to exit the application, defaults to `os.Exit()` + ExitFunc exitFunc } +type exitFunc func(int) + type MutexWrap struct { lock sync.Mutex disabled bool @@ -69,10 +77,12 @@ func (mw *MutexWrap) Disable() { // It's recommended to make this a global instance called `log`. func New() *Logger { return &Logger{ - Out: os.Stderr, - Formatter: new(TextFormatter), - Hooks: make(LevelHooks), - Level: InfoLevel, + Out: os.Stderr, + Formatter: new(TextFormatter), + Hooks: make(LevelHooks), + Level: InfoLevel, + ExitFunc: os.Exit, + ReportCaller: false, } } @@ -85,6 +95,7 @@ func (logger *Logger) newEntry() *Entry { } func (logger *Logger) releaseEntry(entry *Entry) { + entry.Data = map[string]interface{}{} logger.entryPool.Put(entry) } @@ -120,20 +131,24 @@ func (logger *Logger) WithTime(t time.Time) *Entry { return entry.WithTime(t) } -func (logger *Logger) Debugf(format string, args ...interface{}) { - if logger.level() >= DebugLevel { +func (logger *Logger) Logf(level Level, format string, args ...interface{}) { + if logger.IsLevelEnabled(level) { entry := logger.newEntry() - entry.Debugf(format, args...) + entry.Logf(level, format, args...) logger.releaseEntry(entry) } } +func (logger *Logger) Tracef(format string, args ...interface{}) { + logger.Logf(TraceLevel, format, args...) +} + +func (logger *Logger) Debugf(format string, args ...interface{}) { + logger.Logf(DebugLevel, format, args...) +} + func (logger *Logger) Infof(format string, args ...interface{}) { - if logger.level() >= InfoLevel { - entry := logger.newEntry() - entry.Infof(format, args...) - logger.releaseEntry(entry) - } + logger.Logf(InfoLevel, format, args...) } func (logger *Logger) Printf(format string, args ...interface{}) { @@ -143,60 +158,44 @@ func (logger *Logger) Printf(format string, args ...interface{}) { } func (logger *Logger) Warnf(format string, args ...interface{}) { - if logger.level() >= WarnLevel { - entry := logger.newEntry() - entry.Warnf(format, args...) - logger.releaseEntry(entry) - } + logger.Logf(WarnLevel, format, args...) } func (logger *Logger) Warningf(format string, args ...interface{}) { - if logger.level() >= WarnLevel { - entry := logger.newEntry() - entry.Warnf(format, args...) - logger.releaseEntry(entry) - } + logger.Warnf(format, args...) } func (logger *Logger) Errorf(format string, args ...interface{}) { - if logger.level() >= ErrorLevel { - entry := logger.newEntry() - entry.Errorf(format, args...) - logger.releaseEntry(entry) - } + logger.Logf(ErrorLevel, format, args...) } func (logger *Logger) Fatalf(format string, args ...interface{}) { - if logger.level() >= FatalLevel { - entry := logger.newEntry() - entry.Fatalf(format, args...) - logger.releaseEntry(entry) - } - Exit(1) + logger.Logf(FatalLevel, format, args...) + logger.Exit(1) } func (logger *Logger) Panicf(format string, args ...interface{}) { - if logger.level() >= PanicLevel { + logger.Logf(PanicLevel, format, args...) +} + +func (logger *Logger) Log(level Level, args ...interface{}) { + if logger.IsLevelEnabled(level) { entry := logger.newEntry() - entry.Panicf(format, args...) + entry.Log(level, args...) logger.releaseEntry(entry) } } +func (logger *Logger) Trace(args ...interface{}) { + logger.Log(TraceLevel, args...) +} + func (logger *Logger) Debug(args ...interface{}) { - if logger.level() >= DebugLevel { - entry := logger.newEntry() - entry.Debug(args...) - logger.releaseEntry(entry) - } + logger.Log(DebugLevel, args...) } func (logger *Logger) Info(args ...interface{}) { - if logger.level() >= InfoLevel { - entry := logger.newEntry() - entry.Info(args...) - logger.releaseEntry(entry) - } + logger.Log(InfoLevel, args...) } func (logger *Logger) Print(args ...interface{}) { @@ -206,60 +205,44 @@ func (logger *Logger) Print(args ...interface{}) { } func (logger *Logger) Warn(args ...interface{}) { - if logger.level() >= WarnLevel { - entry := logger.newEntry() - entry.Warn(args...) - logger.releaseEntry(entry) - } + logger.Log(WarnLevel, args...) } func (logger *Logger) Warning(args ...interface{}) { - if logger.level() >= WarnLevel { - entry := logger.newEntry() - entry.Warn(args...) - logger.releaseEntry(entry) - } + logger.Warn(args...) } func (logger *Logger) Error(args ...interface{}) { - if logger.level() >= ErrorLevel { - entry := logger.newEntry() - entry.Error(args...) - logger.releaseEntry(entry) - } + logger.Log(ErrorLevel, args...) } func (logger *Logger) Fatal(args ...interface{}) { - if logger.level() >= FatalLevel { - entry := logger.newEntry() - entry.Fatal(args...) - logger.releaseEntry(entry) - } - Exit(1) + logger.Log(FatalLevel, args...) + logger.Exit(1) } func (logger *Logger) Panic(args ...interface{}) { - if logger.level() >= PanicLevel { + logger.Log(PanicLevel, args...) +} + +func (logger *Logger) Logln(level Level, args ...interface{}) { + if logger.IsLevelEnabled(level) { entry := logger.newEntry() - entry.Panic(args...) + entry.Logln(level, args...) logger.releaseEntry(entry) } } +func (logger *Logger) Traceln(args ...interface{}) { + logger.Logln(TraceLevel, args...) +} + func (logger *Logger) Debugln(args ...interface{}) { - if logger.level() >= DebugLevel { - entry := logger.newEntry() - entry.Debugln(args...) - logger.releaseEntry(entry) - } + logger.Logln(DebugLevel, args...) } func (logger *Logger) Infoln(args ...interface{}) { - if logger.level() >= InfoLevel { - entry := logger.newEntry() - entry.Infoln(args...) - logger.releaseEntry(entry) - } + logger.Logln(InfoLevel, args...) } func (logger *Logger) Println(args ...interface{}) { @@ -269,44 +252,32 @@ func (logger *Logger) Println(args ...interface{}) { } func (logger *Logger) Warnln(args ...interface{}) { - if logger.level() >= WarnLevel { - entry := logger.newEntry() - entry.Warnln(args...) - logger.releaseEntry(entry) - } + logger.Logln(WarnLevel, args...) } func (logger *Logger) Warningln(args ...interface{}) { - if logger.level() >= WarnLevel { - entry := logger.newEntry() - entry.Warnln(args...) - logger.releaseEntry(entry) - } + logger.Warn(args...) } func (logger *Logger) Errorln(args ...interface{}) { - if logger.level() >= ErrorLevel { - entry := logger.newEntry() - entry.Errorln(args...) - logger.releaseEntry(entry) - } + logger.Logln(ErrorLevel, args...) } func (logger *Logger) Fatalln(args ...interface{}) { - if logger.level() >= FatalLevel { - entry := logger.newEntry() - entry.Fatalln(args...) - logger.releaseEntry(entry) - } - Exit(1) + logger.Logln(FatalLevel, args...) + logger.Exit(1) } func (logger *Logger) Panicln(args ...interface{}) { - if logger.level() >= PanicLevel { - entry := logger.newEntry() - entry.Panicln(args...) - logger.releaseEntry(entry) + logger.Logln(PanicLevel, args...) +} + +func (logger *Logger) Exit(code int) { + runHandlers() + if logger.ExitFunc == nil { + logger.ExitFunc = os.Exit } + logger.ExitFunc(code) } //When file is opened with appending mode, it's safe to @@ -320,18 +291,53 @@ func (logger *Logger) level() Level { return Level(atomic.LoadUint32((*uint32)(&logger.Level))) } +// SetLevel sets the logger level. func (logger *Logger) SetLevel(level Level) { atomic.StoreUint32((*uint32)(&logger.Level), uint32(level)) } -func (logger *Logger) SetOutput(out io.Writer) { - logger.mu.Lock() - defer logger.mu.Unlock() - logger.Out = out +// GetLevel returns the logger level. +func (logger *Logger) GetLevel() Level { + return logger.level() } +// AddHook adds a hook to the logger hooks. func (logger *Logger) AddHook(hook Hook) { logger.mu.Lock() defer logger.mu.Unlock() logger.Hooks.Add(hook) } + +// IsLevelEnabled checks if the log level of the logger is greater than the level param +func (logger *Logger) IsLevelEnabled(level Level) bool { + return logger.level() >= level +} + +// SetFormatter sets the logger formatter. +func (logger *Logger) SetFormatter(formatter Formatter) { + logger.mu.Lock() + defer logger.mu.Unlock() + logger.Formatter = formatter +} + +// SetOutput sets the logger output. +func (logger *Logger) SetOutput(output io.Writer) { + logger.mu.Lock() + defer logger.mu.Unlock() + logger.Out = output +} + +func (logger *Logger) SetReportCaller(reportCaller bool) { + logger.mu.Lock() + defer logger.mu.Unlock() + logger.ReportCaller = reportCaller +} + +// ReplaceHooks replaces the logger hooks and returns the old ones +func (logger *Logger) ReplaceHooks(hooks LevelHooks) LevelHooks { + logger.mu.Lock() + oldHooks := logger.Hooks + logger.Hooks = hooks + logger.mu.Unlock() + return oldHooks +} diff --git a/vendor/github.com/sirupsen/logrus/logger_bench_test.go b/vendor/github.com/sirupsen/logrus/logger_bench_test.go index dd23a35..f0a7684 100644 --- a/vendor/github.com/sirupsen/logrus/logger_bench_test.go +++ b/vendor/github.com/sirupsen/logrus/logger_bench_test.go @@ -1,6 +1,7 @@ package logrus import ( + "io/ioutil" "os" "testing" ) @@ -59,3 +60,26 @@ func doLoggerBenchmarkNoLock(b *testing.B, out *os.File, formatter Formatter, fi } }) } + +func BenchmarkLoggerJSONFormatter(b *testing.B) { + doLoggerBenchmarkWithFormatter(b, &JSONFormatter{}) +} + +func BenchmarkLoggerTextFormatter(b *testing.B) { + doLoggerBenchmarkWithFormatter(b, &TextFormatter{}) +} + +func doLoggerBenchmarkWithFormatter(b *testing.B, f Formatter) { + b.SetParallelism(100) + log := New() + log.Formatter = f + log.Out = ioutil.Discard + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + log. + WithField("foo1", "bar1"). + WithField("foo2", "bar2"). + Info("this is a dummy log") + } + }) +} diff --git a/vendor/github.com/sirupsen/logrus/logger_test.go b/vendor/github.com/sirupsen/logrus/logger_test.go new file mode 100644 index 0000000..73ba450 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/logger_test.go @@ -0,0 +1,42 @@ +package logrus + +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFieldValueError(t *testing.T) { + buf := &bytes.Buffer{} + l := &Logger{ + Out: buf, + Formatter: new(JSONFormatter), + Hooks: make(LevelHooks), + Level: DebugLevel, + } + l.WithField("func", func() {}).Info("test") + fmt.Println(buf.String()) + var data map[string]interface{} + json.Unmarshal(buf.Bytes(), &data) + _, ok := data[FieldKeyLogrusError] + require.True(t, ok) +} + +func TestNoFieldValueError(t *testing.T) { + buf := &bytes.Buffer{} + l := &Logger{ + Out: buf, + Formatter: new(JSONFormatter), + Hooks: make(LevelHooks), + Level: DebugLevel, + } + l.WithField("str", "str").Info("test") + fmt.Println(buf.String()) + var data map[string]interface{} + json.Unmarshal(buf.Bytes(), &data) + _, ok := data[FieldKeyLogrusError] + require.False(t, ok) +} diff --git a/vendor/github.com/sirupsen/logrus/logrus.go b/vendor/github.com/sirupsen/logrus/logrus.go index dd38999..c1ca889 100644 --- a/vendor/github.com/sirupsen/logrus/logrus.go +++ b/vendor/github.com/sirupsen/logrus/logrus.go @@ -14,22 +14,11 @@ type Level uint32 // Convert the Level to a string. E.g. PanicLevel becomes "panic". func (level Level) String() string { - switch level { - case DebugLevel: - return "debug" - case InfoLevel: - return "info" - case WarnLevel: - return "warning" - case ErrorLevel: - return "error" - case FatalLevel: - return "fatal" - case PanicLevel: - return "panic" + if b, err := level.MarshalText(); err == nil { + return string(b) + } else { + return "unknown" } - - return "unknown" } // ParseLevel takes a string level and returns the Logrus log level constant. @@ -47,12 +36,47 @@ func ParseLevel(lvl string) (Level, error) { return InfoLevel, nil case "debug": return DebugLevel, nil + case "trace": + return TraceLevel, nil } var l Level return l, fmt.Errorf("not a valid logrus Level: %q", lvl) } +// UnmarshalText implements encoding.TextUnmarshaler. +func (level *Level) UnmarshalText(text []byte) error { + l, err := ParseLevel(string(text)) + if err != nil { + return err + } + + *level = Level(l) + + return nil +} + +func (level Level) MarshalText() ([]byte, error) { + switch level { + case TraceLevel: + return []byte("trace"), nil + case DebugLevel: + return []byte("debug"), nil + case InfoLevel: + return []byte("info"), nil + case WarnLevel: + return []byte("warning"), nil + case ErrorLevel: + return []byte("error"), nil + case FatalLevel: + return []byte("fatal"), nil + case PanicLevel: + return []byte("panic"), nil + } + + return nil, fmt.Errorf("not a valid lorus level %q", level) +} + // A constant exposing all logging levels var AllLevels = []Level{ PanicLevel, @@ -61,6 +85,7 @@ var AllLevels = []Level{ WarnLevel, InfoLevel, DebugLevel, + TraceLevel, } // These are the different logging levels. You can set the logging level to log @@ -69,7 +94,7 @@ const ( // PanicLevel level, highest level of severity. Logs and then calls panic with the // message passed to Debug, Info, ... PanicLevel Level = iota - // FatalLevel level. Logs and then calls `os.Exit(1)`. It will exit even if the + // FatalLevel level. Logs and then calls `logger.Exit(1)`. It will exit even if the // logging level is set to Panic. FatalLevel // ErrorLevel level. Logs. Used for errors that should definitely be noted. @@ -82,6 +107,8 @@ const ( InfoLevel // DebugLevel level. Usually only enabled when debugging. Very verbose logging. DebugLevel + // TraceLevel level. Designates finer-grained informational events than the Debug. + TraceLevel ) // Won't compile if StdLogger can't be realized by a log.Logger @@ -140,4 +167,20 @@ type FieldLogger interface { Errorln(args ...interface{}) Fatalln(args ...interface{}) Panicln(args ...interface{}) + + // IsDebugEnabled() bool + // IsInfoEnabled() bool + // IsWarnEnabled() bool + // IsErrorEnabled() bool + // IsFatalEnabled() bool + // IsPanicEnabled() bool +} + +// Ext1FieldLogger (the first extension to FieldLogger) is superfluous, it is +// here for consistancy. Do not use. Use Logger or Entry instead. +type Ext1FieldLogger interface { + FieldLogger + Tracef(format string, args ...interface{}) + Trace(args ...interface{}) + Traceln(args ...interface{}) } diff --git a/vendor/github.com/sirupsen/logrus/logrus_test.go b/vendor/github.com/sirupsen/logrus/logrus_test.go index 78e1301..d857262 100644 --- a/vendor/github.com/sirupsen/logrus/logrus_test.go +++ b/vendor/github.com/sirupsen/logrus/logrus_test.go @@ -1,68 +1,95 @@ -package logrus +package logrus_test import ( "bytes" "encoding/json" - "strconv" - "strings" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" "sync" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + . "github.com/sirupsen/logrus" + . "github.com/sirupsen/logrus/internal/testutils" ) -func LogAndAssertJSON(t *testing.T, log func(*Logger), assertions func(fields Fields)) { +// TestReportCaller verifies that when ReportCaller is set, the 'func' field +// is added, and when it is unset it is not set or modified +// Verify that functions within the Logrus package aren't considered when +// discovering the caller. +func TestReportCallerWhenConfigured(t *testing.T) { + LogAndAssertJSON(t, func(log *Logger) { + log.ReportCaller = false + log.Print("testNoCaller") + }, func(fields Fields) { + assert.Equal(t, "testNoCaller", fields["msg"]) + assert.Equal(t, "info", fields["level"]) + assert.Equal(t, nil, fields["func"]) + }) + + LogAndAssertJSON(t, func(log *Logger) { + log.ReportCaller = true + log.Print("testWithCaller") + }, func(fields Fields) { + assert.Equal(t, "testWithCaller", fields["msg"]) + assert.Equal(t, "info", fields["level"]) + assert.Equal(t, + "github.com/sirupsen/logrus_test.TestReportCallerWhenConfigured.func3", fields["func"]) + }) +} + +func logSomething(t *testing.T, message string) Fields { var buffer bytes.Buffer var fields Fields logger := New() logger.Out = &buffer logger.Formatter = new(JSONFormatter) + logger.ReportCaller = true - log(logger) + entry := logger.WithFields(Fields{ + "foo": "bar", + }) + + entry.Info(message) err := json.Unmarshal(buffer.Bytes(), &fields) assert.Nil(t, err) - assertions(fields) + return fields } -func LogAndAssertText(t *testing.T, log func(*Logger), assertions func(fields map[string]string)) { - var buffer bytes.Buffer +// TestReportCallerHelperDirect - verify reference when logging from a regular function +func TestReportCallerHelperDirect(t *testing.T) { + fields := logSomething(t, "direct") - logger := New() - logger.Out = &buffer - logger.Formatter = &TextFormatter{ - DisableColors: true, - } + assert.Equal(t, "direct", fields["msg"]) + assert.Equal(t, "info", fields["level"]) + assert.Regexp(t, "github.com/.*/logrus_test.logSomething", fields["func"]) +} - log(logger) +// TestReportCallerHelperDirect - verify reference when logging from a function called via pointer +func TestReportCallerHelperViaPointer(t *testing.T) { + fptr := logSomething + fields := fptr(t, "via pointer") - fields := make(map[string]string) - for _, kv := range strings.Split(buffer.String(), " ") { - if !strings.Contains(kv, "=") { - continue - } - kvArr := strings.Split(kv, "=") - key := strings.TrimSpace(kvArr[0]) - val := kvArr[1] - if kvArr[1][0] == '"' { - var err error - val, err = strconv.Unquote(val) - assert.NoError(t, err) - } - fields[key] = val - } - assertions(fields) + assert.Equal(t, "via pointer", fields["msg"]) + assert.Equal(t, "info", fields["level"]) + assert.Regexp(t, "github.com/.*/logrus_test.logSomething", fields["func"]) } func TestPrint(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Print("test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test") - assert.Equal(t, fields["level"], "info") + assert.Equal(t, "test", fields["msg"]) + assert.Equal(t, "info", fields["level"]) }) } @@ -70,8 +97,8 @@ func TestInfo(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Info("test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test") - assert.Equal(t, fields["level"], "info") + assert.Equal(t, "test", fields["msg"]) + assert.Equal(t, "info", fields["level"]) }) } @@ -79,8 +106,17 @@ func TestWarn(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Warn("test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test") - assert.Equal(t, fields["level"], "warning") + assert.Equal(t, "test", fields["msg"]) + assert.Equal(t, "warning", fields["level"]) + }) +} + +func TestLog(t *testing.T) { + LogAndAssertJSON(t, func(log *Logger) { + log.Log(WarnLevel, "test") + }, func(fields Fields) { + assert.Equal(t, "test", fields["msg"]) + assert.Equal(t, "warning", fields["level"]) }) } @@ -88,7 +124,7 @@ func TestInfolnShouldAddSpacesBetweenStrings(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Infoln("test", "test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test test") + assert.Equal(t, "test test", fields["msg"]) }) } @@ -96,7 +132,7 @@ func TestInfolnShouldAddSpacesBetweenStringAndNonstring(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Infoln("test", 10) }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test 10") + assert.Equal(t, "test 10", fields["msg"]) }) } @@ -104,7 +140,7 @@ func TestInfolnShouldAddSpacesBetweenTwoNonStrings(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Infoln(10, 10) }, func(fields Fields) { - assert.Equal(t, fields["msg"], "10 10") + assert.Equal(t, "10 10", fields["msg"]) }) } @@ -112,7 +148,7 @@ func TestInfoShouldAddSpacesBetweenTwoNonStrings(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Infoln(10, 10) }, func(fields Fields) { - assert.Equal(t, fields["msg"], "10 10") + assert.Equal(t, "10 10", fields["msg"]) }) } @@ -120,7 +156,7 @@ func TestInfoShouldNotAddSpacesBetweenStringAndNonstring(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Info("test", 10) }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test10") + assert.Equal(t, "test10", fields["msg"]) }) } @@ -128,7 +164,7 @@ func TestInfoShouldNotAddSpacesBetweenStrings(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.Info("test", "test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "testtest") + assert.Equal(t, "testtest", fields["msg"]) }) } @@ -166,7 +202,7 @@ func TestUserSuppliedFieldDoesNotOverwriteDefaults(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithField("msg", "hello").Info("test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test") + assert.Equal(t, "test", fields["msg"]) }) } @@ -174,8 +210,8 @@ func TestUserSuppliedMsgFieldHasPrefix(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithField("msg", "hello").Info("test") }, func(fields Fields) { - assert.Equal(t, fields["msg"], "test") - assert.Equal(t, fields["fields.msg"], "hello") + assert.Equal(t, "test", fields["msg"]) + assert.Equal(t, "hello", fields["fields.msg"]) }) } @@ -183,7 +219,7 @@ func TestUserSuppliedTimeFieldHasPrefix(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithField("time", "hello").Info("test") }, func(fields Fields) { - assert.Equal(t, fields["fields.time"], "hello") + assert.Equal(t, "hello", fields["fields.time"]) }) } @@ -191,8 +227,8 @@ func TestUserSuppliedLevelFieldHasPrefix(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithField("level", 1).Info("test") }, func(fields Fields) { - assert.Equal(t, fields["level"], "info") - assert.Equal(t, fields["fields.level"], 1.0) // JSON has floats only + assert.Equal(t, "info", fields["level"]) + assert.Equal(t, 1.0, fields["fields.level"]) // JSON has floats only }) } @@ -216,7 +252,7 @@ func TestWithTimeShouldOverrideTime(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithTime(now).Info("foobar") }, func(fields Fields) { - assert.Equal(t, fields["time"], now.Format(defaultTimestampFormat)) + assert.Equal(t, fields["time"], now.Format(time.RFC3339)) }) } @@ -226,7 +262,7 @@ func TestWithTimeShouldNotOverrideFields(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithField("herp", "derp").WithTime(now).Info("blah") }, func(fields Fields) { - assert.Equal(t, fields["time"], now.Format(defaultTimestampFormat)) + assert.Equal(t, fields["time"], now.Format(time.RFC3339)) assert.Equal(t, fields["herp"], "derp") }) } @@ -237,7 +273,7 @@ func TestWithFieldShouldNotOverrideTime(t *testing.T) { LogAndAssertJSON(t, func(log *Logger) { log.WithTime(now).WithField("herp", "derp").Info("blah") }, func(fields Fields) { - assert.Equal(t, fields["time"], now.Format(defaultTimestampFormat)) + assert.Equal(t, fields["time"], now.Format(time.RFC3339)) assert.Equal(t, fields["herp"], "derp") }) } @@ -295,13 +331,119 @@ func TestDoubleLoggingDoesntPrefixPreviousFields(t *testing.T) { err = json.Unmarshal(buffer.Bytes(), &fields) assert.NoError(t, err, "should have decoded second message") assert.Equal(t, len(fields), 4, "should only have msg/time/level/context fields") - assert.Equal(t, fields["msg"], "omg it is!") - assert.Equal(t, fields["context"], "eating raw fish") + assert.Equal(t, "omg it is!", fields["msg"]) + assert.Equal(t, "eating raw fish", fields["context"]) assert.Nil(t, fields["fields.msg"], "should not have prefixed previous `msg` entry") } +func TestNestedLoggingReportsCorrectCaller(t *testing.T) { + var buffer bytes.Buffer + var fields Fields + + logger := New() + logger.Out = &buffer + logger.Formatter = new(JSONFormatter) + logger.ReportCaller = true + + llog := logger.WithField("context", "eating raw fish") + + llog.Info("looks delicious") + _, _, line, _ := runtime.Caller(0) + + err := json.Unmarshal(buffer.Bytes(), &fields) + require.NoError(t, err, "should have decoded first message") + assert.Equal(t, 6, len(fields), "should have msg/time/level/func/context fields") + assert.Equal(t, "looks delicious", fields["msg"]) + assert.Equal(t, "eating raw fish", fields["context"]) + assert.Equal(t, + "github.com/sirupsen/logrus_test.TestNestedLoggingReportsCorrectCaller", fields["func"]) + cwd, err := os.Getwd() + require.NoError(t, err) + assert.Equal(t, filepath.ToSlash(fmt.Sprintf("%s/logrus_test.go:%d", cwd, line-1)), filepath.ToSlash(fields["file"].(string))) + + buffer.Reset() + + logger.WithFields(Fields{ + "Clyde": "Stubblefield", + }).WithFields(Fields{ + "Jab'o": "Starks", + }).WithFields(Fields{ + "uri": "https://www.youtube.com/watch?v=V5DTznu-9v0", + }).WithFields(Fields{ + "func": "y drummer", + }).WithFields(Fields{ + "James": "Brown", + }).Print("The hardest workin' man in show business") + _, _, line, _ = runtime.Caller(0) + + err = json.Unmarshal(buffer.Bytes(), &fields) + assert.NoError(t, err, "should have decoded second message") + assert.Equal(t, 11, len(fields), "should have all builtin fields plus foo,bar,baz,...") + assert.Equal(t, "Stubblefield", fields["Clyde"]) + assert.Equal(t, "Starks", fields["Jab'o"]) + assert.Equal(t, "https://www.youtube.com/watch?v=V5DTznu-9v0", fields["uri"]) + assert.Equal(t, "y drummer", fields["fields.func"]) + assert.Equal(t, "Brown", fields["James"]) + assert.Equal(t, "The hardest workin' man in show business", fields["msg"]) + assert.Nil(t, fields["fields.msg"], "should not have prefixed previous `msg` entry") + assert.Equal(t, + "github.com/sirupsen/logrus_test.TestNestedLoggingReportsCorrectCaller", fields["func"]) + require.NoError(t, err) + assert.Equal(t, filepath.ToSlash(fmt.Sprintf("%s/logrus_test.go:%d", cwd, line-1)), filepath.ToSlash(fields["file"].(string))) + + logger.ReportCaller = false // return to default value +} + +func logLoop(iterations int, reportCaller bool) { + var buffer bytes.Buffer + + logger := New() + logger.Out = &buffer + logger.Formatter = new(JSONFormatter) + logger.ReportCaller = reportCaller + + for i := 0; i < iterations; i++ { + logger.Infof("round %d of %d", i, iterations) + } +} + +// Assertions for upper bounds to reporting overhead +func TestCallerReportingOverhead(t *testing.T) { + iterations := 5000 + before := time.Now() + logLoop(iterations, false) + during := time.Now() + logLoop(iterations, true) + after := time.Now() + + elapsedNotReporting := during.Sub(before).Nanoseconds() + elapsedReporting := after.Sub(during).Nanoseconds() + + maxDelta := 1 * time.Second + assert.WithinDuration(t, during, before, maxDelta, + "%d log calls without caller name lookup takes less than %d second(s) (was %d nanoseconds)", + iterations, maxDelta.Seconds(), elapsedNotReporting) + assert.WithinDuration(t, after, during, maxDelta, + "%d log calls without caller name lookup takes less than %d second(s) (was %d nanoseconds)", + iterations, maxDelta.Seconds(), elapsedReporting) +} + +// benchmarks for both with and without caller-function reporting +func BenchmarkWithoutCallerTracing(b *testing.B) { + for i := 0; i < b.N; i++ { + logLoop(1000, false) + } +} + +func BenchmarkWithCallerTracing(b *testing.B) { + for i := 0; i < b.N; i++ { + logLoop(1000, true) + } +} + func TestConvertLevelToString(t *testing.T) { + assert.Equal(t, "trace", TraceLevel.String()) assert.Equal(t, "debug", DebugLevel.String()) assert.Equal(t, "info", InfoLevel.String()) assert.Equal(t, "warning", WarnLevel.String()) @@ -367,6 +509,14 @@ func TestParseLevel(t *testing.T) { assert.Nil(t, err) assert.Equal(t, DebugLevel, l) + l, err = ParseLevel("trace") + assert.Nil(t, err) + assert.Equal(t, TraceLevel, l) + + l, err = ParseLevel("TRACE") + assert.Nil(t, err) + assert.Equal(t, TraceLevel, l) + l, err = ParseLevel("invalid") assert.Equal(t, "not a valid logrus Level: \"invalid\"", err.Error()) } @@ -403,10 +553,52 @@ func TestLoggingRace(t *testing.T) { wg.Wait() } +func TestLoggingRaceWithHooksOnEntry(t *testing.T) { + logger := New() + hook := new(ModifyHook) + logger.AddHook(hook) + entry := logger.WithField("context", "clue") + + var wg sync.WaitGroup + wg.Add(100) + + for i := 0; i < 100; i++ { + go func() { + entry.Info("info") + wg.Done() + }() + } + wg.Wait() +} + +func TestReplaceHooks(t *testing.T) { + old, cur := &TestHook{}, &TestHook{} + + logger := New() + logger.SetOutput(ioutil.Discard) + logger.AddHook(old) + + hooks := make(LevelHooks) + hooks.Add(cur) + replaced := logger.ReplaceHooks(hooks) + + logger.Info("test") + + assert.Equal(t, old.Fired, false) + assert.Equal(t, cur.Fired, true) + + logger.ReplaceHooks(replaced) + logger.Info("test") + assert.Equal(t, old.Fired, true) +} + // Compile test -func TestLogrusInterface(t *testing.T) { +func TestLogrusInterfaces(t *testing.T) { var buffer bytes.Buffer - fn := func(l FieldLogger) { + // This verifies FieldLogger and Ext1FieldLogger work as designed. + // Please don't use them. Use Logger and Entry directly. + fn := func(xl Ext1FieldLogger) { + var l FieldLogger = xl b := l.WithField("key", "value") b.Debug("Test") } @@ -444,3 +636,81 @@ func TestEntryWriter(t *testing.T) { assert.Equal(t, fields["foo"], "bar") assert.Equal(t, fields["level"], "warning") } + +func TestLogLevelEnabled(t *testing.T) { + log := New() + log.SetLevel(PanicLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, false, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, false, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, false, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, false, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, false, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, false, log.IsLevelEnabled(TraceLevel)) + + log.SetLevel(FatalLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, true, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, false, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, false, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, false, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, false, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, false, log.IsLevelEnabled(TraceLevel)) + + log.SetLevel(ErrorLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, true, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, true, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, false, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, false, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, false, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, false, log.IsLevelEnabled(TraceLevel)) + + log.SetLevel(WarnLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, true, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, true, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, true, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, false, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, false, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, false, log.IsLevelEnabled(TraceLevel)) + + log.SetLevel(InfoLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, true, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, true, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, true, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, true, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, false, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, false, log.IsLevelEnabled(TraceLevel)) + + log.SetLevel(DebugLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, true, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, true, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, true, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, true, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, true, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, false, log.IsLevelEnabled(TraceLevel)) + + log.SetLevel(TraceLevel) + assert.Equal(t, true, log.IsLevelEnabled(PanicLevel)) + assert.Equal(t, true, log.IsLevelEnabled(FatalLevel)) + assert.Equal(t, true, log.IsLevelEnabled(ErrorLevel)) + assert.Equal(t, true, log.IsLevelEnabled(WarnLevel)) + assert.Equal(t, true, log.IsLevelEnabled(InfoLevel)) + assert.Equal(t, true, log.IsLevelEnabled(DebugLevel)) + assert.Equal(t, true, log.IsLevelEnabled(TraceLevel)) +} + +func TestReportCallerOnTextFormatter(t *testing.T) { + l := New() + + l.Formatter.(*TextFormatter).ForceColors = true + l.Formatter.(*TextFormatter).DisableColors = false + l.WithFields(Fields{"func": "func", "file": "file"}).Info("test") + + l.Formatter.(*TextFormatter).ForceColors = false + l.Formatter.(*TextFormatter).DisableColors = true + l.WithFields(Fields{"func": "func", "file": "file"}).Info("test") +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_bsd.go b/vendor/github.com/sirupsen/logrus/terminal_bsd.go deleted file mode 100644 index 4880d13..0000000 --- a/vendor/github.com/sirupsen/logrus/terminal_bsd.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build darwin freebsd openbsd netbsd dragonfly -// +build !appengine,!gopherjs - -package logrus - -import "golang.org/x/sys/unix" - -const ioctlReadTermios = unix.TIOCGETA - -type Termios unix.Termios diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_aix.go b/vendor/github.com/sirupsen/logrus/terminal_check_aix.go new file mode 100644 index 0000000..04fdb7b --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_check_aix.go @@ -0,0 +1,9 @@ +// +build !appengine,!js,!windows,aix + +package logrus + +import "io" + +func checkIfTerminal(w io.Writer) bool { + return false +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_appengine.go b/vendor/github.com/sirupsen/logrus/terminal_check_appengine.go index 3de08e8..2403de9 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_check_appengine.go +++ b/vendor/github.com/sirupsen/logrus/terminal_check_appengine.go @@ -1,4 +1,4 @@ -// +build appengine gopherjs +// +build appengine package logrus diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_js.go b/vendor/github.com/sirupsen/logrus/terminal_check_js.go new file mode 100644 index 0000000..0c20975 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_check_js.go @@ -0,0 +1,11 @@ +// +build js + +package logrus + +import ( + "io" +) + +func checkIfTerminal(w io.Writer) bool { + return false +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_notappengine.go b/vendor/github.com/sirupsen/logrus/terminal_check_notappengine.go index 067047a..d465565 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_check_notappengine.go +++ b/vendor/github.com/sirupsen/logrus/terminal_check_notappengine.go @@ -1,4 +1,4 @@ -// +build !appengine,!gopherjs +// +build !appengine,!js,!windows,!aix package logrus diff --git a/vendor/github.com/sirupsen/logrus/terminal_check_windows.go b/vendor/github.com/sirupsen/logrus/terminal_check_windows.go new file mode 100644 index 0000000..3b9d286 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_check_windows.go @@ -0,0 +1,20 @@ +// +build !appengine,!js,windows + +package logrus + +import ( + "io" + "os" + "syscall" +) + +func checkIfTerminal(w io.Writer) bool { + switch v := w.(type) { + case *os.File: + var mode uint32 + err := syscall.GetConsoleMode(syscall.Handle(v.Fd()), &mode) + return err == nil + default: + return false + } +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_notwindows.go b/vendor/github.com/sirupsen/logrus/terminal_notwindows.go new file mode 100644 index 0000000..3dbd237 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_notwindows.go @@ -0,0 +1,8 @@ +// +build !windows + +package logrus + +import "io" + +func initTerminal(w io.Writer) { +} diff --git a/vendor/github.com/sirupsen/logrus/terminal_windows.go b/vendor/github.com/sirupsen/logrus/terminal_windows.go new file mode 100644 index 0000000..b4ef528 --- /dev/null +++ b/vendor/github.com/sirupsen/logrus/terminal_windows.go @@ -0,0 +1,18 @@ +// +build !appengine,!js,windows + +package logrus + +import ( + "io" + "os" + "syscall" + + sequences "github.com/konsorten/go-windows-terminal-sequences" +) + +func initTerminal(w io.Writer) { + switch v := w.(type) { + case *os.File: + sequences.EnableVirtualTerminalProcessing(syscall.Handle(v.Fd()), true) + } +} diff --git a/vendor/github.com/sirupsen/logrus/text_formatter.go b/vendor/github.com/sirupsen/logrus/text_formatter.go index 3e55040..fb21649 100644 --- a/vendor/github.com/sirupsen/logrus/text_formatter.go +++ b/vendor/github.com/sirupsen/logrus/text_formatter.go @@ -3,6 +3,8 @@ package logrus import ( "bytes" "fmt" + "os" + "runtime" "sort" "strings" "sync" @@ -35,6 +37,9 @@ type TextFormatter struct { // Force disabling colors. DisableColors bool + // Override coloring based on CLICOLOR and CLICOLOR_FORCE. - https://bixense.com/clicolors/ + EnvironmentOverrideColors bool + // Disable timestamp logging. useful when output is redirected to logging // system that already adds timestamps. DisableTimestamp bool @@ -51,6 +56,9 @@ type TextFormatter struct { // be desired. DisableSorting bool + // The keys sorting function, when uninitialized it uses sort.Strings. + SortingFunc func([]string) + // Disables the truncation of the level text to 4 characters. DisableLevelTruncation bool @@ -69,26 +77,77 @@ type TextFormatter struct { // FieldKeyMsg: "@message"}} FieldMap FieldMap - sync.Once + terminalInitOnce sync.Once } func (f *TextFormatter) init(entry *Entry) { if entry.Logger != nil { f.isTerminal = checkIfTerminal(entry.Logger.Out) + + if f.isTerminal { + initTerminal(entry.Logger.Out) + } } } +func (f *TextFormatter) isColored() bool { + isColored := f.ForceColors || (f.isTerminal && (runtime.GOOS != "windows")) + + if f.EnvironmentOverrideColors { + if force, ok := os.LookupEnv("CLICOLOR_FORCE"); ok && force != "0" { + isColored = true + } else if ok && force == "0" { + isColored = false + } else if os.Getenv("CLICOLOR") == "0" { + isColored = false + } + } + + return isColored && !f.DisableColors +} + // Format renders a single log entry func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { - prefixFieldClashes(entry.Data, f.FieldMap) - - keys := make([]string, 0, len(entry.Data)) - for k := range entry.Data { + data := make(Fields) + for k, v := range entry.Data { + data[k] = v + } + prefixFieldClashes(data, f.FieldMap, entry.HasCaller()) + keys := make([]string, 0, len(data)) + for k := range data { keys = append(keys, k) } + fixedKeys := make([]string, 0, 4+len(data)) + if !f.DisableTimestamp { + fixedKeys = append(fixedKeys, f.FieldMap.resolve(FieldKeyTime)) + } + fixedKeys = append(fixedKeys, f.FieldMap.resolve(FieldKeyLevel)) + if entry.Message != "" { + fixedKeys = append(fixedKeys, f.FieldMap.resolve(FieldKeyMsg)) + } + if entry.err != "" { + fixedKeys = append(fixedKeys, f.FieldMap.resolve(FieldKeyLogrusError)) + } + if entry.HasCaller() { + fixedKeys = append(fixedKeys, + f.FieldMap.resolve(FieldKeyFunc), f.FieldMap.resolve(FieldKeyFile)) + } + if !f.DisableSorting { - sort.Strings(keys) + if f.SortingFunc == nil { + sort.Strings(keys) + fixedKeys = append(fixedKeys, keys...) + } else { + if !f.isColored() { + fixedKeys = append(fixedKeys, keys...) + f.SortingFunc(fixedKeys) + } else { + f.SortingFunc(keys) + } + } + } else { + fixedKeys = append(fixedKeys, keys...) } var b *bytes.Buffer @@ -98,26 +157,34 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { b = &bytes.Buffer{} } - f.Do(func() { f.init(entry) }) - - isColored := (f.ForceColors || f.isTerminal) && !f.DisableColors + f.terminalInitOnce.Do(func() { f.init(entry) }) timestampFormat := f.TimestampFormat if timestampFormat == "" { timestampFormat = defaultTimestampFormat } - if isColored { - f.printColored(b, entry, keys, timestampFormat) + if f.isColored() { + f.printColored(b, entry, keys, data, timestampFormat) } else { - if !f.DisableTimestamp { - f.appendKeyValue(b, f.FieldMap.resolve(FieldKeyTime), entry.Time.Format(timestampFormat)) - } - f.appendKeyValue(b, f.FieldMap.resolve(FieldKeyLevel), entry.Level.String()) - if entry.Message != "" { - f.appendKeyValue(b, f.FieldMap.resolve(FieldKeyMsg), entry.Message) - } - for _, key := range keys { - f.appendKeyValue(b, key, entry.Data[key]) + for _, key := range fixedKeys { + var value interface{} + switch { + case key == f.FieldMap.resolve(FieldKeyTime): + value = entry.Time.Format(timestampFormat) + case key == f.FieldMap.resolve(FieldKeyLevel): + value = entry.Level.String() + case key == f.FieldMap.resolve(FieldKeyMsg): + value = entry.Message + case key == f.FieldMap.resolve(FieldKeyLogrusError): + value = entry.err + case key == f.FieldMap.resolve(FieldKeyFunc) && entry.HasCaller(): + value = entry.Caller.Function + case key == f.FieldMap.resolve(FieldKeyFile) && entry.HasCaller(): + value = fmt.Sprintf("%s:%d", entry.Caller.File, entry.Caller.Line) + default: + value = data[key] + } + f.appendKeyValue(b, key, value) } } @@ -125,10 +192,10 @@ func (f *TextFormatter) Format(entry *Entry) ([]byte, error) { return b.Bytes(), nil } -func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []string, timestampFormat string) { +func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []string, data Fields, timestampFormat string) { var levelColor int switch entry.Level { - case DebugLevel: + case DebugLevel, TraceLevel: levelColor = gray case WarnLevel: levelColor = yellow @@ -143,15 +210,26 @@ func (f *TextFormatter) printColored(b *bytes.Buffer, entry *Entry, keys []strin levelText = levelText[0:4] } + // Remove a single newline if it already exists in the message to keep + // the behavior of logrus text_formatter the same as the stdlib log package + entry.Message = strings.TrimSuffix(entry.Message, "\n") + + caller := "" + + if entry.HasCaller() { + caller = fmt.Sprintf("%s:%d %s()", + entry.Caller.File, entry.Caller.Line, entry.Caller.Function) + } + if f.DisableTimestamp { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m %-44s ", levelColor, levelText, entry.Message) + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m%s %-44s ", levelColor, levelText, caller, entry.Message) } else if !f.FullTimestamp { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d] %-44s ", levelColor, levelText, int(entry.Time.Sub(baseTimestamp)/time.Second), entry.Message) + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%04d]%s %-44s ", levelColor, levelText, int(entry.Time.Sub(baseTimestamp)/time.Second), caller, entry.Message) } else { - fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s] %-44s ", levelColor, levelText, entry.Time.Format(timestampFormat), entry.Message) + fmt.Fprintf(b, "\x1b[%dm%s\x1b[0m[%s]%s %-44s ", levelColor, levelText, entry.Time.Format(timestampFormat), caller, entry.Message) } for _, k := range keys { - v := entry.Data[k] + v := data[k] fmt.Fprintf(b, " \x1b[%dm%s\x1b[0m=", levelColor, k) f.appendValue(b, v) } diff --git a/vendor/github.com/sirupsen/logrus/text_formatter_test.go b/vendor/github.com/sirupsen/logrus/text_formatter_test.go index 921d052..9c5e6f0 100644 --- a/vendor/github.com/sirupsen/logrus/text_formatter_test.go +++ b/vendor/github.com/sirupsen/logrus/text_formatter_test.go @@ -4,11 +4,15 @@ import ( "bytes" "errors" "fmt" + "os" + "runtime" + "sort" "strings" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestFormatting(t *testing.T) { @@ -141,7 +145,7 @@ func TestDisableLevelTruncation(t *testing.T) { tf := &TextFormatter{DisableLevelTruncation: disabled} var b bytes.Buffer entry.Level = level - tf.printColored(&b, entry, keys, timestampFormat) + tf.printColored(&b, entry, keys, nil, timestampFormat) logLine := (&b).String() if disabled { expected := strings.ToUpper(level.String()) @@ -177,6 +181,29 @@ func TestDisableTimestampWithColoredOutput(t *testing.T) { } } +func TestNewlineBehavior(t *testing.T) { + tf := &TextFormatter{ForceColors: true} + + // Ensure a single new line is removed as per stdlib log + e := NewEntry(StandardLogger()) + e.Message = "test message\n" + b, _ := tf.Format(e) + if bytes.Contains(b, []byte("test message\n")) { + t.Error("first newline at end of Entry.Message resulted in unexpected 2 newlines in output. Expected newline to be removed.") + } + + // Ensure a double new line is reduced to a single new line + e = NewEntry(StandardLogger()) + e.Message = "test message\n\n" + b, _ = tf.Format(e) + if bytes.Contains(b, []byte("test message\n\n")) { + t.Error("Double newline at end of Entry.Message resulted in unexpected 2 newlines in output. Expected single newline") + } + if !bytes.Contains(b, []byte("test message\n")) { + t.Error("Double newline at end of Entry.Message did not result in a single newline after formatting") + } +} + func TestTextFormatterFieldMap(t *testing.T) { formatter := &TextFormatter{ DisableColors: true, @@ -216,5 +243,243 @@ func TestTextFormatterFieldMap(t *testing.T) { "Formatted output doesn't respect FieldMap") } -// TODO add tests for sorting etc., this requires a parser for the text -// formatter output. +func TestTextFormatterIsColored(t *testing.T) { + params := []struct { + name string + expectedResult bool + isTerminal bool + disableColor bool + forceColor bool + envColor bool + clicolorIsSet bool + clicolorForceIsSet bool + clicolorVal string + clicolorForceVal string + }{ + // Default values + { + name: "testcase1", + expectedResult: false, + isTerminal: false, + disableColor: false, + forceColor: false, + envColor: false, + clicolorIsSet: false, + clicolorForceIsSet: false, + }, + // Output on terminal + { + name: "testcase2", + expectedResult: true, + isTerminal: true, + disableColor: false, + forceColor: false, + envColor: false, + clicolorIsSet: false, + clicolorForceIsSet: false, + }, + // Output on terminal with color disabled + { + name: "testcase3", + expectedResult: false, + isTerminal: true, + disableColor: true, + forceColor: false, + envColor: false, + clicolorIsSet: false, + clicolorForceIsSet: false, + }, + // Output not on terminal with color disabled + { + name: "testcase4", + expectedResult: false, + isTerminal: false, + disableColor: true, + forceColor: false, + envColor: false, + clicolorIsSet: false, + clicolorForceIsSet: false, + }, + // Output not on terminal with color forced + { + name: "testcase5", + expectedResult: true, + isTerminal: false, + disableColor: false, + forceColor: true, + envColor: false, + clicolorIsSet: false, + clicolorForceIsSet: false, + }, + // Output on terminal with clicolor set to "0" + { + name: "testcase6", + expectedResult: false, + isTerminal: true, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: true, + clicolorForceIsSet: false, + clicolorVal: "0", + }, + // Output on terminal with clicolor set to "1" + { + name: "testcase7", + expectedResult: true, + isTerminal: true, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: true, + clicolorForceIsSet: false, + clicolorVal: "1", + }, + // Output not on terminal with clicolor set to "0" + { + name: "testcase8", + expectedResult: false, + isTerminal: false, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: true, + clicolorForceIsSet: false, + clicolorVal: "0", + }, + // Output not on terminal with clicolor set to "1" + { + name: "testcase9", + expectedResult: false, + isTerminal: false, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: true, + clicolorForceIsSet: false, + clicolorVal: "1", + }, + // Output not on terminal with clicolor set to "1" and force color + { + name: "testcase10", + expectedResult: true, + isTerminal: false, + disableColor: false, + forceColor: true, + envColor: true, + clicolorIsSet: true, + clicolorForceIsSet: false, + clicolorVal: "1", + }, + // Output not on terminal with clicolor set to "0" and force color + { + name: "testcase11", + expectedResult: false, + isTerminal: false, + disableColor: false, + forceColor: true, + envColor: true, + clicolorIsSet: true, + clicolorForceIsSet: false, + clicolorVal: "0", + }, + // Output not on terminal with clicolor_force set to "1" + { + name: "testcase12", + expectedResult: true, + isTerminal: false, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: false, + clicolorForceIsSet: true, + clicolorForceVal: "1", + }, + // Output not on terminal with clicolor_force set to "0" + { + name: "testcase13", + expectedResult: false, + isTerminal: false, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: false, + clicolorForceIsSet: true, + clicolorForceVal: "0", + }, + // Output on terminal with clicolor_force set to "0" + { + name: "testcase14", + expectedResult: false, + isTerminal: true, + disableColor: false, + forceColor: false, + envColor: true, + clicolorIsSet: false, + clicolorForceIsSet: true, + clicolorForceVal: "0", + }, + } + + cleanenv := func() { + os.Unsetenv("CLICOLOR") + os.Unsetenv("CLICOLOR_FORCE") + } + + defer cleanenv() + + for _, val := range params { + t.Run("textformatter_"+val.name, func(subT *testing.T) { + tf := TextFormatter{ + isTerminal: val.isTerminal, + DisableColors: val.disableColor, + ForceColors: val.forceColor, + EnvironmentOverrideColors: val.envColor, + } + cleanenv() + if val.clicolorIsSet { + os.Setenv("CLICOLOR", val.clicolorVal) + } + if val.clicolorForceIsSet { + os.Setenv("CLICOLOR_FORCE", val.clicolorForceVal) + } + res := tf.isColored() + if runtime.GOOS == "windows" && !tf.ForceColors && !val.clicolorForceIsSet { + assert.Equal(subT, false, res) + } else { + assert.Equal(subT, val.expectedResult, res) + } + }) + } +} + +func TestCustomSorting(t *testing.T) { + formatter := &TextFormatter{ + DisableColors: true, + SortingFunc: func(keys []string) { + sort.Slice(keys, func(i, j int) bool { + if keys[j] == "prefix" { + return false + } + if keys[i] == "prefix" { + return true + } + return strings.Compare(keys[i], keys[j]) == -1 + }) + }, + } + + entry := &Entry{ + Message: "Testing custom sort function", + Time: time.Now(), + Level: InfoLevel, + Data: Fields{ + "test": "testvalue", + "prefix": "the application prefix", + "blablabla": "blablabla", + }, + } + b, err := formatter.Format(entry) + require.NoError(t, err) + require.True(t, strings.HasPrefix(string(b), "prefix="), "format output is %q", string(b)) +} diff --git a/vendor/github.com/sirupsen/logrus/writer.go b/vendor/github.com/sirupsen/logrus/writer.go index 7bdebed..9e1f751 100644 --- a/vendor/github.com/sirupsen/logrus/writer.go +++ b/vendor/github.com/sirupsen/logrus/writer.go @@ -24,6 +24,8 @@ func (entry *Entry) WriterLevel(level Level) *io.PipeWriter { var printFunc func(args ...interface{}) switch level { + case TraceLevel: + printFunc = entry.Trace case DebugLevel: printFunc = entry.Debug case InfoLevel: diff --git a/vendor/github.com/stretchr/testify/.travis.yml b/vendor/github.com/stretchr/testify/.travis.yml index f408b4c..da6ba0d 100644 --- a/vendor/github.com/stretchr/testify/.travis.yml +++ b/vendor/github.com/stretchr/testify/.travis.yml @@ -2,14 +2,18 @@ language: go sudo: false -go: - - "1.8" - - "1.9" - - "1.10" - - tip - -script: - - ./.travis.gogenerate.sh - - ./.travis.gofmt.sh - - ./.travis.govet.sh - - go test -v -race $(go list ./... | grep -v vendor) +matrix: + include: + - go: "1.8.x" + - go: "1.9.x" + - go: "1.10.x" + - go: "1.11.x" + env: GO111MODULE=off + - go: "1.11.x" + env: GO111MODULE=on + - go: tip + script: + - ./.travis.gogenerate.sh + - ./.travis.gofmt.sh + - ./.travis.govet.sh + - go test -v -race $(go list ./... | grep -v vendor) diff --git a/vendor/github.com/stretchr/testify/LICENSE b/vendor/github.com/stretchr/testify/LICENSE index 473b670..f38ec59 100644 --- a/vendor/github.com/stretchr/testify/LICENSE +++ b/vendor/github.com/stretchr/testify/LICENSE @@ -1,22 +1,21 @@ -Copyright (c) 2012 - 2013 Mat Ryer and Tyler Bunnell +MIT License -Please consider promoting this project if you find it useful. +Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell -Permission is hereby granted, free of charge, to any person -obtaining a copy of this software and associated documentation -files (the "Software"), to deal in the Software without restriction, -including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, -and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included -in all copies or substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES -OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT -OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE -OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/stretchr/testify/README.md b/vendor/github.com/stretchr/testify/README.md index 51b6df3..11951d4 100644 --- a/vendor/github.com/stretchr/testify/README.md +++ b/vendor/github.com/stretchr/testify/README.md @@ -287,8 +287,10 @@ To install Testify, use `go get`: This will then make the following packages available to you: github.com/stretchr/testify/assert + github.com/stretchr/testify/require github.com/stretchr/testify/mock - github.com/stretchr/testify/http + github.com/stretchr/testify/suite + github.com/stretchr/testify/http (deprecated) Import the `testify/assert` package into your code using this template: @@ -319,7 +321,7 @@ To update Testify to the latest version, use `go get -u github.com/stretchr/test Supported go versions ================== -We support the three major Go versions, which are 1.8, 1.9 and 1.10 at the moment. +We support the three major Go versions, which are 1.9, 1.10, and 1.11 at the moment. ------ @@ -329,3 +331,10 @@ Contributing Please feel free to submit issues, fork the repository and send pull requests! When submitting an issue, we ask that you please include a complete test function that demonstrates the issue. Extra credit for those using Testify to write the test code that demonstrates it. + +------ + +License +======= + +This project is licensed under the terms of the MIT license. diff --git a/vendor/github.com/stretchr/testify/assert/assertions.go b/vendor/github.com/stretchr/testify/assert/assertions.go index 5bdec56..9bd4a80 100644 --- a/vendor/github.com/stretchr/testify/assert/assertions.go +++ b/vendor/github.com/stretchr/testify/assert/assertions.go @@ -39,7 +39,7 @@ type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) bool // for table driven tests. type BoolAssertionFunc func(TestingT, bool, ...interface{}) bool -// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful +// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful // for table driven tests. type ErrorAssertionFunc func(TestingT, error, ...interface{}) bool @@ -179,7 +179,11 @@ func messageFromMsgAndArgs(msgAndArgs ...interface{}) string { return "" } if len(msgAndArgs) == 1 { - return msgAndArgs[0].(string) + msg := msgAndArgs[0] + if msgAsStr, ok := msg.(string); ok { + return msgAsStr + } + return fmt.Sprintf("%+v", msg) } if len(msgAndArgs) > 1 { return fmt.Sprintf(msgAndArgs[0].(string), msgAndArgs[1:]...) @@ -415,6 +419,17 @@ func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) bool { return Fail(t, "Expected value not to be nil.", msgAndArgs...) } +// containsKind checks if a specified kind in the slice of kinds. +func containsKind(kinds []reflect.Kind, kind reflect.Kind) bool { + for i := 0; i < len(kinds); i++ { + if kind == kinds[i] { + return true + } + } + + return false +} + // isNil checks if a specified object is nil or not, without Failing. func isNil(object interface{}) bool { if object == nil { @@ -423,7 +438,14 @@ func isNil(object interface{}) bool { value := reflect.ValueOf(object) kind := value.Kind() - if kind >= reflect.Chan && kind <= reflect.Slice && value.IsNil() { + isNilableKind := containsKind( + []reflect.Kind{ + reflect.Chan, reflect.Func, + reflect.Interface, reflect.Map, + reflect.Ptr, reflect.Slice}, + kind) + + if isNilableKind && value.IsNil() { return true } @@ -1327,7 +1349,7 @@ func typeAndKind(v interface{}) (reflect.Type, reflect.Kind) { } // diff returns a diff of both values as long as both are of the same type and -// are a struct, map, slice or array. Otherwise it returns an empty string. +// are a struct, map, slice, array or string. Otherwise it returns an empty string. func diff(expected interface{}, actual interface{}) string { if expected == nil || actual == nil { return "" @@ -1345,7 +1367,7 @@ func diff(expected interface{}, actual interface{}) string { } var e, a string - if ek != reflect.String { + if et != reflect.TypeOf("") { e = spewConfig.Sdump(expected) a = spewConfig.Sdump(actual) } else { diff --git a/vendor/github.com/stretchr/testify/assert/assertions_test.go b/vendor/github.com/stretchr/testify/assert/assertions_test.go index 91b5ee9..00e2d21 100644 --- a/vendor/github.com/stretchr/testify/assert/assertions_test.go +++ b/vendor/github.com/stretchr/testify/assert/assertions_test.go @@ -175,6 +175,8 @@ func TestIsType(t *testing.T) { } +type myType string + func TestEqual(t *testing.T) { mockT := new(testing.T) @@ -200,6 +202,9 @@ func TestEqual(t *testing.T) { if !Equal(mockT, uint64(123), uint64(123)) { t.Error("Equal should return true") } + if !Equal(mockT, myType("1"), myType("1")) { + t.Error("Equal should return true") + } if !Equal(mockT, &struct{}{}, &struct{}{}) { t.Error("Equal should return true (pointer equality is based on equality of underlying value)") } @@ -207,6 +212,9 @@ func TestEqual(t *testing.T) { if Equal(mockT, m["bar"], "something") { t.Error("Equal should return false") } + if Equal(mockT, myType("1"), myType("2")) { + t.Error("Equal should return false") + } } // bufferT implements TestingT. Its implementation of Errorf writes the output that would be produced by @@ -275,6 +283,8 @@ func TestEqualFormatting(t *testing.T) { }{ {equalWant: "want", equalGot: "got", want: "\tassertions.go:\\d+: \n\t+Error Trace:\t\n\t+Error:\\s+Not equal:\\s+\n\\s+expected: \"want\"\n\\s+actual\\s+: \"got\"\n\\s+Diff:\n\\s+-+ Expected\n\\s+\\++ Actual\n\\s+@@ -1 \\+1 @@\n\\s+-want\n\\s+\\+got\n"}, {equalWant: "want", equalGot: "got", msgAndArgs: []interface{}{"hello, %v!", "world"}, want: "\tassertions.go:[0-9]+: \n\t+Error Trace:\t\n\t+Error:\\s+Not equal:\\s+\n\\s+expected: \"want\"\n\\s+actual\\s+: \"got\"\n\\s+Diff:\n\\s+-+ Expected\n\\s+\\++ Actual\n\\s+@@ -1 \\+1 @@\n\\s+-want\n\\s+\\+got\n\\s+Messages:\\s+hello, world!\n"}, + {equalWant: "want", equalGot: "got", msgAndArgs: []interface{}{123}, want: "\tassertions.go:[0-9]+: \n\t+Error Trace:\t\n\t+Error:\\s+Not equal:\\s+\n\\s+expected: \"want\"\n\\s+actual\\s+: \"got\"\n\\s+Diff:\n\\s+-+ Expected\n\\s+\\++ Actual\n\\s+@@ -1 \\+1 @@\n\\s+-want\n\\s+\\+got\n\\s+Messages:\\s+123\n"}, + {equalWant: "want", equalGot: "got", msgAndArgs: []interface{}{struct{ a string }{"hello"}}, want: "\tassertions.go:[0-9]+: \n\t+Error Trace:\t\n\t+Error:\\s+Not equal:\\s+\n\\s+expected: \"want\"\n\\s+actual\\s+: \"got\"\n\\s+Diff:\n\\s+-+ Expected\n\\s+\\++ Actual\n\\s+@@ -1 \\+1 @@\n\\s+-want\n\\s+\\+got\n\\s+Messages:\\s+{a:hello}\n"}, } { mockT := &bufferT{} Equal(mockT, currCase.equalWant, currCase.equalGot, currCase.msgAndArgs...) diff --git a/vendor/github.com/stretchr/testify/go.mod b/vendor/github.com/stretchr/testify/go.mod new file mode 100644 index 0000000..90e5dbe --- /dev/null +++ b/vendor/github.com/stretchr/testify/go.mod @@ -0,0 +1,7 @@ +module github.com/stretchr/testify + +require ( + github.com/davecgh/go-spew v1.1.0 + github.com/pmezard/go-difflib v1.0.0 + github.com/stretchr/objx v0.1.0 +) diff --git a/vendor/github.com/stretchr/testify/go.sum b/vendor/github.com/stretchr/testify/go.sum new file mode 100644 index 0000000..5b98bf3 --- /dev/null +++ b/vendor/github.com/stretchr/testify/go.sum @@ -0,0 +1,6 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/vendor/github.com/stretchr/testify/mock/mock.go b/vendor/github.com/stretchr/testify/mock/mock.go index cc4f642..d6694ed 100644 --- a/vendor/github.com/stretchr/testify/mock/mock.go +++ b/vendor/github.com/stretchr/testify/mock/mock.go @@ -176,6 +176,7 @@ func (c *Call) Maybe() *Call { // Mock. // On("MyMethod", 1).Return(nil). // On("MyOtherMethod", 'a', 'b', 'c').Return(errors.New("Some Error")) +//go:noinline func (c *Call) On(methodName string, arguments ...interface{}) *Call { return c.Parent.On(methodName, arguments...) } @@ -691,7 +692,7 @@ func (args Arguments) Diff(objects []interface{}) (string, int) { output = fmt.Sprintf("%s\t%d: PASS: %s matched by %s\n", output, i, actualFmt, matcher) } else { differences++ - output = fmt.Sprintf("%s\t%d: PASS: %s not matched by %s\n", output, i, actualFmt, matcher) + output = fmt.Sprintf("%s\t%d: FAIL: %s not matched by %s\n", output, i, actualFmt, matcher) } } else if reflect.TypeOf(expected) == reflect.TypeOf((*AnythingOfTypeArgument)(nil)).Elem() { diff --git a/vendor/github.com/stretchr/testify/mock/mock_test.go b/vendor/github.com/stretchr/testify/mock/mock_test.go index 978eae2..2608f5a 100644 --- a/vendor/github.com/stretchr/testify/mock/mock_test.go +++ b/vendor/github.com/stretchr/testify/mock/mock_test.go @@ -32,6 +32,7 @@ func (i *TestExampleImplementation) TheExampleMethod(a, b, c int) (int, error) { return args.Int(0), errors.New("Whoops") } +//go:noinline func (i *TestExampleImplementation) TheExampleMethod2(yesorno bool) { i.Called(yesorno) } @@ -1492,6 +1493,7 @@ func unexpectedCallRegex(method, calledArg, expectedArg, diff string) string { rMethod, calledArg, rMethod, expectedArg, diff) } +//go:noinline func ConcurrencyTestMethod(m *Mock) { m.Called() } diff --git a/vendor/github.com/stretchr/testify/require/requirements.go b/vendor/github.com/stretchr/testify/require/requirements.go index 690583a..6b85c5e 100644 --- a/vendor/github.com/stretchr/testify/require/requirements.go +++ b/vendor/github.com/stretchr/testify/require/requirements.go @@ -22,7 +22,7 @@ type ValueAssertionFunc func(TestingT, interface{}, ...interface{}) // for table driven tests. type BoolAssertionFunc func(TestingT, bool, ...interface{}) -// ValuesAssertionFunc is a common function prototype when validating an error value. Can be useful +// ErrorAssertionFunc is a common function prototype when validating an error value. Can be useful // for table driven tests. type ErrorAssertionFunc func(TestingT, error, ...interface{}) diff --git a/vendor/github.com/stretchr/testify/suite/suite.go b/vendor/github.com/stretchr/testify/suite/suite.go index e20afbc..5cea8f8 100644 --- a/vendor/github.com/stretchr/testify/suite/suite.go +++ b/vendor/github.com/stretchr/testify/suite/suite.go @@ -55,10 +55,32 @@ func (suite *Suite) Assert() *assert.Assertions { return suite.Assertions } +func failOnPanic(t *testing.T) { + r := recover() + if r != nil { + t.Errorf("test panicked: %v", r) + t.FailNow() + } +} + +// Run provides suite functionality around golang subtests. It should be +// called in place of t.Run(name, func(t *testing.T)) in test suite code. +// The passed-in func will be executed as a subtest with a fresh instance of t. +// Provides compatibility with go test pkg -run TestSuite/TestName/SubTestName. +func (suite *Suite) Run(name string, subtest func()) bool { + oldT := suite.T() + defer suite.SetT(oldT) + return oldT.Run(name, func(t *testing.T) { + suite.SetT(t) + subtest() + }) +} + // Run takes a testing suite and runs all of the tests attached // to it. func Run(t *testing.T, suite TestingSuite) { suite.SetT(t) + defer failOnPanic(t) if setupAllSuite, ok := suite.(SetupAllSuite); ok { setupAllSuite.SetupSuite() @@ -84,6 +106,8 @@ func Run(t *testing.T, suite TestingSuite) { F: func(t *testing.T) { parentT := suite.T() suite.SetT(t) + defer failOnPanic(t) + if setupTestSuite, ok := suite.(SetupTestSuite); ok { setupTestSuite.SetupTest() } diff --git a/vendor/github.com/stretchr/testify/suite/suite_test.go b/vendor/github.com/stretchr/testify/suite/suite_test.go index b75fa4a..6dfba08 100644 --- a/vendor/github.com/stretchr/testify/suite/suite_test.go +++ b/vendor/github.com/stretchr/testify/suite/suite_test.go @@ -42,6 +42,99 @@ func (s *SuiteRequireTwice) TestRequireTwo() { r.Equal(1, 2) } +type panickingSuite struct { + Suite + panicInSetupSuite bool + panicInSetupTest bool + panicInBeforeTest bool + panicInTest bool + panicInAfterTest bool + panicInTearDownTest bool + panicInTearDownSuite bool +} + +func (s *panickingSuite) SetupSuite() { + if s.panicInSetupSuite { + panic("oops in setup suite") + } +} + +func (s *panickingSuite) SetupTest() { + if s.panicInSetupTest { + panic("oops in setup test") + } +} + +func (s *panickingSuite) BeforeTest(_, _ string) { + if s.panicInBeforeTest { + panic("oops in before test") + } +} + +func (s *panickingSuite) Test() { + if s.panicInTest { + panic("oops in test") + } +} + +func (s *panickingSuite) AfterTest(_, _ string) { + if s.panicInAfterTest { + panic("oops in after test") + } +} + +func (s *panickingSuite) TearDownTest() { + if s.panicInTearDownTest { + panic("oops in tear down test") + } +} + +func (s *panickingSuite) TearDownSuite() { + if s.panicInTearDownSuite { + panic("oops in tear down suite") + } +} + +func TestSuiteRecoverPanic(t *testing.T) { + ok := true + panickingTests := []testing.InternalTest{ + { + Name: "TestPanicInSetupSuite", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInSetupSuite: true}) }, + }, + { + Name: "TestPanicInSetupTest", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInSetupTest: true}) }, + }, + { + Name: "TestPanicInBeforeTest", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInBeforeTest: true}) }, + }, + { + Name: "TestPanicInTest", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInTest: true}) }, + }, + { + Name: "TestPanicInAfterTest", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInAfterTest: true}) }, + }, + { + Name: "TestPanicInTearDownTest", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInTearDownTest: true}) }, + }, + { + Name: "TestPanicInTearDownSuite", + F: func(t *testing.T) { Run(t, &panickingSuite{panicInTearDownSuite: true}) }, + }, + } + + require.NotPanics(t, func() { + ok = testing.RunTests(allTestsFilter, panickingTests) + }) + + assert.False(t, ok) +} + // This suite is intended to store values to make sure that only // testing-suite-related methods are run. It's also a fully // functional example of a testing suite, using setup/teardown methods @@ -59,6 +152,7 @@ type SuiteTester struct { TearDownTestRunCount int TestOneRunCount int TestTwoRunCount int + TestSubtestRunCount int NonTestMethodRunCount int SuiteNameBefore []string @@ -153,6 +247,27 @@ func (suite *SuiteTester) NonTestMethod() { suite.NonTestMethodRunCount++ } +func (suite *SuiteTester) TestSubtest() { + suite.TestSubtestRunCount++ + + for _, t := range []struct { + testName string + }{ + {"first"}, + {"second"}, + } { + suiteT := suite.T() + suite.Run(t.testName, func() { + // We should get a different *testing.T for subtests, so that + // go test recognizes them as proper subtests for output formatting + // and running individual subtests + subTestT := suite.T() + suite.NotEqual(subTestT, suiteT) + }) + suite.Equal(suiteT, suite.T()) + } +} + // TestRunSuite will be run by the 'go test' command, so within it, we // can run our suite using the Run(*testing.T, TestingSuite) function. func TestRunSuite(t *testing.T) { @@ -168,18 +283,20 @@ func TestRunSuite(t *testing.T) { assert.Equal(t, suiteTester.SetupSuiteRunCount, 1) assert.Equal(t, suiteTester.TearDownSuiteRunCount, 1) - assert.Equal(t, len(suiteTester.SuiteNameAfter), 3) - assert.Equal(t, len(suiteTester.SuiteNameBefore), 3) - assert.Equal(t, len(suiteTester.TestNameAfter), 3) - assert.Equal(t, len(suiteTester.TestNameBefore), 3) + assert.Equal(t, len(suiteTester.SuiteNameAfter), 4) + assert.Equal(t, len(suiteTester.SuiteNameBefore), 4) + assert.Equal(t, len(suiteTester.TestNameAfter), 4) + assert.Equal(t, len(suiteTester.TestNameBefore), 4) assert.Contains(t, suiteTester.TestNameAfter, "TestOne") assert.Contains(t, suiteTester.TestNameAfter, "TestTwo") assert.Contains(t, suiteTester.TestNameAfter, "TestSkip") + assert.Contains(t, suiteTester.TestNameAfter, "TestSubtest") assert.Contains(t, suiteTester.TestNameBefore, "TestOne") assert.Contains(t, suiteTester.TestNameBefore, "TestTwo") assert.Contains(t, suiteTester.TestNameBefore, "TestSkip") + assert.Contains(t, suiteTester.TestNameBefore, "TestSubtest") for _, suiteName := range suiteTester.SuiteNameAfter { assert.Equal(t, "SuiteTester", suiteName) @@ -197,15 +314,16 @@ func TestRunSuite(t *testing.T) { assert.False(t, when.IsZero()) } - // There are three test methods (TestOne, TestTwo, and TestSkip), so + // There are four test methods (TestOne, TestTwo, TestSkip, and TestSubtest), so // the SetupTest and TearDownTest methods (which should be run once for - // each test) should have been run three times. - assert.Equal(t, suiteTester.SetupTestRunCount, 3) - assert.Equal(t, suiteTester.TearDownTestRunCount, 3) + // each test) should have been run four times. + assert.Equal(t, suiteTester.SetupTestRunCount, 4) + assert.Equal(t, suiteTester.TearDownTestRunCount, 4) // Each test should have been run once. assert.Equal(t, suiteTester.TestOneRunCount, 1) assert.Equal(t, suiteTester.TestTwoRunCount, 1) + assert.Equal(t, suiteTester.TestSubtestRunCount, 1) // Methods that don't match the test method identifier shouldn't // have been run at all. diff --git a/vendor/github.com/valyala/fasthttp/.travis.yml b/vendor/github.com/valyala/fasthttp/.travis.yml index 3ed568b..104ead0 100644 --- a/vendor/github.com/valyala/fasthttp/.travis.yml +++ b/vendor/github.com/valyala/fasthttp/.travis.yml @@ -1,10 +1,27 @@ language: go go: + - tip + - 1.11.x + - 1.10.x - 1.9.x - - 1.8.x + +os: + - linux + - osx + +matrix: + allow_failures: + - tip + fast_finish: true + +before_install: + - go get -t -v ./... + # - go get -v golang.org/x/tools/cmd/goimports script: + # TODO(@kirilldanshin) + # - test -z "$(goimports -d $(find . -type f -name '*.go' -not -path "./vendor/*"))" # build test for supported platforms - GOOS=linux go build - GOOS=darwin go build @@ -14,3 +31,6 @@ script: # run tests on a standard platform - go test -v ./... + + # run tests with the race detector as well + - go test -race -v ./... diff --git a/vendor/github.com/valyala/fasthttp/LICENSE b/vendor/github.com/valyala/fasthttp/LICENSE index 22bf00c..b098914 100644 --- a/vendor/github.com/valyala/fasthttp/LICENSE +++ b/vendor/github.com/valyala/fasthttp/LICENSE @@ -1,6 +1,9 @@ The MIT License (MIT) -Copyright (c) 2015-2016 Aliaksandr Valialkin, VertaMedia +Copyright (c) 2015-present Aliaksandr Valialkin, VertaMedia +Copyright (c) 2018-present Kirill Danshin +Copyright (c) 2018-present Erik Dubbelboer +Copyright (c) 2018-present FastHTTP Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/valyala/fasthttp/README.md b/vendor/github.com/valyala/fasthttp/README.md index 684c0bc..1a8bff8 100644 --- a/vendor/github.com/valyala/fasthttp/README.md +++ b/vendor/github.com/valyala/fasthttp/README.md @@ -23,6 +23,8 @@ connections per physical server. [Code examples](examples) +[Awesome fasthttp tools](https://github.com/fasthttp) + [Switching from net/http to fasthttp](#switching-from-nethttp-to-fasthttp) [Fasthttp best practices](#fasthttp-best-practices) @@ -275,10 +277,10 @@ like in net/http. The following code is valid for fasthttp: but there are more powerful third-party routers and web frameworks with fasthttp support: - * [Iris](https://github.com/kataras/iris) * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) * [fasthttprouter](https://github.com/buaazp/fasthttprouter) * [lu](https://github.com/vincentLiuxiang/lu) + * [atreugo](https://github.com/savsgio/atreugo) Net/http code with simple ServeMux is trivially converted to fasthttp code: @@ -478,18 +480,19 @@ uintBuf := fasthttp.AppendUint(nil, 1234) # Related projects - * [fasthttp-contrib](https://github.com/fasthttp-contrib) - various useful + * [fasthttp](https://github.com/fasthttp) - various useful helpers for projects based on fasthttp. - * [iris](https://github.com/kataras/iris) - web application framework built - on top of fasthttp. Features speed and functionality. * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) - fast and powerful routing package for fasthttp servers. * [fasthttprouter](https://github.com/buaazp/fasthttprouter) - a high performance fasthttp request router that scales well. + * [gramework](https://github.com/gramework/gramework) - a web framework made by one of fasthttp maintainers * [lu](https://github.com/vincentLiuxiang/lu) - a high performance go middleware web framework which is based on fasthttp. - * [websocket](https://github.com/leavengood/websocket) - Gorilla-based + * [websocket](https://github.com/fasthttp/websocket) - Gorilla-based websocket implementation for fasthttp. + * [fasthttpsession](https://github.com/phachon/fasthttpsession) - a fast and powerful session package for fasthttp servers. + * [atreugo](https://github.com/savsgio/atreugo) - Micro-framework to make simple the use of routing and middlewares. # FAQ @@ -519,10 +522,9 @@ uintBuf := fasthttp.AppendUint(nil, 1234) * *Why fasthttp doesn't support HTTP/2.0 and WebSockets?* - There are [plans](TODO) for adding HTTP/2.0 and WebSockets support - in the future. - In the mean time, third parties may use [RequestCtx.Hijack](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack) - for implementing these goodies. See [the first third-party websocket implementation on the top of fasthttp](https://github.com/leavengood/websocket). + [HTTP/2.0 support](https://github.com/fasthttp/http2) is in progress. [WebSockets](https://github.com/fasthttp/websockets) has been done already. + Third parties also may use [RequestCtx.Hijack](https://godoc.org/github.com/valyala/fasthttp#RequestCtx.Hijack) + for implementing these goodies. * *Are there known net/http advantages comparing to fasthttp?* @@ -545,8 +547,10 @@ uintBuf := fasthttp.AppendUint(nil, 1234) Go1.5+. Older versions won't be supported, since their standard package [miss useful functions](https://github.com/valyala/fasthttp/issues/5). + + **NOTE**: Go 1.9.7 is the oldest tested version. We recommend you to update as soon as you can. As of 1.11.3 we will drop 1.9.x support. -* *Please provide real benchmark data and sever information* +* *Please provide real benchmark data and server information* See [this issue](https://github.com/valyala/fasthttp/issues/4). @@ -555,11 +559,11 @@ uintBuf := fasthttp.AppendUint(nil, 1234) There are no plans to add request routing into fasthttp. Use third-party routers and web frameworks with fasthttp support: - * [Iris](https://github.com/kataras/iris) * [fasthttp-routing](https://github.com/qiangxue/fasthttp-routing) * [fasthttprouter](https://github.com/buaazp/fasthttprouter) * [gramework](https://github.com/gramework/gramework) * [lu](https://github.com/vincentLiuxiang/lu) + * [atreugo](https://github.com/savsgio/atreugo) See also [this issue](https://github.com/valyala/fasthttp/issues/9) for more info. diff --git a/vendor/github.com/valyala/fasthttp/args.go b/vendor/github.com/valyala/fasthttp/args.go index 5d432f5..c8a1c61 100644 --- a/vendor/github.com/valyala/fasthttp/args.go +++ b/vendor/github.com/valyala/fasthttp/args.go @@ -4,7 +4,15 @@ import ( "bytes" "errors" "io" + "sort" "sync" + + "github.com/valyala/bytebufferpool" +) + +const ( + argsNoValue = true + argsHasValue = false ) // AcquireArgs returns an empty Args object from the pool. @@ -43,8 +51,9 @@ type Args struct { } type argsKV struct { - key []byte - value []byte + key []byte + value []byte + noValue bool } // Reset clears query args. @@ -107,14 +116,29 @@ func (a *Args) QueryString() []byte { return a.buf } +// Sort sorts Args by key and then value using 'f' as comparison function. +// +// For example args.Sort(bytes.Compare) +func (a *Args) Sort(f func(x, y []byte) int) { + sort.SliceStable(a.args, func(i, j int) bool { + n := f(a.args[i].key, a.args[j].key) + if n == 0 { + return f(a.args[i].value, a.args[j].value) == -1 + } + return n == -1 + }) +} + // AppendBytes appends query string to dst and returns the extended dst. func (a *Args) AppendBytes(dst []byte) []byte { for i, n := 0, len(a.args); i < n; i++ { kv := &a.args[i] dst = AppendQuotedArg(dst, kv.key) - if len(kv.value) > 0 { + if !kv.noValue { dst = append(dst, '=') - dst = AppendQuotedArg(dst, kv.value) + if len(kv.value) > 0 { + dst = AppendQuotedArg(dst, kv.value) + } } if i+1 < n { dst = append(dst, '&') @@ -145,48 +169,74 @@ func (a *Args) DelBytes(key []byte) { // // Multiple values for the same key may be added. func (a *Args) Add(key, value string) { - a.args = appendArg(a.args, key, value) + a.args = appendArg(a.args, key, value, argsHasValue) } // AddBytesK adds 'key=value' argument. // // Multiple values for the same key may be added. func (a *Args) AddBytesK(key []byte, value string) { - a.args = appendArg(a.args, b2s(key), value) + a.args = appendArg(a.args, b2s(key), value, argsHasValue) } // AddBytesV adds 'key=value' argument. // // Multiple values for the same key may be added. func (a *Args) AddBytesV(key string, value []byte) { - a.args = appendArg(a.args, key, b2s(value)) + a.args = appendArg(a.args, key, b2s(value), argsHasValue) } // AddBytesKV adds 'key=value' argument. // // Multiple values for the same key may be added. func (a *Args) AddBytesKV(key, value []byte) { - a.args = appendArg(a.args, b2s(key), b2s(value)) + a.args = appendArg(a.args, b2s(key), b2s(value), argsHasValue) +} + +// AddNoValue adds only 'key' as argument without the '='. +// +// Multiple values for the same key may be added. +func (a *Args) AddNoValue(key string) { + a.args = appendArg(a.args, key, "", argsNoValue) +} + +// AddBytesKNoValue adds only 'key' as argument without the '='. +// +// Multiple values for the same key may be added. +func (a *Args) AddBytesKNoValue(key []byte) { + a.args = appendArg(a.args, b2s(key), "", argsNoValue) } // Set sets 'key=value' argument. func (a *Args) Set(key, value string) { - a.args = setArg(a.args, key, value) + a.args = setArg(a.args, key, value, argsHasValue) } // SetBytesK sets 'key=value' argument. func (a *Args) SetBytesK(key []byte, value string) { - a.args = setArg(a.args, b2s(key), value) + a.args = setArg(a.args, b2s(key), value, argsHasValue) } // SetBytesV sets 'key=value' argument. func (a *Args) SetBytesV(key string, value []byte) { - a.args = setArg(a.args, key, b2s(value)) + a.args = setArg(a.args, key, b2s(value), argsHasValue) } // SetBytesKV sets 'key=value' argument. func (a *Args) SetBytesKV(key, value []byte) { - a.args = setArgBytes(a.args, key, value) + a.args = setArgBytes(a.args, key, value, argsHasValue) +} + +// SetNoValue sets only 'key' as argument without the '='. +// +// Only key in argumemt, like key1&key2 +func (a *Args) SetNoValue(key string) { + a.args = setArg(a.args, key, "", argsNoValue) +} + +// SetBytesKNoValue sets 'key' argument. +func (a *Args) SetBytesKNoValue(key []byte) { + a.args = setArg(a.args, b2s(key), "", argsNoValue) } // Peek returns query arg value for the given key. @@ -243,10 +293,10 @@ func (a *Args) GetUint(key string) (int, error) { // SetUint sets uint value for the given key. func (a *Args) SetUint(key string, value int) { - bb := AcquireByteBuffer() + bb := bytebufferpool.Get() bb.B = AppendUint(bb.B[:0], value) a.SetBytesV(key, bb.B) - ReleaseByteBuffer(bb) + bytebufferpool.Put(bb) } // SetUintBytes sets uint value for the given key. @@ -287,11 +337,14 @@ func (a *Args) GetUfloatOrZero(key string) float64 { // GetBool returns boolean value for the given key. // -// true is returned for '1', 'y' and 'yes' values, +// true is returned for "1", "t", "T", "true", "TRUE", "True", "y", "yes", "Y", "YES", "Yes", // otherwise false is returned. func (a *Args) GetBool(key string) bool { - switch string(a.Peek(key)) { - case "1", "y", "yes": + switch b2s(a.Peek(key)) { + // Support the same true cases as strconv.ParseBool + // See: https://github.com/golang/go/blob/4e1b11e2c9bdb0ddea1141eed487be1a626ff5be/src/strconv/atob.go#L12 + // and Y and Yes versions. + case "1", "t", "T", "true", "TRUE", "True", "y", "yes", "Y", "YES", "Yes": return true default: return false @@ -317,7 +370,12 @@ func copyArgs(dst, src []argsKV) []argsKV { dstKV := &dst[i] srcKV := &src[i] dstKV.key = append(dstKV.key[:0], srcKV.key...) - dstKV.value = append(dstKV.value[:0], srcKV.value...) + if srcKV.noValue { + dstKV.value = dstKV.value[:0] + } else { + dstKV.value = append(dstKV.value[:0], srcKV.value...) + } + dstKV.noValue = srcKV.noValue } return dst } @@ -340,31 +398,41 @@ func delAllArgs(args []argsKV, key string) []argsKV { return args } -func setArgBytes(h []argsKV, key, value []byte) []argsKV { - return setArg(h, b2s(key), b2s(value)) +func setArgBytes(h []argsKV, key, value []byte, noValue bool) []argsKV { + return setArg(h, b2s(key), b2s(value), noValue) } -func setArg(h []argsKV, key, value string) []argsKV { +func setArg(h []argsKV, key, value string, noValue bool) []argsKV { n := len(h) for i := 0; i < n; i++ { kv := &h[i] if key == string(kv.key) { - kv.value = append(kv.value[:0], value...) + if noValue { + kv.value = kv.value[:0] + } else { + kv.value = append(kv.value[:0], value...) + } + kv.noValue = noValue return h } } - return appendArg(h, key, value) + return appendArg(h, key, value, noValue) } -func appendArgBytes(h []argsKV, key, value []byte) []argsKV { - return appendArg(h, b2s(key), b2s(value)) +func appendArgBytes(h []argsKV, key, value []byte, noValue bool) []argsKV { + return appendArg(h, b2s(key), b2s(value), noValue) } -func appendArg(args []argsKV, key, value string) []argsKV { +func appendArg(args []argsKV, key, value string, noValue bool) []argsKV { var kv *argsKV args, kv = allocArg(args) kv.key = append(kv.key[:0], key...) - kv.value = append(kv.value[:0], value...) + if noValue { + kv.value = kv.value[:0] + } else { + kv.value = append(kv.value[:0], value...) + } + kv.noValue = noValue return args } @@ -420,6 +488,7 @@ func (s *argsScanner) next(kv *argsKV) bool { if len(s.b) == 0 { return false } + kv.noValue = argsHasValue isKey := true k := 0 @@ -435,6 +504,7 @@ func (s *argsScanner) next(kv *argsKV) bool { if isKey { kv.key = decodeArgAppend(kv.key[:0], s.b[:i]) kv.value = kv.value[:0] + kv.noValue = argsNoValue } else { kv.value = decodeArgAppend(kv.value[:0], s.b[k:i]) } @@ -446,6 +516,7 @@ func (s *argsScanner) next(kv *argsKV) bool { if isKey { kv.key = decodeArgAppend(kv.key[:0], s.b) kv.value = kv.value[:0] + kv.noValue = argsNoValue } else { kv.value = decodeArgAppend(kv.value[:0], s.b[k:]) } @@ -486,7 +557,7 @@ func decodeArgAppend(dst, src []byte) []byte { // decodeArgAppendNoPlus is almost identical to decodeArgAppend, but it doesn't // substitute '+' with ' '. // -// The function is copy-pasted from decodeArgAppend due to the preformance +// The function is copy-pasted from decodeArgAppend due to the performance // reasons only. func decodeArgAppendNoPlus(dst, src []byte) []byte { if bytes.IndexByte(src, '%') < 0 { diff --git a/vendor/github.com/valyala/fasthttp/args_test.go b/vendor/github.com/valyala/fasthttp/args_test.go index d5b768f..db44750 100644 --- a/vendor/github.com/valyala/fasthttp/args_test.go +++ b/vendor/github.com/valyala/fasthttp/args_test.go @@ -1,11 +1,14 @@ package fasthttp import ( + "bytes" "fmt" "reflect" "strings" "testing" "time" + + "github.com/valyala/bytebufferpool" ) func TestDecodeArgAppend(t *testing.T) { @@ -35,22 +38,31 @@ func TestArgsAdd(t *testing.T) { a.Add("foo", "baz") a.Add("foo", "1") a.Add("ba", "23") - if a.Len() != 4 { - t.Fatalf("unexpected number of elements: %d. Expecting 4", a.Len()) + a.Add("foo", "") + a.AddNoValue("foo") + if a.Len() != 6 { + t.Fatalf("unexpected number of elements: %d. Expecting 6", a.Len()) } s := a.String() - expectedS := "foo=bar&foo=baz&foo=1&ba=23" + expectedS := "foo=bar&foo=baz&foo=1&ba=23&foo=&foo" if s != expectedS { t.Fatalf("unexpected result: %q. Expecting %q", s, expectedS) } - var a1 Args - a1.Parse(s) - if a1.Len() != 4 { - t.Fatalf("unexpected number of elements: %d. Expecting 4", a.Len()) + a.Sort(bytes.Compare) + ss := a.String() + expectedSS := "ba=23&foo=&foo&foo=1&foo=bar&foo=baz" + if ss != expectedSS { + t.Fatalf("unexpected result: %q. Expecting %q", ss, expectedSS) } - var barFound, bazFound, oneFound, baFound bool + var a1 Args + a1.Parse(s) + if a1.Len() != 6 { + t.Fatalf("unexpected number of elements: %d. Expecting 6", a.Len()) + } + + var barFound, bazFound, oneFound, emptyFound1, emptyFound2, baFound bool a1.VisitAll(func(k, v []byte) { switch string(k) { case "foo": @@ -61,6 +73,12 @@ func TestArgsAdd(t *testing.T) { bazFound = true case "1": oneFound = true + case "": + if emptyFound1 { + emptyFound2 = true + } else { + emptyFound1 = true + } default: t.Fatalf("unexpected value %q", v) } @@ -73,8 +91,8 @@ func TestArgsAdd(t *testing.T) { t.Fatalf("unexpected key found %q", k) } }) - if !barFound || !bazFound || !oneFound || !baFound { - t.Fatalf("something is missing: %v, %v, %v, %v", barFound, bazFound, oneFound, baFound) + if !barFound || !bazFound || !oneFound || !emptyFound1 || !emptyFound2 || !baFound { + t.Fatalf("something is missing: %v, %v, %v, %v, %v, %v", barFound, bazFound, oneFound, emptyFound1, emptyFound2, baFound) } } @@ -171,7 +189,7 @@ func TestArgsWriteTo(t *testing.T) { var a Args a.Parse(s) - var w ByteBuffer + var w bytebufferpool.ByteBuffer n, err := a.WriteTo(&w) if err != nil { t.Fatalf("unexpected error: %s", err) @@ -301,10 +319,12 @@ func TestArgsStringCompose(t *testing.T) { a.Set("foo", "bar") a.Set("aa", "bbb") a.Set("привет", "мир") + a.SetNoValue("bb") a.Set("", "xxxx") a.Set("cvx", "") + a.SetNoValue("novalue") - expectedS := "foo=bar&aa=bbb&%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=%D0%BC%D0%B8%D1%80&=xxxx&cvx" + expectedS := "foo=bar&aa=bbb&%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82=%D0%BC%D0%B8%D1%80&bb&=xxxx&cvx=&novalue" s := a.String() if s != expectedS { t.Fatalf("Unexpected string %q. Exected %q", s, expectedS) diff --git a/vendor/github.com/valyala/fasthttp/bytebuffer.go b/vendor/github.com/valyala/fasthttp/bytebuffer.go deleted file mode 100644 index f965172..0000000 --- a/vendor/github.com/valyala/fasthttp/bytebuffer.go +++ /dev/null @@ -1,64 +0,0 @@ -package fasthttp - -import ( - "github.com/valyala/bytebufferpool" -) - -// ByteBuffer provides byte buffer, which can be used with fasthttp API -// in order to minimize memory allocations. -// -// ByteBuffer may be used with functions appending data to the given []byte -// slice. See example code for details. -// -// Use AcquireByteBuffer for obtaining an empty byte buffer. -// -// ByteBuffer is deprecated. Use github.com/valyala/bytebufferpool instead. -type ByteBuffer bytebufferpool.ByteBuffer - -// Write implements io.Writer - it appends p to ByteBuffer.B -func (b *ByteBuffer) Write(p []byte) (int, error) { - return bb(b).Write(p) -} - -// WriteString appends s to ByteBuffer.B -func (b *ByteBuffer) WriteString(s string) (int, error) { - return bb(b).WriteString(s) -} - -// Set sets ByteBuffer.B to p -func (b *ByteBuffer) Set(p []byte) { - bb(b).Set(p) -} - -// SetString sets ByteBuffer.B to s -func (b *ByteBuffer) SetString(s string) { - bb(b).SetString(s) -} - -// Reset makes ByteBuffer.B empty. -func (b *ByteBuffer) Reset() { - bb(b).Reset() -} - -// AcquireByteBuffer returns an empty byte buffer from the pool. -// -// Acquired byte buffer may be returned to the pool via ReleaseByteBuffer call. -// This reduces the number of memory allocations required for byte buffer -// management. -func AcquireByteBuffer() *ByteBuffer { - return (*ByteBuffer)(defaultByteBufferPool.Get()) -} - -// ReleaseByteBuffer returns byte buffer to the pool. -// -// ByteBuffer.B mustn't be touched after returning it to the pool. -// Otherwise data races occur. -func ReleaseByteBuffer(b *ByteBuffer) { - defaultByteBufferPool.Put(bb(b)) -} - -func bb(b *ByteBuffer) *bytebufferpool.ByteBuffer { - return (*bytebufferpool.ByteBuffer)(b) -} - -var defaultByteBufferPool bytebufferpool.Pool diff --git a/vendor/github.com/valyala/fasthttp/bytebuffer_example_test.go b/vendor/github.com/valyala/fasthttp/bytebuffer_example_test.go deleted file mode 100644 index b89542b..0000000 --- a/vendor/github.com/valyala/fasthttp/bytebuffer_example_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package fasthttp_test - -import ( - "fmt" - - "github.com/valyala/fasthttp" -) - -func ExampleByteBuffer() { - // This request handler sets 'Your-IP' response header - // to 'Your IP is '. It uses ByteBuffer for constructing response - // header value with zero memory allocations. - yourIPRequestHandler := func(ctx *fasthttp.RequestCtx) { - b := fasthttp.AcquireByteBuffer() - b.B = append(b.B, "Your IP is <"...) - b.B = fasthttp.AppendIPv4(b.B, ctx.RemoteIP()) - b.B = append(b.B, ">"...) - ctx.Response.Header.SetBytesV("Your-IP", b.B) - - fmt.Fprintf(ctx, "Check response headers - they must contain 'Your-IP: %s'", b.B) - - // It is safe to release byte buffer now, since it is - // no longer used. - fasthttp.ReleaseByteBuffer(b) - } - - // Start fasthttp server returning your ip in response headers. - fasthttp.ListenAndServe(":8080", yourIPRequestHandler) -} diff --git a/vendor/github.com/valyala/fasthttp/bytebuffer_test.go b/vendor/github.com/valyala/fasthttp/bytebuffer_test.go deleted file mode 100644 index 2ecbd77..0000000 --- a/vendor/github.com/valyala/fasthttp/bytebuffer_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package fasthttp - -import ( - "fmt" - "testing" - "time" -) - -func TestByteBufferAcquireReleaseSerial(t *testing.T) { - testByteBufferAcquireRelease(t) -} - -func TestByteBufferAcquireReleaseConcurrent(t *testing.T) { - concurrency := 10 - ch := make(chan struct{}, concurrency) - for i := 0; i < concurrency; i++ { - go func() { - testByteBufferAcquireRelease(t) - ch <- struct{}{} - }() - } - - for i := 0; i < concurrency; i++ { - select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timeout!") - } - } -} - -func testByteBufferAcquireRelease(t *testing.T) { - for i := 0; i < 10; i++ { - b := AcquireByteBuffer() - b.B = append(b.B, "num "...) - b.B = AppendUint(b.B, i) - expectedS := fmt.Sprintf("num %d", i) - if string(b.B) != expectedS { - t.Fatalf("unexpected result: %q. Expecting %q", b.B, expectedS) - } - ReleaseByteBuffer(b) - } -} diff --git a/vendor/github.com/valyala/fasthttp/bytebuffer_timing_test.go b/vendor/github.com/valyala/fasthttp/bytebuffer_timing_test.go deleted file mode 100644 index 92bbafa..0000000 --- a/vendor/github.com/valyala/fasthttp/bytebuffer_timing_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package fasthttp - -import ( - "bytes" - "testing" -) - -func BenchmarkByteBufferWrite(b *testing.B) { - s := []byte("foobarbaz") - b.RunParallel(func(pb *testing.PB) { - var buf ByteBuffer - for pb.Next() { - for i := 0; i < 100; i++ { - buf.Write(s) - } - buf.Reset() - } - }) -} - -func BenchmarkBytesBufferWrite(b *testing.B) { - s := []byte("foobarbaz") - b.RunParallel(func(pb *testing.PB) { - var buf bytes.Buffer - for pb.Next() { - for i := 0; i < 100; i++ { - buf.Write(s) - } - buf.Reset() - } - }) -} diff --git a/vendor/github.com/valyala/fasthttp/bytesconv.go b/vendor/github.com/valyala/fasthttp/bytesconv.go index 004d28d..9631e9c 100644 --- a/vendor/github.com/valyala/fasthttp/bytesconv.go +++ b/vendor/github.com/valyala/fasthttp/bytesconv.go @@ -164,7 +164,7 @@ func ParseUint(buf []byte) (int, error) { var ( errEmptyInt = errors.New("empty integer") errUnexpectedFirstChar = errors.New("unexpected first char found. Expecting 0-9") - errUnexpectedTrailingChar = errors.New("unexpected traling char found. Expecting 0-9") + errUnexpectedTrailingChar = errors.New("unexpected trailing char found. Expecting 0-9") errTooLongInt = errors.New("too long int") ) @@ -183,7 +183,8 @@ func parseUintBuf(b []byte) (int, int, error) { } return v, i, nil } - if i >= maxIntChars { + // Test for overflow. + if v*10 < v { return -1, i, errTooLongInt } v = 10*v + int(k) @@ -416,8 +417,17 @@ func AppendQuotedArg(dst, src []byte) []byte { func appendQuotedPath(dst, src []byte) []byte { for _, c := range src { + // From the spec: http://tools.ietf.org/html/rfc3986#section-3.3 + // an path can contain zero or more of pchar that is defined as follows: + // pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + // pct-encoded = "%" HEXDIG HEXDIG + // unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + // sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + // / "*" / "+" / "," / ";" / "=" if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || - c == '/' || c == '.' || c == ',' || c == '=' || c == ':' || c == '&' || c == '~' || c == '-' || c == '_' { + c == '-' || c == '.' || c == '_' || c == '~' || c == '!' || c == '$' || + c == '&' || c == '\'' || c == '(' || c == ')' || c == '*' || c == '+' || + c == ',' || c == ';' || c == '=' || c == ':' || c == '@' || c == '/' { dst = append(dst, c) } else { dst = append(dst, '%', hexCharUpper(c>>4), hexCharUpper(c&15)) @@ -431,7 +441,7 @@ func appendQuotedPath(dst, src []byte) []byte { // This function has no performance benefits comparing to string(b) == s. // It is left here for backwards compatibility only. // -// This function is deperecated and may be deleted soon. +// Deprecated: may be deleted soon. func EqualBytesStr(b []byte, s string) bool { return string(b) == s } @@ -441,7 +451,7 @@ func EqualBytesStr(b []byte, s string) bool { // This function has no performance benefits comparing to append(dst, src...). // It is left here for backwards compatibility only. // -// This function is deprecated and may be deleted soon. +// Deprecated: may be deleted soon. func AppendBytesStr(dst []byte, src string) []byte { return append(dst, src...) } diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_32.go b/vendor/github.com/valyala/fasthttp/bytesconv_32.go index 1437754..7fd6f5f 100644 --- a/vendor/github.com/valyala/fasthttp/bytesconv_32.go +++ b/vendor/github.com/valyala/fasthttp/bytesconv_32.go @@ -3,6 +3,5 @@ package fasthttp const ( - maxIntChars = 9 maxHexIntChars = 7 ) diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_64.go b/vendor/github.com/valyala/fasthttp/bytesconv_64.go index 09d07ef..edf7309 100644 --- a/vendor/github.com/valyala/fasthttp/bytesconv_64.go +++ b/vendor/github.com/valyala/fasthttp/bytesconv_64.go @@ -3,6 +3,5 @@ package fasthttp const ( - maxIntChars = 18 maxHexIntChars = 15 ) diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_64_test.go b/vendor/github.com/valyala/fasthttp/bytesconv_64_test.go index 52d9e3e..b4165a6 100644 --- a/vendor/github.com/valyala/fasthttp/bytesconv_64_test.go +++ b/vendor/github.com/valyala/fasthttp/bytesconv_64_test.go @@ -38,4 +38,5 @@ func TestParseUintSuccess(t *testing.T) { testParseUintSuccess(t, "123", 123) testParseUintSuccess(t, "1234567890", 1234567890) testParseUintSuccess(t, "123456789012345678", 123456789012345678) + testParseUintSuccess(t, "9223372036854775807", 9223372036854775807) } diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_test.go b/vendor/github.com/valyala/fasthttp/bytesconv_test.go index b5da44d..483bc67 100644 --- a/vendor/github.com/valyala/fasthttp/bytesconv_test.go +++ b/vendor/github.com/valyala/fasthttp/bytesconv_test.go @@ -7,6 +7,8 @@ import ( "net" "testing" "time" + + "github.com/valyala/bytebufferpool" ) func TestAppendHTMLEscape(t *testing.T) { @@ -92,7 +94,7 @@ func testAppendUint(t *testing.T, n int) { } func testWriteHexInt(t *testing.T, n int, expectedS string) { - var w ByteBuffer + var w bytebufferpool.ByteBuffer bw := bufio.NewWriter(&w) if err := writeHexInt(bw, n); err != nil { t.Fatalf("unexpected error when writing hex %x: %s", n, err) @@ -174,6 +176,7 @@ func TestParseUintError(t *testing.T) { // too big num testParseUintError(t, "12345678901234567890") + testParseUintError(t, "1234567890123456789012") } func TestParseUfloatSuccess(t *testing.T) { diff --git a/vendor/github.com/valyala/fasthttp/bytesconv_timing_test.go b/vendor/github.com/valyala/fasthttp/bytesconv_timing_test.go index 24b6a49..f817cf4 100644 --- a/vendor/github.com/valyala/fasthttp/bytesconv_timing_test.go +++ b/vendor/github.com/valyala/fasthttp/bytesconv_timing_test.go @@ -5,6 +5,8 @@ import ( "html" "net" "testing" + + "github.com/valyala/bytebufferpool" ) func BenchmarkAppendHTMLEscape(b *testing.B) { @@ -77,7 +79,7 @@ func BenchmarkInt2HexByte(b *testing.B) { func BenchmarkWriteHexInt(b *testing.B) { b.RunParallel(func(pb *testing.PB) { - var w ByteBuffer + var w bytebufferpool.ByteBuffer bw := bufio.NewWriter(&w) i := 0 for pb.Next() { diff --git a/vendor/github.com/valyala/fasthttp/client.go b/vendor/github.com/valyala/fasthttp/client.go index 302af8a..92a5b07 100644 --- a/vendor/github.com/valyala/fasthttp/client.go +++ b/vendor/github.com/valyala/fasthttp/client.go @@ -60,6 +60,11 @@ func Do(req *Request, resp *Response) error { // // It is recommended obtaining req and resp via AcquireRequest // and AcquireResponse in performance-critical code. +// +// Warning: DoTimeout does not terminate the request itself. The request will +// continue in the background and the response will be discarded. +// If requests take too long and the connection pool gets filled up please +// try using a Client and setting a ReadTimeout. func DoTimeout(req *Request, resp *Response, timeout time.Duration) error { return defaultClient.DoTimeout(req, resp, timeout) } @@ -91,33 +96,36 @@ func DoDeadline(req *Request, resp *Response, deadline time.Time) error { return defaultClient.DoDeadline(req, resp, deadline) } -// Get appends url contents to dst and returns it as body. +// Get returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. -// -// New body buffer is allocated if dst is nil. func Get(dst []byte, url string) (statusCode int, body []byte, err error) { return defaultClient.Get(dst, url) } -// GetTimeout appends url contents to dst and returns it as body. +// GetTimeout returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // ErrTimeout error is returned if url contents couldn't be fetched // during the given timeout. func GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { return defaultClient.GetTimeout(dst, url, timeout) } -// GetDeadline appends url contents to dst and returns it as body. +// GetDeadline returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // ErrTimeout error is returned if url contents couldn't be fetched // until the given deadline. func GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { @@ -126,12 +134,11 @@ func GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, bo // Post sends POST request to the given url with the given POST arguments. // -// Response body is appended to dst, which is returned as body. +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // Empty POST body is sent if postArgs is nil. func Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { return defaultClient.Post(dst, url, postArgs) @@ -152,6 +159,10 @@ type Client struct { // Default client name is used if not set. Name string + // NoDefaultUserAgentHeader when set to true, causes the default + // User-Agent header to be excluded from the Request. + NoDefaultUserAgentHeader bool + // Callback for establishing new connections to hosts. // // Default Dial is used if not set. @@ -182,6 +193,11 @@ type Client struct { // after DefaultMaxIdleConnDuration. MaxIdleConnDuration time.Duration + // Maximum number of attempts for idempotent calls + // + // DefaultMaxIdemponentCallAttempts is used if not set. + MaxIdemponentCallAttempts int + // Per-connection buffer size for responses' reading. // This also limits the maximum header size. // @@ -234,33 +250,36 @@ type Client struct { ms map[string]*HostClient } -// Get appends url contents to dst and returns it as body. +// Get returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. -// -// New body buffer is allocated if dst is nil. func (c *Client) Get(dst []byte, url string) (statusCode int, body []byte, err error) { return clientGetURL(dst, url, c) } -// GetTimeout appends url contents to dst and returns it as body. +// GetTimeout returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // ErrTimeout error is returned if url contents couldn't be fetched // during the given timeout. func (c *Client) GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { return clientGetURLTimeout(dst, url, timeout, c) } -// GetDeadline appends url contents to dst and returns it as body. +// GetDeadline returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // ErrTimeout error is returned if url contents couldn't be fetched // until the given deadline. func (c *Client) GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { @@ -269,12 +288,11 @@ func (c *Client) GetDeadline(dst []byte, url string, deadline time.Time) (status // Post sends POST request to the given url with the given POST arguments. // -// Response body is appended to dst, which is returned as body. +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // Empty POST body is sent if postArgs is nil. func (c *Client) Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { return clientPostURL(dst, url, postArgs, c) @@ -303,6 +321,11 @@ func (c *Client) Post(dst []byte, url string, postArgs *Args) (statusCode int, b // // It is recommended obtaining req and resp via AcquireRequest // and AcquireResponse in performance-critical code. +// +// Warning: DoTimeout does not terminate the request itself. The request will +// continue in the background and the response will be discarded. +// If requests take too long and the connection pool gets filled up please +// try setting a ReadTimeout. func (c *Client) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { return clientDoTimeout(req, resp, timeout, c) } @@ -385,12 +408,14 @@ func (c *Client) Do(req *Request, resp *Response) error { hc = &HostClient{ Addr: addMissingPort(string(host), isTLS), Name: c.Name, + NoDefaultUserAgentHeader: c.NoDefaultUserAgentHeader, Dial: c.Dial, DialDualStack: c.DialDualStack, IsTLS: isTLS, TLSConfig: c.TLSConfig, MaxConns: c.MaxConnsPerHost, MaxIdleConnDuration: c.MaxIdleConnDuration, + MaxIdemponentCallAttempts: c.MaxIdemponentCallAttempts, ReadBufferSize: c.ReadBufferSize, WriteBufferSize: c.WriteBufferSize, ReadTimeout: c.ReadTimeout, @@ -414,11 +439,15 @@ func (c *Client) Do(req *Request, resp *Response) error { func (c *Client) mCleaner(m map[string]*HostClient) { mustStop := false + for { - t := time.Now() c.mLock.Lock() for k, v := range m { - if t.Sub(v.LastUseTime()) > time.Minute { + v.connsLock.Lock() + shouldRemove := v.connsCount == 0 + v.connsLock.Unlock() + + if shouldRemove { delete(m, k) } } @@ -443,6 +472,9 @@ const DefaultMaxConnsPerHost = 512 // connection is closed. const DefaultMaxIdleConnDuration = 10 * time.Second +// DefaultMaxIdemponentCallAttempts is the default idempotent calls attempts count. +const DefaultMaxIdemponentCallAttempts = 5 + // DialFunc must establish connection to addr. // // There is no need in establishing TLS (SSL) connection for https. @@ -484,6 +516,10 @@ type HostClient struct { // Client name. Used in User-Agent request header. Name string + // NoDefaultUserAgentHeader when set to true, causes the default + // User-Agent header to be excluded from the Request. + NoDefaultUserAgentHeader bool + // Callback for establishing new connection to the host. // // Default Dial is used if not set. @@ -522,6 +558,11 @@ type HostClient struct { // after DefaultMaxIdleConnDuration. MaxIdleConnDuration time.Duration + // Maximum number of attempts for idempotent calls + // + // DefaultMaxIdemponentCallAttempts is used if not set. + MaxIdemponentCallAttempts int + // Per-connection buffer size for responses' reading. // This also limits the maximum header size. // @@ -586,7 +627,7 @@ type HostClient struct { readerPool sync.Pool writerPool sync.Pool - pendingRequests uint64 + pendingRequests int32 connsCleanerRun bool } @@ -596,9 +637,6 @@ type clientConn struct { createdTime time.Time lastUseTime time.Time - - lastReadDeadlineTime time.Time - lastWriteDeadlineTime time.Time } var startTimeUnix = time.Now().Unix() @@ -609,33 +647,36 @@ func (c *HostClient) LastUseTime() time.Time { return time.Unix(startTimeUnix+int64(n), 0) } -// Get appends url contents to dst and returns it as body. +// Get returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. -// -// New body buffer is allocated if dst is nil. func (c *HostClient) Get(dst []byte, url string) (statusCode int, body []byte, err error) { return clientGetURL(dst, url, c) } -// GetTimeout appends url contents to dst and returns it as body. +// GetTimeout returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // ErrTimeout error is returned if url contents couldn't be fetched // during the given timeout. func (c *HostClient) GetTimeout(dst []byte, url string, timeout time.Duration) (statusCode int, body []byte, err error) { return clientGetURLTimeout(dst, url, timeout, c) } -// GetDeadline appends url contents to dst and returns it as body. +// GetDeadline returns the status code and body of url. +// +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // ErrTimeout error is returned if url contents couldn't be fetched // until the given deadline. func (c *HostClient) GetDeadline(dst []byte, url string, deadline time.Time) (statusCode int, body []byte, err error) { @@ -644,12 +685,11 @@ func (c *HostClient) GetDeadline(dst []byte, url string, deadline time.Time) (st // Post sends POST request to the given url with the given POST arguments. // -// Response body is appended to dst, which is returned as body. +// The contents of dst will be replaced by the body and returned, if the dst +// is too small a new slice will be allocated. // // The function follows redirects. Use Do* for manually handling redirects. // -// New body buffer is allocated if dst is nil. -// // Empty POST body is sent if postArgs is nil. func (c *HostClient) Post(dst []byte, url string, postArgs *Args) (statusCode int, body []byte, err error) { return clientPostURL(dst, url, postArgs, c) @@ -756,9 +796,24 @@ func doRequestFollowRedirects(req *Request, dst []byte, url string, c clientDoer resp.keepBodyBuffer = true oldBody := bodyBuf.B bodyBuf.B = dst + scheme := req.uri.Scheme() + req.schemaUpdate = false redirectsCount := 0 for { + // In case redirect to different scheme + if redirectsCount > 0 && !bytes.Equal(scheme, req.uri.Scheme()) { + if strings.HasPrefix(url, string(strHTTPS)) { + req.isTLS = true + req.uri.SetSchemeBytes(strHTTPS) + } else { + req.isTLS = false + req.uri.SetSchemeBytes(strHTTP) + } + scheme = req.uri.Scheme() + req.schemaUpdate = true + } + req.parsedURI = false req.Header.host = req.Header.host[:0] req.SetRequestURI(url) @@ -767,7 +822,11 @@ func doRequestFollowRedirects(req *Request, dst []byte, url string, c clientDoer break } statusCode = resp.Header.StatusCode() - if statusCode != StatusMovedPermanently && statusCode != StatusFound && statusCode != StatusSeeOther { + if statusCode != StatusMovedPermanently && + statusCode != StatusFound && + statusCode != StatusSeeOther && + statusCode != StatusTemporaryRedirect && + statusCode != StatusPermanentRedirect { break } @@ -868,6 +927,11 @@ func ReleaseResponse(resp *Response) { // // It is recommended obtaining req and resp via AcquireRequest // and AcquireResponse in performance-critical code. +// +// Warning: DoTimeout does not terminate the request itself. The request will +// continue in the background and the response will be discarded. +// If requests take too long and the connection pool gets filled up please +// try setting a ReadTimeout. func (c *HostClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { return clientDoTimeout(req, resp, timeout, c) } @@ -926,8 +990,17 @@ func clientDoDeadline(req *Request, resp *Response, deadline time.Time, c client // Without this 'hack' the load on slow host could exceed MaxConns* // concurrent requests, since timed out requests on client side // usually continue execution on the host. + + var cleanup int32 go func() { - ch <- c.Do(reqCopy, respCopy) + errDo := c.Do(reqCopy, respCopy) + if atomic.LoadInt32(&cleanup) == 1 { + ReleaseResponse(respCopy) + ReleaseRequest(reqCopy) + errorChPool.Put(chv) + } else { + ch <- errDo + } }() tc := acquireTimer(timeout) @@ -943,6 +1016,7 @@ func clientDoDeadline(req *Request, resp *Response, deadline time.Time, c client ReleaseRequest(reqCopy) errorChPool.Put(chv) case <-tc.C: + atomic.StoreInt32(&cleanup, 1) err = ErrTimeout } releaseTimer(tc) @@ -969,10 +1043,13 @@ var errorChPool sync.Pool func (c *HostClient) Do(req *Request, resp *Response) error { var err error var retry bool - const maxAttempts = 5 + maxAttempts := c.MaxIdemponentCallAttempts + if maxAttempts <= 0 { + maxAttempts = DefaultMaxIdemponentCallAttempts + } attempts := 0 - atomic.AddUint64(&c.pendingRequests, 1) + atomic.AddInt32(&c.pendingRequests, 1) for { retry, err = c.do(req, resp) if err == nil || !retry { @@ -996,7 +1073,7 @@ func (c *HostClient) Do(req *Request, resp *Response) error { break } } - atomic.AddUint64(&c.pendingRequests, ^uint64(0)) + atomic.AddInt32(&c.pendingRequests, -1) if err == io.EOF { err = ErrConnectionClosed @@ -1010,7 +1087,7 @@ func (c *HostClient) Do(req *Request, resp *Response) error { // This function may be used for balancing load among multiple HostClient // instances. func (c *HostClient) PendingRequests() int { - return int(atomic.LoadUint64(&c.pendingRequests)) + return int(atomic.LoadInt32(&c.pendingRequests)) } func isIdempotent(req *Request) bool { @@ -1041,12 +1118,22 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) panic("BUG: resp cannot be nil") } - atomic.StoreUint32(&c.lastUseTime, uint32(CoarseTimeNow().Unix()-startTimeUnix)) + atomic.StoreUint32(&c.lastUseTime, uint32(time.Now().Unix()-startTimeUnix)) // Free up resources occupied by response before sending the request, // so the GC may reclaim these resources (e.g. response body). resp.Reset() + // If we detected a redirect to another schema + if req.schemaUpdate { + c.IsTLS = bytes.Equal(req.URI().Scheme(), strHTTPS) + c.Addr = addMissingPort(string(req.Host()), c.IsTLS) + c.addrIdx = 0 + c.addrs = nil + req.schemaUpdate = false + req.SetConnectionClose() + } + cc, err := c.acquireConn() if err != nil { return false, err @@ -1054,16 +1141,12 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) conn := cc.c if c.WriteTimeout > 0 { - // Optimization: update write deadline only if more than 25% - // of the last write deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() - if currentTime.Sub(cc.lastWriteDeadlineTime) > (c.WriteTimeout >> 2) { - if err = conn.SetWriteDeadline(currentTime.Add(c.WriteTimeout)); err != nil { - c.closeConn(cc) - return true, err - } - cc.lastWriteDeadlineTime = currentTime + // Set Deadline every time, since golang has fixed the performance issue + // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details + currentTime := time.Now() + if err = conn.SetWriteDeadline(currentTime.Add(c.WriteTimeout)); err != nil { + c.closeConn(cc) + return true, err } } @@ -1079,9 +1162,6 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) } bw := c.acquireWriter(conn) err = req.Write(bw) - if len(userAgentOld) == 0 { - req.Header.userAgent = userAgentOld - } if resetConnection { req.Header.ResetConnectionClose() @@ -1098,16 +1178,12 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) c.releaseWriter(bw) if c.ReadTimeout > 0 { - // Optimization: update read deadline only if more than 25% - // of the last read deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() - if currentTime.Sub(cc.lastReadDeadlineTime) > (c.ReadTimeout >> 2) { - if err = conn.SetReadDeadline(currentTime.Add(c.ReadTimeout)); err != nil { - c.closeConn(cc) - return true, err - } - cc.lastReadDeadlineTime = currentTime + // Set Deadline every time, since golang has fixed the performance issue + // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details + currentTime := time.Now() + if err = conn.SetReadDeadline(currentTime.Add(c.ReadTimeout)); err != nil { + c.closeConn(cc) + return true, err } } @@ -1122,7 +1198,9 @@ func (c *HostClient) doNonNilReqResp(req *Request, resp *Response) (bool, error) if err = resp.ReadLimitBody(br, c.MaxResponseBodySize); err != nil { c.releaseReader(br) c.closeConn(cc) - return true, err + // Don't retry in case of ErrBodyTooLarge since we will just get the same again. + retry := err != ErrBodyTooLarge + return retry, err } c.releaseReader(br) @@ -1226,6 +1304,12 @@ func (c *HostClient) connsCleaner() { for i < n && currentTime.Sub(conns[i].lastUseTime) > maxIdleConnDuration { i++ } + sleepFor := maxIdleConnDuration + if i < n { + // + 1 so we actually sleep past the expiration time and not up to it. + // Otherwise the > check above would still fail. + sleepFor = maxIdleConnDuration - currentTime.Sub(conns[i].lastUseTime) + 1 + } scratch = append(scratch[:0], conns[:i]...) if i > 0 { m := copy(conns, conns[i:]) @@ -1253,7 +1337,7 @@ func (c *HostClient) connsCleaner() { break } - time.Sleep(maxIdleConnDuration) + time.Sleep(sleepFor) } } @@ -1276,19 +1360,20 @@ func acquireClientConn(conn net.Conn) *clientConn { } cc := v.(*clientConn) cc.c = conn - cc.createdTime = CoarseTimeNow() + cc.createdTime = time.Now() return cc } func releaseClientConn(cc *clientConn) { - cc.c = nil + // Reset all fields. + *cc = clientConn{} clientConnPool.Put(cc) } var clientConnPool sync.Pool func (c *HostClient) releaseConn(cc *clientConn) { - cc.lastUseTime = CoarseTimeNow() + cc.lastUseTime = time.Now() c.connsLock.Lock() c.conns = append(c.conns, cc) c.connsLock.Unlock() @@ -1482,7 +1567,7 @@ func (c *HostClient) getClientName() []byte { var clientName []byte if v == nil { clientName = []byte(c.Name) - if len(clientName) == 0 { + if len(clientName) == 0 && !c.NoDefaultUserAgentHeader { clientName = defaultUserAgent } c.clientName.Store(clientName) @@ -1524,7 +1609,7 @@ type PipelineClient struct { // The maximum number of concurrent connections to the Addr. // - // A sinle connection is used by default. + // A single connection is used by default. MaxConns int // The maximum number of pending pipelined requests over @@ -1649,6 +1734,11 @@ type pipelineWork struct { // // It is recommended obtaining req and resp via AcquireRequest // and AcquireResponse in performance-critical code. +// +// Warning: DoTimeout does not terminate the request itself. The request will +// continue in the background and the response will be discarded. +// If requests take too long and the connection pool gets filled up please +// try setting a ReadTimeout. func (c *PipelineClient) DoTimeout(req *Request, resp *Response, timeout time.Duration) error { return c.DoDeadline(req, resp, time.Now().Add(timeout)) } @@ -1954,8 +2044,6 @@ func (c *pipelineConnClient) writer(conn net.Conn, stopCh <-chan struct{}) error w *pipelineWork err error - - lastWriteDeadlineTime time.Time ) close(instantTimerCh) for { @@ -1988,17 +2076,13 @@ func (c *pipelineConnClient) writer(conn net.Conn, stopCh <-chan struct{}) error } if writeTimeout > 0 { - // Optimization: update write deadline only if more than 25% - // of the last write deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() - if currentTime.Sub(lastWriteDeadlineTime) > (writeTimeout >> 2) { - if err = conn.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - lastWriteDeadlineTime = currentTime + // Set Deadline every time, since golang has fixed the performance issue + // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details + currentTime := time.Now() + if err = conn.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { + w.err = err + w.done <- struct{}{} + return err } } if err = w.req.Write(bw); err != nil { @@ -2052,8 +2136,6 @@ func (c *pipelineConnClient) reader(conn net.Conn, stopCh <-chan struct{}) error var ( w *pipelineWork err error - - lastReadDeadlineTime time.Time ) for { select { @@ -2069,17 +2151,13 @@ func (c *pipelineConnClient) reader(conn net.Conn, stopCh <-chan struct{}) error } if readTimeout > 0 { - // Optimization: update read deadline only if more than 25% - // of the last read deadline exceeded. - // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() - if currentTime.Sub(lastReadDeadlineTime) > (readTimeout >> 2) { - if err = conn.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { - w.err = err - w.done <- struct{}{} - return err - } - lastReadDeadlineTime = currentTime + // Set Deadline every time, since golang has fixed the performance issue + // See https://github.com/golang/go/issues/15133#issuecomment-271571395 for details + currentTime := time.Now() + if err = conn.SetReadDeadline(currentTime.Add(readTimeout)); err != nil { + w.err = err + w.done <- struct{}{} + return err } } if err = w.resp.Read(br); err != nil { diff --git a/vendor/github.com/valyala/fasthttp/client_test.go b/vendor/github.com/valyala/fasthttp/client_test.go index fc29b41..a6310a0 100644 --- a/vendor/github.com/valyala/fasthttp/client_test.go +++ b/vendor/github.com/valyala/fasthttp/client_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "net/url" "os" "runtime" "strings" @@ -17,6 +18,386 @@ import ( "github.com/valyala/fasthttp/fasthttputil" ) +func TestClientPostArgs(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + s := &Server{ + Handler: func(ctx *RequestCtx) { + body := ctx.Request.Body() + if len(body) == 0 { + return + } + ctx.Write(body) + }, + } + go s.Serve(ln) + c := &Client{ + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + } + req, res := AcquireRequest(), AcquireResponse() + args := req.PostArgs() + args.Add("addhttp2", "support") + args.Add("fast", "http") + req.Header.SetMethod("POST") + req.SetRequestURI("http://make.fasthttp.great?again") + err := c.Do(req, res) + if err != nil { + t.Fatal(err) + } + if len(res.Body()) == 0 { + t.Fatal("cannot set args as body") + } +} + +func TestClientRedirectSameSchema(t *testing.T) { + + listenHTTPS1 := testClientRedirectListener(t, true) + defer listenHTTPS1.Close() + + listenHTTPS2 := testClientRedirectListener(t, true) + defer listenHTTPS2.Close() + + sHTTPS1 := testClientRedirectChangingSchemaServer(t, listenHTTPS1, listenHTTPS1, true) + defer sHTTPS1.Stop() + + sHTTPS2 := testClientRedirectChangingSchemaServer(t, listenHTTPS2, listenHTTPS2, false) + defer sHTTPS2.Stop() + + destURL := fmt.Sprintf("https://%s/baz", listenHTTPS1.Addr().String()) + + urlParsed, err := url.Parse(destURL) + if err != nil { + fmt.Println(err) + return + } + + reqClient := &HostClient{ + IsTLS: true, + Addr: urlParsed.Host, + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + statusCode, _, err := reqClient.GetTimeout(nil, destURL, 4000*time.Millisecond) + if err != nil { + t.Fatalf("HostClient error: %s", err) + return + } + + if statusCode != 200 { + t.Fatalf("HostClient error code response %d", statusCode) + return + } + +} + +func TestClientRedirectChangingSchemaHttp2Https(t *testing.T) { + + listenHTTPS := testClientRedirectListener(t, true) + defer listenHTTPS.Close() + + listenHTTP := testClientRedirectListener(t, false) + defer listenHTTP.Close() + + sHTTPS := testClientRedirectChangingSchemaServer(t, listenHTTPS, listenHTTP, true) + defer sHTTPS.Stop() + + sHTTP := testClientRedirectChangingSchemaServer(t, listenHTTPS, listenHTTP, false) + defer sHTTP.Stop() + + destURL := fmt.Sprintf("http://%s/baz", listenHTTP.Addr().String()) + + urlParsed, err := url.Parse(destURL) + if err != nil { + fmt.Println(err) + return + } + + reqClient := &HostClient{ + Addr: urlParsed.Host, + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + statusCode, _, err := reqClient.GetTimeout(nil, destURL, 4000*time.Millisecond) + if err != nil { + t.Fatalf("HostClient error: %s", err) + return + } + + if statusCode != 200 { + t.Fatalf("HostClient error code response %d", statusCode) + return + } + +} + +func testClientRedirectListener(t *testing.T, isTLS bool) net.Listener { + + var ln net.Listener + var err error + var tlsConfig *tls.Config + + if isTLS { + certFile := "./ssl-cert-snakeoil.pem" + keyFile := "./ssl-cert-snakeoil.key" + cert, err1 := tls.LoadX509KeyPair(certFile, keyFile) + if err1 != nil { + t.Fatalf("Cannot load TLS certificate: %s", err1) + } + tlsConfig = &tls.Config{ + Certificates: []tls.Certificate{cert}, + } + ln, err = tls.Listen("tcp", "localhost:0", tlsConfig) + } else { + ln, err = net.Listen("tcp", "localhost:0") + } + + if err != nil { + t.Fatalf("cannot listen isTLS %v: %s", isTLS, err) + } + + return ln +} + +func testClientRedirectChangingSchemaServer(t *testing.T, https, http net.Listener, isTLS bool) *testEchoServer { + s := &Server{ + Handler: func(ctx *RequestCtx) { + if ctx.IsTLS() { + ctx.SetStatusCode(200) + } else { + ctx.Redirect(fmt.Sprintf("https://%s/baz", https.Addr().String()), 301) + } + }, + } + + var ln net.Listener + if isTLS { + ln = https + } else { + ln = http + } + + ch := make(chan struct{}) + go func() { + err := s.Serve(ln) + if err != nil { + t.Fatalf("unexpected error returned from Serve(): %s", err) + } + close(ch) + }() + return &testEchoServer{ + s: s, + ln: ln, + ch: ch, + t: t, + } +} + +func TestClientHeaderCase(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + defer ln.Close() + + go func() { + c, err := ln.Accept() + if err != nil { + t.Fatal(err) + } + c.Write([]byte("HTTP/1.1 200 OK\r\n" + + "content-type: text/plain\r\n" + + "transfer-encoding: chunked\r\n\r\n" + + "24\r\nThis is the data in the first chunk \r\n" + + "1B\r\nand this is the second one \r\n" + + "0\r\n\r\n", + )) + }() + + c := &Client{ + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + ReadTimeout: time.Millisecond * 10, + + // Even without name normalizing we should parse headers correctly. + DisableHeaderNamesNormalizing: true, + } + + code, body, err := c.Get(nil, "http://example.com") + if err != nil { + t.Error(err) + } else if code != 200 { + t.Errorf("expected status code 200 got %d", code) + } else if string(body) != "This is the data in the first chunk and this is the second one " { + t.Errorf("wrong body: %q", body) + } +} + +func TestClientReadTimeout(t *testing.T) { + // This test is rather slow and increase the total test time + // from 2.5 seconds to 6.5 seconds. + if testing.Short() { + t.Skip("skipping test in short mode") + } + + ln := fasthttputil.NewInmemoryListener() + + timeout := false + s := &Server{ + Handler: func(ctx *RequestCtx) { + if timeout { + time.Sleep(time.Minute) + } else { + timeout = true + } + }, + Logger: &customLogger{}, // Don't print closed pipe errors. + } + go s.Serve(ln) + + c := &HostClient{ + ReadTimeout: time.Second * 4, + MaxIdemponentCallAttempts: 1, + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + } + + req := AcquireRequest() + res := AcquireResponse() + + req.SetRequestURI("http://localhost") + + // Setting Connection: Close will make the connection be + // returned to the pool. + req.SetConnectionClose() + + if err := c.Do(req, res); err != nil { + t.Fatal(err) + } + + ReleaseRequest(req) + ReleaseResponse(res) + + done := make(chan struct{}) + go func() { + req := AcquireRequest() + res := AcquireResponse() + + req.SetRequestURI("http://localhost") + req.SetConnectionClose() + + c.Do(req, res) + + ReleaseRequest(req) + ReleaseResponse(res) + close(done) + }() + + select { + case <-done: + // This shouldn't take longer than the timeout times the number of requests it is going to try to do. + // Give it 2 seconds extra seconds just to be sure. + case <-time.After(c.ReadTimeout*time.Duration(c.MaxIdemponentCallAttempts) + time.Second*2): + t.Fatal("Client.ReadTimeout didn't work") + } +} + +func TestClientDefaultUserAgent(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + + userAgentSeen := "" + s := &Server{ + Handler: func(ctx *RequestCtx) { + userAgentSeen = string(ctx.UserAgent()) + }, + } + go s.Serve(ln) + + c := &Client{ + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + } + req := AcquireRequest() + res := AcquireResponse() + + req.SetRequestURI("http://example.com") + + err := c.Do(req, res) + if err != nil { + t.Fatal(err) + } + if userAgentSeen != string(defaultUserAgent) { + t.Fatalf("User-Agent defers %q != %q", userAgentSeen, defaultUserAgent) + } +} + +func TestClientSetUserAgent(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + + userAgentSeen := "" + s := &Server{ + Handler: func(ctx *RequestCtx) { + userAgentSeen = string(ctx.UserAgent()) + }, + } + go s.Serve(ln) + + userAgent := "I'm not fasthttp" + c := &Client{ + Name: userAgent, + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + } + req := AcquireRequest() + res := AcquireResponse() + + req.SetRequestURI("http://example.com") + + err := c.Do(req, res) + if err != nil { + t.Fatal(err) + } + if userAgentSeen != userAgent { + t.Fatalf("User-Agent defers %q != %q", userAgentSeen, userAgent) + } +} + +func TestClientNoUserAgent(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + + userAgentSeen := "" + s := &Server{ + Handler: func(ctx *RequestCtx) { + userAgentSeen = string(ctx.UserAgent()) + }, + } + go s.Serve(ln) + + c := &Client{ + NoDefaultUserAgentHeader: true, + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + } + req := AcquireRequest() + res := AcquireResponse() + + req.SetRequestURI("http://example.com") + + err := c.Do(req, res) + if err != nil { + t.Fatal(err) + } + if userAgentSeen != "" { + t.Fatalf("User-Agent wrong %q != %q", userAgentSeen, "") + } +} + func TestClientDoWithCustomHeaders(t *testing.T) { // make sure that the client sends all the request headers and body. ln := fasthttputil.NewInmemoryListener() @@ -562,7 +943,6 @@ func TestHostClientMultipleAddrs(t *testing.T) { } func TestClientFollowRedirects(t *testing.T) { - addr := "127.0.0.1:55234" s := &Server{ Handler: func(ctx *RequestCtx) { switch string(ctx.Path()) { @@ -579,10 +959,7 @@ func TestClientFollowRedirects(t *testing.T) { } }, } - ln, err := net.Listen("tcp4", addr) - if err != nil { - t.Fatalf("unexpected error: %s", err) - } + ln := fasthttputil.NewInmemoryListener() serverStopCh := make(chan struct{}) go func() { @@ -592,9 +969,15 @@ func TestClientFollowRedirects(t *testing.T) { close(serverStopCh) }() - uri := fmt.Sprintf("http://%s/foo", addr) + c := &HostClient{ + Addr: "xxx", + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + } + for i := 0; i < 10; i++ { - statusCode, body, err := GetTimeout(nil, uri, time.Second) + statusCode, body, err := c.GetTimeout(nil, "http://xxx/foo", time.Second) if err != nil { t.Fatalf("unexpected error: %s", err) } @@ -606,9 +989,8 @@ func TestClientFollowRedirects(t *testing.T) { } } - uri = fmt.Sprintf("http://%s/aaab/sss", addr) for i := 0; i < 10; i++ { - statusCode, body, err := Get(nil, uri) + statusCode, body, err := c.Get(nil, "http://xxx/aaab/sss") if err != nil { t.Fatalf("unexpected error: %s", err) } @@ -622,52 +1004,44 @@ func TestClientFollowRedirects(t *testing.T) { } func TestClientGetTimeoutSuccess(t *testing.T) { - addr := "127.0.0.1:56889" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr - testClientGetTimeoutSuccess(t, &defaultClient, addr, 100) + testClientGetTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) } func TestClientGetTimeoutSuccessConcurrent(t *testing.T) { - addr := "127.0.0.1:56989" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() - testClientGetTimeoutSuccess(t, &defaultClient, addr, 100) + testClientGetTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) }() } wg.Wait() } func TestClientDoTimeoutSuccess(t *testing.T) { - addr := "127.0.0.1:63897" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr - testClientDoTimeoutSuccess(t, &defaultClient, addr, 100) + testClientDoTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) } func TestClientDoTimeoutSuccessConcurrent(t *testing.T) { - addr := "127.0.0.1:63898" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) go func() { defer wg.Done() - testClientDoTimeoutSuccess(t, &defaultClient, addr, 100) + testClientDoTimeoutSuccess(t, &defaultClient, "http://"+s.Addr(), 100) }() } wg.Wait() @@ -937,16 +1311,13 @@ func (r *singleReadConn) Close() error { } func TestClientHTTPSInvalidServerName(t *testing.T) { - addrHTTPS := "127.0.0.1:57794" - sHTTPS := startEchoServerTLS(t, "tcp", addrHTTPS) + sHTTPS := startEchoServerTLS(t, "tcp", "127.0.0.1:") defer sHTTPS.Stop() var c Client - addr := "https://" + addrHTTPS - for i := 0; i < 10; i++ { - _, _, err := c.GetTimeout(nil, addr, time.Second) + _, _, err := c.GetTimeout(nil, "https://"+sHTTPS.Addr(), time.Second) if err == nil { t.Fatalf("expecting TLS error") } @@ -954,12 +1325,10 @@ func TestClientHTTPSInvalidServerName(t *testing.T) { } func TestClientHTTPSConcurrent(t *testing.T) { - addrHTTP := "127.0.0.1:56793" - sHTTP := startEchoServer(t, "tcp", addrHTTP) + sHTTP := startEchoServer(t, "tcp", "127.0.0.1:") defer sHTTP.Stop() - addrHTTPS := "127.0.0.1:56794" - sHTTPS := startEchoServerTLS(t, "tcp", addrHTTPS) + sHTTPS := startEchoServerTLS(t, "tcp", "127.0.0.1:") defer sHTTPS.Stop() c := &Client{ @@ -971,9 +1340,9 @@ func TestClientHTTPSConcurrent(t *testing.T) { var wg sync.WaitGroup for i := 0; i < 4; i++ { wg.Add(1) - addr := "http://" + addrHTTP + addr := "http://" + sHTTP.Addr() if i&1 != 0 { - addr = "https://" + addrHTTPS + addr = "https://" + sHTTPS.Addr() } go func() { defer wg.Done() @@ -987,10 +1356,9 @@ func TestClientHTTPSConcurrent(t *testing.T) { func TestClientManyServers(t *testing.T) { var addrs []string for i := 0; i < 10; i++ { - addr := fmt.Sprintf("127.0.0.1:%d", 56904+i) - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addrs = append(addrs, addr) + addrs = append(addrs, s.Addr()) } var wg sync.WaitGroup @@ -1007,29 +1375,24 @@ func TestClientManyServers(t *testing.T) { } func TestClientGet(t *testing.T) { - addr := "127.0.0.1:56789" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr - testClientGet(t, &defaultClient, addr, 100) + testClientGet(t, &defaultClient, "http://"+s.Addr(), 100) } func TestClientPost(t *testing.T) { - addr := "127.0.0.1:56798" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr - testClientPost(t, &defaultClient, addr, 100) + testClientPost(t, &defaultClient, "http://"+s.Addr(), 100) } func TestClientConcurrent(t *testing.T) { - addr := "127.0.0.1:55780" - s := startEchoServer(t, "tcp", addr) + s := startEchoServer(t, "tcp", "127.0.0.1:") defer s.Stop() - addr = "http://" + addr + addr := "http://" + s.Addr() var wg sync.WaitGroup for i := 0; i < 10; i++ { wg.Add(1) @@ -1219,6 +1582,10 @@ func (s *testEchoServer) Stop() { } } +func (s *testEchoServer) Addr() string { + return s.ln.Addr().String() +} + func startEchoServerTLS(t *testing.T, network, addr string) *testEchoServer { return startEchoServerExt(t, network, addr, true) } @@ -1259,6 +1626,7 @@ func startEchoServerExt(t *testing.T, network, addr string, isTLS bool) *testEch ctx.PostArgs().WriteTo(ctx) } }, + Logger: &customLogger{}, // Ignore log output. } ch := make(chan struct{}) go func() { diff --git a/vendor/github.com/valyala/fasthttp/coarseTime.go b/vendor/github.com/valyala/fasthttp/coarseTime.go index 03c14f3..4679df6 100644 --- a/vendor/github.com/valyala/fasthttp/coarseTime.go +++ b/vendor/github.com/valyala/fasthttp/coarseTime.go @@ -1,28 +1,13 @@ package fasthttp import ( - "sync/atomic" "time" ) // CoarseTimeNow returns the current time truncated to the nearest second. // -// This is a faster alternative to time.Now(). +// Deprecated: This is slower than calling time.Now() directly. +// This is now time.Now().Truncate(time.Second) shortcut. func CoarseTimeNow() time.Time { - tp := coarseTime.Load().(*time.Time) - return *tp + return time.Now().Truncate(time.Second) } - -func init() { - t := time.Now().Truncate(time.Second) - coarseTime.Store(&t) - go func() { - for { - time.Sleep(time.Second) - t := time.Now().Truncate(time.Second) - coarseTime.Store(&t) - } - }() -} - -var coarseTime atomic.Value diff --git a/vendor/github.com/valyala/fasthttp/compress.go b/vendor/github.com/valyala/fasthttp/compress.go index 221472a..73a40d3 100644 --- a/vendor/github.com/valyala/fasthttp/compress.go +++ b/vendor/github.com/valyala/fasthttp/compress.go @@ -152,7 +152,6 @@ func WriteGzipLevel(w io.Writer, p []byte, level int) (int, error) { switch w.(type) { case *byteSliceWriter, *bytes.Buffer, - *ByteBuffer, *bytebufferpool.ByteBuffer: // These writers don't block, so we can just use stacklessWriteGzip ctx := &compressCtx{ @@ -249,7 +248,6 @@ func WriteDeflateLevel(w io.Writer, p []byte, level int) (int, error) { switch w.(type) { case *byteSliceWriter, *bytes.Buffer, - *ByteBuffer, *bytebufferpool.ByteBuffer: // These writers don't block, so we can just use stacklessWriteDeflate ctx := &compressCtx{ @@ -409,7 +407,7 @@ func isFileCompressible(f *os.File, minCompressRatio float64) bool { // Try compressing the first 4kb of of the file // and see if it can be compressed by more than // the given minCompressRatio. - b := AcquireByteBuffer() + b := bytebufferpool.Get() zw := acquireStacklessGzipWriter(b, CompressDefaultCompression) lr := &io.LimitedReader{ R: f, @@ -424,7 +422,7 @@ func isFileCompressible(f *os.File, minCompressRatio float64) bool { n := 4096 - lr.N zn := len(b.B) - ReleaseByteBuffer(b) + bytebufferpool.Put(b) return float64(zn) < float64(n)*minCompressRatio } diff --git a/vendor/github.com/valyala/fasthttp/cookie.go b/vendor/github.com/valyala/fasthttp/cookie.go index 17ecc62..1d9861d 100644 --- a/vendor/github.com/valyala/fasthttp/cookie.go +++ b/vendor/github.com/valyala/fasthttp/cookie.go @@ -18,6 +18,20 @@ var ( CookieExpireUnlimited = zeroTime ) +// CookieSameSite is an enum for the mode in which the SameSite flag should be set for the given cookie. +// See https://tools.ietf.org/html/draft-ietf-httpbis-cookie-same-site-00 for details. +type CookieSameSite int +const ( + // CookieSameSiteDisabled removes the SameSite flag + CookieSameSiteDisabled CookieSameSite = iota + // CookieSameSiteDefaultMode sets the SameSite flag + CookieSameSiteDefaultMode + // CookieSameSiteLaxMode sets the SameSite flag with the "Lax" parameter + CookieSameSiteLaxMode + // CookieSameSiteStrictMode sets the SameSite flag with the "Strict" parameter + CookieSameSiteStrictMode +) + // AcquireCookie returns an empty Cookie object from the pool. // // The returned object may be returned back to the pool with ReleaseCookie. @@ -52,11 +66,13 @@ type Cookie struct { key []byte value []byte expire time.Time + maxAge int domain []byte path []byte httpOnly bool secure bool + sameSite CookieSameSite bufKV argsKV buf []byte @@ -68,10 +84,12 @@ func (c *Cookie) CopyTo(src *Cookie) { c.key = append(c.key[:0], src.key...) c.value = append(c.value[:0], src.value...) c.expire = src.expire + c.maxAge = src.maxAge c.domain = append(c.domain[:0], src.domain...) c.path = append(c.path[:0], src.path...) c.httpOnly = src.httpOnly c.secure = src.secure + c.sameSite = src.sameSite } // HTTPOnly returns true if the cookie is http only. @@ -94,6 +112,16 @@ func (c *Cookie) SetSecure(secure bool) { c.secure = secure } +// SameSite returns the SameSite mode. +func (c *Cookie) SameSite() CookieSameSite { + return c.sameSite +} + +// SetSameSite sets the cookie's SameSite flag to the given value. +func (c *Cookie) SetSameSite(mode CookieSameSite) { + c.sameSite = mode +} + // Path returns cookie path. func (c *Cookie) Path() []byte { return c.path @@ -128,6 +156,20 @@ func (c *Cookie) SetDomainBytes(domain []byte) { c.domain = append(c.domain[:0], domain...) } +// MaxAge returns the seconds until the cookie is meant to expire or 0 +// if no max age. +func (c *Cookie) MaxAge() int { + return c.maxAge +} + +// SetMaxAge sets cookie expiration time based on seconds. This takes precedence +// over any absolute expiry set on the cookie +// +// Set max age to 0 to unset +func (c *Cookie) SetMaxAge(seconds int) { + c.maxAge = seconds +} + // Expire returns cookie expiration time. // // CookieExpireUnlimited is returned if cookie doesn't expire @@ -188,10 +230,12 @@ func (c *Cookie) Reset() { c.key = c.key[:0] c.value = c.value[:0] c.expire = zeroTime + c.maxAge = 0 c.domain = c.domain[:0] c.path = c.path[:0] c.httpOnly = false c.secure = false + c.sameSite = CookieSameSiteDisabled } // AppendBytes appends cookie representation to dst and returns @@ -203,7 +247,12 @@ func (c *Cookie) AppendBytes(dst []byte) []byte { } dst = append(dst, c.value...) - if !c.expire.IsZero() { + if c.maxAge > 0 { + dst = append(dst, ';', ' ') + dst = append(dst, strCookieMaxAge...) + dst = append(dst, '=') + dst = AppendUint(dst, c.maxAge) + } else if !c.expire.IsZero() { c.bufKV.value = AppendHTTPDate(c.bufKV.value[:0], c.expire) dst = append(dst, ';', ' ') dst = append(dst, strCookieExpires...) @@ -224,6 +273,21 @@ func (c *Cookie) AppendBytes(dst []byte) []byte { dst = append(dst, ';', ' ') dst = append(dst, strCookieSecure...) } + switch c.sameSite { + case CookieSameSiteDefaultMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + case CookieSameSiteLaxMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + dst = append(dst, '=') + dst = append(dst, strCookieSameSiteLax...) + case CookieSameSiteStrictMode: + dst = append(dst, ';', ' ') + dst = append(dst, strCookieSameSite...) + dst = append(dst, '=') + dst = append(dst, strCookieSameSiteStrict...) + } return dst } @@ -272,29 +336,75 @@ func (c *Cookie) ParseBytes(src []byte) error { c.value = append(c.value[:0], kv.value...) for s.next(kv) { - if len(kv.key) == 0 && len(kv.value) == 0 { - continue - } - switch string(kv.key) { - case "expires": - v := b2s(kv.value) - exptime, err := time.ParseInLocation(time.RFC1123, v, time.UTC) - if err != nil { - return err + if len(kv.key) != 0 { + // Case insensitive switch on first char + switch kv.key[0] | 0x20 { + case 'm': + if caseInsensitiveCompare(strCookieMaxAge, kv.key) { + maxAge, err := ParseUint(kv.value) + if err != nil { + return err + } + c.maxAge = maxAge + } + + case 'e': // "expires" + if caseInsensitiveCompare(strCookieExpires, kv.key) { + v := b2s(kv.value) + // Try the same two formats as net/http + // See: https://github.com/golang/go/blob/00379be17e63a5b75b3237819392d2dc3b313a27/src/net/http/cookie.go#L133-L135 + exptime, err := time.ParseInLocation(time.RFC1123, v, time.UTC) + if err != nil { + exptime, err = time.Parse("Mon, 02-Jan-2006 15:04:05 MST", v) + if err != nil { + return err + } + } + c.expire = exptime + } + + case 'd': // "domain" + if caseInsensitiveCompare(strCookieDomain, kv.key) { + c.domain = append(c.domain[:0], kv.value...) + } + + case 'p': // "path" + if caseInsensitiveCompare(strCookiePath, kv.key) { + c.path = append(c.path[:0], kv.value...) + } + + case 's': // "samesite" + if caseInsensitiveCompare(strCookieSameSite, kv.key) { + // Case insensitive switch on first char + switch kv.value[0] | 0x20 { + case 'l': // "lax" + if caseInsensitiveCompare(strCookieSameSiteLax, kv.value) { + c.sameSite = CookieSameSiteLaxMode + } + case 's': // "strict" + if caseInsensitiveCompare(strCookieSameSiteStrict, kv.value) { + c.sameSite = CookieSameSiteStrictMode + } + } + } } - c.expire = exptime - case "domain": - c.domain = append(c.domain[:0], kv.value...) - case "path": - c.path = append(c.path[:0], kv.value...) - case "": - switch string(kv.value) { - case "HttpOnly": - c.httpOnly = true - case "secure": - c.secure = true + + } else if len(kv.value) != 0 { + // Case insensitive switch on first char + switch kv.value[0] | 0x20 { + case 'h': // "httponly" + if caseInsensitiveCompare(strCookieHTTPOnly, kv.value) { + c.httpOnly = true + } + + case 's': // "secure" + if caseInsensitiveCompare(strCookieSecure, kv.value) { + c.secure = true + } else if caseInsensitiveCompare(strCookieSameSite, kv.value) { + c.sameSite = CookieSameSiteDefaultMode + } } - } + } // else empty or no match } return nil } @@ -329,6 +439,19 @@ func appendRequestCookieBytes(dst []byte, cookies []argsKV) []byte { return dst } +// For Response we can not use the above function as response cookies +// already contain the key= in the value. +func appendResponseCookieBytes(dst []byte, cookies []argsKV) []byte { + for i, n := 0, len(cookies); i < n; i++ { + kv := &cookies[i] + dst = append(dst, kv.value...) + if i+1 < n { + dst = append(dst, ';', ' ') + } + } + return dst +} + func parseRequestCookies(cookies []argsKV, src []byte) []argsKV { var s cookieScanner s.b = src @@ -394,3 +517,17 @@ func decodeCookieArg(dst, src []byte, skipQuotes bool) []byte { } return append(dst[:0], src...) } + +// caseInsensitiveCompare does a case insensitive equality comparison of +// two []byte. Assumes only letters need to be matched. +func caseInsensitiveCompare(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := 0; i < len(a); i++ { + if a[i]|0x20 != b[i]|0x20 { + return false + } + } + return true +} diff --git a/vendor/github.com/valyala/fasthttp/cookie_test.go b/vendor/github.com/valyala/fasthttp/cookie_test.go index 85f1569..50c2ad3 100644 --- a/vendor/github.com/valyala/fasthttp/cookie_test.go +++ b/vendor/github.com/valyala/fasthttp/cookie_test.go @@ -76,7 +76,7 @@ func TestCookieSecure(t *testing.T) { if err := c.Parse("foo=bar"); err != nil { t.Fatalf("unexpected error: %s", err) } - if c.HTTPOnly() { + if c.Secure() { t.Fatalf("Unexpected secure flag set") } s = c.String() @@ -85,6 +85,94 @@ func TestCookieSecure(t *testing.T) { } } +func TestCookieSameSite(t *testing.T) { + var c Cookie + + if err := c.Parse("foo=bar; samesite"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if c.SameSite() != CookieSameSiteDefaultMode { + t.Fatalf("SameSite must be set") + } + s := c.String() + if !strings.Contains(s, "; SameSite") { + t.Fatalf("missing SameSite flag in cookie %q", s) + } + + if err := c.Parse("foo=bar; samesite=lax"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if c.SameSite() != CookieSameSiteLaxMode { + t.Fatalf("SameSite Lax Mode must be set") + } + s = c.String() + if !strings.Contains(s, "; SameSite=Lax") { + t.Fatalf("missing SameSite flag in cookie %q", s) + } + + if err := c.Parse("foo=bar; samesite=strict"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if c.SameSite() != CookieSameSiteStrictMode { + t.Fatalf("SameSite Strict Mode must be set") + } + s = c.String() + if !strings.Contains(s, "; SameSite=Strict") { + t.Fatalf("missing SameSite flag in cookie %q", s) + } + + if err := c.Parse("foo=bar"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if c.SameSite() != CookieSameSiteDisabled { + t.Fatalf("Unexpected SameSite flag set") + } + s = c.String() + if strings.Contains(s, "SameSite") { + t.Fatalf("unexpected SameSite flag in cookie %q", s) + } +} + +func TestCookieMaxAge(t *testing.T) { + var c Cookie + + maxAge := 100 + if err := c.Parse("foo=bar; max-age=100"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if maxAge != c.MaxAge() { + t.Fatalf("max-age must be set") + } + s := c.String() + if !strings.Contains(s, "; max-age=100") { + t.Fatalf("missing max-age flag in cookie %q", s) + } + + if err := c.Parse("foo=bar; expires=Tue, 10 Nov 2009 23:00:00 GMT; max-age=100;"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if maxAge != c.MaxAge() { + t.Fatalf("max-age ignored") + } + s = c.String() + if s != "foo=bar; max-age=100" { + t.Fatalf("missing max-age in cookie %q", s) + } + + expires := time.Unix(100, 0) + c.SetExpire(expires) + s = c.String() + if s != "foo=bar; max-age=100" { + t.Fatalf("expires should be ignored due to max-age: %q", s) + } + + c.SetMaxAge(0) + s = c.String() + if s != "foo=bar; expires=Thu, 01 Jan 1970 00:01:40 GMT" { + t.Fatalf("missing expires %q", s) + } +} + func TestCookieHttpOnly(t *testing.T) { var c Cookie @@ -175,7 +263,8 @@ func TestCookieParse(t *testing.T) { testCookieParse(t, "foo=", "foo=") testCookieParse(t, `foo="bar"`, "foo=bar") testCookieParse(t, `"foo"=bar`, `"foo"=bar`) - testCookieParse(t, "foo=bar; domain=aaa.com; path=/foo/bar", "foo=bar; domain=aaa.com; path=/foo/bar") + testCookieParse(t, "foo=bar; Domain=aaa.com; PATH=/foo/bar", "foo=bar; domain=aaa.com; path=/foo/bar") + testCookieParse(t, "foo=bar; max-age= 101 ; expires= Tue, 10 Nov 2009 23:00:00 GMT", "foo=bar; max-age=101") testCookieParse(t, " xxx = yyy ; path=/a/b;;;domain=foobar.com ; expires= Tue, 10 Nov 2009 23:00:00 GMT ; ;;", "xxx=yyy; expires=Tue, 10 Nov 2009 23:00:00 GMT; domain=foobar.com; path=/a/b") } @@ -244,8 +333,9 @@ func TestAppendRequestCookieBytes(t *testing.T) { } func testAppendRequestCookieBytes(t *testing.T, s, expectedS string) { - var cookies []argsKV - for _, ss := range strings.Split(s, "&") { + kvs := strings.Split(s, "&") + cookies := make([]argsKV, 0, len(kvs)) + for _, ss := range kvs { tmp := strings.SplitN(ss, "=", 2) if len(tmp) != 2 { t.Fatalf("Cannot find '=' in %q, part of %q", ss, s) diff --git a/vendor/github.com/valyala/fasthttp/doc.go b/vendor/github.com/valyala/fasthttp/doc.go index 501eff6..efcd4a0 100644 --- a/vendor/github.com/valyala/fasthttp/doc.go +++ b/vendor/github.com/valyala/fasthttp/doc.go @@ -7,9 +7,6 @@ Fasthttp provides the following features: concurrent keep-alive connections on modern hardware. * Optimized for low memory usage. * Easy 'Connection: Upgrade' support via RequestCtx.Hijack. - * Server supports requests' pipelining. Multiple requests may be read from - a single network packet and multiple responses may be sent in a single - network packet. This may be useful for highly loaded REST services. * Server provides the following anti-DoS limits: * The number of concurrent connections. diff --git a/vendor/github.com/valyala/fasthttp/examples/multidomain/Makefile b/vendor/github.com/valyala/fasthttp/examples/multidomain/Makefile new file mode 100644 index 0000000..0787e50 --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/examples/multidomain/Makefile @@ -0,0 +1,6 @@ +writer: clean + go get -u github.com/valyala/fasthttp + go build + +clean: + rm -f multidomain diff --git a/vendor/github.com/valyala/fasthttp/examples/multidomain/README.md b/vendor/github.com/valyala/fasthttp/examples/multidomain/README.md new file mode 100644 index 0000000..12c09ec --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/examples/multidomain/README.md @@ -0,0 +1,15 @@ +# Multidomain using SSL certs example + +* Prints two messages depending on visited host. + +# How to build + +``` +make +``` + +# How to run + +``` +./multidomain +``` diff --git a/vendor/github.com/valyala/fasthttp/examples/multidomain/multidomain.go b/vendor/github.com/valyala/fasthttp/examples/multidomain/multidomain.go new file mode 100644 index 0000000..598dda7 --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/examples/multidomain/multidomain.go @@ -0,0 +1,118 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" + + "github.com/valyala/fasthttp" +) + +var domains = make(map[string]fasthttp.RequestHandler) + +func main() { + server := &fasthttp.Server{ + // You can check the access using openssl command: + // $ openssl s_client -connect localhost:8080 << EOF + // > GET / + // > Host: localhost + // > EOF + // + // $ openssl s_client -connect localhost:8080 << EOF + // > GET / + // > Host: 127.0.0.1:8080 + // > EOF + // + Handler: func(ctx *fasthttp.RequestCtx) { + h, ok := domains[string(ctx.Host())] + if !ok { + ctx.NotFound() + return + } + h(ctx) + }, + } + + // preparing first host + cert, priv, err := GenerateCert("localhost:8080") + if err != nil { + panic(err) + } + domains["localhost:8080"] = func(ctx *fasthttp.RequestCtx) { + ctx.Write([]byte("You are accessing to localhost:8080\n")) + } + + err = server.AppendCertEmbed(cert, priv) + if err != nil { + panic(err) + } + + // preparing second host + cert, priv, err = GenerateCert("127.0.0.1") + if err != nil { + panic(err) + } + domains["127.0.0.1:8080"] = func(ctx *fasthttp.RequestCtx) { + ctx.Write([]byte("You are accessing to 127.0.0.1:8080\n")) + } + + err = server.AppendCertEmbed(cert, priv) + if err != nil { + panic(err) + } + + fmt.Println(server.ListenAndServeTLS(":8080", "", "")) +} + +func GenerateCert(host string) ([]byte, []byte, error) { + priv, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, err + } + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, err + } + + cert := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"I have your data"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + SignatureAlgorithm: x509.SHA256WithRSA, + DNSNames: []string{host}, + BasicConstraintsValid: true, + IsCA: true, + } + + certBytes, err := x509.CreateCertificate( + rand.Reader, cert, cert, &priv.PublicKey, priv, + ) + + p := pem.EncodeToMemory( + &pem.Block{ + Type: "PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(priv), + }, + ) + + b := pem.EncodeToMemory( + &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }, + ) + + return b, p, err +} diff --git a/vendor/github.com/valyala/fasthttp/fasthttpproxy/socks5.go b/vendor/github.com/valyala/fasthttp/fasthttpproxy/socks5.go new file mode 100644 index 0000000..77d965d --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/fasthttpproxy/socks5.go @@ -0,0 +1,25 @@ +package fasthttpproxy + +import ( + "net" + + "github.com/valyala/fasthttp" + "golang.org/x/net/proxy" +) + +// FasthttpSocksDialer returns a fasthttp.DialFunc that dials using +// the provided SOCKS5 proxy. +// +// Example usage: +// c := &fasthttp.Client{ +// Dial: fasthttpproxy.FasthttpSocksDialer("localhost:9050"), +// } +func FasthttpSocksDialer(proxyAddr string) fasthttp.DialFunc { + return func(addr string) (net.Conn, error) { + dialer, err := proxy.SOCKS5("tcp", proxyAddr, nil, proxy.Direct) + if err != nil { + return nil, err + } + return dialer.Dial("tcp", addr) + } +} diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go b/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go index d6bcca4..1b1a5f3 100644 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go +++ b/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener.go @@ -8,18 +8,23 @@ import ( // InmemoryListener provides in-memory dialer<->net.Listener implementation. // -// It may be used either for fast in-process client<->server communcations +// It may be used either for fast in-process client<->server communications // without network stack overhead or for client<->server tests. type InmemoryListener struct { lock sync.Mutex closed bool - conns chan net.Conn + conns chan acceptConn +} + +type acceptConn struct { + conn net.Conn + accepted chan struct{} } // NewInmemoryListener returns new in-memory dialer<->net.Listener. func NewInmemoryListener() *InmemoryListener { return &InmemoryListener{ - conns: make(chan net.Conn, 1024), + conns: make(chan acceptConn, 1024), } } @@ -33,7 +38,8 @@ func (ln *InmemoryListener) Accept() (net.Conn, error) { if !ok { return nil, fmt.Errorf("InmemoryListener is already closed: use of closed network connection") } - return c, nil + close(c.accepted) + return c.conn, nil } // Close implements net.Listener's Close. @@ -59,8 +65,9 @@ func (ln *InmemoryListener) Addr() net.Addr { } } -// Dial creates new client<->server connection, enqueues server side -// of the connection to Accept and returns client side of the connection. +// Dial creates new client<->server connection. +// Just like a real Dial it only returns once the server +// has accepted the connection. // // It is safe calling Dial from concurrently running goroutines. func (ln *InmemoryListener) Dial() (net.Conn, error) { @@ -68,8 +75,11 @@ func (ln *InmemoryListener) Dial() (net.Conn, error) { cConn := pc.Conn1() sConn := pc.Conn2() ln.lock.Lock() + accepted := make(chan struct{}) if !ln.closed { - ln.conns <- sConn + ln.conns <- acceptConn{sConn, accepted} + // Wait until the connection has been accepted. + <-accepted } else { sConn.Close() cConn.Close() diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener_timing_test.go b/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener_timing_test.go index 0b7ec0e..2691813 100644 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener_timing_test.go +++ b/vendor/github.com/valyala/fasthttp/fasthttputil/inmemory_listener_timing_test.go @@ -39,7 +39,7 @@ func BenchmarkTLSStreaming(b *testing.B) { // It re-establishes new TLS connection per each http request. func BenchmarkTLSHandshakeRSAWithClientSessionCache(b *testing.B) { bc := &benchConfig{ - IsTLS: true, + IsTLS: true, DisableClientSessionCache: false, } benchmarkExt(b, handshakeHandler, bc) @@ -47,7 +47,7 @@ func BenchmarkTLSHandshakeRSAWithClientSessionCache(b *testing.B) { func BenchmarkTLSHandshakeRSAWithoutClientSessionCache(b *testing.B) { bc := &benchConfig{ - IsTLS: true, + IsTLS: true, DisableClientSessionCache: true, } benchmarkExt(b, handshakeHandler, bc) @@ -55,7 +55,7 @@ func BenchmarkTLSHandshakeRSAWithoutClientSessionCache(b *testing.B) { func BenchmarkTLSHandshakeECDSAWithClientSessionCache(b *testing.B) { bc := &benchConfig{ - IsTLS: true, + IsTLS: true, DisableClientSessionCache: false, UseECDSA: true, } @@ -64,7 +64,7 @@ func BenchmarkTLSHandshakeECDSAWithClientSessionCache(b *testing.B) { func BenchmarkTLSHandshakeECDSAWithoutClientSessionCache(b *testing.B) { bc := &benchConfig{ - IsTLS: true, + IsTLS: true, DisableClientSessionCache: true, UseECDSA: true, } @@ -73,7 +73,7 @@ func BenchmarkTLSHandshakeECDSAWithoutClientSessionCache(b *testing.B) { func BenchmarkTLSHandshakeECDSAWithCurvesWithClientSessionCache(b *testing.B) { bc := &benchConfig{ - IsTLS: true, + IsTLS: true, DisableClientSessionCache: false, UseCurves: true, UseECDSA: true, @@ -83,7 +83,7 @@ func BenchmarkTLSHandshakeECDSAWithCurvesWithClientSessionCache(b *testing.B) { func BenchmarkTLSHandshakeECDSAWithCurvesWithoutClientSessionCache(b *testing.B) { bc := &benchConfig{ - IsTLS: true, + IsTLS: true, DisableClientSessionCache: true, UseCurves: true, UseECDSA: true, diff --git a/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go b/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go index e5a0235..aa92b6f 100644 --- a/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go +++ b/vendor/github.com/valyala/fasthttp/fasthttputil/pipeconns.go @@ -8,7 +8,7 @@ import ( "time" ) -// NewPipeConns returns new bi-directonal connection pipe. +// NewPipeConns returns new bi-directional connection pipe. func NewPipeConns() *PipeConns { ch1 := make(chan *byteBuffer, 4) ch2 := make(chan *byteBuffer, 4) diff --git a/vendor/github.com/valyala/fasthttp/fs.go b/vendor/github.com/valyala/fasthttp/fs.go index 0d83b60..1e9b4ab 100644 --- a/vendor/github.com/valyala/fasthttp/fs.go +++ b/vendor/github.com/valyala/fasthttp/fs.go @@ -17,6 +17,7 @@ import ( "time" "github.com/klauspost/compress/gzip" + "github.com/valyala/bytebufferpool" ) // ServeFileBytesUncompressed returns HTTP response containing file contents @@ -139,12 +140,12 @@ func NewVHostPathRewriter(slashesCount int) PathRewriteFunc { if len(host) == 0 { host = strInvalidHost } - b := AcquireByteBuffer() + b := bytebufferpool.Get() b.B = append(b.B, '/') b.B = append(b.B, host...) b.B = append(b.B, path...) ctx.URI().SetPathBytes(b.B) - ReleaseByteBuffer(b) + bytebufferpool.Put(b) return ctx.Path() } @@ -225,7 +226,7 @@ type FS struct { // It adds CompressedFileSuffix suffix to the original file name and // tries saving the resulting compressed file under the new file name. // So it is advisable to give the server write access to Root - // and to all inner folders in order to minimze CPU usage when serving + // and to all inner folders in order to minimize CPU usage when serving // compressed responses. // // Transparent compression is disabled by default. @@ -241,6 +242,14 @@ type FS struct { // By default request path is not modified. PathRewrite PathRewriteFunc + // PathNotFound fires when file is not found in filesystem + // this functions tries to replace "Cannot open requested path" + // server response giving to the programmer the control of server flow. + // + // By default PathNotFound returns + // "Cannot open requested path" + PathNotFound RequestHandler + // Expiration duration for inactive file handlers. // // FSHandlerCacheDuration is used by default. @@ -343,6 +352,7 @@ func (fs *FS) initRequestHandler() { pathRewrite: fs.PathRewrite, generateIndexPages: fs.GenerateIndexPages, compress: fs.Compress, + pathNotFound: fs.PathNotFound, acceptByteRange: fs.AcceptByteRange, cacheDuration: cacheDuration, compressedFileSuffix: compressedFileSuffix, @@ -365,6 +375,7 @@ type fsHandler struct { root string indexNames []string pathRewrite PathRewriteFunc + pathNotFound RequestHandler generateIndexPages bool compress bool acceptByteRange bool @@ -726,7 +737,12 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) { } } else if err != nil { ctx.Logger().Printf("cannot open file %q: %s", filePath, err) - ctx.Error("Cannot open requested path", StatusNotFound) + if h.pathNotFound == nil { + ctx.Error("Cannot open requested path", StatusNotFound) + } else { + ctx.SetStatusCode(StatusNotFound) + h.pathNotFound(ctx) + } return } @@ -808,7 +824,10 @@ func (h *fsHandler) handleRequest(ctx *RequestCtx) { } } } - ctx.SetContentType(ff.contentType) + hdr.noDefaultContentType = true + if len(hdr.ContentType()) == 0 { + ctx.SetContentType(ff.contentType) + } ctx.SetStatusCode(statusCode) } @@ -897,7 +916,7 @@ var ( ) func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) (*fsFile, error) { - w := &ByteBuffer{} + w := &bytebufferpool.ByteBuffer{} basePathEscaped := html.EscapeString(string(base.Path())) fmt.Fprintf(w, "%s", basePathEscaped) @@ -924,7 +943,7 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) } fm := make(map[string]os.FileInfo, len(fileinfos)) - var filenames []string + filenames := make([]string, 0, len(fileinfos)) for _, fi := range fileinfos { name := fi.Name() if strings.HasSuffix(name, h.compressedFileSuffix) { @@ -939,7 +958,7 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) base.CopyTo(&u) u.Update(string(u.Path()) + "/") - sort.Sort(sort.StringSlice(filenames)) + sort.Strings(filenames) for _, name := range filenames { u.Update(name) pathEscaped := html.EscapeString(string(u.Path())) @@ -957,7 +976,7 @@ func (h *fsHandler) createDirIndex(base *URI, dirPath string, mustCompress bool) fmt.Fprintf(w, "") if mustCompress { - var zbuf ByteBuffer + var zbuf bytebufferpool.ByteBuffer zbuf.B = AppendGzipBytesLevel(zbuf.B, w.B, CompressDefaultCompression) w = &zbuf } diff --git a/vendor/github.com/valyala/fasthttp/fs_test.go b/vendor/github.com/valyala/fasthttp/fs_test.go index 9eda008..cc348b5 100644 --- a/vendor/github.com/valyala/fasthttp/fs_test.go +++ b/vendor/github.com/valyala/fasthttp/fs_test.go @@ -14,6 +14,14 @@ import ( "time" ) +type TestLogger struct { + t *testing.T +} + +func (t TestLogger) Printf(format string, args ...interface{}) { + t.t.Logf(format, args...) +} + func TestNewVHostPathRewriter(t *testing.T) { var ctx RequestCtx var req Request @@ -53,6 +61,43 @@ func TestNewVHostPathRewriterMaliciousHost(t *testing.T) { } } +func testPathNotFound(t *testing.T, pathNotFoundFunc RequestHandler) { + var ctx RequestCtx + var req Request + req.SetRequestURI("http//some.url/file") + ctx.Init(&req, nil, TestLogger{t}) + + fs := &FS{ + Root: "./", + PathNotFound: pathNotFoundFunc, + } + fs.NewRequestHandler()(&ctx) + + if pathNotFoundFunc == nil { + // different to ... + if !bytes.Equal(ctx.Response.Body(), + []byte("Cannot open requested path")) { + t.Fatalf("response defers. Response: %q", ctx.Response.Body()) + } + } else { + // Equals to ... + if bytes.Equal(ctx.Response.Body(), + []byte("Cannot open requested path")) { + t.Fatalf("respones defers. Response: %q", ctx.Response.Body()) + } + } +} + +func TestPathNotFound(t *testing.T) { + testPathNotFound(t, nil) +} + +func TestPathNotFoundFunc(t *testing.T) { + testPathNotFound(t, func(ctx *RequestCtx) { + ctx.WriteString("Not found hehe") + }) +} + func TestServeFileHead(t *testing.T) { var ctx RequestCtx var req Request @@ -127,7 +172,7 @@ func TestServeFileSmallNoReadFrom(t *testing.T) { t.Fatalf("expected %d bytes, got %d bytes", len(teststr), n) } - body := string(buf.Bytes()) + body := buf.String() if body != teststr { t.Fatalf("expected '%s'", teststr) } @@ -489,7 +534,7 @@ func TestFSHandlerSingleThread(t *testing.T) { if err != nil { t.Fatalf("cannot read dirnames in cwd: %s", err) } - sort.Sort(sort.StringSlice(filenames)) + sort.Strings(filenames) for i := 0; i < 3; i++ { fsHandlerTest(t, requestHandler, filenames) @@ -509,7 +554,7 @@ func TestFSHandlerConcurrent(t *testing.T) { if err != nil { t.Fatalf("cannot read dirnames in cwd: %s", err) } - sort.Sort(sort.StringSlice(filenames)) + sort.Strings(filenames) concurrency := 10 ch := make(chan struct{}, concurrency) @@ -637,3 +682,25 @@ func testFileExtension(t *testing.T, path string, compressed bool, compressedFil t.Fatalf("unexpected file extension for file %q: %q. Expecting %q", path, ext, expectedExt) } } + +func TestServeFileContentType(t *testing.T) { + var ctx RequestCtx + var req Request + req.Header.SetMethod("GET") + req.SetRequestURI("http://foobar.com/baz") + ctx.Init(&req, nil, nil) + + ServeFile(&ctx, "testdata/test.png") + + var resp Response + s := ctx.Response.String() + br := bufio.NewReader(bytes.NewBufferString(s)) + if err := resp.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + expected := []byte("image/png") + if !bytes.Equal(resp.Header.ContentType(), expected) { + t.Fatalf("Unexpected Content-Type, expected: %q got %q", expected, resp.Header.ContentType()) + } +} diff --git a/vendor/github.com/valyala/fasthttp/go.mod b/vendor/github.com/valyala/fasthttp/go.mod new file mode 100644 index 0000000..8434ca1 --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/go.mod @@ -0,0 +1,9 @@ +module github.com/valyala/fasthttp + +require ( + github.com/klauspost/compress v1.4.0 + github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e // indirect + github.com/valyala/bytebufferpool v1.0.0 + github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a + golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 +) diff --git a/vendor/github.com/valyala/fasthttp/go.sum b/vendor/github.com/valyala/fasthttp/go.sum new file mode 100644 index 0000000..93f38fc --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/go.sum @@ -0,0 +1,10 @@ +github.com/klauspost/compress v1.4.0 h1:8nsMz3tWa9SWWPL60G1V6CUsf4lLjWLTNEtibhe8gh8= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e h1:+lIPJOWl+jSiJOc70QXJ07+2eg2Jy2EC7Mi11BWujeM= +github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc= +github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3 h1:czFLhve3vsQetD6JOJ8NZZvGQIXlnN3/yXxbT6/awxI= +golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/vendor/github.com/valyala/fasthttp/header.go b/vendor/github.com/valyala/fasthttp/header.go index fdc4851..3176f94 100644 --- a/vendor/github.com/valyala/fasthttp/header.go +++ b/vendor/github.com/valyala/fasthttp/header.go @@ -20,9 +20,10 @@ import ( type ResponseHeader struct { noCopy noCopy - disableNormalizing bool - noHTTP11 bool - connectionClose bool + disableNormalizing bool + noHTTP11 bool + connectionClose bool + noDefaultContentType bool statusCode int contentLength int @@ -50,7 +51,6 @@ type RequestHeader struct { disableNormalizing bool noHTTP11 bool connectionClose bool - isGet bool // These two fields have been moved close to other bool fields // for reducing RequestHeader object size. @@ -72,6 +72,10 @@ type RequestHeader struct { cookies []argsKV rawHeaders []byte + + // stores an immutable copy of headers as they were received from the + // wire. + rawHeadersCopy []byte } // SetContentRange sets 'Content-Range: bytes startPos-endPos/contentLength' @@ -157,12 +161,6 @@ func (h *RequestHeader) ConnectionClose() bool { return h.connectionClose } -func (h *RequestHeader) connectionCloseFast() bool { - // h.parseRawHeaders() isn't called for performance reasons. - // Use ConnectionClose for triggering raw headers parsing. - return h.connectionClose -} - // SetConnectionClose sets 'Connection: close' header. func (h *RequestHeader) SetConnectionClose() { // h.parseRawHeaders() isn't called for performance reasons. @@ -189,6 +187,11 @@ func (h *RequestHeader) ConnectionUpgrade() bool { return hasHeaderValue(h.Peek("Connection"), strUpgrade) } +// PeekCookie is able to returns cookie by a given key from response. +func (h *ResponseHeader) PeekCookie(key string) []byte { + return peekArgStr(h.cookies, key) +} + // ContentLength returns Content-Length header value. // // It may be negative: @@ -218,7 +221,7 @@ func (h *ResponseHeader) SetContentLength(contentLength int) { h.SetConnectionClose() value = strIdentity } - h.h = setArgBytes(h.h, strTransferEncoding, value) + h.h = setArgBytes(h.h, strTransferEncoding, value, argsHasValue) } } @@ -241,9 +244,15 @@ func (h *ResponseHeader) mustSkipContentLength() bool { // It may be negative: // -1 means Transfer-Encoding: chunked. func (h *RequestHeader) ContentLength() int { - if h.noBody() { + if h.ignoreBody() { return 0 } + return h.realContentLength() +} + +// realContentLength returns the actual Content-Length set in the request, +// including positive lengths for GET/HEAD requests. +func (h *RequestHeader) realContentLength() int { h.parseRawHeaders() return h.contentLength } @@ -259,7 +268,7 @@ func (h *RequestHeader) SetContentLength(contentLength int) { h.h = delAllArgsBytes(h.h, strTransferEncoding) } else { h.contentLengthBytes = h.contentLengthBytes[:0] - h.h = setArgBytes(h.h, strTransferEncoding, strChunked) + h.h = setArgBytes(h.h, strTransferEncoding, strChunked, argsHasValue) } } @@ -272,7 +281,7 @@ func (h *ResponseHeader) isCompressibleContentType() bool { // ContentType returns Content-Type header value. func (h *ResponseHeader) ContentType() []byte { contentType := h.contentType - if len(h.contentType) == 0 { + if !h.noDefaultContentType && len(h.contentType) == 0 { contentType = defaultContentType } return contentType @@ -504,14 +513,10 @@ func (h *RequestHeader) SetRequestURIBytes(requestURI []byte) { // IsGet returns true if request method is GET. func (h *RequestHeader) IsGet() bool { - // Optimize fast path for GET requests. - if !h.isGet { - h.isGet = bytes.Equal(h.Method(), strGet) - } - return h.isGet + return bytes.Equal(h.Method(), strGet) } -// IsPost returns true if request methos is POST. +// IsPost returns true if request method is POST. func (h *RequestHeader) IsPost() bool { return bytes.Equal(h.Method(), strPost) } @@ -523,10 +528,6 @@ func (h *RequestHeader) IsPut() bool { // IsHead returns true if request method is HEAD. func (h *RequestHeader) IsHead() bool { - // Fast path - if h.isGet { - return false - } return bytes.Equal(h.Method(), strHead) } @@ -535,6 +536,26 @@ func (h *RequestHeader) IsDelete() bool { return bytes.Equal(h.Method(), strDelete) } +// IsConnect returns true if request method is CONNECT. +func (h *RequestHeader) IsConnect() bool { + return bytes.Equal(h.Method(), strConnect) +} + +// IsOptions returns true if request method is OPTIONS. +func (h *RequestHeader) IsOptions() bool { + return bytes.Equal(h.Method(), strOptions) +} + +// IsTrace returns true if request method is TRACE. +func (h *RequestHeader) IsTrace() bool { + return bytes.Equal(h.Method(), strTrace) +} + +// IsPatch returns true if request method is PATCH. +func (h *RequestHeader) IsPatch() bool { + return bytes.Equal(h.Method(), strPatch) +} + // IsHTTP11 returns true if the request is HTTP/1.1. func (h *RequestHeader) IsHTTP11() bool { return !h.noHTTP11 @@ -621,6 +642,7 @@ func (h *ResponseHeader) DisableNormalizing() { // Reset clears response header. func (h *ResponseHeader) Reset() { h.disableNormalizing = false + h.noDefaultContentType = false h.resetSkipNormalize() } @@ -648,7 +670,6 @@ func (h *RequestHeader) Reset() { func (h *RequestHeader) resetSkipNormalize() { h.noHTTP11 = false h.connectionClose = false - h.isGet = false h.contentLength = 0 h.contentLengthBytes = h.contentLengthBytes[:0] @@ -674,6 +695,7 @@ func (h *ResponseHeader) CopyTo(dst *ResponseHeader) { dst.disableNormalizing = h.disableNormalizing dst.noHTTP11 = h.noHTTP11 dst.connectionClose = h.connectionClose + dst.noDefaultContentType = h.noDefaultContentType dst.statusCode = h.statusCode dst.contentLength = h.contentLength @@ -691,7 +713,6 @@ func (h *RequestHeader) CopyTo(dst *RequestHeader) { dst.disableNormalizing = h.disableNormalizing dst.noHTTP11 = h.noHTTP11 dst.connectionClose = h.connectionClose - dst.isGet = h.isGet dst.contentLength = h.contentLength dst.contentLengthBytes = append(dst.contentLengthBytes[:0], h.contentLengthBytes...) @@ -758,6 +779,8 @@ func (h *RequestHeader) VisitAllCookie(f func(key, value []byte)) { // // f must not retain references to key and/or value after returning. // Copy key and/or value contents before returning if you need retaining them. +// +// To get the headers in order they were received use VisitAllInOrder. func (h *RequestHeader) VisitAll(f func(key, value []byte)) { h.parseRawHeaders() host := h.Host() @@ -787,6 +810,25 @@ func (h *RequestHeader) VisitAll(f func(key, value []byte)) { } } +// VisitAllInOrder calls f for each header in the order they were received. +// +// f must not retain references to key and/or value after returning. +// Copy key and/or value contents before returning if you need retaining them. +// +// This function is slightly slower than VisitAll because it has to reparse the +// raw headers to get the order. +func (h *RequestHeader) VisitAllInOrder(f func(key, value []byte)) { + h.parseRawHeaders() + var s headerScanner + s.b = h.rawHeaders + s.disableNormalizing = h.disableNormalizing + for s.next() { + if len(s.key) > 0 { + f(s.key, s.value) + } + } +} + // Del deletes header with the given key. func (h *ResponseHeader) Del(key string) { k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) @@ -857,7 +899,7 @@ func (h *RequestHeader) del(key []byte) { // Use Set for setting a single header for the given key. func (h *ResponseHeader) Add(key, value string) { k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value) + h.h = appendArg(h.h, b2s(k), value, argsHasValue) } // AddBytesK adds the given 'key: value' header. @@ -940,14 +982,14 @@ func (h *ResponseHeader) SetCanonical(key, value []byte) { h.SetConnectionClose() } else { h.ResetConnectionClose() - h.h = setArgBytes(h.h, key, value) + h.h = setArgBytes(h.h, key, value, argsHasValue) } case "Transfer-Encoding": // Transfer-Encoding is managed automatically. case "Date": // Date is managed automatically. default: - h.h = setArgBytes(h.h, key, value) + h.h = setArgBytes(h.h, key, value, argsHasValue) } } @@ -955,14 +997,14 @@ func (h *ResponseHeader) SetCanonical(key, value []byte) { // // It is save re-using the cookie after the function returns. func (h *ResponseHeader) SetCookie(cookie *Cookie) { - h.cookies = setArgBytes(h.cookies, cookie.Key(), cookie.Cookie()) + h.cookies = setArgBytes(h.cookies, cookie.Key(), cookie.Cookie(), argsHasValue) } // SetCookie sets 'key: value' cookies. func (h *RequestHeader) SetCookie(key, value string) { h.parseRawHeaders() h.collectCookies() - h.cookies = setArg(h.cookies, key, value) + h.cookies = setArg(h.cookies, key, value, argsHasValue) } // SetCookieBytesK sets 'key: value' cookies. @@ -1041,7 +1083,7 @@ func (h *RequestHeader) DelAllCookies() { // Use Set for setting a single header for the given key. func (h *RequestHeader) Add(key, value string) { k := getHeaderKeyBytes(&h.bufKV, key, h.disableNormalizing) - h.h = appendArg(h.h, b2s(k), value) + h.h = appendArg(h.h, b2s(k), value, argsHasValue) } // AddBytesK adds the given 'key: value' header. @@ -1125,12 +1167,12 @@ func (h *RequestHeader) SetCanonical(key, value []byte) { h.SetConnectionClose() } else { h.ResetConnectionClose() - h.h = setArgBytes(h.h, key, value) + h.h = setArgBytes(h.h, key, value, argsHasValue) } case "Transfer-Encoding": // Transfer-Encoding is managed automatically. default: - h.h = setArgBytes(h.h, key, value) + h.h = setArgBytes(h.h, key, value, argsHasValue) } } @@ -1185,6 +1227,8 @@ func (h *ResponseHeader) peek(key []byte) []byte { return peekArgBytes(h.h, key) case "Content-Length": return h.contentLengthBytes + case "Set-Cookie": + return appendResponseCookieBytes(nil, h.cookies) default: return peekArgBytes(h.h, key) } @@ -1206,6 +1250,12 @@ func (h *RequestHeader) peek(key []byte) []byte { return peekArgBytes(h.h, key) case "Content-Length": return h.contentLengthBytes + case "Cookie": + if h.cookiesCollected { + return appendRequestCookieBytes(nil, h.cookies) + } else { + return peekArgBytes(h.h, key) + } default: return peekArgBytes(h.h, key) } @@ -1330,9 +1380,12 @@ func (h *RequestHeader) tryRead(r *bufio.Reader, n int) error { h.resetSkipNormalize() b, err := r.Peek(n) if len(b) == 0 { - // treat all errors on the first byte read as EOF - if n == 1 || err == io.EOF { - return io.EOF + if err == io.EOF { + return err + } + + if err == nil { + panic("bufio.Reader.Peek() returned nil, nil") } // This is for go 1.6 bug. See https://github.com/golang/go/issues/14121 . @@ -1342,6 +1395,11 @@ func (h *RequestHeader) tryRead(r *bufio.Reader, n int) error { } } + if n == 1 { + // We didn't read a single byte. + return errNothingRead + } + return fmt.Errorf("error when reading request headers: %s", err) } b = mustPeekBuffered(r) @@ -1431,10 +1489,9 @@ func (h *ResponseHeader) AppendBytes(dst []byte) []byte { dst = append(dst, statusLine(statusCode)...) server := h.Server() - if len(server) == 0 { - server = defaultServerName + if len(server) != 0 { + dst = appendHeaderLine(dst, strServer, server) } - dst = appendHeaderLine(dst, strServer, server) dst = appendHeaderLine(dst, strDate, serverDate.Load().([]byte)) // Append Content-Type only for non-zero responses @@ -1492,6 +1549,20 @@ func (h *RequestHeader) Header() []byte { return h.bufKV.value } +// RawHeaders returns raw header key/value bytes. +// +// Depending on server configuration, header keys may be normalized to +// capital-case in place. +// +// This copy is set aside during parsing, so empty slice is returned for all +// cases where parsing did not happen. Similarly, request line is not stored +// during parsing and can not be returned. +// +// The slice is not safe to use after the handler returns. +func (h *RequestHeader) RawHeaders() []byte { + return h.rawHeadersCopy +} + // String returns request header representation. func (h *RequestHeader) String() string { return string(h.Header()) @@ -1513,10 +1584,9 @@ func (h *RequestHeader) AppendBytes(dst []byte) []byte { } userAgent := h.UserAgent() - if len(userAgent) == 0 { - userAgent = defaultUserAgent + if len(userAgent) > 0 { + dst = appendHeaderLine(dst, strUserAgent, userAgent) } - dst = appendHeaderLine(dst, strUserAgent, userAgent) host := h.Host() if len(host) > 0 { @@ -1524,7 +1594,7 @@ func (h *RequestHeader) AppendBytes(dst []byte) []byte { } contentType := h.ContentType() - if !h.noBody() { + if !h.ignoreBody() { if len(contentType) == 0 { contentType = strPostArgsContentType } @@ -1578,7 +1648,7 @@ func (h *ResponseHeader) parse(buf []byte) (int, error) { return m + n, nil } -func (h *RequestHeader) noBody() bool { +func (h *RequestHeader) ignoreBody() bool { return h.IsGet() || h.IsHead() } @@ -1589,18 +1659,20 @@ func (h *RequestHeader) parse(buf []byte) (int, error) { } var n int - if !h.noBody() || h.noHTTP11 { + var rawHeaders []byte + rawHeaders, n, err = readRawHeaders(h.rawHeaders[:0], buf[m:]) + if err != nil { + return 0, err + } + h.rawHeadersCopy = append(h.rawHeadersCopy[:0], rawHeaders...) + if !h.ignoreBody() || h.noHTTP11 { n, err = h.parseHeaders(buf[m:]) if err != nil { return 0, err } + h.rawHeaders = append(h.rawHeaders[:0], buf[m:m+n]...) h.rawHeadersParsed = true } else { - var rawHeaders []byte - rawHeaders, n, err = readRawHeaders(h.rawHeaders[:0], buf[m:]) - if err != nil { - return 0, err - } h.rawHeaders = rawHeaders } return m + n, nil @@ -1738,37 +1810,53 @@ func (h *ResponseHeader) parseHeaders(buf []byte) (int, error) { var err error var kv *argsKV for s.next() { - switch string(s.key) { - case "Content-Type": - h.contentType = append(h.contentType[:0], s.value...) - case "Server": - h.server = append(h.server[:0], s.value...) - case "Content-Length": - if h.contentLength != -1 { - if h.contentLength, err = parseContentLength(s.value); err != nil { - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) + if len(s.key) > 0 { + switch s.key[0] | 0x20 { + case 'c': + if caseInsensitiveCompare(s.key, strContentType) { + h.contentType = append(h.contentType[:0], s.value...) + continue + } + if caseInsensitiveCompare(s.key, strContentLength) { + if h.contentLength != -1 { + if h.contentLength, err = parseContentLength(s.value); err != nil { + h.contentLength = -2 + } else { + h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) + } + } + continue + } + if caseInsensitiveCompare(s.key, strConnection) { + if bytes.Equal(s.value, strClose) { + h.connectionClose = true + } else { + h.connectionClose = false + h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) + } + continue + } + case 's': + if caseInsensitiveCompare(s.key, strServer) { + h.server = append(h.server[:0], s.value...) + continue + } + if caseInsensitiveCompare(s.key, strSetCookie) { + h.cookies, kv = allocArg(h.cookies) + kv.key = getCookieKey(kv.key, s.value) + kv.value = append(kv.value[:0], s.value...) + continue + } + case 't': + if caseInsensitiveCompare(s.key, strTransferEncoding) { + if !bytes.Equal(s.value, strIdentity) { + h.contentLength = -1 + h.h = setArgBytes(h.h, strTransferEncoding, strChunked, argsHasValue) + } + continue } } - case "Transfer-Encoding": - if !bytes.Equal(s.value, strIdentity) { - h.contentLength = -1 - h.h = setArgBytes(h.h, strTransferEncoding, strChunked) - } - case "Set-Cookie": - h.cookies, kv = allocArg(h.cookies) - kv.key = getCookieKey(kv.key, s.value) - kv.value = append(kv.value[:0], s.value...) - case "Connection": - if bytes.Equal(s.value, strClose) { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArgBytes(h.h, s.key, s.value) - } - default: - h.h = appendArgBytes(h.h, s.key, s.value) + h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) } } if s.err != nil { @@ -1780,7 +1868,7 @@ func (h *ResponseHeader) parseHeaders(buf []byte) (int, error) { h.contentLengthBytes = h.contentLengthBytes[:0] } if h.contentLength == -2 && !h.ConnectionUpgrade() && !h.mustSkipContentLength() { - h.h = setArgBytes(h.h, strTransferEncoding, strIdentity) + h.h = setArgBytes(h.h, strTransferEncoding, strIdentity, argsHasValue) h.connectionClose = true } if h.noHTTP11 && !h.connectionClose { @@ -1800,36 +1888,53 @@ func (h *RequestHeader) parseHeaders(buf []byte) (int, error) { s.disableNormalizing = h.disableNormalizing var err error for s.next() { - switch string(s.key) { - case "Host": - h.host = append(h.host[:0], s.value...) - case "User-Agent": - h.userAgent = append(h.userAgent[:0], s.value...) - case "Content-Type": - h.contentType = append(h.contentType[:0], s.value...) - case "Content-Length": - if h.contentLength != -1 { - if h.contentLength, err = parseContentLength(s.value); err != nil { - h.contentLength = -2 - } else { - h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) + if len(s.key) > 0 { + switch s.key[0] | 0x20 { + case 'h': + if caseInsensitiveCompare(s.key, strHost) { + h.host = append(h.host[:0], s.value...) + continue + } + case 'u': + if caseInsensitiveCompare(s.key, strUserAgent) { + h.userAgent = append(h.userAgent[:0], s.value...) + continue + } + case 'c': + if caseInsensitiveCompare(s.key, strContentType) { + h.contentType = append(h.contentType[:0], s.value...) + continue + } + if caseInsensitiveCompare(s.key, strContentLength) { + if h.contentLength != -1 { + if h.contentLength, err = parseContentLength(s.value); err != nil { + h.contentLength = -2 + } else { + h.contentLengthBytes = append(h.contentLengthBytes[:0], s.value...) + } + } + continue + } + if caseInsensitiveCompare(s.key, strConnection) { + if bytes.Equal(s.value, strClose) { + h.connectionClose = true + } else { + h.connectionClose = false + h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) + } + continue + } + case 't': + if caseInsensitiveCompare(s.key, strTransferEncoding) { + if !bytes.Equal(s.value, strIdentity) { + h.contentLength = -1 + h.h = setArgBytes(h.h, strTransferEncoding, strChunked, argsHasValue) + } + continue } } - case "Transfer-Encoding": - if !bytes.Equal(s.value, strIdentity) { - h.contentLength = -1 - h.h = setArgBytes(h.h, strTransferEncoding, strChunked) - } - case "Connection": - if bytes.Equal(s.value, strClose) { - h.connectionClose = true - } else { - h.connectionClose = false - h.h = appendArgBytes(h.h, s.key, s.value) - } - default: - h.h = appendArgBytes(h.h, s.key, s.value) } + h.h = appendArgBytes(h.h, s.key, s.value, argsHasValue) } if s.err != nil { h.connectionClose = true @@ -1839,17 +1944,12 @@ func (h *RequestHeader) parseHeaders(buf []byte) (int, error) { if h.contentLength < 0 { h.contentLengthBytes = h.contentLengthBytes[:0] } - if h.noBody() { - h.contentLength = 0 - h.contentLengthBytes = h.contentLengthBytes[:0] - } if h.noHTTP11 && !h.connectionClose { // close connection for non-http/1.1 request unless 'Connection: keep-alive' is set. v := peekArgBytes(h.h, strConnection) h.connectionClose = !hasHeaderValue(v, strKeepAlive) && !hasHeaderValue(v, strKeepAliveCamelCase) } - - return len(buf) - len(s.b), nil + return s.hLen, nil } func (h *RequestHeader) parseRawHeaders() { @@ -1900,6 +2000,9 @@ type headerScanner struct { value []byte err error + // hLen stores header subslice len + hLen int + disableNormalizing bool } @@ -1907,10 +2010,12 @@ func (s *headerScanner) next() bool { bLen := len(s.b) if bLen >= 2 && s.b[0] == '\r' && s.b[1] == '\n' { s.b = s.b[2:] + s.hLen += 2 return false } if bLen >= 1 && s.b[0] == '\n' { s.b = s.b[1:] + s.hLen++ return false } n := bytes.IndexByte(s.b, ':') @@ -1924,6 +2029,7 @@ func (s *headerScanner) next() bool { for len(s.b) > n && s.b[n] == ' ' { n++ } + s.hLen += n s.b = s.b[n:] n = bytes.IndexByte(s.b, '\n') if n < 0 { @@ -1931,6 +2037,7 @@ func (s *headerScanner) next() bool { return false } s.value = s.b[:n] + s.hLen += n + 1 s.b = s.b[n+1:] if n > 0 && s.value[n-1] == '\r' { @@ -1978,7 +2085,7 @@ func hasHeaderValue(s, value []byte) bool { var vs headerValueScanner vs.b = s for vs.next() { - if bytes.Equal(vs.value, value) { + if caseInsensitiveCompare(vs.value, value) { return true } } @@ -2065,6 +2172,7 @@ func AppendNormalizedHeaderKeyBytes(dst, key []byte) []byte { var ( errNeedMore = errors.New("need more data: cannot find trailing lf") errSmallBuffer = errors.New("small read buffer. Increase ReadBufferSize") + errNothingRead = errors.New("read timeout with nothing read") ) // ErrSmallBuffer is returned when the provided buffer size is too small diff --git a/vendor/github.com/valyala/fasthttp/header_test.go b/vendor/github.com/valyala/fasthttp/header_test.go index 112faa8..e0f567f 100644 --- a/vendor/github.com/valyala/fasthttp/header_test.go +++ b/vendor/github.com/valyala/fasthttp/header_test.go @@ -110,6 +110,97 @@ func TestRequestHeaderEmptyValueFromString(t *testing.T) { } } +func TestRequestRawHeaders(t *testing.T) { + kvs := "host: foobar\r\n" + + "value: b\r\n" + + "\r\n" + t.Run("normalized", func(t *testing.T) { + s := "GET / HTTP/1.1\r\n" + kvs + exp := kvs + var h RequestHeader + br := bufio.NewReader(bytes.NewBufferString(s)) + if err := h.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if string(h.Host()) != "foobar" { + t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") + } + v2 := h.Peek("Value") + if !bytes.Equal(v2, []byte{'b'}) { + t.Fatalf("expecting non empty value. Got %q", v2) + } + if raw := h.RawHeaders(); string(raw) != exp { + t.Fatalf("expected header %q, got %q", exp, raw) + } + }) + for _, n := range []int{0, 1, 4, 8} { + t.Run(fmt.Sprintf("post-%dk", n), func(t *testing.T) { + l := 1024 * n + body := make([]byte, l) + for i := range body { + body[i] = 'a' + } + cl := fmt.Sprintf("Content-Length: %d\r\n", l) + s := "POST / HTTP/1.1\r\n" + cl + kvs + string(body) + exp := cl + kvs + var h RequestHeader + br := bufio.NewReader(bytes.NewBufferString(s)) + if err := h.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if string(h.Host()) != "foobar" { + t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") + } + v2 := h.Peek("Value") + if !bytes.Equal(v2, []byte{'b'}) { + t.Fatalf("expecting non empty value. Got %q", v2) + } + if raw := h.RawHeaders(); string(raw) != exp { + t.Fatalf("expected header %q, got %q", exp, raw) + } + }) + } + t.Run("http10", func(t *testing.T) { + s := "GET / HTTP/1.0\r\n" + kvs + exp := kvs + var h RequestHeader + br := bufio.NewReader(bytes.NewBufferString(s)) + if err := h.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if string(h.Host()) != "foobar" { + t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "foobar") + } + v2 := h.Peek("Value") + if !bytes.Equal(v2, []byte{'b'}) { + t.Fatalf("expecting non empty value. Got %q", v2) + } + if raw := h.RawHeaders(); string(raw) != exp { + t.Fatalf("expected header %q, got %q", exp, raw) + } + }) + t.Run("no-kvs", func(t *testing.T) { + s := "GET / HTTP/1.1\r\n\r\n" + exp := "" + var h RequestHeader + h.DisableNormalizing() + br := bufio.NewReader(bytes.NewBufferString(s)) + if err := h.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if string(h.Host()) != "" { + t.Fatalf("unexpected host: %q. Expecting %q", h.Host(), "") + } + v1 := h.Peek("NoKey") + if len(v1) > 0 { + t.Fatalf("expecting empty value. Got %q", v1) + } + if raw := h.RawHeaders(); string(raw) != exp { + t.Fatalf("expected header %q, got %q", exp, raw) + } + }) +} + func TestRequestHeaderSetCookieWithSpecialChars(t *testing.T) { var h RequestHeader h.Set("Cookie", "ID&14") @@ -431,9 +522,6 @@ func TestRequestHeaderHTTP10ConnectionClose(t *testing.T) { t.Fatalf("unexpected error: %s", err) } - if !h.connectionCloseFast() { - t.Fatalf("expecting 'Connection: close' request header") - } if !h.ConnectionClose() { t.Fatalf("expecting 'Connection: close' request header") } @@ -896,20 +984,6 @@ func TestResponseHeaderFirstByteReadEOF(t *testing.T) { } } -func TestRequestHeaderFirstByteReadEOF(t *testing.T) { - var h RequestHeader - - r := &errorReader{fmt.Errorf("non-eof error")} - br := bufio.NewReader(r) - err := h.Read(br) - if err == nil { - t.Fatalf("expecting error") - } - if err != io.EOF { - t.Fatalf("unexpected error %s. Expecting %s", err, io.EOF) - } -} - type errorReader struct { err error } @@ -924,15 +998,6 @@ func TestRequestHeaderEmptyMethod(t *testing.T) { if !h.IsGet() { t.Fatalf("empty method must be equivalent to GET") } - if h.IsPost() { - t.Fatalf("empty method cannot be POST") - } - if h.IsHead() { - t.Fatalf("empty method cannot be HEAD") - } - if h.IsDelete() { - t.Fatalf("empty method cannot be DELETE") - } } func TestResponseHeaderHTTPVer(t *testing.T) { @@ -1212,6 +1277,47 @@ func TestRequestHeaderVisitAll(t *testing.T) { } } +func TestResponseHeaderVisitAllInOrder(t *testing.T) { + var h RequestHeader + + r := bytes.NewBufferString("GET / HTTP/1.1\r\nContent-Type: aa\r\nCookie: a=b\r\nHost: example.com\r\nUser-Agent: xxx\r\n\r\n") + br := bufio.NewReader(r) + if err := h.Read(br); err != nil { + t.Fatalf("Unepxected error: %s", err) + } + + if h.Len() != 4 { + t.Fatalf("Unexpected number of headers: %d. Expected 4", h.Len()) + } + + order := []string{ + "Content-Type", + "Cookie", + "Host", + "User-Agent", + } + values := []string{ + "aa", + "a=b", + "example.com", + "xxx", + } + + h.VisitAllInOrder(func(key, value []byte) { + if len(order) == 0 { + t.Fatalf("no more headers expected, got %q", key) + } + if order[0] != string(key) { + t.Fatalf("expected header %q got %q", order[0], key) + } + if values[0] != string(value) { + t.Fatalf("expected header value %q got %q", values[0], value) + } + order = order[1:] + values = values[1:] + }) +} + func TestResponseHeaderCookie(t *testing.T) { var h ResponseHeader var c Cookie @@ -1393,6 +1499,90 @@ func TestRequestHeaderCookie(t *testing.T) { } } +func TestResponseHeaderCookieIssue4(t *testing.T) { + var h ResponseHeader + + c := AcquireCookie() + c.SetKey("foo") + c.SetValue("bar") + h.SetCookie(c) + + if string(h.Peek("Set-Cookie")) != "foo=bar" { + t.Fatalf("Unexpected Set-Cookie header %q. Expected %q", h.Peek("Set-Cookie"), "foo=bar") + } + cookieSeen := false + h.VisitAll(func(key, value []byte) { + switch string(key) { + case "Set-Cookie": + cookieSeen = true + } + }) + if !cookieSeen { + t.Fatalf("Set-Cookie not present in VisitAll") + } + + c = AcquireCookie() + c.SetKey("foo") + h.Cookie(c) + if string(c.Value()) != "bar" { + t.Fatalf("Unexpected cookie value %q. Exepcted %q", c.Value(), "bar") + } + + if string(h.Peek("Set-Cookie")) != "foo=bar" { + t.Fatalf("Unexpected Set-Cookie header %q. Expected %q", h.Peek("Set-Cookie"), "foo=bar") + } + cookieSeen = false + h.VisitAll(func(key, value []byte) { + switch string(key) { + case "Set-Cookie": + cookieSeen = true + } + }) + if !cookieSeen { + t.Fatalf("Set-Cookie not present in VisitAll") + } +} + +func TestRequestHeaderCookieIssue313(t *testing.T) { + var h RequestHeader + h.SetRequestURI("/") + h.Set("Host", "foobar.com") + + h.SetCookie("foo", "bar") + + if string(h.Peek("Cookie")) != "foo=bar" { + t.Fatalf("Unexpected Cookie header %q. Expected %q", h.Peek("Cookie"), "foo=bar") + } + cookieSeen := false + h.VisitAll(func(key, value []byte) { + switch string(key) { + case "Cookie": + cookieSeen = true + } + }) + if !cookieSeen { + t.Fatalf("Cookie not present in VisitAll") + } + + if string(h.Cookie("foo")) != "bar" { + t.Fatalf("Unexpected cookie value %q. Exepcted %q", h.Cookie("foo"), "bar") + } + + if string(h.Peek("Cookie")) != "foo=bar" { + t.Fatalf("Unexpected Cookie header %q. Expected %q", h.Peek("Cookie"), "foo=bar") + } + cookieSeen = false + h.VisitAll(func(key, value []byte) { + switch string(key) { + case "Cookie": + cookieSeen = true + } + }) + if !cookieSeen { + t.Fatalf("Cookie not present in VisitAll") + } +} + func TestRequestHeaderMethod(t *testing.T) { // common http methods testRequestHeaderMethod(t, "GET") @@ -1793,6 +1983,12 @@ func TestResponseHeaderReadSuccess(t *testing.T) { testResponseHeaderReadSuccess(t, h, "HTTP/1.1 400 OK\r\nContent-Length: 123\r\n\r\nfoiaaa", 400, 123, string(defaultContentType), "foiaaa") + // no content-type and no default + h.noDefaultContentType = true + testResponseHeaderReadSuccess(t, h, "HTTP/1.1 400 OK\r\nContent-Length: 123\r\n\r\nfoiaaa", + 400, 123, "", "foiaaa") + h.noDefaultContentType = false + // no headers testResponseHeaderReadSuccess(t, h, "HTTP/1.1 200 OK\r\n\r\naaaabbb", 200, -2, string(defaultContentType), "aaaabbb") diff --git a/vendor/github.com/valyala/fasthttp/header_timing_test.go b/vendor/github.com/valyala/fasthttp/header_timing_test.go index a6f7a11..55f5df9 100644 --- a/vendor/github.com/valyala/fasthttp/header_timing_test.go +++ b/vendor/github.com/valyala/fasthttp/header_timing_test.go @@ -5,6 +5,8 @@ import ( "bytes" "io" "testing" + + "github.com/valyala/bytebufferpool" ) var strFoobar = []byte("foobar.com") @@ -65,7 +67,7 @@ func BenchmarkRequestHeaderWrite(b *testing.B) { h.SetHost("foobar.com") h.SetUserAgent("aaa.bbb") h.SetReferer("http://google.com/aaa/bbb") - var w ByteBuffer + var w bytebufferpool.ByteBuffer for pb.Next() { if _, err := h.WriteTo(&w); err != nil { b.Fatalf("unexpected error when writing header: %s", err) @@ -83,7 +85,7 @@ func BenchmarkResponseHeaderWrite(b *testing.B) { h.SetContentLength(1256) h.SetServer("aaa 1/2.3") h.Set("Test", "1.2.3") - var w ByteBuffer + var w bytebufferpool.ByteBuffer for pb.Next() { if _, err := h.WriteTo(&w); err != nil { b.Fatalf("unexpected error when writing header: %s", err) diff --git a/vendor/github.com/valyala/fasthttp/http.go b/vendor/github.com/valyala/fasthttp/http.go index 5fc3bee..1458602 100644 --- a/vendor/github.com/valyala/fasthttp/http.go +++ b/vendor/github.com/valyala/fasthttp/http.go @@ -44,6 +44,9 @@ type Request struct { keepBodyBuffer bool isTLS bool + + // To detect scheme changes in redirects + schemaUpdate bool } // Response represents HTTP response. @@ -346,7 +349,7 @@ func (resp *Response) BodyGunzip() ([]byte, error) { } func gunzipData(p []byte) ([]byte, error) { - var bb ByteBuffer + var bb bytebufferpool.ByteBuffer _, err := WriteGunzip(&bb, p) if err != nil { return nil, err @@ -373,7 +376,7 @@ func (resp *Response) BodyInflate() ([]byte, error) { } func inflateData(p []byte) ([]byte, error) { - var bb ByteBuffer + var bb bytebufferpool.ByteBuffer _, err := WriteInflate(&bb, p) if err != nil { return nil, err @@ -711,7 +714,7 @@ func (req *Request) MultipartForm() (*multipart.Form, error) { } func marshalMultipartForm(f *multipart.Form, boundary string) ([]byte, error) { - var buf ByteBuffer + var buf bytebufferpool.ByteBuffer if err := WriteMultipartForm(&buf, f, boundary); err != nil { return nil, err } @@ -722,7 +725,7 @@ func marshalMultipartForm(f *multipart.Form, boundary string) ([]byte, error) { // boundary to w. func WriteMultipartForm(w io.Writer, f *multipart.Form, boundary string) error { // Do not care about memory allocations here, since multipart - // form processing is slooow. + // form processing is slow. if len(boundary) == 0 { panic("BUG: form boundary cannot be empty") } @@ -744,7 +747,7 @@ func WriteMultipartForm(w io.Writer, f *multipart.Form, boundary string) error { // marshal files for k, fvv := range f.File { for _, fv := range fvv { - vw, err := mw.CreateFormFile(k, fv.Filename) + vw, err := mw.CreatePart(fv.Header) if err != nil { return fmt.Errorf("cannot create form file %q (%q): %s", k, fv.Filename, err) } @@ -844,7 +847,9 @@ func (req *Request) Read(r *bufio.Reader) error { const defaultMaxInMemoryFileSize = 16 * 1024 * 1024 -var errGetOnly = errors.New("non-GET request received") +// ErrGetOnly is returned when server expects only GET requests, +// but some other type of request came (Server.GetOnly option is true). +var ErrGetOnly = errors.New("non-GET request received") // ReadLimitBody reads request from the given r, limiting the body size. // @@ -878,11 +883,7 @@ func (req *Request) readLimitBody(r *bufio.Reader, maxBodySize int, getOnly bool return err } if getOnly && !req.Header.IsGet() { - return errGetOnly - } - - if req.Header.noBody() { - return nil + return ErrGetOnly } if req.MayContinue() { @@ -918,7 +919,7 @@ func (req *Request) MayContinue() bool { // then ErrBodyTooLarge is returned. func (req *Request) ContinueReadBody(r *bufio.Reader, maxBodySize int) error { var err error - contentLength := req.Header.ContentLength() + contentLength := req.Header.realContentLength() if contentLength > 0 { if maxBodySize > 0 && contentLength > maxBodySize { return ErrBodyTooLarge @@ -988,7 +989,6 @@ func (resp *Response) ReadLimitBody(r *bufio.Reader, maxBodySize int) error { bodyBuf.Reset() bodyBuf.B, err = readBody(r, resp.Header.ContentLength(), maxBodySize, bodyBuf.B) if err != nil { - resp.Reset() return err } resp.Header.SetContentLength(len(bodyBuf.B)) @@ -1109,8 +1109,11 @@ func (req *Request) Write(w *bufio.Writer) error { req.Header.SetMultipartFormBoundary(req.multipartFormBoundary) } - hasBody := !req.Header.noBody() + hasBody := !req.Header.ignoreBody() if hasBody { + if len(body) == 0 { + body = req.postArgs.QueryString() + } req.Header.SetContentLength(len(body)) } if err = req.Header.Write(w); err != nil { @@ -1458,7 +1461,7 @@ func (resp *Response) String() string { } func getHTTPString(hw httpWriter) string { - w := AcquireByteBuffer() + w := bytebufferpool.Get() bw := bufio.NewWriter(w) if err := hw.Write(bw); err != nil { return err.Error() @@ -1467,7 +1470,7 @@ func getHTTPString(hw httpWriter) string { return err.Error() } s := string(w.B) - ReleaseByteBuffer(w) + bytebufferpool.Put(w) return s } @@ -1650,6 +1653,11 @@ func appendBodyFixedSize(r *bufio.Reader, dst []byte, n int) ([]byte, error) { } } +// ErrBrokenChunk is returned when server receives a broken chunked body (Transfer-Encoding: chunked). +type ErrBrokenChunk struct { + error +} + func readBodyChunked(r *bufio.Reader, maxBodySize int, dst []byte) ([]byte, error) { if len(dst) > 0 { panic("BUG: expected zero-length buffer") @@ -1669,7 +1677,9 @@ func readBodyChunked(r *bufio.Reader, maxBodySize int, dst []byte) ([]byte, erro return dst, err } if !bytes.Equal(dst[len(dst)-strCRLFLen:], strCRLF) { - return dst, fmt.Errorf("cannot find crlf at the end of chunk") + return dst, ErrBrokenChunk{ + error: fmt.Errorf("cannot find crlf at the end of chunk"), + } } dst = dst[:len(dst)-strCRLFLen] if chunkSize == 0 { @@ -1683,19 +1693,34 @@ func parseChunkSize(r *bufio.Reader) (int, error) { if err != nil { return -1, err } + for { + c, err := r.ReadByte() + if err != nil { + return -1, ErrBrokenChunk{ + error: fmt.Errorf("cannot read '\r' char at the end of chunk size: %s", err), + } + } + // Skip any trailing whitespace after chunk size. + if c == ' ' { + continue + } + if c != '\r' { + return -1, ErrBrokenChunk{ + error: fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\r'), + } + } + break + } c, err := r.ReadByte() if err != nil { - return -1, fmt.Errorf("cannot read '\r' char at the end of chunk size: %s", err) - } - if c != '\r' { - return -1, fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\r') - } - c, err = r.ReadByte() - if err != nil { - return -1, fmt.Errorf("cannot read '\n' char at the end of chunk size: %s", err) + return -1, ErrBrokenChunk{ + error: fmt.Errorf("cannot read '\n' char at the end of chunk size: %s", err), + } } if c != '\n' { - return -1, fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\n') + return -1, ErrBrokenChunk{ + error: fmt.Errorf("unexpected char %q at the end of chunk size. Expected %q", c, '\n'), + } } return n, nil } diff --git a/vendor/github.com/valyala/fasthttp/http_test.go b/vendor/github.com/valyala/fasthttp/http_test.go index fe91ca4..16a101a 100644 --- a/vendor/github.com/valyala/fasthttp/http_test.go +++ b/vendor/github.com/valyala/fasthttp/http_test.go @@ -10,6 +10,8 @@ import ( "strings" "testing" "time" + + "github.com/valyala/bytebufferpool" ) func TestResponseBodyStreamDeflate(t *testing.T) { @@ -393,7 +395,7 @@ func TestRequestUpdateURI(t *testing.T) { if !strings.HasPrefix(s, "GET /123/432.html?aaa=bcse") { t.Fatalf("cannot find %q in %q", "GET /123/432.html?aaa=bcse", s) } - if strings.Index(s, "\r\nHost: foobar.com\r\n") < 0 { + if !strings.Contains(s, "\r\nHost: foobar.com\r\n") { t.Fatalf("cannot find %q in %q", "\r\nHost: foobar.com\r\n", s) } } @@ -490,7 +492,7 @@ type bodyWriterTo interface { } func testBodyWriteTo(t *testing.T, bw bodyWriterTo, expectedS string, isRetainedBody bool) { - var buf ByteBuffer + var buf bytebufferpool.ByteBuffer if err := bw.BodyWriteTo(&buf); err != nil { t.Fatalf("unexpected error: %s", err) } @@ -564,7 +566,7 @@ func TestResponseWriteTo(t *testing.T) { r.SetBodyString("foobar") s := r.String() - var buf ByteBuffer + var buf bytebufferpool.ByteBuffer n, err := r.WriteTo(&buf) if err != nil { t.Fatalf("unexpected error: %s", err) @@ -583,7 +585,7 @@ func TestRequestWriteTo(t *testing.T) { r.SetRequestURI("http://foobar.com/aaa/bbb") s := r.String() - var buf ByteBuffer + var buf bytebufferpool.ByteBuffer n, err := r.WriteTo(&buf) if err != nil { t.Fatalf("unexpected error: %s", err) @@ -1100,7 +1102,7 @@ func TestRequestString(t *testing.T) { var r Request r.SetRequestURI("http://foobar.com/aaa") s := r.String() - expectedS := "GET /aaa HTTP/1.1\r\nUser-Agent: fasthttp\r\nHost: foobar.com\r\n\r\n" + expectedS := "GET /aaa HTTP/1.1\r\nHost: foobar.com\r\n\r\n" if s != expectedS { t.Fatalf("unexpected request: %q. Expecting %q", s, expectedS) } @@ -1300,6 +1302,23 @@ func TestRequestReadChunked(t *testing.T) { verifyTrailer(t, rb, "trail") } +// See: https://github.com/erikdubbelboer/fasthttp/issues/34 +func TestRequestChunkedWhitespace(t *testing.T) { + var req Request + + s := "POST /foo HTTP/1.1\r\nHost: google.com\r\nTransfer-Encoding: chunked\r\nContent-Type: aa/bb\r\n\r\n3 \r\nabc\r\n0\r\n\r\n" + r := bytes.NewBufferString(s) + rb := bufio.NewReader(r) + err := req.Read(rb) + if err != nil { + t.Fatalf("Unexpected error when reading chunked request: %s", err) + } + expectedBody := "abc" + if string(req.Body()) != expectedBody { + t.Fatalf("Unexpected body %q. Expected %q", req.Body(), expectedBody) + } +} + func TestResponseReadWithoutBody(t *testing.T) { var resp Response @@ -1372,7 +1391,7 @@ func TestResponseSuccess(t *testing.T) { // response with missing server testResponseSuccess(t, 500, "aaa", "", "aaadfsd", - 500, "aaa", string(defaultServerName)) + 500, "aaa", "") // empty body testResponseSuccess(t, 200, "bbb", "qwer", "", @@ -1440,7 +1459,7 @@ func testRequestWriteError(t *testing.T, method, requestURI, host, userAgent, bo req.Header.Set("User-Agent", userAgent) req.SetBody([]byte(body)) - w := &ByteBuffer{} + w := &bytebufferpool.ByteBuffer{} bw := bufio.NewWriter(w) err := req.Write(bw) if err == nil { @@ -1489,9 +1508,6 @@ func testRequestSuccess(t *testing.T, method, requestURI, host, userAgent, body, if string(req1.Header.Peek("Host")) != host { t.Fatalf("Unexpected host: %q. Expected %q", req1.Header.Peek("Host"), host) } - if len(userAgent) == 0 { - userAgent = string(defaultUserAgent) - } if string(req1.Header.Peek("User-Agent")) != userAgent { t.Fatalf("Unexpected user-agent: %q. Expected %q", req1.Header.Peek("User-Agent"), userAgent) } @@ -1792,3 +1808,32 @@ func createChunkedBody(body []byte) []byte { } return append(b, []byte("0\r\n\r\n")...) } + +func TestWriteMultipartForm(t *testing.T) { + var w bytes.Buffer + s := strings.Replace(`--foo +Content-Disposition: form-data; name="key" + +value +--foo +Content-Disposition: form-data; name="file"; filename="test.json" +Content-Type: application/json + +{"foo": "bar"} +--foo-- +`, "\n", "\r\n", -1) + mr := multipart.NewReader(strings.NewReader(s), "foo") + form, err := mr.ReadForm(1024) + + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if err := WriteMultipartForm(&w, form, "foo"); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + if w.String() != s { + t.Fatalf("unexpected output %q", w.Bytes()) + } +} diff --git a/vendor/github.com/valyala/fasthttp/lbclient.go b/vendor/github.com/valyala/fasthttp/lbclient.go index 41fe727..12418b6 100644 --- a/vendor/github.com/valyala/fasthttp/lbclient.go +++ b/vendor/github.com/valyala/fasthttp/lbclient.go @@ -59,7 +59,7 @@ type LBClient struct { // DefaultLBClientTimeout is the default request timeout used by LBClient // when calling LBClient.Do. // -// The timeout may be overriden via LBClient.Timeout. +// The timeout may be overridden via LBClient.Timeout. const DefaultLBClientTimeout = time.Second // DoDeadline calls DoDeadline on the least loaded client diff --git a/vendor/github.com/valyala/fasthttp/nocopy.go b/vendor/github.com/valyala/fasthttp/nocopy.go index 32af52e..8e9b89a 100644 --- a/vendor/github.com/valyala/fasthttp/nocopy.go +++ b/vendor/github.com/valyala/fasthttp/nocopy.go @@ -4,6 +4,8 @@ package fasthttp // so `go vet` gives a warning if this struct is copied. // // See https://github.com/golang/go/issues/8005#issuecomment-190753527 for details. +// and also: https://stackoverflow.com/questions/52494458/nocopy-minimal-example type noCopy struct{} -func (*noCopy) Lock() {} +func (*noCopy) Lock() {} +func (*noCopy) Unlock() {} diff --git a/vendor/github.com/valyala/fasthttp/pprofhandler/pprof.go b/vendor/github.com/valyala/fasthttp/pprofhandler/pprof.go new file mode 100644 index 0000000..c94d2c1 --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/pprofhandler/pprof.go @@ -0,0 +1,44 @@ +package pprofhandler + +import ( + "net/http/pprof" + rtp "runtime/pprof" + "strings" + + "github.com/valyala/fasthttp" + "github.com/valyala/fasthttp/fasthttpadaptor" +) + +var ( + cmdline = fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Cmdline) + profile = fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Profile) + symbol = fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Symbol) + trace = fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Trace) + index = fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Index) +) + +// PprofHandler serves server runtime profiling data in the format expected by the pprof visualization tool. +// +// See https://golang.org/pkg/net/http/pprof/ for details. +func PprofHandler(ctx *fasthttp.RequestCtx) { + ctx.Response.Header.Set("Content-Type", "text/html") + if strings.HasPrefix(string(ctx.Path()), "/debug/pprof/cmdline") { + cmdline(ctx) + } else if strings.HasPrefix(string(ctx.Path()), "/debug/pprof/profile") { + profile(ctx) + } else if strings.HasPrefix(string(ctx.Path()), "/debug/pprof/symbol") { + symbol(ctx) + } else if strings.HasPrefix(string(ctx.Path()), "/debug/pprof/trace") { + trace(ctx) + } else { + for _, v := range rtp.Profiles() { + ppName := v.Name() + if strings.HasPrefix(string(ctx.Path()), "/debug/pprof/"+ppName) { + namedHandler := fasthttpadaptor.NewFastHTTPHandlerFunc(pprof.Handler(ppName).ServeHTTP) + namedHandler(ctx) + return + } + } + index(ctx) + } +} diff --git a/vendor/github.com/valyala/fasthttp/reuseport/reuseport.go b/vendor/github.com/valyala/fasthttp/reuseport/reuseport.go index fe80eae..191fe64 100644 --- a/vendor/github.com/valyala/fasthttp/reuseport/reuseport.go +++ b/vendor/github.com/valyala/fasthttp/reuseport/reuseport.go @@ -10,9 +10,10 @@ package reuseport import ( "fmt" - "github.com/valyala/tcplisten" "net" "strings" + + "github.com/valyala/tcplisten" ) // ErrNoReusePort is returned if the OS doesn't support SO_REUSEPORT. diff --git a/vendor/github.com/valyala/fasthttp/reuseport/reuseport_test.go b/vendor/github.com/valyala/fasthttp/reuseport/reuseport_test.go index 0c70e83..e781ff1 100644 --- a/vendor/github.com/valyala/fasthttp/reuseport/reuseport_test.go +++ b/vendor/github.com/valyala/fasthttp/reuseport/reuseport_test.go @@ -73,7 +73,7 @@ func testNewListener(t *testing.T, network, addr string, serversCount, requestsC select { case <-ch: case <-time.After(200 * time.Millisecond): - t.Fatalf("%d. timeout when waiting for response: %s", i, err) + t.Fatalf("%d. timeout when waiting for response", i) } if string(resp) != req { diff --git a/vendor/github.com/valyala/fasthttp/server.go b/vendor/github.com/valyala/fasthttp/server.go index 28627db..8007575 100644 --- a/vendor/github.com/valyala/fasthttp/server.go +++ b/vendor/github.com/valyala/fasthttp/server.go @@ -16,6 +16,14 @@ import ( "time" ) +var errNoCertOrKeyProvided = errors.New("Cert or key has not provided") + +var ( + // ErrAlreadyServing is returned when calling Serve on a Server + // that is already serving connections. + ErrAlreadyServing = errors.New("Server is already serving connections") +) + // ServeConn serves HTTP requests from the given connection // using the given handler. // @@ -126,6 +134,9 @@ func ListenAndServeTLSEmbed(addr string, certData, keyData []byte, handler Reque // must be limited. type RequestHandler func(ctx *RequestCtx) +// ServeHandler must process tls.Config.NextProto negotiated requests. +type ServeHandler func(c net.Conn) error + // Server implements HTTP server. // // Default Server settings should satisfy the majority of Server users. @@ -139,8 +150,22 @@ type Server struct { noCopy noCopy // Handler for processing incoming requests. + // + // Take into account that no `panic` recovery is done by `fasthttp` (thus any `panic` will take down the entire server). + // Instead the user should use `recover` to handle these situations. Handler RequestHandler + // ErrorHandler for returning a response in case of an error while receiving or parsing the request. + // + // The following is a non-exhaustive list of errors that can be expected as argument: + // * io.EOF + // * io.ErrUnexpectedEOF + // * ErrGetOnly + // * ErrSmallBuffer + // * ErrBodyTooLarge + // * ErrBrokenChunks + ErrorHandler func(ctx *RequestCtx, err error) + // Server name for sending in response headers. // // Default server name is used if left blank. @@ -211,6 +236,18 @@ type Server struct { // By default keep-alive connection lifetime is unlimited. MaxKeepaliveDuration time.Duration + // Whether to enable tcp keep-alive connections. + // + // Whether the operating system should send tcp keep-alive messages on the tcp connection. + // + // By default tcp keep-alive connections are disabled. + TCPKeepalive bool + + // Period between tcp keep-alive messages. + // + // TCP keep-alive period is determined by operation system by default. + TCPKeepalivePeriod time.Duration + // Maximum request body size. // // The server rejects requests with bodies exceeding this limit. @@ -265,11 +302,40 @@ type Server struct { // * cONTENT-lenGTH -> Content-Length DisableHeaderNamesNormalizing bool + // SleepWhenConcurrencyLimitsExceeded is a duration to be slept of if + // the concurrency limit in exceeded (default [when is 0]: don't sleep + // and accept new connections immidiatelly). + SleepWhenConcurrencyLimitsExceeded time.Duration + + // NoDefaultServerHeader, when set to true, causes the default Server header + // to be excluded from the Response. + // + // The default Server header value is the value of the Name field or an + // internal default value in its absence. With this option set to true, + // the only time a Server header will be sent is if a non-zero length + // value is explicitly provided during a request. + NoDefaultServerHeader bool + + // NoDefaultContentType, when set to true, causes the default Content-Type + // header to be excluded from the Response. + // + // The default Content-Type header value is the internal default value. When + // set to true, the Content-Type will not be present. + NoDefaultContentType bool + + // ConnState specifies an optional callback function that is + // called when a client connection changes state. See the + // ConnState type and associated constants for details. + ConnState func(net.Conn, ConnState) + // Logger, which is used by RequestCtx.Logger(). // // By default standard logger from log package is used. Logger Logger + tlsConfig *tls.Config + nextProtos map[string]ServeHandler + concurrency uint32 concurrencyCh chan struct{} perIPConnCounter perIPConnCounter @@ -280,6 +346,14 @@ type Server struct { writerPool sync.Pool hijackConnPool sync.Pool bytePool sync.Pool + + // We need to know our listener so we can close it in Shutdown(). + ln net.Listener + + mu sync.Mutex + open int32 + stop int32 + done chan struct{} } // TimeoutHandler creates RequestHandler, which returns StatusRequestTimeout @@ -344,12 +418,6 @@ func CompressHandler(h RequestHandler) RequestHandler { func CompressHandlerLevel(h RequestHandler, level int) RequestHandler { return func(ctx *RequestCtx) { h(ctx) - ce := ctx.Response.Header.PeekBytes(strContentEncoding) - if len(ce) > 0 { - // Do not compress responses with non-empty - // Content-Encoding. - return - } if ctx.Request.Header.HasAcceptEncodingBytes(strGzip) { ctx.Response.gzipBody(level) } else if ctx.Request.Header.HasAcceptEncodingBytes(strDeflate) { @@ -498,6 +566,7 @@ func (ctx *RequestCtx) VisitUserValues(visitor func([]byte, interface{})) { } type connTLSer interface { + Handshake() error ConnectionState() tls.ConnectionState } @@ -532,6 +601,15 @@ func (ctx *RequestCtx) TLSConnectionState() *tls.ConnectionState { return &state } +// Conn returns a reference to the underlying net.Conn. +// +// WARNING: Only use this method if you know what you are doing! +// +// Reading from or writing to the returned connection will end badly! +func (ctx *RequestCtx) Conn() net.Conn { + return ctx.c +} + type firstByteReader struct { c net.Conn ch byte @@ -598,18 +676,13 @@ func (ctx *RequestCtx) ConnID() uint64 { return ctx.connID } -// Time returns RequestHandler call time truncated to the nearest second. -// -// Call time.Now() at the beginning of RequestHandler in order to obtain -// percise RequestHandler call time. +// Time returns RequestHandler call time. func (ctx *RequestCtx) Time() time.Time { return ctx.time } -// ConnTime returns the time server starts serving the connection +// ConnTime returns the time the server started serving the connection // the current request came from. -// -// The returned time is truncated to the nearest second. func (ctx *RequestCtx) ConnTime() time.Time { return ctx.connTime } @@ -758,12 +831,28 @@ func SaveMultipartFile(fh *multipart.FileHeader, path string) error { if err != nil { return err } - defer f.Close() if ff, ok := f.(*os.File); ok { - return os.Rename(ff.Name(), path) + // Windows can't rename files that are opened. + if err := f.Close(); err != nil { + return err + } + + // If renaming fails we try the normal copying method. + // Renaming could fail if the files are on different devices. + if os.Rename(ff.Name(), path) == nil { + return nil + } + + // Reopen f for the code below. + f, err = fh.Open() + if err != nil { + return err + } } + defer f.Close() + ff, err := os.Create(path) if err != nil { return err @@ -827,6 +916,26 @@ func (ctx *RequestCtx) IsDelete() bool { return ctx.Request.Header.IsDelete() } +// IsConnect returns true if request method is CONNECT. +func (ctx *RequestCtx) IsConnect() bool { + return ctx.Request.Header.IsConnect() +} + +// IsOptions returns true if request method is OPTIONS. +func (ctx *RequestCtx) IsOptions() bool { + return ctx.Request.Header.IsOptions() +} + +// IsTrace returns true if request method is TRACE. +func (ctx *RequestCtx) IsTrace() bool { + return ctx.Request.Header.IsTrace() +} + +// IsPatch returns true if request method is PATCH. +func (ctx *RequestCtx) IsPatch() bool { + return ctx.Request.Header.IsPatch() +} + // Method return request method. // // Returned value is valid until returning from RequestHandler. @@ -918,11 +1027,18 @@ func (ctx *RequestCtx) SuccessString(contentType, body string) { // * StatusFound (302) // * StatusSeeOther (303) // * StatusTemporaryRedirect (307) +// * StatusPermanentRedirect (308) // // All other statusCode values are replaced by StatusFound (302). // // The redirect uri may be either absolute or relative to the current -// request uri. +// request uri. Fasthttp will always send an absolute uri back to the client. +// To send a relative uri you can use the following code: +// +// strLocation = []byte("Location") // Put this with your top level var () declarations. +// ctx.Response.Header.SetCanonical(strLocation, "/relative?uri") +// ctx.Response.SetStatusCode(fasthttp.StatusMovedPermanently) +// func (ctx *RequestCtx) Redirect(uri string, statusCode int) { u := AcquireURI() ctx.URI().CopyTo(u) @@ -940,11 +1056,18 @@ func (ctx *RequestCtx) Redirect(uri string, statusCode int) { // * StatusFound (302) // * StatusSeeOther (303) // * StatusTemporaryRedirect (307) +// * StatusPermanentRedirect (308) // // All other statusCode values are replaced by StatusFound (302). // // The redirect uri may be either absolute or relative to the current -// request uri. +// request uri. Fasthttp will always send an absolute uri back to the client. +// To send a relative uri you can use the following code: +// +// strLocation = []byte("Location") // Put this with your top level var () declarations. +// ctx.Response.Header.SetCanonical(strLocation, "/relative?uri") +// ctx.Response.SetStatusCode(fasthttp.StatusMovedPermanently) +// func (ctx *RequestCtx) RedirectBytes(uri []byte, statusCode int) { s := b2s(uri) ctx.Redirect(s, statusCode) @@ -958,7 +1081,8 @@ func (ctx *RequestCtx) redirect(uri []byte, statusCode int) { func getRedirectStatusCode(statusCode int) int { if statusCode == StatusMovedPermanently || statusCode == StatusFound || - statusCode == StatusSeeOther || statusCode == StatusTemporaryRedirect { + statusCode == StatusSeeOther || statusCode == StatusTemporaryRedirect || + statusCode == StatusPermanentRedirect { return statusCode } return StatusFound @@ -1155,15 +1279,69 @@ func (ctx *RequestCtx) TimeoutErrorWithResponse(resp *Response) { ctx.timeoutResponse = respCopy } +// NextProto adds nph to be processed when key is negotiated when TLS +// connection is established. +// +// This function can only be called before the server is started. +func (s *Server) NextProto(key string, nph ServeHandler) { + if s.nextProtos == nil { + s.nextProtos = make(map[string]ServeHandler) + } + s.configTLS() + s.tlsConfig.NextProtos = append(s.tlsConfig.NextProtos, key) + s.nextProtos[key] = nph +} + +func (s *Server) getNextProto(c net.Conn) (proto string, err error) { + if tlsConn, ok := c.(connTLSer); ok { + err = tlsConn.Handshake() + if err == nil { + proto = tlsConn.ConnectionState().NegotiatedProtocol + } + } + return +} + +// tcpKeepAliveListener sets TCP keep-alive timeouts on accepted +// connections. It's used by ListenAndServe, ListenAndServeTLS and +// ListenAndServeTLSEmbed so dead TCP connections (e.g. closing laptop mid-download) +// eventually go away. +type tcpKeepaliveListener struct { + *net.TCPListener + keepalivePeriod time.Duration +} + +func (ln tcpKeepaliveListener) Accept() (net.Conn, error) { + tc, err := ln.AcceptTCP() + if err != nil { + return nil, err + } + tc.SetKeepAlive(true) + if ln.keepalivePeriod > 0 { + tc.SetKeepAlivePeriod(ln.keepalivePeriod) + } + return tc, nil +} + // ListenAndServe serves HTTP requests from the given TCP4 addr. // // Pass custom listener to Serve if you need listening on non-TCP4 media // such as IPv6. +// +// Accepted connections are configured to enable TCP keep-alives. func (s *Server) ListenAndServe(addr string) error { ln, err := net.Listen("tcp4", addr) if err != nil { return err } + if s.TCPKeepalive { + if tcpln, ok := ln.(*net.TCPListener); ok { + return s.Serve(tcpKeepaliveListener{ + TCPListener: tcpln, + keepalivePeriod: s.TCPKeepalivePeriod, + }) + } + } return s.Serve(ln) } @@ -1192,11 +1370,24 @@ func (s *Server) ListenAndServeUNIX(addr string, mode os.FileMode) error { // // Pass custom listener to Serve if you need listening on non-TCP4 media // such as IPv6. +// +// If the certFile or keyFile has not been provided to the server structure, +// the function will use the previously added TLS configuration. +// +// Accepted connections are configured to enable TCP keep-alives. func (s *Server) ListenAndServeTLS(addr, certFile, keyFile string) error { ln, err := net.Listen("tcp4", addr) if err != nil { return err } + if s.TCPKeepalive { + if tcpln, ok := ln.(*net.TCPListener); ok { + return s.ServeTLS(tcpKeepaliveListener{ + TCPListener: tcpln, + keepalivePeriod: s.TCPKeepalivePeriod, + }, certFile, keyFile) + } + } return s.ServeTLS(ln, certFile, keyFile) } @@ -1206,59 +1397,113 @@ func (s *Server) ListenAndServeTLS(addr, certFile, keyFile string) error { // // Pass custom listener to Serve if you need listening on arbitrary media // such as IPv6. +// +// If the certFile or keyFile has not been provided the server structure, +// the function will use previously added TLS configuration. +// +// Accepted connections are configured to enable TCP keep-alives. func (s *Server) ListenAndServeTLSEmbed(addr string, certData, keyData []byte) error { ln, err := net.Listen("tcp4", addr) if err != nil { return err } + if s.TCPKeepalive { + if tcpln, ok := ln.(*net.TCPListener); ok { + return s.ServeTLSEmbed(tcpKeepaliveListener{ + TCPListener: tcpln, + keepalivePeriod: s.TCPKeepalivePeriod, + }, certData, keyData) + } + } return s.ServeTLSEmbed(ln, certData, keyData) } // ServeTLS serves HTTPS requests from the given listener. // // certFile and keyFile are paths to TLS certificate and key files. +// +// If the certFile or keyFile has not been provided the server structure, +// the function will use previously added TLS configuration. func (s *Server) ServeTLS(ln net.Listener, certFile, keyFile string) error { - lnTLS, err := newTLSListener(ln, certFile, keyFile) - if err != nil { + err := s.AppendCert(certFile, keyFile) + if err != nil && err != errNoCertOrKeyProvided { return err } - return s.Serve(lnTLS) + if s.tlsConfig == nil { + return errNoCertOrKeyProvided + } + s.tlsConfig.BuildNameToCertificate() + + return s.Serve( + tls.NewListener(ln, s.tlsConfig), + ) } // ServeTLSEmbed serves HTTPS requests from the given listener. // // certData and keyData must contain valid TLS certificate and key data. +// +// If the certFile or keyFile has not been provided the server structure, +// the function will use previously added TLS configuration. func (s *Server) ServeTLSEmbed(ln net.Listener, certData, keyData []byte) error { - lnTLS, err := newTLSListenerEmbed(ln, certData, keyData) - if err != nil { + err := s.AppendCertEmbed(certData, keyData) + if err != nil && err != errNoCertOrKeyProvided { return err } - return s.Serve(lnTLS) + if s.tlsConfig == nil { + return errNoCertOrKeyProvided + } + s.tlsConfig.BuildNameToCertificate() + + return s.Serve( + tls.NewListener(ln, s.tlsConfig), + ) } -func newTLSListener(ln net.Listener, certFile, keyFile string) (net.Listener, error) { +// AppendCert appends certificate and keyfile to TLS Configuration. +// +// This function allows programmer to handle multiple domains +// in one server structure. See examples/multidomain +func (s *Server) AppendCert(certFile, keyFile string) error { + if len(certFile) == 0 && len(keyFile) == 0 { + return errNoCertOrKeyProvided + } + cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { - return nil, fmt.Errorf("cannot load TLS key pair from certFile=%q and keyFile=%q: %s", certFile, keyFile, err) + return fmt.Errorf("cannot load TLS key pair from certFile=%q and keyFile=%q: %s", certFile, keyFile, err) } - return newCertListener(ln, &cert), nil + + s.configTLS() + + s.tlsConfig.Certificates = append(s.tlsConfig.Certificates, cert) + return nil } -func newTLSListenerEmbed(ln net.Listener, certData, keyData []byte) (net.Listener, error) { +// AppendCertEmbed does the same as AppendCert but using in-memory data. +func (s *Server) AppendCertEmbed(certData, keyData []byte) error { + if len(certData) == 0 && len(keyData) == 0 { + return errNoCertOrKeyProvided + } + cert, err := tls.X509KeyPair(certData, keyData) if err != nil { - return nil, fmt.Errorf("cannot load TLS key pair from the provided certData(%d) and keyData(%d): %s", + return fmt.Errorf("cannot load TLS key pair from the provided certData(%d) and keyData(%d): %s", len(certData), len(keyData), err) } - return newCertListener(ln, &cert), nil + + s.configTLS() + + s.tlsConfig.Certificates = append(s.tlsConfig.Certificates, cert) + return nil } -func newCertListener(ln net.Listener, cert *tls.Certificate) net.Listener { - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{*cert}, - PreferServerCipherSuites: true, +func (s *Server) configTLS() { + if s.tlsConfig == nil { + s.tlsConfig = &tls.Config{ + PreferServerCipherSuites: true, + } } - return tls.NewListener(ln, tlsConfig) } // DefaultConcurrency is the maximum number of concurrent connections @@ -1274,6 +1519,18 @@ func (s *Server) Serve(ln net.Listener) error { var c net.Conn var err error + s.mu.Lock() + { + if s.ln != nil { + s.mu.Unlock() + return ErrAlreadyServing + } + + s.ln = ln + s.done = make(chan struct{}) + } + s.mu.Unlock() + maxWorkersCount := s.getConcurrency() s.concurrencyCh = make(chan struct{}, maxWorkersCount) wp := &workerPool{ @@ -1281,9 +1538,17 @@ func (s *Server) Serve(ln net.Listener) error { MaxWorkersCount: maxWorkersCount, LogAllErrors: s.LogAllErrors, Logger: s.logger(), + connState: s.setState, } wp.Start() + // Count our waiting to accept a connection as an open connection. + // This way we can't get into any weird state where just after accepting + // a connection Shutdown is called which reads open as 0 because it isn't + // incremented yet. + atomic.AddInt32(&s.open, 1) + defer atomic.AddInt32(&s.open, -1) + for { if c, err = acceptConn(s, ln, &lastPerIPErrorTime); err != nil { wp.Stop() @@ -1292,14 +1557,18 @@ func (s *Server) Serve(ln net.Listener) error { } return err } + s.setState(c, StateNew) + atomic.AddInt32(&s.open, 1) if !wp.Serve(c) { + atomic.AddInt32(&s.open, -1) s.writeFastError(c, StatusServiceUnavailable, "The connection cannot be served because Server.Concurrency limit exceeded") c.Close() + s.setState(c, StateClosed) if time.Since(lastOverflowErrorTime) > time.Minute { s.logger().Printf("The incoming connection cannot be served, because %d concurrent connections are served. "+ "Try increasing Server.Concurrency", maxWorkersCount) - lastOverflowErrorTime = CoarseTimeNow() + lastOverflowErrorTime = time.Now() } // The current server reached concurrency limit, @@ -1308,12 +1577,59 @@ func (s *Server) Serve(ln net.Listener) error { // // There is a hope other servers didn't reach their // concurrency limits yet :) - time.Sleep(100 * time.Millisecond) + // + // See also: https://github.com/valyala/fasthttp/pull/485#discussion_r239994990 + if s.SleepWhenConcurrencyLimitsExceeded > 0 { + time.Sleep(s.SleepWhenConcurrencyLimitsExceeded) + } } c = nil } } +// Shutdown gracefully shuts down the server without interrupting any active connections. +// Shutdown works by first closing all open listeners and then waiting indefinitely for all connections to return to idle and then shut down. +// +// When Shutdown is called, Serve, ListenAndServe, and ListenAndServeTLS immediately return nil. +// Make sure the program doesn't exit and waits instead for Shutdown to return. +// +// Shutdown does not close keepalive connections so its recommended to set ReadTimeout to something else than 0. +func (s *Server) Shutdown() error { + s.mu.Lock() + defer s.mu.Unlock() + + atomic.StoreInt32(&s.stop, 1) + defer atomic.StoreInt32(&s.stop, 0) + + if s.ln == nil { + return nil + } + + if err := s.ln.Close(); err != nil { + return err + } + + if s.done != nil { + close(s.done) + } + + // Closing the listener will make Serve() call Stop on the worker pool. + // Setting .stop to 1 will make serveConn() break out of its loop. + // Now we just have to wait until all workers are done. + for { + if open := atomic.LoadInt32(&s.open); open == 0 { + break + } + // This is not an optimal solution but using a sync.WaitGroup + // here causes data races as it's hard to prevent Add() to be called + // while Wait() is waiting. + time.Sleep(time.Millisecond * 100) + } + + s.ln = nil + return nil +} + func acceptConn(s *Server, ln net.Listener, lastPerIPErrorTime *time.Time) (net.Conn, error) { for { c, err := ln.Accept() @@ -1341,7 +1657,7 @@ func acceptConn(s *Server, ln net.Listener, lastPerIPErrorTime *time.Time) (net. if time.Since(*lastPerIPErrorTime) > time.Minute { s.logger().Printf("The number of connections from %s exceeds MaxConnsPerIP=%d", getConnIP4(c), s.MaxConnsPerIP) - *lastPerIPErrorTime = CoarseTimeNow() + *lastPerIPErrorTime = time.Now() } continue } @@ -1381,8 +1697,8 @@ var ( ErrPerIPConnLimit = errors.New("too many connections per ip") // ErrConcurrencyLimit may be returned from ServeConn if the number - // of concurrenty served connections exceeds Server.Concurrency. - ErrConcurrencyLimit = errors.New("canot serve the connection because Server.Concurrency concurrent connections are served") + // of concurrently served connections exceeds Server.Concurrency. + ErrConcurrencyLimit = errors.New("cannot serve the connection because Server.Concurrency concurrent connections are served") // ErrKeepaliveTimeout is returned from ServeConn // if the connection lifetime exceeds MaxKeepaliveDuration. @@ -1415,23 +1731,42 @@ func (s *Server) ServeConn(c net.Conn) error { return ErrConcurrencyLimit } + atomic.AddInt32(&s.open, 1) + err := s.serveConn(c) atomic.AddUint32(&s.concurrency, ^uint32(0)) if err != errHijacked { err1 := c.Close() + s.setState(c, StateClosed) if err == nil { err = err1 } } else { err = nil + s.setState(c, StateHijacked) } return err } var errHijacked = errors.New("connection has been hijacked") +// GetCurrentConcurrency returns a number of currently served +// connections. +// +// This function is intended be used by monitoring systems +func (s *Server) GetCurrentConcurrency() uint32 { + return atomic.LoadUint32(&s.concurrency) +} + +// GetOpenConnectionsCount returns a number of opened connections. +// +// This function is intended be used by monitoring systems +func (s *Server) GetOpenConnectionsCount() int32 { + return atomic.LoadInt32(&s.open) - 1 +} + func (s *Server) getConcurrency() int { n := s.Concurrency if n <= 0 { @@ -1453,10 +1788,24 @@ func nextConnID() uint64 { const DefaultMaxRequestBodySize = 4 * 1024 * 1024 func (s *Server) serveConn(c net.Conn) error { - serverName := s.getServerName() + defer atomic.AddInt32(&s.open, -1) + + if proto, err := s.getNextProto(c); err != nil { + return err + } else { + handler, ok := s.nextProtos[proto] + if ok { + return handler(c) + } + } + + var serverName []byte + if !s.NoDefaultServerHeader { + serverName = s.getServerName() + } connRequestNum := uint64(0) connID := nextConnID() - currentTime := CoarseTimeNow() + currentTime := time.Now() connTime := currentTime maxRequestBodySize := s.MaxRequestBodySize if maxRequestBodySize <= 0 { @@ -1500,68 +1849,84 @@ func (s *Server) serveConn(c net.Conn) error { br, err = acquireByteReader(&ctx) } ctx.Request.isTLS = isTLS + ctx.Response.Header.noDefaultContentType = s.NoDefaultContentType if err == nil { if s.DisableHeaderNamesNormalizing { ctx.Request.Header.DisableNormalizing() ctx.Response.Header.DisableNormalizing() } + // reading Headers and Body err = ctx.Request.readLimitBody(br, maxRequestBodySize, s.GetOnly) - if br.Buffered() == 0 || err != nil { + if err == nil { + // If we read any bytes off the wire, we're active. + s.setState(c, StateActive) + } + if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil { releaseReader(s, br) br = nil } } - currentTime = CoarseTimeNow() + currentTime = time.Now() ctx.lastReadDuration = currentTime.Sub(ctx.time) if err != nil { if err == io.EOF { err = nil + } else if connRequestNum > 1 && err == errNothingRead { + // This is not the first request and we haven't read a single byte + // of a new request yet. This means it's just a keep-alive connection + // closing down either because the remote closed it or because + // or a read timeout on our side. Either way just close the connection + // and don't return any error response. + err = nil } else { - bw = writeErrorResponse(bw, ctx, err) + bw = s.writeErrorResponse(bw, ctx, serverName, err) } break } // 'Expect: 100-continue' request handling. // See http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html for details. - if !ctx.Request.Header.noBody() && ctx.Request.MayContinue() { + if !ctx.Request.Header.ignoreBody() && ctx.Request.MayContinue() { // Send 'HTTP/1.1 100 Continue' response. if bw == nil { bw = acquireWriter(ctx) } bw.Write(strResponseContinue) err = bw.Flush() - releaseWriter(s, bw) - bw = nil if err != nil { break } + if s.ReduceMemoryUsage { + releaseWriter(s, bw) + bw = nil + } // Read request body. if br == nil { br = acquireReader(ctx) } err = ctx.Request.ContinueReadBody(br, maxRequestBodySize) - if br.Buffered() == 0 || err != nil { + if (s.ReduceMemoryUsage && br.Buffered() == 0) || err != nil { releaseReader(s, br) br = nil } if err != nil { - bw = writeErrorResponse(bw, ctx, err) + bw = s.writeErrorResponse(bw, ctx, serverName, err) break } } - connectionClose = s.DisableKeepalive || ctx.Request.Header.connectionCloseFast() + connectionClose = s.DisableKeepalive || ctx.Request.Header.ConnectionClose() isHTTP11 = ctx.Request.Header.IsHTTP11() - ctx.Response.Header.SetServerBytes(serverName) + if serverName != nil { + ctx.Response.Header.SetServerBytes(serverName) + } ctx.connID = connID ctx.connRequestNum = connRequestNum - ctx.connTime = connTime ctx.time = currentTime s.Handler(ctx) @@ -1593,9 +1958,7 @@ func (s *Server) serveConn(c net.Conn) error { lastWriteDeadlineTime = s.updateWriteDeadline(c, ctx, lastWriteDeadlineTime) } - // Verify Request.Header.connectionCloseFast() again, - // since request handler might trigger full headers' parsing. - connectionClose = connectionClose || ctx.Request.Header.connectionCloseFast() || ctx.Response.ConnectionClose() + connectionClose = connectionClose || ctx.Response.ConnectionClose() if connectionClose { ctx.Response.Header.SetCanonical(strConnection, strClose) } else if !isHTTP11 { @@ -1605,7 +1968,7 @@ func (s *Server) serveConn(c net.Conn) error { ctx.Response.Header.SetCanonical(strConnection, strKeepAlive) } - if len(ctx.Response.Header.Server()) == 0 { + if serverName != nil && len(ctx.Response.Header.Server()) == 0 { ctx.Response.Header.SetServerBytes(serverName) } @@ -1616,21 +1979,16 @@ func (s *Server) serveConn(c net.Conn) error { break } - if br == nil || connectionClose { - err = bw.Flush() - releaseWriter(s, bw) - bw = nil - if err != nil { - break - } - if connectionClose { - break - } + err = bw.Flush() + if err != nil { + break + } + if connectionClose { + break } if hijackHandler != nil { - var hjr io.Reader - hjr = c + var hjr io.Reader = c if br != nil { hjr = br br = nil @@ -1654,7 +2012,13 @@ func (s *Server) serveConn(c net.Conn) error { break } - currentTime = CoarseTimeNow() + currentTime = time.Now() + s.setState(c, StateIdle) + + if atomic.LoadInt32(&s.stop) == 1 { + err = nil + break + } } if br != nil { @@ -1667,6 +2031,12 @@ func (s *Server) serveConn(c net.Conn) error { return err } +func (s *Server) setState(nc net.Conn, state ConnState) { + if hook := s.ConnState; hook != nil { + hook(nc, state) + } +} + func (s *Server) updateReadDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTime time.Time) time.Time { readTimeout := s.ReadTimeout currentTime := ctx.time @@ -1710,7 +2080,7 @@ func (s *Server) updateWriteDeadline(c net.Conn, ctx *RequestCtx, lastDeadlineTi // Optimization: update write deadline only if more than 25% // of the last write deadline exceeded. // See https://github.com/golang/go/issues/15133 for details. - currentTime := CoarseTimeNow() + currentTime := time.Now() if currentTime.Sub(lastDeadlineTime) > (writeTimeout >> 2) { if err := c.SetWriteDeadline(currentTime.Add(writeTimeout)); err != nil { panic(fmt.Sprintf("BUG: error in SetWriteDeadline(%s): %s", writeTimeout, err)) @@ -1864,9 +2234,8 @@ func releaseWriter(s *Server, w *bufio.Writer) { s.writerPool.Put(w) } -func (s *Server) acquireCtx(c net.Conn) *RequestCtx { +func (s *Server) acquireCtx(c net.Conn) (ctx *RequestCtx) { v := s.ctxPool.Get() - var ctx *RequestCtx if v == nil { ctx = &RequestCtx{ s: s, @@ -1878,7 +2247,7 @@ func (s *Server) acquireCtx(c net.Conn) *RequestCtx { ctx = v.(*RequestCtx) } ctx.c = c - return ctx + return } // Init2 prepares ctx for passing to RequestHandler. @@ -1893,7 +2262,7 @@ func (ctx *RequestCtx) Init2(conn net.Conn, logger Logger, reduceMemoryUsage boo ctx.connID = nextConnID() ctx.s = fakeServer ctx.connRequestNum = 0 - ctx.connTime = CoarseTimeNow() + ctx.connTime = time.Now() ctx.time = ctx.connTime keepBodyBuffer := !reduceMemoryUsage @@ -1921,6 +2290,49 @@ func (ctx *RequestCtx) Init(req *Request, remoteAddr net.Addr, logger Logger) { req.CopyTo(&ctx.Request) } +// Deadline returns the time when work done on behalf of this context +// should be canceled. Deadline returns ok==false when no deadline is +// set. Successive calls to Deadline return the same results. +// +// This method always returns 0, false and is only present to make +// RequestCtx implement the context interface. +func (ctx *RequestCtx) Deadline() (deadline time.Time, ok bool) { + return +} + +// Done returns a channel that's closed when work done on behalf of this +// context should be canceled. Done may return nil if this context can +// never be canceled. Successive calls to Done return the same value. +func (ctx *RequestCtx) Done() <-chan struct{} { + return ctx.s.done +} + +// Err returns a non-nil error value after Done is closed, +// successive calls to Err return the same error. +// If Done is not yet closed, Err returns nil. +// If Done is closed, Err returns a non-nil error explaining why: +// Canceled if the context was canceled +// or DeadlineExceeded if the context's deadline passed. +// +// This method always returns nil and is only present to make +// RequestCtx implement the context interface. +func (ctx *RequestCtx) Err() error { + return nil +} + +// Value returns the value associated with this context for key, or nil +// if no value is associated with key. Successive calls to Value with +// the same key returns the same result. +// +// This method is present to make RequestCtx implement the context interface. +// This method is the same as calling ctx.UserValue(key) +func (ctx *RequestCtx) Value(key interface{}) interface{} { + if keyString, ok := key.(string); ok { + return ctx.UserValue(keyString) + } + return nil +} + var fakeServer = &Server{ // Initialize concurrencyCh for TimeoutHandler concurrencyCh: make(chan struct{}, DefaultConcurrency), @@ -1978,22 +2390,41 @@ func (s *Server) getServerName() []byte { func (s *Server) writeFastError(w io.Writer, statusCode int, msg string) { w.Write(statusLine(statusCode)) + + server := "" + if !s.NoDefaultServerHeader { + server = fmt.Sprintf("Server: %s\r\n", s.getServerName()) + } + fmt.Fprintf(w, "Connection: close\r\n"+ - "Server: %s\r\n"+ + server+ "Date: %s\r\n"+ "Content-Type: text/plain\r\n"+ "Content-Length: %d\r\n"+ "\r\n"+ "%s", - s.getServerName(), serverDate.Load(), len(msg), msg) + serverDate.Load(), len(msg), msg) } -func writeErrorResponse(bw *bufio.Writer, ctx *RequestCtx, err error) *bufio.Writer { +func defaultErrorHandler(ctx *RequestCtx, err error) { if _, ok := err.(*ErrSmallBuffer); ok { ctx.Error("Too big request header", StatusRequestHeaderFieldsTooLarge) } else { ctx.Error("Error when parsing request", StatusBadRequest) } +} + +func (s *Server) writeErrorResponse(bw *bufio.Writer, ctx *RequestCtx, serverName []byte, err error) *bufio.Writer { + errorHandler := defaultErrorHandler + if s.ErrorHandler != nil { + errorHandler = s.ErrorHandler + } + + errorHandler(ctx, err) + + if serverName != nil { + ctx.Response.Header.SetServerBytes(serverName) + } ctx.SetConnectionClose() if bw == nil { bw = acquireWriter(ctx) @@ -2002,3 +2433,55 @@ func writeErrorResponse(bw *bufio.Writer, ctx *RequestCtx, err error) *bufio.Wri bw.Flush() return bw } + +// A ConnState represents the state of a client connection to a server. +// It's used by the optional Server.ConnState hook. +type ConnState int + +const ( + // StateNew represents a new connection that is expected to + // send a request immediately. Connections begin at this + // state and then transition to either StateActive or + // StateClosed. + StateNew ConnState = iota + + // StateActive represents a connection that has read 1 or more + // bytes of a request. The Server.ConnState hook for + // StateActive fires before the request has entered a handler + // and doesn't fire again until the request has been + // handled. After the request is handled, the state + // transitions to StateClosed, StateHijacked, or StateIdle. + // For HTTP/2, StateActive fires on the transition from zero + // to one active request, and only transitions away once all + // active requests are complete. That means that ConnState + // cannot be used to do per-request work; ConnState only notes + // the overall state of the connection. + StateActive + + // StateIdle represents a connection that has finished + // handling a request and is in the keep-alive state, waiting + // for a new request. Connections transition from StateIdle + // to either StateActive or StateClosed. + StateIdle + + // StateHijacked represents a hijacked connection. + // This is a terminal state. It does not transition to StateClosed. + StateHijacked + + // StateClosed represents a closed connection. + // This is a terminal state. Hijacked connections do not + // transition to StateClosed. + StateClosed +) + +var stateName = map[ConnState]string{ + StateNew: "new", + StateActive: "active", + StateIdle: "idle", + StateHijacked: "hijacked", + StateClosed: "closed", +} + +func (c ConnState) String() string { + return stateName[c] +} diff --git a/vendor/github.com/valyala/fasthttp/server_test.go b/vendor/github.com/valyala/fasthttp/server_test.go index 5b3ffa4..372ef40 100644 --- a/vendor/github.com/valyala/fasthttp/server_test.go +++ b/vendor/github.com/valyala/fasthttp/server_test.go @@ -3,12 +3,15 @@ package fasthttp import ( "bufio" "bytes" + "context" "crypto/tls" "fmt" "io" "io/ioutil" + "mime/multipart" "net" "os" + "reflect" "strings" "sync" "testing" @@ -17,6 +20,201 @@ import ( "github.com/valyala/fasthttp/fasthttputil" ) +// Make sure RequestCtx implements context.Context +var _ context.Context = &RequestCtx{} + +func TestServerConnState(t *testing.T) { + states := make([]string, 0) + s := &Server{ + Handler: func(ctx *RequestCtx) {}, + ConnState: func(conn net.Conn, state ConnState) { + states = append(states, state.String()) + }, + } + + ln := fasthttputil.NewInmemoryListener() + + serverCh := make(chan struct{}) + go func() { + if err := s.Serve(ln); err != nil { + t.Fatalf("unexpected error: %s", err) + } + close(serverCh) + }() + + clientCh := make(chan struct{}) + go func() { + c, err := ln.Dial() + if err != nil { + t.Fatalf("unexpected error: %s", err) + } + br := bufio.NewReader(c) + // Send 2 requests on the same connection. + for i := 0; i < 2; i++ { + if _, err = c.Write([]byte("GET / HTTP/1.1\r\nHost: aa\r\n\r\n")); err != nil { + t.Fatalf("unexpected error: %s", err) + } + var resp Response + if err := resp.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + if resp.StatusCode() != StatusOK { + t.Fatalf("unexpected status code: %d. Expecting %d", resp.StatusCode(), StatusOK) + } + } + if err := c.Close(); err != nil { + t.Fatalf("unexpected error: %s", err) + } + // Give the server a little bit of time to transition the connection to the close state. + time.Sleep(time.Millisecond * 100) + close(clientCh) + }() + + select { + case <-clientCh: + case <-time.After(time.Second): + t.Fatalf("timeout") + } + + if err := ln.Close(); err != nil { + t.Fatalf("unexpected error: %s", err) + } + + select { + case <-serverCh: + case <-time.After(time.Second): + t.Fatalf("timeout") + } + + // 2 requests so we go to active and idle twice. + expected := []string{"new", "active", "idle", "active", "idle", "closed"} + + if !reflect.DeepEqual(expected, states) { + t.Fatalf("wrong state, expected %s, got %s", expected, states) + } +} + +func TestSaveMultipartFile(t *testing.T) { + filea := "This is a test file." + fileb := strings.Repeat("test", 64) + + mr := multipart.NewReader(strings.NewReader(""+ + "--foo\r\n"+ + "Content-Disposition: form-data; name=\"filea\"; filename=\"filea.txt\"\r\n"+ + "Content-Type: text/plain\r\n"+ + "\r\n"+ + filea+"\r\n"+ + "--foo\r\n"+ + "Content-Disposition: form-data; name=\"fileb\"; filename=\"fileb.txt\"\r\n"+ + "Content-Type: text/plain\r\n"+ + "\r\n"+ + fileb+"\r\n"+ + "--foo--\r\n", + ), "foo") + + f, err := mr.ReadForm(64) + if err != nil { + t.Fatal(err) + } + + if err := SaveMultipartFile(f.File["filea"][0], "filea.txt"); err != nil { + t.Fatal(err) + } + defer os.Remove("filea.txt") + + if c, err := ioutil.ReadFile("filea.txt"); err != nil { + t.Fatal(err) + } else if string(c) != filea { + t.Fatalf("filea changed expeced %q got %q", filea, c) + } + + // Make sure fileb was saved to a file. + if ff, err := f.File["fileb"][0].Open(); err != nil { + t.Fatalf("expected FileHeader.Open to work") + } else if _, ok := ff.(*os.File); !ok { + t.Fatalf("expected fileb to be an os.File") + } else { + ff.Close() + } + + if err := SaveMultipartFile(f.File["fileb"][0], "fileb.txt"); err != nil { + t.Fatal(err) + } + defer os.Remove("fileb.txt") + + if c, err := ioutil.ReadFile("fileb.txt"); err != nil { + t.Fatal(err) + } else if string(c) != fileb { + t.Fatalf("fileb changed expeced %q got %q", fileb, c) + } +} + +func TestServerName(t *testing.T) { + s := &Server{ + Handler: func(ctx *RequestCtx) { + }, + } + + getReponse := func() []byte { + rw := &readWriter{} + rw.r.WriteString("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n") + + ch := make(chan error) + go func() { + ch <- s.ServeConn(rw) + }() + + select { + case err := <-ch: + if err != nil { + t.Fatalf("Unexpected error from serveConn: %s", err) + } + case <-time.After(100 * time.Millisecond): + t.Fatalf("timeout") + } + + resp, err := ioutil.ReadAll(&rw.w) + if err != nil { + t.Fatalf("Unexpected error from ReadAll: %s", err) + } + + return resp + } + + resp := getReponse() + if !bytes.Contains(resp, []byte("\r\nServer: "+string(defaultServerName)+"\r\n")) { + t.Fatalf("Unexpected response %q expected Server: "+string(defaultServerName), resp) + } + + // We can't just overwrite s.Name as fasthttp caches the name in an atomic.Value + s = &Server{ + Handler: func(ctx *RequestCtx) { + }, + Name: "foobar", + } + + resp = getReponse() + if !bytes.Contains(resp, []byte("\r\nServer: foobar\r\n")) { + t.Fatalf("Unexpected response %q expected Server: foobar", resp) + } + + s = &Server{ + Handler: func(ctx *RequestCtx) { + }, + NoDefaultServerHeader: true, + NoDefaultContentType: true, + } + + resp = getReponse() + if bytes.Contains(resp, []byte("\r\nServer: ")) { + t.Fatalf("Unexpected response %q expected no Server header", resp) + } + + if bytes.Contains(resp, []byte("\r\nContent-Type: ")) { + t.Fatalf("Unexpected response %q expected no Content-Type header", resp) + } +} + func TestRequestCtxString(t *testing.T) { var ctx RequestCtx @@ -35,80 +233,48 @@ func TestRequestCtxString(t *testing.T) { } func TestServerErrSmallBuffer(t *testing.T) { - logger := &customLogger{} s := &Server{ Handler: func(ctx *RequestCtx) { ctx.WriteString("shouldn't be never called") }, ReadBufferSize: 20, - Logger: logger, } - ln := fasthttputil.NewInmemoryListener() - serverCh := make(chan error, 1) + rw := &readWriter{} + rw.r.WriteString("GET / HTTP/1.1\r\nHost: aabb.com\r\nVERY-long-Header: sdfdfsd dsf dsaf dsf df fsd\r\n\r\n") + + ch := make(chan error) go func() { - err := s.Serve(ln) - serverCh <- err + ch <- s.ServeConn(rw) }() - clientCh := make(chan error, 1) - go func() { - c, err := ln.Dial() - if err != nil { - clientCh <- fmt.Errorf("unexpected error: %s", err) - return - } - _, err = c.Write([]byte("GET / HTTP/1.1\r\nHost: aabb.com\r\nVERY-long-Header: sdfdfsd dsf dsaf dsf df fsd\r\n\r\n")) - if err != nil { - clientCh <- fmt.Errorf("unexpected error when sending request: %s", err) - return - } - br := bufio.NewReader(c) - var resp Response - if err = resp.Read(br); err != nil { - clientCh <- fmt.Errorf("unexpected error: %s", err) - return - } - statusCode := resp.StatusCode() - if statusCode != StatusRequestHeaderFieldsTooLarge { - clientCh <- fmt.Errorf("unexpected status code: %d. Expecting %d", statusCode, StatusRequestHeaderFieldsTooLarge) - return - } - if !resp.ConnectionClose() { - clientCh <- fmt.Errorf("missing 'Connection: close' response header") - return - } - clientCh <- nil - }() - - var err error - - // wait for the client + var serverErr error select { - case <-time.After(time.Second): - t.Fatalf("timeout when waiting for the client. Server log: %q", logger.out) - case err = <-clientCh: - if err != nil { - t.Fatalf("unexpected client error: %s. Server log: %q", err, logger.out) - } + case serverErr = <-ch: + case <-time.After(100 * time.Millisecond): + t.Fatalf("timeout") } - // wait for the server - if err := ln.Close(); err != nil { - t.Fatalf("unexpected error: %s. Server log: %q", err, logger.out) + if serverErr == nil { + t.Fatalf("expected error") } - select { - case <-time.After(time.Second): - t.Fatalf("timeout when waiting for the server. Server log: %q", logger.out) - case err = <-serverCh: - if err != nil { - t.Fatalf("unexpected server error: %s. Server log: %q", err, logger.out) - } + + br := bufio.NewReader(&rw.w) + var resp Response + if err := resp.Read(br); err != nil { + t.Fatalf("unexpected error: %s", err) + } + statusCode := resp.StatusCode() + if statusCode != StatusRequestHeaderFieldsTooLarge { + t.Fatalf("unexpected status code: %d. Expecting %d", statusCode, StatusRequestHeaderFieldsTooLarge) + } + if !resp.ConnectionClose() { + t.Fatalf("missing 'Connection: close' response header") } expectedErr := errSmallBuffer.Error() - if !strings.Contains(logger.out, expectedErr) { - t.Fatalf("unexpected log output: %q. Expecting %q", logger.out, expectedErr) + if !strings.Contains(serverErr.Error(), expectedErr) { + t.Fatalf("unexpected log output: %v. Expecting %q", serverErr, expectedErr) } } @@ -631,6 +797,52 @@ func TestServerWriteFastError(t *testing.T) { } } +func TestServerTLS(t *testing.T) { + text := []byte("Make fasthttp great again") + ln := fasthttputil.NewInmemoryListener() + + certFile := "./ssl-cert-snakeoil.pem" + keyFile := "./ssl-cert-snakeoil.key" + + s := &Server{ + Handler: func(ctx *RequestCtx) { + ctx.Write(text) + }, + } + + err := s.AppendCert(certFile, keyFile) + if err != nil { + t.Fatal(err) + } + go func() { + err = s.ServeTLS(ln, "", "") + if err != nil { + t.Fatal(err) + } + }() + + c := &Client{ + ReadTimeout: time.Second * 2, + Dial: func(addr string) (net.Conn, error) { + return ln.Dial() + }, + TLSConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + + req, res := AcquireRequest(), AcquireResponse() + req.SetRequestURI("https://some.url") + + err = c.Do(req, res) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(text, res.Body()) { + t.Fatal("error transmitting information") + } +} + func TestServerServeTLSEmbed(t *testing.T) { ln := fasthttputil.NewInmemoryListener() @@ -828,6 +1040,37 @@ Connection: close } } +func TestServerGetWithContent(t *testing.T) { + h := func(ctx *RequestCtx) { + ctx.Success("foo/bar", []byte("success")) + } + s := &Server{ + Handler: h, + } + + rw := &readWriter{} + rw.r.WriteString("GET / HTTP/1.1\r\nHost: mm.com\r\nContent-Length: 5\r\n\r\nabcde") + + ch := make(chan error) + go func() { + ch <- s.ServeConn(rw) + }() + + select { + case err := <-ch: + if err != nil { + t.Fatalf("Unexpected error from serveConn: %s.", err) + } + case <-time.After(100 * time.Millisecond): + t.Fatalf("timeout") + } + + resp := rw.w.String() + if !strings.HasSuffix(resp, "success") { + t.Fatalf("unexpected response %s.", resp) + } +} + func TestServerDisableHeaderNamesNormalizing(t *testing.T) { headerName := "CASE-senSITive-HEAder-NAME" headerNameLower := strings.ToLower(headerName) @@ -1822,7 +2065,7 @@ func TestTimeoutHandlerTimeout(t *testing.T) { go func() { conn, err := ln.Dial() if err != nil { - t.Fatalf("unexepcted error: %s", err) + t.Fatalf("unexpected error: %s", err) } if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { t.Fatalf("unexpected error: %s", err) @@ -1886,8 +2129,8 @@ func TestServerGetOnly(t *testing.T) { if err == nil { t.Fatalf("expecting error") } - if err != errGetOnly { - t.Fatalf("Unexpected error from serveConn: %s. Expecting %s", err, errGetOnly) + if err != ErrGetOnly { + t.Fatalf("Unexpected error from serveConn: %s. Expecting %s", err, ErrGetOnly) } case <-time.After(100 * time.Millisecond): t.Fatalf("timeout") @@ -2477,6 +2720,143 @@ func TestServeConnMultiRequests(t *testing.T) { verifyResponse(t, br, 200, "aaa", "requestURI=/abc, host=foobar.com") } +func TestShutdown(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + s := &Server{ + Handler: func(ctx *RequestCtx) { + time.Sleep(time.Millisecond * 500) + ctx.Success("aaa/bbb", []byte("real response")) + }, + } + serveCh := make(chan struct{}) + go func() { + if err := s.Serve(ln); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + _, err := ln.Dial() + if err == nil { + t.Fatalf("server is still listening") + } + serveCh <- struct{}{} + }() + clientCh := make(chan struct{}) + go func() { + conn, err := ln.Dial() + if err != nil { + t.Fatalf("unexepcted error: %s", err) + } + if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { + t.Fatalf("unexpected error: %s", err) + } + br := bufio.NewReader(conn) + verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") + clientCh <- struct{}{} + }() + time.Sleep(time.Millisecond * 100) + shutdownCh := make(chan struct{}) + go func() { + if err := s.Shutdown(); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + shutdownCh <- struct{}{} + }() + done := 0 + for { + select { + case <-time.After(time.Second): + t.Fatalf("shutdown took too long") + case <-serveCh: + done++ + case <-clientCh: + done++ + case <-shutdownCh: + done++ + } + if done == 3 { + return + } + } +} + +func TestShutdownReuse(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + s := &Server{ + Handler: func(ctx *RequestCtx) { + ctx.Success("aaa/bbb", []byte("real response")) + }, + ReadTimeout: time.Second, + Logger: &customLogger{}, // Ignore log output. + } + go func() { + if err := s.Serve(ln); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + }() + conn, err := ln.Dial() + if err != nil { + t.Fatalf("unexepcted error: %s", err) + } + if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { + t.Fatalf("unexpected error: %s", err) + } + br := bufio.NewReader(conn) + verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") + if err := s.Shutdown(); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + ln = fasthttputil.NewInmemoryListener() + go func() { + if err := s.Serve(ln); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + }() + conn, err = ln.Dial() + if err != nil { + t.Fatalf("unexepcted error: %s", err) + } + if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { + t.Fatalf("unexpected error: %s", err) + } + br = bufio.NewReader(conn) + verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") + if err := s.Shutdown(); err != nil { + t.Fatalf("unexepcted error: %s", err) + } +} + +func TestShutdownDone(t *testing.T) { + ln := fasthttputil.NewInmemoryListener() + s := &Server{ + Handler: func(ctx *RequestCtx) { + <-ctx.Done() + ctx.Success("aaa/bbb", []byte("real response")) + }, + } + go func() { + if err := s.Serve(ln); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + }() + conn, err := ln.Dial() + if err != nil { + t.Fatalf("unexepcted error: %s", err) + } + if _, err = conn.Write([]byte("GET / HTTP/1.1\r\nHost: google.com\r\n\r\n")); err != nil { + t.Fatalf("unexpected error: %s", err) + } + go func() { + // Shutdown won't return if the connection doesn't close, + // which doesn't happen until we read the response. + if err := s.Shutdown(); err != nil { + t.Fatalf("unexepcted error: %s", err) + } + }() + // We can only reach this point and get a valid response + // if reading from ctx.Done() returned. + br := bufio.NewReader(conn) + verifyResponse(t, br, StatusOK, "aaa/bbb", "real response") +} + func verifyResponse(t *testing.T, r *bufio.Reader, expectedStatusCode int, expectedContentType, expectedBody string) { var resp Response if err := resp.Read(r); err != nil { diff --git a/vendor/github.com/valyala/fasthttp/stackless/writer.go b/vendor/github.com/valyala/fasthttp/stackless/writer.go index 9b9ff09..c2053f9 100644 --- a/vendor/github.com/valyala/fasthttp/stackless/writer.go +++ b/vendor/github.com/valyala/fasthttp/stackless/writer.go @@ -3,8 +3,9 @@ package stackless import ( "errors" "fmt" - "github.com/valyala/bytebufferpool" "io" + + "github.com/valyala/bytebufferpool" ) // Writer is an interface stackless writer must conform to. diff --git a/vendor/github.com/valyala/fasthttp/strings.go b/vendor/github.com/valyala/fasthttp/strings.go index ebfa3ed..0f05af5 100644 --- a/vendor/github.com/valyala/fasthttp/strings.go +++ b/vendor/github.com/valyala/fasthttp/strings.go @@ -22,11 +22,15 @@ var ( strResponseContinue = []byte("HTTP/1.1 100 Continue\r\n\r\n") - strGet = []byte("GET") - strHead = []byte("HEAD") - strPost = []byte("POST") - strPut = []byte("PUT") - strDelete = []byte("DELETE") + strGet = []byte("GET") + strHead = []byte("HEAD") + strPost = []byte("POST") + strPut = []byte("PUT") + strDelete = []byte("DELETE") + strConnect = []byte("CONNECT") + strOptions = []byte("OPTIONS") + strTrace = []byte("TRACE") + strPatch = []byte("PATCH") strExpect = []byte("Expect") strConnection = []byte("Connection") @@ -49,12 +53,16 @@ var ( strRange = []byte("Range") strContentRange = []byte("Content-Range") - strCookieExpires = []byte("expires") - strCookieDomain = []byte("domain") - strCookiePath = []byte("path") - strCookieHTTPOnly = []byte("HttpOnly") - strCookieSecure = []byte("secure") - + strCookieExpires = []byte("expires") + strCookieDomain = []byte("domain") + strCookiePath = []byte("path") + strCookieHTTPOnly = []byte("HttpOnly") + strCookieSecure = []byte("secure") + strCookieMaxAge = []byte("max-age") + strCookieSameSite = []byte("SameSite") + strCookieSameSiteLax = []byte("Lax") + strCookieSameSiteStrict = []byte("Strict") + strClose = []byte("close") strGzip = []byte("gzip") strDeflate = []byte("deflate") diff --git a/vendor/github.com/valyala/fasthttp/testdata/test.png b/vendor/github.com/valyala/fasthttp/testdata/test.png new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/vendor/github.com/valyala/fasthttp/testdata/test.png @@ -0,0 +1 @@ + diff --git a/vendor/github.com/valyala/fasthttp/uri.go b/vendor/github.com/valyala/fasthttp/uri.go index 37572f5..d536f59 100644 --- a/vendor/github.com/valyala/fasthttp/uri.go +++ b/vendor/github.com/valyala/fasthttp/uri.go @@ -180,7 +180,7 @@ func (u *URI) Reset() { u.parsedQueryArgs = false // There is no need in u.fullURI = u.fullURI[:0], since full uri - // is calucalted on each call to FullURI(). + // is calculated on each call to FullURI(). // There is no need in u.requestURI = u.requestURI[:0], since requestURI // is calculated on each call to RequestURI(). diff --git a/vendor/github.com/valyala/fasthttp/uri_test.go b/vendor/github.com/valyala/fasthttp/uri_test.go index beb4f7d..dcba56e 100644 --- a/vendor/github.com/valyala/fasthttp/uri_test.go +++ b/vendor/github.com/valyala/fasthttp/uri_test.go @@ -275,7 +275,7 @@ func TestURIParse(t *testing.T) { // encoded path testURIParse(t, &u, "aa.com", "/Test%20+%20%D0%BF%D1%80%D0%B8?asdf=%20%20&s=12#sdf", - "http://aa.com/Test%20%2B%20%D0%BF%D1%80%D0%B8?asdf=%20%20&s=12#sdf", "aa.com", "/Test + при", "/Test%20+%20%D0%BF%D1%80%D0%B8", "asdf=%20%20&s=12", "sdf") + "http://aa.com/Test%20+%20%D0%BF%D1%80%D0%B8?asdf=%20%20&s=12#sdf", "aa.com", "/Test + при", "/Test%20+%20%D0%BF%D1%80%D0%B8", "asdf=%20%20&s=12", "sdf") // host in uppercase testURIParse(t, &u, "FOObar.COM", "/bC?De=F#Gh", diff --git a/vendor/github.com/valyala/fasthttp/workerpool.go b/vendor/github.com/valyala/fasthttp/workerpool.go index cf602e0..bfd297c 100644 --- a/vendor/github.com/valyala/fasthttp/workerpool.go +++ b/vendor/github.com/valyala/fasthttp/workerpool.go @@ -16,7 +16,7 @@ import ( type workerPool struct { // Function for serving server connections. // It must leave c unclosed. - WorkerFunc func(c net.Conn) error + WorkerFunc ServeHandler MaxWorkersCount int @@ -35,6 +35,8 @@ type workerPool struct { stopCh chan struct{} workerChanPool sync.Pool + + connState func(net.Conn, ConnState) } type workerChan struct { @@ -187,7 +189,7 @@ func (wp *workerPool) getCh() *workerChan { } func (wp *workerPool) release(ch *workerChan) bool { - ch.lastUseTime = CoarseTimeNow() + ch.lastUseTime = time.Now() wp.lock.Lock() if wp.mustStop { wp.lock.Unlock() @@ -211,12 +213,16 @@ func (wp *workerPool) workerFunc(ch *workerChan) { errStr := err.Error() if wp.LogAllErrors || !(strings.Contains(errStr, "broken pipe") || strings.Contains(errStr, "reset by peer") || + strings.Contains(errStr, "request headers: small read buffer") || strings.Contains(errStr, "i/o timeout")) { wp.Logger.Printf("error when serving connection %q<->%q: %s", c.LocalAddr(), c.RemoteAddr(), err) } } - if err != errHijacked { + if err == errHijacked { + wp.connState(c, StateHijacked) + } else { c.Close() + wp.connState(c, StateClosed) } c = nil diff --git a/vendor/github.com/valyala/fasthttp/workerpool_test.go b/vendor/github.com/valyala/fasthttp/workerpool_test.go index 6f63f2c..05e1be0 100644 --- a/vendor/github.com/valyala/fasthttp/workerpool_test.go +++ b/vendor/github.com/valyala/fasthttp/workerpool_test.go @@ -94,6 +94,7 @@ func testWorkerPoolMaxWorkersCount(t *testing.T) { }, MaxWorkersCount: 10, Logger: defaultLogger, + connState: func(net.Conn, ConnState) {}, } wp.Start() diff --git a/vendor/github.com/xeipuuv/gojsonschema/.travis.yml b/vendor/github.com/xeipuuv/gojsonschema/.travis.yml index 37ec167..09ee789 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/.travis.yml +++ b/vendor/github.com/xeipuuv/gojsonschema/.travis.yml @@ -1,6 +1,11 @@ language: go go: - - 1.5 + - "1.5" + - "1.6" + - "1.7" + - "1.8" + - "1.9" + - "1.10" before_install: - go get github.com/xeipuuv/gojsonreference - go get github.com/xeipuuv/gojsonpointer diff --git a/vendor/github.com/xeipuuv/gojsonschema/README.md b/vendor/github.com/xeipuuv/gojsonschema/README.md index b890eee..24ca347 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/README.md +++ b/vendor/github.com/xeipuuv/gojsonschema/README.md @@ -55,7 +55,6 @@ func main() { fmt.Printf("- %s\n", desc) } } - } @@ -149,6 +148,87 @@ To check the result : } ``` + +## Loading local schemas + +By default `file` and `http(s)` references to external schemas are loaded automatically via the file system or via http(s). An external schema can also be loaded using a `SchemaLoader`. + +```go + sl := gojsonschema.NewSchemaLoader() + loader1 := gojsonschema.NewStringLoader(`{ "type" : "string" }`) + err := sl.AddSchema("http://some_host.com/string.json", loader1) +``` + +Alternatively if your schema already has an `$id` you can use the `AddSchemas` function +```go + loader2 := gojsonschema.NewStringLoader(`{ + "$id" : "http://some_host.com/maxlength.json", + "maxLength" : 5 + }`) + err = sl.AddSchemas(loader2) +``` + +The main schema should be passed to the `Compile` function. This main schema can then directly reference the added schemas without needing to download them. +```go + loader3 := gojsonschema.NewStringLoader(`{ + "$id" : "http://some_host.com/main.json", + "allOf" : [ + { "$ref" : "http://some_host.com/string.json" }, + { "$ref" : "http://some_host.com/maxlength.json" } + ] + }`) + + schema, err := sl.Compile(loader3) + + documentLoader := gojsonschema.NewStringLoader(`"hello world"`) + + result, err := schema.Validate(documentLoader) +``` + +It's also possible to pass a `ReferenceLoader` to the `Compile` function that references a loaded schema. + +```go +err = sl.AddSchemas(loader3) +schema, err := sl.Compile(gojsonschema.NewReferenceLoader("http://some_host.com/main.json")) +``` + +Schemas added by `AddSchema` and `AddSchemas` are only validated when the entire schema is compiled, unless meta-schema validation is used. + +## Using a specific draft +By default `gojsonschema` will try to detect the draft of a schema by using the `$schema` keyword and parse it in a strict draft-04, draft-06 or draft-07 mode. If `$schema` is missing, or the draft version is not explicitely set, a hybrid mode is used which merges together functionality of all drafts into one mode. + +Autodectection can be turned off with the `AutoDetect` property. Specific draft versions can be specified with the `Draft` property. + +```go +sl := gojsonschema.NewSchemaLoader() +sl.Draft = gojsonschema.Draft7 +sl.AutoDetect = false +``` + +If autodetection is on (default), a draft-07 schema can savely reference draft-04 schemas and vice-versa, as long as `$schema` is specified in all schemas. + +## Meta-schema validation +Schemas that are added using the `AddSchema`, `AddSchemas` and `Compile` can be validated against their meta-schema by setting the `Validate` property. + +The following example will produce an error as `multipleOf` must be a number. If `Validate` is off (default), this error is only returned at the `Compile` step. + +```go +sl := gojsonschema.NewSchemaLoader() +sl.Validate = true +err := sl.AddSchemas(gojsonschema.NewStringLoader(`{ + $id" : "http://some_host.com/invalid.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "multipleOf" : true +}`)) + ``` +``` + ``` + +Errors returned by meta-schema validation are more readable and contain more information, which helps significantly if you are developing a schema. + +Meta-schema validation also works with a custom `$schema`. In case `$schema` is missing, or `AutoDetect` is set to `false`, the meta-schema of the used draft is used. + + ## Working with Errors The library handles string error codes which you can customize by creating your own gojsonschema.locale and setting it @@ -192,6 +272,8 @@ Note: An error of RequiredType has an err.Type() return value of "required" "number_gt": NumberGTError "number_lte": NumberLTEError "number_lt": NumberLTError + "condition_then" : ConditionThenError + "condition_else" : ConditionElseError **err.Value()**: *interface{}* Returns the value given @@ -233,10 +315,35 @@ Learn more about what types of template functions you can use in `ErrorTemplateF ## Formats JSON Schema allows for optional "format" property to validate instances against well-known formats. gojsonschema ships with all of the formats defined in the spec that you can use like this: + ````json {"type": "string", "format": "email"} ```` -Available formats: date-time, hostname, email, ipv4, ipv6, uri, uri-reference, uuid, regex. Some of the new formats in draft-06 and draft-07 are not yet implemented. + +Not all formats defined in draft-07 are available. Implemented formats are: + +* `date` +* `time` +* `date-time` +* `hostname`. Subdomains that start with a number are also supported, but this means that it doesn't strictly follow [RFC1034](http://tools.ietf.org/html/rfc1034#section-3.5) and has the implication that ipv4 addresses are also recognized as valid hostnames. +* `email`. Go's email parser deviates slightly from [RFC5322](https://tools.ietf.org/html/rfc5322). Includes unicode support. +* `idn-email`. Same caveat as `email`. +* `ipv4` +* `ipv6` +* `uri`. Includes unicode support. +* `uri-reference`. Includes unicode support. +* `iri` +* `iri-reference` +* `uri-template` +* `uuid` +* `regex`. Go uses the [RE2](https://github.com/google/re2/wiki/Syntax) engine and is not [ECMA262](http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf) compatible. +* `json-pointer` +* `relative-json-pointer` + +`email`, `uri` and `uri-reference` use the same validation code as their unicode counterparts `idn-email`, `iri` and `iri-reference`. If you rely on unicode support you should use the specific +unicode enabled formats for the sake of interoperability as other implementations might not support unicode in the regular formats. + +The validation code for `uri`, `idn-email` and their relatives use mostly standard library code. Go 1.5 and 1.6 contain some minor bugs with handling URIs and unicode. You are encouraged to use Go 1.7+ if you rely on these formats. For repetitive or more complex formats, you can create custom format checkers and add them to gojsonschema like this: @@ -293,6 +400,13 @@ func (f ValidUserIdFormatChecker) IsFormat(input interface{}) bool { gojsonschema.FormatCheckers.Add("ValidUserId", ValidUserIdFormatChecker{}) ```` +Formats can also be removed, for example if you want to override one of the formats that is defined by default. + +```go +gojsonschema.FormatCheckers.Remove("hostname") +``` + + ## Additional custom validation After the validation has run and you have the results, you may add additional errors using `Result.AddError`. This is useful to maintain the same format within the resultset instead diff --git a/vendor/github.com/xeipuuv/gojsonschema/draft.go b/vendor/github.com/xeipuuv/gojsonschema/draft.go new file mode 100644 index 0000000..bfde4a2 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/draft.go @@ -0,0 +1,118 @@ +// Copyright 2018 johandorland ( https://github.com/johandorland ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gojsonschema + +import ( + "errors" + "math" + "reflect" + + "github.com/xeipuuv/gojsonreference" +) + +type Draft int + +const ( + Draft4 Draft = 4 + Draft6 Draft = 6 + Draft7 Draft = 7 + Hybrid Draft = math.MaxInt32 +) + +type draftConfig struct { + Version Draft + MetaSchemaURL string + MetaSchema string +} +type draftConfigs []draftConfig + +var drafts draftConfigs + +func init() { + drafts = []draftConfig{ + draftConfig{ + Version: Draft4, + MetaSchemaURL: "http://json-schema.org/draft-04/schema", + MetaSchema: `{"id":"http://json-schema.org/draft-04/schema#","$schema":"http://json-schema.org/draft-04/schema#","description":"Core schema meta-schema","definitions":{"schemaArray":{"type":"array","minItems":1,"items":{"$ref":"#"}},"positiveInteger":{"type":"integer","minimum":0},"positiveIntegerDefault0":{"allOf":[{"$ref":"#/definitions/positiveInteger"},{"default":0}]},"simpleTypes":{"enum":["array","boolean","integer","null","number","object","string"]},"stringArray":{"type":"array","items":{"type":"string"},"minItems":1,"uniqueItems":true}},"type":"object","properties":{"id":{"type":"string"},"$schema":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"default":{},"multipleOf":{"type":"number","minimum":0,"exclusiveMinimum":true},"maximum":{"type":"number"},"exclusiveMaximum":{"type":"boolean","default":false},"minimum":{"type":"number"},"exclusiveMinimum":{"type":"boolean","default":false},"maxLength":{"$ref":"#/definitions/positiveInteger"},"minLength":{"$ref":"#/definitions/positiveIntegerDefault0"},"pattern":{"type":"string","format":"regex"},"additionalItems":{"anyOf":[{"type":"boolean"},{"$ref":"#"}],"default":{}},"items":{"anyOf":[{"$ref":"#"},{"$ref":"#/definitions/schemaArray"}],"default":{}},"maxItems":{"$ref":"#/definitions/positiveInteger"},"minItems":{"$ref":"#/definitions/positiveIntegerDefault0"},"uniqueItems":{"type":"boolean","default":false},"maxProperties":{"$ref":"#/definitions/positiveInteger"},"minProperties":{"$ref":"#/definitions/positiveIntegerDefault0"},"required":{"$ref":"#/definitions/stringArray"},"additionalProperties":{"anyOf":[{"type":"boolean"},{"$ref":"#"}],"default":{}},"definitions":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"properties":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"patternProperties":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"dependencies":{"type":"object","additionalProperties":{"anyOf":[{"$ref":"#"},{"$ref":"#/definitions/stringArray"}]}},"enum":{"type":"array","minItems":1,"uniqueItems":true},"type":{"anyOf":[{"$ref":"#/definitions/simpleTypes"},{"type":"array","items":{"$ref":"#/definitions/simpleTypes"},"minItems":1,"uniqueItems":true}]},"format":{"type":"string"},"allOf":{"$ref":"#/definitions/schemaArray"},"anyOf":{"$ref":"#/definitions/schemaArray"},"oneOf":{"$ref":"#/definitions/schemaArray"},"not":{"$ref":"#"}},"dependencies":{"exclusiveMaximum":["maximum"],"exclusiveMinimum":["minimum"]},"default":{}}`, + }, + draftConfig{ + Version: Draft6, + MetaSchemaURL: "http://json-schema.org/draft-06/schema", + MetaSchema: `{"$schema":"http://json-schema.org/draft-06/schema#","$id":"http://json-schema.org/draft-06/schema#","title":"Core schema meta-schema","definitions":{"schemaArray":{"type":"array","minItems":1,"items":{"$ref":"#"}},"nonNegativeInteger":{"type":"integer","minimum":0},"nonNegativeIntegerDefault0":{"allOf":[{"$ref":"#/definitions/nonNegativeInteger"},{"default":0}]},"simpleTypes":{"enum":["array","boolean","integer","null","number","object","string"]},"stringArray":{"type":"array","items":{"type":"string"},"uniqueItems":true,"default":[]}},"type":["object","boolean"],"properties":{"$id":{"type":"string","format":"uri-reference"},"$schema":{"type":"string","format":"uri"},"$ref":{"type":"string","format":"uri-reference"},"title":{"type":"string"},"description":{"type":"string"},"default":{},"examples":{"type":"array","items":{}},"multipleOf":{"type":"number","exclusiveMinimum":0},"maximum":{"type":"number"},"exclusiveMaximum":{"type":"number"},"minimum":{"type":"number"},"exclusiveMinimum":{"type":"number"},"maxLength":{"$ref":"#/definitions/nonNegativeInteger"},"minLength":{"$ref":"#/definitions/nonNegativeIntegerDefault0"},"pattern":{"type":"string","format":"regex"},"additionalItems":{"$ref":"#"},"items":{"anyOf":[{"$ref":"#"},{"$ref":"#/definitions/schemaArray"}],"default":{}},"maxItems":{"$ref":"#/definitions/nonNegativeInteger"},"minItems":{"$ref":"#/definitions/nonNegativeIntegerDefault0"},"uniqueItems":{"type":"boolean","default":false},"contains":{"$ref":"#"},"maxProperties":{"$ref":"#/definitions/nonNegativeInteger"},"minProperties":{"$ref":"#/definitions/nonNegativeIntegerDefault0"},"required":{"$ref":"#/definitions/stringArray"},"additionalProperties":{"$ref":"#"},"definitions":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"properties":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"patternProperties":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"dependencies":{"type":"object","additionalProperties":{"anyOf":[{"$ref":"#"},{"$ref":"#/definitions/stringArray"}]}},"propertyNames":{"$ref":"#"},"const":{},"enum":{"type":"array","minItems":1,"uniqueItems":true},"type":{"anyOf":[{"$ref":"#/definitions/simpleTypes"},{"type":"array","items":{"$ref":"#/definitions/simpleTypes"},"minItems":1,"uniqueItems":true}]},"format":{"type":"string"},"allOf":{"$ref":"#/definitions/schemaArray"},"anyOf":{"$ref":"#/definitions/schemaArray"},"oneOf":{"$ref":"#/definitions/schemaArray"},"not":{"$ref":"#"}},"default":{}}`, + }, + draftConfig{ + Version: Draft7, + MetaSchemaURL: "http://json-schema.org/draft-07/schema", + MetaSchema: `{"$schema":"http://json-schema.org/draft-07/schema#","$id":"http://json-schema.org/draft-07/schema#","title":"Core schema meta-schema","definitions":{"schemaArray":{"type":"array","minItems":1,"items":{"$ref":"#"}},"nonNegativeInteger":{"type":"integer","minimum":0},"nonNegativeIntegerDefault0":{"allOf":[{"$ref":"#/definitions/nonNegativeInteger"},{"default":0}]},"simpleTypes":{"enum":["array","boolean","integer","null","number","object","string"]},"stringArray":{"type":"array","items":{"type":"string"},"uniqueItems":true,"default":[]}},"type":["object","boolean"],"properties":{"$id":{"type":"string","format":"uri-reference"},"$schema":{"type":"string","format":"uri"},"$ref":{"type":"string","format":"uri-reference"},"$comment":{"type":"string"},"title":{"type":"string"},"description":{"type":"string"},"default":true,"readOnly":{"type":"boolean","default":false},"examples":{"type":"array","items":true},"multipleOf":{"type":"number","exclusiveMinimum":0},"maximum":{"type":"number"},"exclusiveMaximum":{"type":"number"},"minimum":{"type":"number"},"exclusiveMinimum":{"type":"number"},"maxLength":{"$ref":"#/definitions/nonNegativeInteger"},"minLength":{"$ref":"#/definitions/nonNegativeIntegerDefault0"},"pattern":{"type":"string","format":"regex"},"additionalItems":{"$ref":"#"},"items":{"anyOf":[{"$ref":"#"},{"$ref":"#/definitions/schemaArray"}],"default":true},"maxItems":{"$ref":"#/definitions/nonNegativeInteger"},"minItems":{"$ref":"#/definitions/nonNegativeIntegerDefault0"},"uniqueItems":{"type":"boolean","default":false},"contains":{"$ref":"#"},"maxProperties":{"$ref":"#/definitions/nonNegativeInteger"},"minProperties":{"$ref":"#/definitions/nonNegativeIntegerDefault0"},"required":{"$ref":"#/definitions/stringArray"},"additionalProperties":{"$ref":"#"},"definitions":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"properties":{"type":"object","additionalProperties":{"$ref":"#"},"default":{}},"patternProperties":{"type":"object","additionalProperties":{"$ref":"#"},"propertyNames":{"format":"regex"},"default":{}},"dependencies":{"type":"object","additionalProperties":{"anyOf":[{"$ref":"#"},{"$ref":"#/definitions/stringArray"}]}},"propertyNames":{"$ref":"#"},"const":true,"enum":{"type":"array","items":true,"minItems":1,"uniqueItems":true},"type":{"anyOf":[{"$ref":"#/definitions/simpleTypes"},{"type":"array","items":{"$ref":"#/definitions/simpleTypes"},"minItems":1,"uniqueItems":true}]},"format":{"type":"string"},"contentMediaType":{"type":"string"},"contentEncoding":{"type":"string"},"if":{"$ref":"#"},"then":{"$ref":"#"},"else":{"$ref":"#"},"allOf":{"$ref":"#/definitions/schemaArray"},"anyOf":{"$ref":"#/definitions/schemaArray"},"oneOf":{"$ref":"#/definitions/schemaArray"},"not":{"$ref":"#"}},"default":true}`, + }, + } +} + +func (dc draftConfigs) GetMetaSchema(url string) string { + for _, config := range dc { + if config.MetaSchemaURL == url { + return config.MetaSchema + } + } + return "" +} +func (dc draftConfigs) GetDraftVersion(url string) *Draft { + for _, config := range dc { + if config.MetaSchemaURL == url { + return &config.Version + } + } + return nil +} +func (dc draftConfigs) GetSchemaURL(draft Draft) string { + for _, config := range dc { + if config.Version == draft { + return config.MetaSchemaURL + } + } + return "" +} + +func parseSchemaURL(documentNode interface{}) (string, *Draft, error) { + + if isKind(documentNode, reflect.Bool) { + return "", nil, nil + } + m := documentNode.(map[string]interface{}) + + if existsMapKey(m, KEY_SCHEMA) { + if !isKind(m[KEY_SCHEMA], reflect.String) { + return "", nil, errors.New(formatErrorDescription( + Locale.MustBeOfType(), + ErrorDetails{ + "key": KEY_SCHEMA, + "type": TYPE_STRING, + }, + )) + } + + schemaReference, err := gojsonreference.NewJsonReference(m[KEY_SCHEMA].(string)) + + if err != nil { + return "", nil, err + } + + schema := schemaReference.String() + + return schema, drafts.GetDraftVersion(schema), nil + } + + return "", nil, nil +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/errors.go b/vendor/github.com/xeipuuv/gojsonschema/errors.go index 2f326a2..2f01a1c 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/errors.go +++ b/vendor/github.com/xeipuuv/gojsonschema/errors.go @@ -81,7 +81,7 @@ type ( ResultErrorFields } - // ItemsMustBeUniqueError. ErrorDetails: type + // ItemsMustBeUniqueError. ErrorDetails: type, i, j ItemsMustBeUniqueError struct { ResultErrorFields } diff --git a/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go b/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go index 20ead1f..26217fc 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go +++ b/vendor/github.com/xeipuuv/gojsonschema/format_checkers.go @@ -2,9 +2,11 @@ package gojsonschema import ( "net" + "net/mail" "net/url" "regexp" "strings" + "sync" "time" ) @@ -51,12 +53,19 @@ type ( // http://tools.ietf.org/html/rfc3339#section-5.6 DateTimeFormatChecker struct{} + DateFormatChecker struct{} + + TimeFormatChecker struct{} + // URIFormatChecker validates a URI with a valid Scheme per RFC3986 URIFormatChecker struct{} // URIReferenceFormatChecker validates a URI or relative-reference per RFC3986 URIReferenceFormatChecker struct{} + // URITemplateFormatChecker validates a URI template per RFC6570 + URITemplateFormatChecker struct{} + // HostnameFormatChecker validates a hostname is in the correct format HostnameFormatChecker struct{} @@ -65,6 +74,12 @@ type ( // RegexFormatChecker validates a regex is in the correct format RegexFormatChecker struct{} + + // JSONPointerFormatChecker validates a JSON Pointer per RFC6901 + JSONPointerFormatChecker struct{} + + // RelativeJSONPointerFormatChecker validates a relative JSON Pointer is in the correct format + RelativeJSONPointerFormatChecker struct{} ) var ( @@ -72,45 +87,65 @@ var ( // so library users can add custom formatters FormatCheckers = FormatCheckerChain{ formatters: map[string]FormatChecker{ - "date-time": DateTimeFormatChecker{}, - "hostname": HostnameFormatChecker{}, - "email": EmailFormatChecker{}, - "ipv4": IPV4FormatChecker{}, - "ipv6": IPV6FormatChecker{}, - "uri": URIFormatChecker{}, - "uri-reference": URIReferenceFormatChecker{}, - "uuid": UUIDFormatChecker{}, - "regex": RegexFormatChecker{}, + "date": DateFormatChecker{}, + "time": TimeFormatChecker{}, + "date-time": DateTimeFormatChecker{}, + "hostname": HostnameFormatChecker{}, + "email": EmailFormatChecker{}, + "idn-email": EmailFormatChecker{}, + "ipv4": IPV4FormatChecker{}, + "ipv6": IPV6FormatChecker{}, + "uri": URIFormatChecker{}, + "uri-reference": URIReferenceFormatChecker{}, + "iri": URIFormatChecker{}, + "iri-reference": URIReferenceFormatChecker{}, + "uri-template": URITemplateFormatChecker{}, + "uuid": UUIDFormatChecker{}, + "regex": RegexFormatChecker{}, + "json-pointer": JSONPointerFormatChecker{}, + "relative-json-pointer": RelativeJSONPointerFormatChecker{}, }, } - // Regex credit: https://github.com/asaskevich/govalidator - rxEmail = regexp.MustCompile("^(((([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+(\\.([a-zA-Z]|\\d|[!#\\$%&'\\*\\+\\-\\/=\\?\\^_`{\\|}~]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])+)*)|((\\x22)((((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(([\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x7f]|\\x21|[\\x23-\\x5b]|[\\x5d-\\x7e]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(\\([\\x01-\\x09\\x0b\\x0c\\x0d-\\x7f]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}]))))*(((\\x20|\\x09)*(\\x0d\\x0a))?(\\x20|\\x09)+)?(\\x22)))@((([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|\\d|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.)+(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])|(([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])([a-zA-Z]|\\d|-|\\.|_|~|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])*([a-zA-Z]|[\\x{00A0}-\\x{D7FF}\\x{F900}-\\x{FDCF}\\x{FDF0}-\\x{FFEF}])))\\.?$") - // Regex credit: https://www.socketloop.com/tutorials/golang-validate-hostname rxHostname = regexp.MustCompile(`^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`) + // Use a regex to make sure curly brackets are balanced properly after validating it as a AURI + rxURITemplate = regexp.MustCompile("^([^{]*({[^}]*})?)*$") + rxUUID = regexp.MustCompile("^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$") + + rxJSONPointer = regexp.MustCompile("^(?:/(?:[^~/]|~0|~1)*)*$") + + rxRelJSONPointer = regexp.MustCompile("^(?:0|[1-9][0-9]*)(?:#|(?:/(?:[^~/]|~0|~1)*)*)$") + + lock = new(sync.Mutex) ) // Add adds a FormatChecker to the FormatCheckerChain // The name used will be the value used for the format key in your json schema func (c *FormatCheckerChain) Add(name string, f FormatChecker) *FormatCheckerChain { + lock.Lock() c.formatters[name] = f + lock.Unlock() return c } // Remove deletes a FormatChecker from the FormatCheckerChain (if it exists) func (c *FormatCheckerChain) Remove(name string) *FormatCheckerChain { + lock.Lock() delete(c.formatters, name) + lock.Unlock() return c } // Has checks to see if the FormatCheckerChain holds a FormatChecker with the given name func (c *FormatCheckerChain) Has(name string) bool { + lock.Lock() _, ok := c.formatters[name] + lock.Unlock() return ok } @@ -134,7 +169,9 @@ func (f EmailFormatChecker) IsFormat(input interface{}) bool { return false } - return rxEmail.MatchString(asString) + _, err := mail.ParseAddress(asString) + + return err == nil } // Credit: https://github.com/asaskevich/govalidator @@ -185,6 +222,29 @@ func (f DateTimeFormatChecker) IsFormat(input interface{}) bool { return false } +func (f DateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + _, err := time.Parse("2006-01-02", asString) + return err == nil +} + +func (f TimeFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + if _, err := time.Parse("15:04:05Z07:00", asString); err == nil { + return true + } + + _, err := time.Parse("15:04:05", asString) + return err == nil +} + func (f URIFormatChecker) IsFormat(input interface{}) bool { asString, ok := input.(string) @@ -193,11 +253,12 @@ func (f URIFormatChecker) IsFormat(input interface{}) bool { } u, err := url.Parse(asString) + if err != nil || u.Scheme == "" { return false } - return true + return !strings.Contains(asString, `\`) } func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool { @@ -208,7 +269,21 @@ func (f URIReferenceFormatChecker) IsFormat(input interface{}) bool { } _, err := url.Parse(asString) - return err == nil + return err == nil && !strings.Contains(asString, `\`) +} + +func (f URITemplateFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + u, err := url.Parse(asString) + if err != nil || strings.Contains(asString, `\`) { + return false + } + + return rxURITemplate.MatchString(u.Path) } func (f HostnameFormatChecker) IsFormat(input interface{}) bool { @@ -248,3 +323,21 @@ func (f RegexFormatChecker) IsFormat(input interface{}) bool { } return true } + +func (f JSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxJSONPointer.MatchString(asString) +} + +func (f RelativeJSONPointerFormatChecker) IsFormat(input interface{}) bool { + asString, ok := input.(string) + if ok == false { + return false + } + + return rxRelJSONPointer.MatchString(asString) +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go b/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go index 5557025..cfa5f6a 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go +++ b/vendor/github.com/xeipuuv/gojsonschema/jsonLoader.go @@ -107,7 +107,7 @@ func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory { } // NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system. -func NewReferenceLoader(source string) *jsonReferenceLoader { +func NewReferenceLoader(source string) JSONLoader { return &jsonReferenceLoader{ fs: osFS, source: source, @@ -115,7 +115,7 @@ func NewReferenceLoader(source string) *jsonReferenceLoader { } // NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system. -func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) *jsonReferenceLoader { +func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) JSONLoader { return &jsonReferenceLoader{ fs: fs, source: source, @@ -166,6 +166,12 @@ func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) { func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) { + // returned cached versions for metaschemas for drafts 4, 6 and 7 + // for performance and allow for easier offline use + if metaSchema := drafts.GetMetaSchema(address); metaSchema != "" { + return decodeJsonUsingNumber(strings.NewReader(metaSchema)) + } + resp, err := http.Get(address) if err != nil { return nil, err @@ -182,7 +188,6 @@ func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) } return decodeJsonUsingNumber(bytes.NewReader(bodyBuff)) - } func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) { @@ -219,7 +224,7 @@ func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory { return &DefaultJSONLoaderFactory{} } -func NewStringLoader(source string) *jsonStringLoader { +func NewStringLoader(source string) JSONLoader { return &jsonStringLoader{source: source} } @@ -247,7 +252,7 @@ func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory { return &DefaultJSONLoaderFactory{} } -func NewBytesLoader(source []byte) *jsonBytesLoader { +func NewBytesLoader(source []byte) JSONLoader { return &jsonBytesLoader{source: source} } @@ -274,7 +279,7 @@ func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory { return &DefaultJSONLoaderFactory{} } -func NewGoLoader(source interface{}) *jsonGoLoader { +func NewGoLoader(source interface{}) JSONLoader { return &jsonGoLoader{source: source} } @@ -295,12 +300,12 @@ type jsonIOLoader struct { buf *bytes.Buffer } -func NewReaderLoader(source io.Reader) (*jsonIOLoader, io.Reader) { +func NewReaderLoader(source io.Reader) (JSONLoader, io.Reader) { buf := &bytes.Buffer{} return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf) } -func NewWriterLoader(source io.Writer) (*jsonIOLoader, io.Writer) { +func NewWriterLoader(source io.Writer) (JSONLoader, io.Writer) { buf := &bytes.Buffer{} return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf) } diff --git a/vendor/github.com/xeipuuv/gojsonschema/jsonschema_test.go b/vendor/github.com/xeipuuv/gojsonschema/jsonschema_test.go index baf4910..5fac0c2 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/jsonschema_test.go +++ b/vendor/github.com/xeipuuv/gojsonschema/jsonschema_test.go @@ -17,10 +17,12 @@ package gojsonschema import ( "encoding/json" "fmt" + "io/ioutil" "net/http" "os" "path/filepath" "regexp" + "runtime" "strings" "testing" ) @@ -39,6 +41,84 @@ type jsonSchemaTestCase struct { Valid bool `json:"valid"` } +//Skip any directories not named appropiately +// filepath.Walk will also visit files in the root of the test directory +var testDirectories = regexp.MustCompile(`(draft\d+)`) +var draftMapping = map[string]Draft{ + "draft4": Draft4, + "draft6": Draft6, + "draft7": Draft7, +} + +func executeTests(t *testing.T, path string) error { + file, err := os.Open(path) + if err != nil { + t.Errorf("Error (%s)\n", err.Error()) + } + fmt.Println(file.Name()) + + var tests []jsonSchemaTest + d := json.NewDecoder(file) + d.UseNumber() + err = d.Decode(&tests) + + if err != nil { + t.Errorf("Error (%s)\n", err.Error()) + } + + draft := Hybrid + if m := testDirectories.FindString(path); m != "" { + draft = draftMapping[m] + } + + for _, test := range tests { + fmt.Println(" " + test.Description) + + if test.Disabled { + continue + } + + testSchemaLoader := NewRawLoader(test.Schema) + sl := NewSchemaLoader() + sl.Draft = draft + sl.Validate = true + testSchema, err := sl.Compile(testSchemaLoader) + + if err != nil { + t.Errorf("Error (%s)\n", err.Error()) + } + + for _, testCase := range test.Tests { + testDataLoader := NewRawLoader(testCase.Data) + result, err := testSchema.Validate(testDataLoader) + + if err != nil { + t.Errorf("Error (%s)\n", err.Error()) + } + + if result.Valid() != testCase.Valid { + schemaString, _ := marshalToJsonString(test.Schema) + testCaseString, _ := marshalToJsonString(testCase.Data) + + t.Errorf("Test failed : %s\n"+ + "%s.\n"+ + "%s.\n"+ + "expects: %t, given %t\n"+ + "Schema: %s\n"+ + "Data: %s\n", + file.Name(), + test.Description, + testCase.Description, + testCase.Valid, + result.Valid(), + *schemaString, + *testCaseString) + } + } + } + return nil +} + func TestSuite(t *testing.T) { wd, err := os.Getwd() @@ -55,10 +135,6 @@ func TestSuite(t *testing.T) { } }() - //Skip any directories not named appropiately - // filepath.Walk will also visit files in the root of the test directory - testDirectories := regexp.MustCompile(`^draft\d+$`) - err = filepath.Walk(wd, func(path string, fileInfo os.FileInfo, err error) error { if fileInfo.IsDir() && path != wd && !testDirectories.MatchString(fileInfo.Name()) { return filepath.SkipDir @@ -66,66 +142,45 @@ func TestSuite(t *testing.T) { if !strings.HasSuffix(fileInfo.Name(), ".json") { return nil } - file, err := os.Open(path) - if err != nil { - t.Errorf("Error (%s)\n", err.Error()) - } - fmt.Println(file.Name()) - - var tests []jsonSchemaTest - d := json.NewDecoder(file) - d.UseNumber() - err = d.Decode(&tests) - - if err != nil { - t.Errorf("Error (%s)\n", err.Error()) - } - - for _, test := range tests { - fmt.Println(" " + test.Description) - - if test.Disabled { - continue - } - - testSchemaLoader := NewRawLoader(test.Schema) - testSchema, err := NewSchema(testSchemaLoader) - - if err != nil { - t.Errorf("Error (%s)\n", err.Error()) - } - - for _, testCase := range test.Tests { - testDataLoader := NewRawLoader(testCase.Data) - result, err := testSchema.Validate(testDataLoader) - - if err != nil { - t.Errorf("Error (%s)\n", err.Error()) - } - - if result.Valid() != testCase.Valid { - schemaString, _ := marshalToJsonString(test.Schema) - testCaseString, _ := marshalToJsonString(testCase.Data) - - t.Errorf("Test failed : %s\n"+ - "%s.\n"+ - "%s.\n"+ - "expects: %t, given %t\n"+ - "Schema: %s\n"+ - "Data: %s\n", - file.Name(), - test.Description, - testCase.Description, - testCase.Valid, - result.Valid(), - *schemaString, - *testCaseString) - } - } - } - return nil + return executeTests(t, path) }) if err != nil { t.Errorf("Error (%s)\n", err.Error()) } } + +func TestFormats(t *testing.T) { + // Go 1.5 and 1.6 contain minor bugs in parsing URIs and unicode which makes the test suite fail uri and idn-email formats + // Therefore disable these tests for these go versions + if strings.HasPrefix(runtime.Version(), "go1.5") || strings.HasPrefix(runtime.Version(), "go1.6") { + return + } + + wd, err := os.Getwd() + if err != nil { + panic(err.Error()) + } + wd = filepath.Join(wd, "testdata") + + dirs, err := ioutil.ReadDir(wd) + + if err != nil { + panic(err.Error()) + } + + for _, dir := range dirs { + if testDirectories.MatchString(dir.Name()) { + formatsDirectory := filepath.Join(wd, dir.Name(), "optional", "format") + err = filepath.Walk(formatsDirectory, func(path string, fileInfo os.FileInfo, err error) error { + if fileInfo == nil || !strings.HasSuffix(fileInfo.Name(), ".json") { + return nil + } + return executeTests(t, path) + }) + + if err != nil { + t.Errorf("Error (%s)\n", err.Error()) + } + } + } +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/locales.go b/vendor/github.com/xeipuuv/gojsonschema/locales.go index 7bd9a9d..9b4570f 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/locales.go +++ b/vendor/github.com/xeipuuv/gojsonschema/locales.go @@ -147,7 +147,7 @@ func (l DefaultLocale) ArrayMaxItems() string { } func (l DefaultLocale) Unique() string { - return `{{.type}} items must be unique` + return `{{.type}} items[{{.i}},{{.j}}] must be unique` } func (l DefaultLocale) ArrayContains() string { diff --git a/vendor/github.com/xeipuuv/gojsonschema/result.go b/vendor/github.com/xeipuuv/gojsonschema/result.go index 7896f31..040d35c 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/result.go +++ b/vendor/github.com/xeipuuv/gojsonschema/result.go @@ -76,12 +76,6 @@ type ( // Field outputs the field name without the root context // i.e. firstName or person.firstName instead of (root).firstName or (root).person.firstName func (v *ResultErrorFields) Field() string { - if p, ok := v.Details()["property"]; ok { - if str, isString := p.(string); isString { - return str - } - } - return strings.TrimPrefix(v.context.String(), STRING_ROOT_SCHEMA_PROPERTY+".") } diff --git a/vendor/github.com/xeipuuv/gojsonschema/schema.go b/vendor/github.com/xeipuuv/gojsonschema/schema.go index 0746a59..323fe85 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/schema.go +++ b/vendor/github.com/xeipuuv/gojsonschema/schema.go @@ -46,41 +46,7 @@ var ( ) func NewSchema(l JSONLoader) (*Schema, error) { - ref, err := l.JsonReference() - if err != nil { - return nil, err - } - - d := Schema{} - d.pool = newSchemaPool(l.LoaderFactory()) - d.documentReference = ref - d.referencePool = newSchemaReferencePool() - - var spd *schemaPoolDocument - var doc interface{} - if ref.String() != "" { - // Get document from schema pool - spd, err = d.pool.GetDocument(d.documentReference) - if err != nil { - return nil, err - } - doc = spd.Document - } else { - // Load JSON directly - doc, err = l.LoadJSON() - if err != nil { - return nil, err - } - } - - d.pool.ParseReferences(doc, ref) - - err = d.parse(doc) - if err != nil { - return nil, err - } - - return &d, nil + return NewSchemaLoader().Compile(l) } type Schema struct { @@ -90,8 +56,8 @@ type Schema struct { referencePool *schemaReferencePool } -func (d *Schema) parse(document interface{}) error { - d.rootSchema = &subSchema{property: STRING_ROOT_SCHEMA_PROPERTY} +func (d *Schema) parse(document interface{}, draft Draft) error { + d.rootSchema = &subSchema{property: STRING_ROOT_SCHEMA_PROPERTY, draft: &draft} return d.parseSchema(document, d.rootSchema) } @@ -107,8 +73,15 @@ func (d *Schema) SetRootSchemaName(name string) { // func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) error { + if currentSchema.draft == nil { + if currentSchema.parent == nil { + return errors.New("Draft not set") + } + currentSchema.draft = currentSchema.parent.draft + } + // As of draft 6 "true" is equivalent to an empty schema "{}" and false equals "{"not":{}}" - if isKind(documentNode, reflect.Bool) { + if *currentSchema.draft >= Draft6 && isKind(documentNode, reflect.Bool) { b := documentNode.(bool) if b { documentNode = map[string]interface{}{} @@ -131,18 +104,6 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) if currentSchema.parent == nil { currentSchema.ref = &d.documentReference currentSchema.id = &d.documentReference - - if existsMapKey(m, KEY_SCHEMA) && false { - if !isKind(m[KEY_SCHEMA], reflect.String) { - return errors.New(formatErrorDescription( - Locale.InvalidType(), - ErrorDetails{ - "expected": TYPE_STRING, - "given": KEY_SCHEMA, - }, - )) - } - } } if currentSchema.id == nil && currentSchema.parent != nil { @@ -150,10 +111,19 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) } // In draft 6 the id keyword was renamed to $id - // Use the old id by default - keyID := KEY_ID_NEW - if existsMapKey(m, KEY_ID) { + // Hybrid mode uses the old id by default + var keyID string + + switch *currentSchema.draft { + case Draft4: keyID = KEY_ID + case Hybrid: + keyID = KEY_ID_NEW + if existsMapKey(m, KEY_ID) { + keyID = KEY_ID + } + default: + keyID = KEY_ID_NEW } if existsMapKey(m, keyID) && !isKind(m[keyID], reflect.String) { return errors.New(formatErrorDescription( @@ -378,7 +348,7 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) } // propertyNames - if existsMapKey(m, KEY_PROPERTY_NAMES) { + if existsMapKey(m, KEY_PROPERTY_NAMES) && *currentSchema.draft >= Draft6 { if isKind(m[KEY_PROPERTY_NAMES], reflect.Map, reflect.Bool) { newSchema := &subSchema{property: KEY_PROPERTY_NAMES, parent: currentSchema, ref: currentSchema.ref} currentSchema.propertyNames = newSchema @@ -483,7 +453,7 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) }, )) } - if multipleOfValue.Cmp(big.NewFloat(0)) <= 0 { + if multipleOfValue.Cmp(big.NewRat(0, 1)) <= 0 { return errors.New(formatErrorDescription( Locale.GreaterThanZero(), ErrorDetails{"number": KEY_MULTIPLE_OF}, @@ -504,25 +474,62 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) } if existsMapKey(m, KEY_EXCLUSIVE_MINIMUM) { - if isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + switch *currentSchema.draft { + case Draft4: + if !isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN, + "given": KEY_EXCLUSIVE_MINIMUM, + }, + )) + } if currentSchema.minimum == nil { return errors.New(formatErrorDescription( Locale.CannotBeUsedWithout(), ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": KEY_MINIMUM}, )) } - exclusiveMinimumValue := m[KEY_EXCLUSIVE_MINIMUM].(bool) - currentSchema.exclusiveMinimum = exclusiveMinimumValue - } else if isJsonNumber(m[KEY_EXCLUSIVE_MINIMUM]) { - minimumValue := mustBeNumber(m[KEY_EXCLUSIVE_MINIMUM]) - - currentSchema.minimum = minimumValue - currentSchema.exclusiveMinimum = true - } else { - return errors.New(formatErrorDescription( - Locale.InvalidType(), - ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MINIMUM}, - )) + if m[KEY_EXCLUSIVE_MINIMUM].(bool) { + currentSchema.exclusiveMinimum = currentSchema.minimum + currentSchema.minimum = nil + } + case Hybrid: + if isKind(m[KEY_EXCLUSIVE_MINIMUM], reflect.Bool) { + if currentSchema.minimum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MINIMUM, "y": KEY_MINIMUM}, + )) + } + if m[KEY_EXCLUSIVE_MINIMUM].(bool) { + currentSchema.exclusiveMinimum = currentSchema.minimum + currentSchema.minimum = nil + } + } else if isJsonNumber(m[KEY_EXCLUSIVE_MINIMUM]) { + currentSchema.exclusiveMinimum = mustBeNumber(m[KEY_EXCLUSIVE_MINIMUM]) + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + TYPE_NUMBER, + "given": KEY_EXCLUSIVE_MINIMUM, + }, + )) + } + default: + if isJsonNumber(m[KEY_EXCLUSIVE_MINIMUM]) { + currentSchema.exclusiveMinimum = mustBeNumber(m[KEY_EXCLUSIVE_MINIMUM]) + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_NUMBER, + "given": KEY_EXCLUSIVE_MINIMUM, + }, + )) + } } } @@ -538,34 +545,62 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) } if existsMapKey(m, KEY_EXCLUSIVE_MAXIMUM) { - if isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + switch *currentSchema.draft { + case Draft4: + if !isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN, + "given": KEY_EXCLUSIVE_MAXIMUM, + }, + )) + } if currentSchema.maximum == nil { return errors.New(formatErrorDescription( Locale.CannotBeUsedWithout(), ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": KEY_MAXIMUM}, )) } - exclusiveMaximumValue := m[KEY_EXCLUSIVE_MAXIMUM].(bool) - currentSchema.exclusiveMaximum = exclusiveMaximumValue - } else if isJsonNumber(m[KEY_EXCLUSIVE_MAXIMUM]) { - maximumValue := mustBeNumber(m[KEY_EXCLUSIVE_MAXIMUM]) - - currentSchema.maximum = maximumValue - currentSchema.exclusiveMaximum = true - } else { - return errors.New(formatErrorDescription( - Locale.InvalidType(), - ErrorDetails{"expected": TYPE_BOOLEAN + ", " + TYPE_NUMBER, "given": KEY_EXCLUSIVE_MAXIMUM}, - )) - } - } - - if currentSchema.minimum != nil && currentSchema.maximum != nil { - if currentSchema.minimum.Cmp(currentSchema.maximum) == 1 { - return errors.New(formatErrorDescription( - Locale.CannotBeGT(), - ErrorDetails{"x": KEY_MINIMUM, "y": KEY_MAXIMUM}, - )) + if m[KEY_EXCLUSIVE_MAXIMUM].(bool) { + currentSchema.exclusiveMaximum = currentSchema.maximum + currentSchema.maximum = nil + } + case Hybrid: + if isKind(m[KEY_EXCLUSIVE_MAXIMUM], reflect.Bool) { + if currentSchema.maximum == nil { + return errors.New(formatErrorDescription( + Locale.CannotBeUsedWithout(), + ErrorDetails{"x": KEY_EXCLUSIVE_MAXIMUM, "y": KEY_MAXIMUM}, + )) + } + if m[KEY_EXCLUSIVE_MAXIMUM].(bool) { + currentSchema.exclusiveMaximum = currentSchema.maximum + currentSchema.maximum = nil + } + } else if isJsonNumber(m[KEY_EXCLUSIVE_MAXIMUM]) { + currentSchema.exclusiveMaximum = mustBeNumber(m[KEY_EXCLUSIVE_MAXIMUM]) + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_BOOLEAN + "/" + TYPE_NUMBER, + "given": KEY_EXCLUSIVE_MAXIMUM, + }, + )) + } + default: + if isJsonNumber(m[KEY_EXCLUSIVE_MAXIMUM]) { + currentSchema.exclusiveMaximum = mustBeNumber(m[KEY_EXCLUSIVE_MAXIMUM]) + } else { + return errors.New(formatErrorDescription( + Locale.InvalidType(), + ErrorDetails{ + "expected": TYPE_NUMBER, + "given": KEY_EXCLUSIVE_MAXIMUM, + }, + )) + } } } @@ -755,7 +790,7 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) } } - if existsMapKey(m, KEY_CONTAINS) { + if existsMapKey(m, KEY_CONTAINS) && *currentSchema.draft >= Draft6 { newSchema := &subSchema{property: KEY_CONTAINS, parent: currentSchema, ref: currentSchema.ref} currentSchema.contains = newSchema err := d.parseSchema(m[KEY_CONTAINS], newSchema) @@ -766,7 +801,7 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) // validation : all - if existsMapKey(m, KEY_CONST) { + if existsMapKey(m, KEY_CONST) && *currentSchema.draft >= Draft6 { err := currentSchema.AddConst(m[KEY_CONST]) if err != nil { return err @@ -861,51 +896,53 @@ func (d *Schema) parseSchema(documentNode interface{}, currentSchema *subSchema) } } - if existsMapKey(m, KEY_IF) { - if isKind(m[KEY_IF], reflect.Map, reflect.Bool) { - newSchema := &subSchema{property: KEY_IF, parent: currentSchema, ref: currentSchema.ref} - currentSchema.SetIf(newSchema) - err := d.parseSchema(m[KEY_IF], newSchema) - if err != nil { - return err + if *currentSchema.draft >= Draft7 { + if existsMapKey(m, KEY_IF) { + if isKind(m[KEY_IF], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_IF, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetIf(newSchema) + err := d.parseSchema(m[KEY_IF], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_IF, "y": TYPE_OBJECT}, + )) } - } else { - return errors.New(formatErrorDescription( - Locale.MustBeOfAn(), - ErrorDetails{"x": KEY_IF, "y": TYPE_OBJECT}, - )) } - } - if existsMapKey(m, KEY_THEN) { - if isKind(m[KEY_THEN], reflect.Map, reflect.Bool) { - newSchema := &subSchema{property: KEY_THEN, parent: currentSchema, ref: currentSchema.ref} - currentSchema.SetThen(newSchema) - err := d.parseSchema(m[KEY_THEN], newSchema) - if err != nil { - return err + if existsMapKey(m, KEY_THEN) { + if isKind(m[KEY_THEN], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_THEN, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetThen(newSchema) + err := d.parseSchema(m[KEY_THEN], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_THEN, "y": TYPE_OBJECT}, + )) } - } else { - return errors.New(formatErrorDescription( - Locale.MustBeOfAn(), - ErrorDetails{"x": KEY_THEN, "y": TYPE_OBJECT}, - )) } - } - if existsMapKey(m, KEY_ELSE) { - if isKind(m[KEY_ELSE], reflect.Map, reflect.Bool) { - newSchema := &subSchema{property: KEY_ELSE, parent: currentSchema, ref: currentSchema.ref} - currentSchema.SetElse(newSchema) - err := d.parseSchema(m[KEY_ELSE], newSchema) - if err != nil { - return err + if existsMapKey(m, KEY_ELSE) { + if isKind(m[KEY_ELSE], reflect.Map, reflect.Bool) { + newSchema := &subSchema{property: KEY_ELSE, parent: currentSchema, ref: currentSchema.ref} + currentSchema.SetElse(newSchema) + err := d.parseSchema(m[KEY_ELSE], newSchema) + if err != nil { + return err + } + } else { + return errors.New(formatErrorDescription( + Locale.MustBeOfAn(), + ErrorDetails{"x": KEY_ELSE, "y": TYPE_OBJECT}, + )) } - } else { - return errors.New(formatErrorDescription( - Locale.MustBeOfAn(), - ErrorDetails{"x": KEY_ELSE, "y": TYPE_OBJECT}, - )) } } @@ -930,6 +967,7 @@ func (d *Schema) parseReference(documentNode interface{}, currentSchema *subSche newSchema.id = currentSchema.ref refdDocumentNode = dsp.Document + newSchema.draft = dsp.Draft if err != nil { return err diff --git a/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go b/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go new file mode 100644 index 0000000..95e0568 --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/schemaLoader.go @@ -0,0 +1,203 @@ +// Copyright 2018 johandorland ( https://github.com/johandorland ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gojsonschema + +import ( + "bytes" + "errors" + + "github.com/xeipuuv/gojsonreference" +) + +type SchemaLoader struct { + pool *schemaPool + AutoDetect bool + Validate bool + Draft Draft +} + +func NewSchemaLoader() *SchemaLoader { + + ps := &SchemaLoader{ + pool: &schemaPool{ + schemaPoolDocuments: make(map[string]*schemaPoolDocument), + }, + AutoDetect: true, + Validate: false, + Draft: Hybrid, + } + ps.pool.autoDetect = &ps.AutoDetect + + return ps +} + +func (sl *SchemaLoader) validateMetaschema(documentNode interface{}) error { + + var ( + schema string + err error + ) + if sl.AutoDetect { + schema, _, err = parseSchemaURL(documentNode) + if err != nil { + return err + } + } + + // If no explicit "$schema" is used, use the default metaschema associated with the draft used + if schema == "" { + if sl.Draft == Hybrid { + return nil + } + schema = drafts.GetSchemaURL(sl.Draft) + } + + //Disable validation when loading the metaschema to prevent an infinite recursive loop + sl.Validate = false + + metaSchema, err := sl.Compile(NewReferenceLoader(schema)) + + if err != nil { + return err + } + + sl.Validate = true + + result := metaSchema.validateDocument(documentNode) + + if !result.Valid() { + var res bytes.Buffer + for _, err := range result.Errors() { + res.WriteString(err.String()) + res.WriteString("\n") + } + return errors.New(res.String()) + } + + return nil +} + +// AddSchemas adds an arbritrary amount of schemas to the schema cache. As this function does not require +// an explicit URL, every schema should contain an $id, so that it can be referenced by the main schema +func (sl *SchemaLoader) AddSchemas(loaders ...JSONLoader) error { + emptyRef, _ := gojsonreference.NewJsonReference("") + + for _, loader := range loaders { + doc, err := loader.LoadJSON() + + if err != nil { + return err + } + + if sl.Validate { + if err := sl.validateMetaschema(doc); err != nil { + return err + } + } + + // Directly use the Recursive function, so that it get only added to the schema pool by $id + // and not by the ref of the document as it's empty + if err = sl.pool.parseReferences(doc, emptyRef, false); err != nil { + return err + } + } + + return nil +} + +//AddSchema adds a schema under the provided URL to the schema cache +func (sl *SchemaLoader) AddSchema(url string, loader JSONLoader) error { + + ref, err := gojsonreference.NewJsonReference(url) + + if err != nil { + return err + } + + doc, err := loader.LoadJSON() + + if err != nil { + return err + } + + if sl.Validate { + if err := sl.validateMetaschema(doc); err != nil { + return err + } + } + + return sl.pool.parseReferences(doc, ref, true) +} + +func (sl *SchemaLoader) Compile(rootSchema JSONLoader) (*Schema, error) { + + ref, err := rootSchema.JsonReference() + + if err != nil { + return nil, err + } + + d := Schema{} + d.pool = sl.pool + d.pool.jsonLoaderFactory = rootSchema.LoaderFactory() + d.documentReference = ref + d.referencePool = newSchemaReferencePool() + + var doc interface{} + if ref.String() != "" { + // Get document from schema pool + spd, err := d.pool.GetDocument(d.documentReference) + if err != nil { + return nil, err + } + doc = spd.Document + } else { + // Load JSON directly + doc, err = rootSchema.LoadJSON() + if err != nil { + return nil, err + } + // References need only be parsed if loading JSON directly + // as pool.GetDocument already does this for us if loading by reference + err = sl.pool.parseReferences(doc, ref, true) + if err != nil { + return nil, err + } + } + + if sl.Validate { + if err := sl.validateMetaschema(doc); err != nil { + return nil, err + } + } + + draft := sl.Draft + if sl.AutoDetect { + _, detectedDraft, err := parseSchemaURL(doc) + if err != nil { + return nil, err + } + if detectedDraft != nil { + draft = *detectedDraft + } + } + + err = d.parse(doc, draft) + if err != nil { + return nil, err + } + + return &d, nil +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schemaLoader_test.go b/vendor/github.com/xeipuuv/gojsonschema/schemaLoader_test.go new file mode 100644 index 0000000..8a2e9ed --- /dev/null +++ b/vendor/github.com/xeipuuv/gojsonschema/schemaLoader_test.go @@ -0,0 +1,162 @@ +// Copyright 2018 johandorland ( https://github.com/johandorland ) +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gojsonschema + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSchemaLoaderWithReferenceToAddedSchema(t *testing.T) { + sl := NewSchemaLoader() + err := sl.AddSchemas(NewStringLoader(`{ + "$id" : "http://localhost:1234/test1.json", + "type" : "integer" + }`)) + + assert.Nil(t, err) + schema, err := sl.Compile(NewReferenceLoader("http://localhost:1234/test1.json")) + assert.Nil(t, err) + result, err := schema.Validate(NewStringLoader(`"hello"`)) + assert.Nil(t, err) + if len(result.Errors()) != 1 || result.Errors()[0].Type() != "invalid_type" { + t.Errorf("Expected invalid type erorr, instead got %v", result.Errors()) + } +} + +func TestCrossReference(t *testing.T) { + schema1 := NewStringLoader(`{ + "$ref" : "http://localhost:1234/test3.json", + "definitions" : { + "foo" : { + "type" : "integer" + } + } + }`) + schema2 := NewStringLoader(`{ + "$ref" : "http://localhost:1234/test2.json#/definitions/foo" + }`) + + sl := NewSchemaLoader() + err := sl.AddSchema("http://localhost:1234/test2.json", schema1) + assert.Nil(t, err) + err = sl.AddSchema("http://localhost:1234/test3.json", schema2) + assert.Nil(t, err) + schema, err := sl.Compile(NewStringLoader(`{"$ref" : "http://localhost:1234/test2.json"}`)) + assert.Nil(t, err) + result, err := schema.Validate(NewStringLoader(`"hello"`)) + assert.Nil(t, err) + if len(result.Errors()) != 1 || result.Errors()[0].Type() != "invalid_type" { + t.Errorf("Expected invalid type erorr, instead got %v", result.Errors()) + } +} + +// Multiple schemas identifying under the same $id should throw an error +func TestDoubleIDReference(t *testing.T) { + sl := NewSchemaLoader() + err := sl.AddSchema("http://localhost:1234/test4.json", NewStringLoader("{}")) + assert.Nil(t, err) + err = sl.AddSchemas(NewStringLoader(`{ "$id" : "http://localhost:1234/test4.json"}`)) + assert.NotNil(t, err) +} + +func TestCustomMetaSchema(t *testing.T) { + + loader := NewStringLoader(`{ + "$id" : "http://localhost:1234/test5.json", + "properties" : { + "multipleOf" : false + } + }`) + + // Test a custom metaschema in which we disallow the use of the keyword "multipleOf" + sl := NewSchemaLoader() + sl.Validate = true + + err := sl.AddSchemas(loader) + assert.Nil(t, err) + _, err = sl.Compile(NewStringLoader(`{ + "$id" : "http://localhost:1234/test6.json", + "$schema" : "http://localhost:1234/test5.json", + "type" : "string" + }`)) + assert.Nil(t, err) + + sl = NewSchemaLoader() + sl.Validate = true + err = sl.AddSchemas(loader) + _, err = sl.Compile(NewStringLoader(`{ + "$id" : "http://localhost:1234/test7.json", + "$schema" : "http://localhost:1234/test5.json", + "multipleOf" : 5 + }`)) + assert.NotNil(t, err) +} + +func TestSchemaDetection(t *testing.T) { + loader := NewStringLoader(`{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "exclusiveMinimum" : 5 + }`) + + // The schema should produce an error in draft-04 mode + _, err := NewSchema(loader) + assert.NotNil(t, err) + + // With schema detection disabled the schema should not produce an error in hybrid mode + sl := NewSchemaLoader() + sl.AutoDetect = false + + _, err = sl.Compile(loader) + assert.Nil(t, err) +} + +func TestDraftCrossReferencing(t *testing.T) { + + // Tests the following cross referencing with any combination + // of autodetection and preset draft version. + + loader1 := NewStringLoader(`{ + "$schema" : "http://json-schema.org/draft-04/schema#", + "id" : "http://localhost:1234/file.json", + "$id" : "http://localhost:1234/file.json", + "exclusiveMinimum" : 5 + }`) + loader2 := NewStringLoader(`{ + "$schema" : "http://json-schema.org/draft-07/schema#", + "id" : "http://localhost:1234/main.json", + "$id" : "http://localhost:1234/main.json", + "$ref" : "file.json" + }`) + + for _, b := range []bool{true, false} { + for _, draft := range []Draft{Draft4, Draft6, Draft7} { + sl := NewSchemaLoader() + sl.Draft = draft + sl.AutoDetect = b + + err := sl.AddSchemas(loader1) + assert.Nil(t, err) + _, err = sl.Compile(loader2) + + // It will always fail with autodetection on as "exclusiveMinimum" : 5 + // is only valid since draft-06. With autodetection off it will pass if + // draft-06 or newer is used + + assert.Equal(t, err == nil, !b && draft >= Draft6) + } + } +} diff --git a/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go b/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go index 65c91e4..f124e03 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go +++ b/vendor/github.com/xeipuuv/gojsonschema/schemaPool.go @@ -28,6 +28,7 @@ package gojsonschema import ( "errors" + "fmt" "reflect" "github.com/xeipuuv/gojsonreference" @@ -35,29 +36,44 @@ import ( type schemaPoolDocument struct { Document interface{} + Draft *Draft } type schemaPool struct { schemaPoolDocuments map[string]*schemaPoolDocument jsonLoaderFactory JSONLoaderFactory + autoDetect *bool } -func newSchemaPool(f JSONLoaderFactory) *schemaPool { +func (p *schemaPool) parseReferences(document interface{}, ref gojsonreference.JsonReference, pooled bool) error { - p := &schemaPool{} - p.schemaPoolDocuments = make(map[string]*schemaPoolDocument) - p.jsonLoaderFactory = f + var ( + draft *Draft + err error + reference = ref.String() + ) + // Only the root document should be added to the schema pool if pooled is true + if _, ok := p.schemaPoolDocuments[reference]; pooled && ok { + return fmt.Errorf("Reference already exists: \"%s\"", reference) + } - return p + if *p.autoDetect { + _, draft, err = parseSchemaURL(document) + if err != nil { + return err + } + } + + err = p.parseReferencesRecursive(document, ref, draft) + + if pooled { + p.schemaPoolDocuments[reference] = &schemaPoolDocument{Document: document, Draft: draft} + } + + return err } -func (p *schemaPool) ParseReferences(document interface{}, ref gojsonreference.JsonReference) { - // Only the root document should be added to the schema pool - p.schemaPoolDocuments[ref.String()] = &schemaPoolDocument{Document: document} - p.parseReferencesRecursive(document, ref) -} - -func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonreference.JsonReference) { +func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonreference.JsonReference, draft *Draft) error { // parseReferencesRecursive parses a JSON document and resolves all $id and $ref references. // For $ref references it takes into account the $id scope it is in and replaces // the reference by the absolute resolved reference @@ -67,7 +83,7 @@ func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonre switch m := document.(type) { case []interface{}: for _, v := range m { - p.parseReferencesRecursive(v, ref) + p.parseReferencesRecursive(v, ref, draft) } case map[string]interface{}: localRef := &ref @@ -81,7 +97,10 @@ func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonre if err == nil { localRef, err = ref.Inherits(jsonReference) if err == nil { - p.schemaPoolDocuments[localRef.String()] = &schemaPoolDocument{Document: document} + if _, ok := p.schemaPoolDocuments[localRef.String()]; ok { + return fmt.Errorf("Reference already exists: \"%s\"", localRef.String()) + } + p.schemaPoolDocuments[localRef.String()] = &schemaPoolDocument{Document: document, Draft: draft} } } } @@ -106,22 +125,24 @@ func (p *schemaPool) parseReferencesRecursive(document interface{}, ref gojsonre if k == KEY_PROPERTIES || k == KEY_DEPENDENCIES || k == KEY_PATTERN_PROPERTIES { if child, ok := v.(map[string]interface{}); ok { for _, v := range child { - p.parseReferencesRecursive(v, *localRef) + p.parseReferencesRecursive(v, *localRef, draft) } } } else { - p.parseReferencesRecursive(v, *localRef) + p.parseReferencesRecursive(v, *localRef, draft) } } } + return nil } func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) { var ( - spd *schemaPoolDocument - ok bool - err error + spd *schemaPoolDocument + draft *Draft + ok bool + err error ) if internalLogEnabled { @@ -157,7 +178,7 @@ func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*sche internalLog(" From pool") } - spd = &schemaPoolDocument{Document: document} + spd = &schemaPoolDocument{Document: document, Draft: cachedSpd.Draft} p.schemaPoolDocuments[reference.String()] = spd return spd, nil @@ -179,7 +200,9 @@ func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*sche } // add the whole document to the pool for potential re-use - p.ParseReferences(document, refToUrl) + p.parseReferences(document, refToUrl, true) + + _, draft, _ = parseSchemaURL(document) // resolve the potential fragment and also cache it document, _, err = reference.GetPointer().Get(document) @@ -188,5 +211,5 @@ func (p *schemaPool) GetDocument(reference gojsonreference.JsonReference) (*sche return nil, err } - return &schemaPoolDocument{Document: document}, nil + return &schemaPoolDocument{Document: document, Draft: draft}, nil } diff --git a/vendor/github.com/xeipuuv/gojsonschema/schema_test.go b/vendor/github.com/xeipuuv/gojsonschema/schema_test.go index d35415e..9a7bab4 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/schema_test.go +++ b/vendor/github.com/xeipuuv/gojsonschema/schema_test.go @@ -245,7 +245,7 @@ func TestFragmentLoader(t *testing.T) { fileName := filepath.Join(wd, "testdata", "extra", "fragment_schema.json") - schemaLoader := NewReferenceLoader("file://" + fileName + "#/definitions/x") + schemaLoader := NewReferenceLoader("file://" + filepath.ToSlash(fileName) + "#/definitions/x") schema, err := NewSchema(schemaLoader) if err != nil { diff --git a/vendor/github.com/xeipuuv/gojsonschema/subSchema.go b/vendor/github.com/xeipuuv/gojsonschema/subSchema.go index ea792a1..362d86c 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/subSchema.go +++ b/vendor/github.com/xeipuuv/gojsonschema/subSchema.go @@ -79,6 +79,7 @@ const ( ) type subSchema struct { + draft *Draft // basic subSchema meta properties id *gojsonreference.JsonReference @@ -102,11 +103,11 @@ type subSchema struct { propertiesChildren []*subSchema // validation : number / integer - multipleOf *big.Float - maximum *big.Float - exclusiveMaximum bool - minimum *big.Float - exclusiveMinimum bool + multipleOf *big.Rat + maximum *big.Rat + exclusiveMaximum *big.Rat + minimum *big.Rat + exclusiveMinimum *big.Rat // validation : string minLength *int diff --git a/vendor/github.com/xeipuuv/gojsonschema/testdata/draft7/optional/format/idn-hostname.json b/vendor/github.com/xeipuuv/gojsonschema/testdata/draft7/optional/format/idn-hostname.json index 3291820..6a228e6 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/testdata/draft7/optional/format/idn-hostname.json +++ b/vendor/github.com/xeipuuv/gojsonschema/testdata/draft7/optional/format/idn-hostname.json @@ -2,6 +2,7 @@ { "description": "validation of internationalized host names", "schema": {"format": "idn-hostname"}, + "disabled" : true, "tests": [ { "description": "a valid host name (example.test in Hangul)", diff --git a/vendor/github.com/xeipuuv/gojsonschema/utils.go b/vendor/github.com/xeipuuv/gojsonschema/utils.go index cc3100a..88d223f 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/utils.go +++ b/vendor/github.com/xeipuuv/gojsonschema/utils.go @@ -62,6 +62,16 @@ func isStringInSlice(s []string, what string) bool { return false } +// indexStringInSlice returns the index of the first instance of 'what' in s or -1 if it is not found in s. +func indexStringInSlice(s []string, what string) int { + for i := range s { + if s[i] == what { + return i + } + } + return -1 +} + func marshalToJsonString(value interface{}) (*string, error) { mBytes, err := json.Marshal(value) @@ -110,7 +120,7 @@ func checkJsonInteger(what interface{}) (isInt bool) { jsonNumber := what.(json.Number) - bigFloat, isValidNumber := new(big.Float).SetString(string(jsonNumber)) + bigFloat, isValidNumber := new(big.Rat).SetString(string(jsonNumber)) return isValidNumber && bigFloat.IsInt() @@ -158,11 +168,11 @@ func mustBeInteger(what interface{}) *int { return nil } -func mustBeNumber(what interface{}) *big.Float { +func mustBeNumber(what interface{}) *big.Rat { if isJsonNumber(what) { number := what.(json.Number) - float64Value, success := new(big.Float).SetString(string(number)) + float64Value, success := new(big.Rat).SetString(string(number)) if success { return float64Value } else { diff --git a/vendor/github.com/xeipuuv/gojsonschema/validation.go b/vendor/github.com/xeipuuv/gojsonschema/validation.go index 2f1e7df..090c11e 100644 --- a/vendor/github.com/xeipuuv/gojsonschema/validation.go +++ b/vendor/github.com/xeipuuv/gojsonschema/validation.go @@ -517,17 +517,17 @@ func (v *subSchema) validateArray(currentSubSchema *subSchema, value []interface // uniqueItems: if currentSubSchema.uniqueItems { var stringifiedItems []string - for _, v := range value { + for j, v := range value { vString, err := marshalWithoutNumber(v) if err != nil { result.addInternalError(new(InternalError), context, value, ErrorDetails{"err": err}) } - if isStringInSlice(stringifiedItems, *vString) { + if i := indexStringInSlice(stringifiedItems, *vString); i > -1 { result.addInternalError( new(ItemsMustBeUniqueError), context, value, - ErrorDetails{"type": TYPE_ARRAY}, + ErrorDetails{"type": TYPE_ARRAY, "i": i, "j": j}, ) } stringifiedItems = append(stringifiedItems, *vString) @@ -842,73 +842,70 @@ func (v *subSchema) validateNumber(currentSubSchema *subSchema, value interface{ } number := value.(json.Number) - float64Value, _ := new(big.Float).SetString(string(number)) + float64Value, _ := new(big.Rat).SetString(string(number)) // multipleOf: if currentSubSchema.multipleOf != nil { - - if q := new(big.Float).Quo(float64Value, currentSubSchema.multipleOf); !q.IsInt() { + if q := new(big.Rat).Quo(float64Value, currentSubSchema.multipleOf); !q.IsInt() { result.addInternalError( new(MultipleOfError), context, resultErrorFormatJsonNumber(number), - ErrorDetails{"multiple": currentSubSchema.multipleOf}, + ErrorDetails{"multiple": new(big.Float).SetRat(currentSubSchema.multipleOf)}, ) } } //maximum & exclusiveMaximum: if currentSubSchema.maximum != nil { - if currentSubSchema.exclusiveMaximum { - if float64Value.Cmp(currentSubSchema.maximum) >= 0 { - result.addInternalError( - new(NumberLTError), - context, - resultErrorFormatJsonNumber(number), - ErrorDetails{ - "max": currentSubSchema.maximum, - }, - ) - } - } else { - if float64Value.Cmp(currentSubSchema.maximum) == 1 { - result.addInternalError( - new(NumberLTEError), - context, - resultErrorFormatJsonNumber(number), - ErrorDetails{ - "max": currentSubSchema.maximum, - }, - ) - } + if float64Value.Cmp(currentSubSchema.maximum) == 1 { + result.addInternalError( + new(NumberLTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.maximum, + }, + ) + } + } + if currentSubSchema.exclusiveMaximum != nil { + if float64Value.Cmp(currentSubSchema.exclusiveMaximum) >= 0 { + result.addInternalError( + new(NumberLTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "max": currentSubSchema.exclusiveMaximum, + }, + ) } } //minimum & exclusiveMinimum: if currentSubSchema.minimum != nil { - if currentSubSchema.exclusiveMinimum { - if float64Value.Cmp(currentSubSchema.minimum) <= 0 { - // if float64Value <= *currentSubSchema.minimum { - result.addInternalError( - new(NumberGTError), - context, - resultErrorFormatJsonNumber(number), - ErrorDetails{ - "min": currentSubSchema.minimum, - }, - ) - } - } else { - if float64Value.Cmp(currentSubSchema.minimum) == -1 { - result.addInternalError( - new(NumberGTEError), - context, - resultErrorFormatJsonNumber(number), - ErrorDetails{ - "min": currentSubSchema.minimum, - }, - ) - } + if float64Value.Cmp(currentSubSchema.minimum) == -1 { + result.addInternalError( + new(NumberGTEError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.minimum, + }, + ) + } + } + if currentSubSchema.exclusiveMinimum != nil { + if float64Value.Cmp(currentSubSchema.exclusiveMinimum) <= 0 { + // if float64Value <= *currentSubSchema.minimum { + result.addInternalError( + new(NumberGTError), + context, + resultErrorFormatJsonNumber(number), + ErrorDetails{ + "min": currentSubSchema.exclusiveMinimum, + }, + ) } } diff --git a/vendor/golang.org/x/crypto/acme/acme.go b/vendor/golang.org/x/crypto/acme/acme.go index 7df6476..edd1c82 100644 --- a/vendor/golang.org/x/crypto/acme/acme.go +++ b/vendor/golang.org/x/crypto/acme/acme.go @@ -77,6 +77,10 @@ const ( type Client struct { // Key is the account key used to register with a CA and sign requests. // Key.Public() must return a *rsa.PublicKey or *ecdsa.PublicKey. + // + // The following algorithms are supported: + // RS256, ES256, ES384 and ES512. + // See RFC7518 for more details about the algorithms. Key crypto.Signer // HTTPClient optionally specifies an HTTP client to use @@ -319,6 +323,20 @@ func (c *Client) UpdateReg(ctx context.Context, a *Account) (*Account, error) { // a valid authorization (Authorization.Status is StatusValid). If so, the caller // need not fulfill any challenge and can proceed to requesting a certificate. func (c *Client) Authorize(ctx context.Context, domain string) (*Authorization, error) { + return c.authorize(ctx, "dns", domain) +} + +// AuthorizeIP is the same as Authorize but requests IP address authorization. +// Clients which successfully obtain such authorization may request to issue +// a certificate for IP addresses. +// +// See the ACME spec extension for more details about IP address identifiers: +// https://tools.ietf.org/html/draft-ietf-acme-ip. +func (c *Client) AuthorizeIP(ctx context.Context, ipaddr string) (*Authorization, error) { + return c.authorize(ctx, "ip", ipaddr) +} + +func (c *Client) authorize(ctx context.Context, typ, val string) (*Authorization, error) { if _, err := c.Discover(ctx); err != nil { return nil, err } @@ -332,7 +350,7 @@ func (c *Client) Authorize(ctx context.Context, domain string) (*Authorization, Identifier authzID `json:"identifier"` }{ Resource: "new-authz", - Identifier: authzID{Type: "dns", Value: domain}, + Identifier: authzID{Type: typ, Value: val}, } res, err := c.post(ctx, c.Key, c.dir.AuthzURL, req, wantStatus(http.StatusCreated)) if err != nil { diff --git a/vendor/golang.org/x/crypto/acme/acme_test.go b/vendor/golang.org/x/crypto/acme/acme_test.go index ef1fe47..29d7abb 100644 --- a/vendor/golang.org/x/crypto/acme/acme_test.go +++ b/vendor/golang.org/x/crypto/acme/acme_test.go @@ -288,106 +288,127 @@ func TestGetReg(t *testing.T) { } func TestAuthorize(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method == "HEAD" { - w.Header().Set("Replay-Nonce", "test-nonce") - return - } - if r.Method != "POST" { - t.Errorf("r.Method = %q; want POST", r.Method) - } - - var j struct { - Resource string - Identifier struct { - Type string - Value string - } - } - decodeJWSRequest(t, &j, r) - - // Test request - if j.Resource != "new-authz" { - t.Errorf("j.Resource = %q; want new-authz", j.Resource) - } - if j.Identifier.Type != "dns" { - t.Errorf("j.Identifier.Type = %q; want dns", j.Identifier.Type) - } - if j.Identifier.Value != "example.com" { - t.Errorf("j.Identifier.Value = %q; want example.com", j.Identifier.Value) - } - - w.Header().Set("Location", "https://ca.tld/acme/auth/1") - w.WriteHeader(http.StatusCreated) - fmt.Fprintf(w, `{ - "identifier": {"type":"dns","value":"example.com"}, - "status":"pending", - "challenges":[ - { - "type":"http-01", - "status":"pending", - "uri":"https://ca.tld/acme/challenge/publickey/id1", - "token":"token1" - }, - { - "type":"tls-sni-01", - "status":"pending", - "uri":"https://ca.tld/acme/challenge/publickey/id2", - "token":"token2" + tt := []struct{ typ, value string }{ + {"dns", "example.com"}, + {"ip", "1.2.3.4"}, + } + for _, test := range tt { + t.Run(test.typ, func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == "HEAD" { + w.Header().Set("Replay-Nonce", "test-nonce") + return + } + if r.Method != "POST" { + t.Errorf("r.Method = %q; want POST", r.Method) } - ], - "combinations":[[0],[1]]}`) - })) - defer ts.Close() - cl := Client{Key: testKeyEC, dir: &Directory{AuthzURL: ts.URL}} - auth, err := cl.Authorize(context.Background(), "example.com") - if err != nil { - t.Fatal(err) - } + var j struct { + Resource string + Identifier struct { + Type string + Value string + } + } + decodeJWSRequest(t, &j, r) - if auth.URI != "https://ca.tld/acme/auth/1" { - t.Errorf("URI = %q; want https://ca.tld/acme/auth/1", auth.URI) - } - if auth.Status != "pending" { - t.Errorf("Status = %q; want pending", auth.Status) - } - if auth.Identifier.Type != "dns" { - t.Errorf("Identifier.Type = %q; want dns", auth.Identifier.Type) - } - if auth.Identifier.Value != "example.com" { - t.Errorf("Identifier.Value = %q; want example.com", auth.Identifier.Value) - } + // Test request + if j.Resource != "new-authz" { + t.Errorf("j.Resource = %q; want new-authz", j.Resource) + } + if j.Identifier.Type != test.typ { + t.Errorf("j.Identifier.Type = %q; want %q", j.Identifier.Type, test.typ) + } + if j.Identifier.Value != test.value { + t.Errorf("j.Identifier.Value = %q; want %q", j.Identifier.Value, test.value) + } - if n := len(auth.Challenges); n != 2 { - t.Fatalf("len(auth.Challenges) = %d; want 2", n) - } + w.Header().Set("Location", "https://ca.tld/acme/auth/1") + w.WriteHeader(http.StatusCreated) + fmt.Fprintf(w, `{ + "identifier": {"type":%q,"value":%q}, + "status":"pending", + "challenges":[ + { + "type":"http-01", + "status":"pending", + "uri":"https://ca.tld/acme/challenge/publickey/id1", + "token":"token1" + }, + { + "type":"tls-sni-01", + "status":"pending", + "uri":"https://ca.tld/acme/challenge/publickey/id2", + "token":"token2" + } + ], + "combinations":[[0],[1]] + }`, test.typ, test.value) + })) + defer ts.Close() - c := auth.Challenges[0] - if c.Type != "http-01" { - t.Errorf("c.Type = %q; want http-01", c.Type) - } - if c.URI != "https://ca.tld/acme/challenge/publickey/id1" { - t.Errorf("c.URI = %q; want https://ca.tld/acme/challenge/publickey/id1", c.URI) - } - if c.Token != "token1" { - t.Errorf("c.Token = %q; want token1", c.Token) - } + var ( + auth *Authorization + err error + ) + cl := Client{Key: testKeyEC, dir: &Directory{AuthzURL: ts.URL}} + switch test.typ { + case "dns": + auth, err = cl.Authorize(context.Background(), test.value) + case "ip": + auth, err = cl.AuthorizeIP(context.Background(), test.value) + default: + t.Fatalf("unknown identifier type: %q", test.typ) + } + if err != nil { + t.Fatal(err) + } - c = auth.Challenges[1] - if c.Type != "tls-sni-01" { - t.Errorf("c.Type = %q; want tls-sni-01", c.Type) - } - if c.URI != "https://ca.tld/acme/challenge/publickey/id2" { - t.Errorf("c.URI = %q; want https://ca.tld/acme/challenge/publickey/id2", c.URI) - } - if c.Token != "token2" { - t.Errorf("c.Token = %q; want token2", c.Token) - } + if auth.URI != "https://ca.tld/acme/auth/1" { + t.Errorf("URI = %q; want https://ca.tld/acme/auth/1", auth.URI) + } + if auth.Status != "pending" { + t.Errorf("Status = %q; want pending", auth.Status) + } + if auth.Identifier.Type != test.typ { + t.Errorf("Identifier.Type = %q; want %q", auth.Identifier.Type, test.typ) + } + if auth.Identifier.Value != test.value { + t.Errorf("Identifier.Value = %q; want %q", auth.Identifier.Value, test.value) + } - combs := [][]int{{0}, {1}} - if !reflect.DeepEqual(auth.Combinations, combs) { - t.Errorf("auth.Combinations: %+v\nwant: %+v\n", auth.Combinations, combs) + if n := len(auth.Challenges); n != 2 { + t.Fatalf("len(auth.Challenges) = %d; want 2", n) + } + + c := auth.Challenges[0] + if c.Type != "http-01" { + t.Errorf("c.Type = %q; want http-01", c.Type) + } + if c.URI != "https://ca.tld/acme/challenge/publickey/id1" { + t.Errorf("c.URI = %q; want https://ca.tld/acme/challenge/publickey/id1", c.URI) + } + if c.Token != "token1" { + t.Errorf("c.Token = %q; want token1", c.Token) + } + + c = auth.Challenges[1] + if c.Type != "tls-sni-01" { + t.Errorf("c.Type = %q; want tls-sni-01", c.Type) + } + if c.URI != "https://ca.tld/acme/challenge/publickey/id2" { + t.Errorf("c.URI = %q; want https://ca.tld/acme/challenge/publickey/id2", c.URI) + } + if c.Token != "token2" { + t.Errorf("c.Token = %q; want token2", c.Token) + } + + combs := [][]int{{0}, {1}} + if !reflect.DeepEqual(auth.Combinations, combs) { + t.Errorf("auth.Combinations: %+v\nwant: %+v\n", auth.Combinations, combs) + } + + }) } } diff --git a/vendor/golang.org/x/crypto/acme/autocert/autocert.go b/vendor/golang.org/x/crypto/acme/autocert/autocert.go index 4c2fc07..a50d9bf 100644 --- a/vendor/golang.org/x/crypto/acme/autocert/autocert.go +++ b/vendor/golang.org/x/crypto/acme/autocert/autocert.go @@ -69,7 +69,7 @@ func HostWhitelist(hosts ...string) HostPolicy { } return func(_ context.Context, host string) error { if !whitelist[host] { - return errors.New("acme/autocert: host not configured") + return fmt.Errorf("acme/autocert: host %q not configured in HostWhitelist", host) } return nil } diff --git a/vendor/golang.org/x/crypto/acme/autocert/example_test.go b/vendor/golang.org/x/crypto/acme/autocert/example_test.go index 89e2d83..d4225e5 100644 --- a/vendor/golang.org/x/crypto/acme/autocert/example_test.go +++ b/vendor/golang.org/x/crypto/acme/autocert/example_test.go @@ -24,7 +24,7 @@ func ExampleManager() { m := &autocert.Manager{ Cache: autocert.DirCache("secret-dir"), Prompt: autocert.AcceptTOS, - HostPolicy: autocert.HostWhitelist("example.org"), + HostPolicy: autocert.HostWhitelist("example.org", "www.example.org"), } s := &http.Server{ Addr: ":https", diff --git a/vendor/golang.org/x/crypto/acme/jws.go b/vendor/golang.org/x/crypto/acme/jws.go index 6cbca25..1093b50 100644 --- a/vendor/golang.org/x/crypto/acme/jws.go +++ b/vendor/golang.org/x/crypto/acme/jws.go @@ -25,7 +25,7 @@ func jwsEncodeJSON(claimset interface{}, key crypto.Signer, nonce string) ([]byt if err != nil { return nil, err } - alg, sha := jwsHasher(key) + alg, sha := jwsHasher(key.Public()) if alg == "" || !sha.Available() { return nil, ErrUnsupportedKey } @@ -97,13 +97,16 @@ func jwkEncode(pub crypto.PublicKey) (string, error) { } // jwsSign signs the digest using the given key. -// It returns ErrUnsupportedKey if the key type is unknown. -// The hash is used only for RSA keys. +// The hash is unused for ECDSA keys. +// +// Note: non-stdlib crypto.Signer implementations are expected to return +// the signature in the format as specified in RFC7518. +// See https://tools.ietf.org/html/rfc7518 for more details. func jwsSign(key crypto.Signer, hash crypto.Hash, digest []byte) ([]byte, error) { - switch key := key.(type) { - case *rsa.PrivateKey: - return key.Sign(rand.Reader, digest, hash) - case *ecdsa.PrivateKey: + if key, ok := key.(*ecdsa.PrivateKey); ok { + // The key.Sign method of ecdsa returns ASN1-encoded signature. + // So, we use the package Sign function instead + // to get R and S values directly and format the result accordingly. r, s, err := ecdsa.Sign(rand.Reader, key, digest) if err != nil { return nil, err @@ -118,18 +121,18 @@ func jwsSign(key crypto.Signer, hash crypto.Hash, digest []byte) ([]byte, error) copy(sig[size*2-len(sb):], sb) return sig, nil } - return nil, ErrUnsupportedKey + return key.Sign(rand.Reader, digest, hash) } // jwsHasher indicates suitable JWS algorithm name and a hash function // to use for signing a digest with the provided key. // It returns ("", 0) if the key is not supported. -func jwsHasher(key crypto.Signer) (string, crypto.Hash) { - switch key := key.(type) { - case *rsa.PrivateKey: +func jwsHasher(pub crypto.PublicKey) (string, crypto.Hash) { + switch pub := pub.(type) { + case *rsa.PublicKey: return "RS256", crypto.SHA256 - case *ecdsa.PrivateKey: - switch key.Params().Name { + case *ecdsa.PublicKey: + switch pub.Params().Name { case "P-256": return "ES256", crypto.SHA256 case "P-384": diff --git a/vendor/golang.org/x/crypto/acme/jws_test.go b/vendor/golang.org/x/crypto/acme/jws_test.go index 0ff0fb5..ee30b1e 100644 --- a/vendor/golang.org/x/crypto/acme/jws_test.go +++ b/vendor/golang.org/x/crypto/acme/jws_test.go @@ -5,6 +5,7 @@ package acme import ( + "crypto" "crypto/ecdsa" "crypto/elliptic" "crypto/rsa" @@ -13,6 +14,7 @@ import ( "encoding/json" "encoding/pem" "fmt" + "io" "math/big" "testing" ) @@ -241,6 +243,79 @@ func TestJWSEncodeJSONEC(t *testing.T) { } } +type customTestSigner struct { + sig []byte + pub crypto.PublicKey +} + +func (s *customTestSigner) Public() crypto.PublicKey { return s.pub } +func (s *customTestSigner) Sign(io.Reader, []byte, crypto.SignerOpts) ([]byte, error) { + return s.sig, nil +} + +func TestJWSEncodeJSONCustom(t *testing.T) { + claims := struct{ Msg string }{"hello"} + const ( + // printf '{"Msg":"hello"}' | base64 | tr -d '=' | tr '/+' '_-' + payload = "eyJNc2ciOiJoZWxsbyJ9" + // printf 'testsig' | base64 | tr -d '=' + testsig = "dGVzdHNpZw" + + // printf '{"alg":"ES256","jwk":{"crv":"P-256","kty":"EC","x":,"y":,"nonce":"nonce"}' | \ + // base64 | tr -d '=' | tr '/+' '_-' + es256phead = "eyJhbGciOiJFUzI1NiIsImp3ayI6eyJjcnYiOiJQLTI1NiIsImt0eSI6IkVDIiwieCI6IjVsaEV1" + + "ZzV4SzR4QkRaMm5BYmF4THRhTGl2ODVieEo3ZVBkMWRrTzIzSFEiLCJ5IjoiNGFpSzcyc0JlVUFH" + + "a3YwVGFMc213b2tZVVl5TnhHc1M1RU1JS3dzTklLayJ9LCJub25jZSI6Im5vbmNlIn0" + + // {"alg":"RS256","jwk":{"e":"AQAB","kty":"RSA","n":"..."},"nonce":"nonce"} + rs256phead = "eyJhbGciOiJSUzI1NiIsImp3ayI6eyJlIjoiQVFBQiIsImt0eSI6" + + "IlJTQSIsIm4iOiI0eGdaM2VSUGt3b1J2eTdxZVJVYm1NRGUwVi14" + + "SDllV0xkdTBpaGVlTGxybUQybXFXWGZQOUllU0tBcGJuMzRnOFR1" + + "QVM5ZzV6aHE4RUxRM2ttanItS1Y4NkdBTWdJNlZBY0dscTNRcnpw" + + "VENmXzMwQWI3LXphd3JmUmFGT05hMUh3RXpQWTFLSG5HVmt4SmM4" + + "NWdOa3dZSTlTWTJSSFh0dmxuM3pzNXdJVE5yZG9zcUVYZWFJa1ZZ" + + "QkVoYmhOdTU0cHAza3hvNlR1V0xpOWU2cFhlV2V0RXdtbEJ3dFda" + + "bFBvaWIyajNUeExCa3NLWmZveUZ5ZWszODBtSGdKQXVtUV9JMmZq" + + "ajk4Xzk3bWszaWhPWTRBZ1ZkQ0RqMXpfR0NvWmtHNVJxN25iQ0d5" + + "b3N5S1d5RFgwMFpzLW5OcVZob0xlSXZYQzRubldkSk1aNnJvZ3h5" + + "UVEifSwibm9uY2UiOiJub25jZSJ9" + ) + + tt := []struct { + alg, phead string + pub crypto.PublicKey + }{ + {"RS256", rs256phead, testKey.Public()}, + {"ES256", es256phead, testKeyEC.Public()}, + } + for _, tc := range tt { + tc := tc + t.Run(tc.alg, func(t *testing.T) { + signer := &customTestSigner{ + sig: []byte("testsig"), + pub: tc.pub, + } + b, err := jwsEncodeJSON(claims, signer, "nonce") + if err != nil { + t.Fatal(err) + } + var j struct{ Protected, Payload, Signature string } + if err := json.Unmarshal(b, &j); err != nil { + t.Fatal(err) + } + if j.Protected != tc.phead { + t.Errorf("j.Protected = %q\nwant %q", j.Protected, tc.phead) + } + if j.Payload != payload { + t.Errorf("j.Payload = %q\nwant %q", j.Payload, payload) + } + if j.Signature != testsig { + t.Errorf("j.Signature = %q\nwant %q", j.Signature, testsig) + } + }) + } +} + func TestJWKThumbprintRSA(t *testing.T) { // Key example from RFC 7638 const base64N = "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAt" + diff --git a/vendor/golang.org/x/crypto/bcrypt/bcrypt_test.go b/vendor/golang.org/x/crypto/bcrypt/bcrypt_test.go index aecf759..b7162d8 100644 --- a/vendor/golang.org/x/crypto/bcrypt/bcrypt_test.go +++ b/vendor/golang.org/x/crypto/bcrypt/bcrypt_test.go @@ -209,19 +209,19 @@ func TestMinorNotRequired(t *testing.T) { func BenchmarkEqual(b *testing.B) { b.StopTimer() passwd := []byte("somepasswordyoulike") - hash, _ := GenerateFromPassword(passwd, 10) + hash, _ := GenerateFromPassword(passwd, DefaultCost) b.StartTimer() for i := 0; i < b.N; i++ { CompareHashAndPassword(hash, passwd) } } -func BenchmarkGeneration(b *testing.B) { +func BenchmarkDefaultCost(b *testing.B) { b.StopTimer() passwd := []byte("mylongpassword1234") b.StartTimer() for i := 0; i < b.N; i++ { - GenerateFromPassword(passwd, 10) + GenerateFromPassword(passwd, DefaultCost) } } diff --git a/vendor/golang.org/x/crypto/blake2b/blake2b.go b/vendor/golang.org/x/crypto/blake2b/blake2b.go index 58ea875..c160e1a 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2b.go +++ b/vendor/golang.org/x/crypto/blake2b/blake2b.go @@ -75,19 +75,19 @@ func Sum256(data []byte) [Size256]byte { } // New512 returns a new hash.Hash computing the BLAKE2b-512 checksum. A non-nil -// key turns the hash into a MAC. The key must between zero and 64 bytes long. +// key turns the hash into a MAC. The key must be between zero and 64 bytes long. func New512(key []byte) (hash.Hash, error) { return newDigest(Size, key) } // New384 returns a new hash.Hash computing the BLAKE2b-384 checksum. A non-nil -// key turns the hash into a MAC. The key must between zero and 64 bytes long. +// key turns the hash into a MAC. The key must be between zero and 64 bytes long. func New384(key []byte) (hash.Hash, error) { return newDigest(Size384, key) } // New256 returns a new hash.Hash computing the BLAKE2b-256 checksum. A non-nil -// key turns the hash into a MAC. The key must between zero and 64 bytes long. +// key turns the hash into a MAC. The key must be between zero and 64 bytes long. func New256(key []byte) (hash.Hash, error) { return newDigest(Size256, key) } // New returns a new hash.Hash computing the BLAKE2b checksum with a custom length. -// A non-nil key turns the hash into a MAC. The key must between zero and 64 bytes long. +// A non-nil key turns the hash into a MAC. The key must be between zero and 64 bytes long. // The hash size can be a value between 1 and 64 but it is highly recommended to use // values equal or greater than: // - 32 if BLAKE2b is used as a hash function (The key is zero bytes long). diff --git a/vendor/golang.org/x/crypto/blake2b/blake2x.go b/vendor/golang.org/x/crypto/blake2b/blake2x.go index c814496..52c414d 100644 --- a/vendor/golang.org/x/crypto/blake2b/blake2x.go +++ b/vendor/golang.org/x/crypto/blake2b/blake2x.go @@ -29,7 +29,7 @@ type XOF interface { } // OutputLengthUnknown can be used as the size argument to NewXOF to indicate -// the the length of the output is not known in advance. +// the length of the output is not known in advance. const OutputLengthUnknown = 0 // magicUnknownOutputLength is a magic value for the output size that indicates diff --git a/vendor/golang.org/x/crypto/blake2s/blake2x.go b/vendor/golang.org/x/crypto/blake2s/blake2x.go index eaff2a7..828749f 100644 --- a/vendor/golang.org/x/crypto/blake2s/blake2x.go +++ b/vendor/golang.org/x/crypto/blake2s/blake2x.go @@ -29,7 +29,7 @@ type XOF interface { } // OutputLengthUnknown can be used as the size argument to NewXOF to indicate -// the the length of the output is not known in advance. +// the length of the output is not known in advance. const OutputLengthUnknown = 0 // magicUnknownOutputLength is a magic value for the output size that indicates diff --git a/vendor/golang.org/x/crypto/bn256/gfp12.go b/vendor/golang.org/x/crypto/bn256/gfp12.go index f084edd..2b0151e 100644 --- a/vendor/golang.org/x/crypto/bn256/gfp12.go +++ b/vendor/golang.org/x/crypto/bn256/gfp12.go @@ -125,8 +125,8 @@ func (e *gfP12) Mul(a, b *gfP12, pool *bnPool) *gfP12 { } func (e *gfP12) MulScalar(a *gfP12, b *gfP6, pool *bnPool) *gfP12 { - e.x.Mul(e.x, b, pool) - e.y.Mul(e.y, b, pool) + e.x.Mul(a.x, b, pool) + e.y.Mul(a.y, b, pool) return e } diff --git a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go index ec13d13..2aa4fd8 100644 --- a/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go +++ b/vendor/golang.org/x/crypto/chacha20poly1305/chacha20poly1305_amd64.go @@ -20,7 +20,6 @@ func chacha20Poly1305Open(dst []byte, key []uint32, src, ad []byte) bool func chacha20Poly1305Seal(dst []byte, key []uint32, src, ad []byte) var ( - useASM = cpu.X86.HasSSSE3 useAVX2 = cpu.X86.HasAVX2 && cpu.X86.HasBMI2 ) @@ -48,7 +47,7 @@ func setupState(state *[16]uint32, key *[8]uint32, nonce []byte) { } func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) []byte { - if !useASM { + if !cpu.X86.HasSSSE3 { return c.sealGeneric(dst, nonce, plaintext, additionalData) } @@ -64,7 +63,7 @@ func (c *chacha20poly1305) seal(dst, nonce, plaintext, additionalData []byte) [] } func (c *chacha20poly1305) open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) { - if !useASM { + if !cpu.X86.HasSSSE3 { return c.openGeneric(dst, nonce, ciphertext, additionalData) } diff --git a/vendor/golang.org/x/crypto/cryptobyte/builder.go b/vendor/golang.org/x/crypto/cryptobyte/builder.go index 29b4c76..ca7b1db 100644 --- a/vendor/golang.org/x/crypto/cryptobyte/builder.go +++ b/vendor/golang.org/x/crypto/cryptobyte/builder.go @@ -50,8 +50,14 @@ func NewFixedBuilder(buffer []byte) *Builder { } } +// SetError sets the value to be returned as the error from Bytes. Writes +// performed after calling SetError are ignored. +func (b *Builder) SetError(err error) { + b.err = err +} + // Bytes returns the bytes written by the builder or an error if one has -// occurred during during building. +// occurred during building. func (b *Builder) Bytes() ([]byte, error) { if b.err != nil { return nil, b.err @@ -94,7 +100,7 @@ func (b *Builder) AddBytes(v []byte) { b.add(v...) } -// BuilderContinuation is continuation-passing interface for building +// BuilderContinuation is a continuation-passing interface for building // length-prefixed byte sequences. Builder methods for length-prefixed // sequences (AddUint8LengthPrefixed etc) will invoke the BuilderContinuation // supplied to them. The child builder passed to the continuation can be used @@ -268,9 +274,11 @@ func (b *Builder) flushChild() { return } - if !b.fixedSize { - b.result = child.result // In case child reallocated result. + if b.fixedSize && &b.result[0] != &child.result[0] { + panic("cryptobyte: BuilderContinuation reallocated a fixed-size buffer") } + + b.result = child.result } func (b *Builder) add(bytes ...byte) { @@ -278,7 +286,7 @@ func (b *Builder) add(bytes ...byte) { return } if b.child != nil { - panic("attempted write while child is pending") + panic("cryptobyte: attempted write while child is pending") } if len(b.result)+len(bytes) < len(bytes) { b.err = errors.New("cryptobyte: length overflow") @@ -290,6 +298,26 @@ func (b *Builder) add(bytes ...byte) { b.result = append(b.result, bytes...) } +// Unwrite rolls back n bytes written directly to the Builder. An attempt by a +// child builder passed to a continuation to unwrite bytes from its parent will +// panic. +func (b *Builder) Unwrite(n int) { + if b.err != nil { + return + } + if b.child != nil { + panic("cryptobyte: attempted unwrite while child is pending") + } + length := len(b.result) - b.pendingLenLen - b.offset + if length < 0 { + panic("cryptobyte: internal error") + } + if n > length { + panic("cryptobyte: attempted to unwrite more than was written") + } + b.result = b.result[:len(b.result)-n] +} + // A MarshalingValue marshals itself into a Builder. type MarshalingValue interface { // Marshal is called by Builder.AddValue. It receives a pointer to a builder diff --git a/vendor/golang.org/x/crypto/cryptobyte/cryptobyte_test.go b/vendor/golang.org/x/crypto/cryptobyte/cryptobyte_test.go index f294dd5..fb63709 100644 --- a/vendor/golang.org/x/crypto/cryptobyte/cryptobyte_test.go +++ b/vendor/golang.org/x/crypto/cryptobyte/cryptobyte_test.go @@ -327,12 +327,14 @@ func TestWriteWithPendingChild(t *testing.T) { var b Builder b.AddUint8LengthPrefixed(func(c *Builder) { c.AddUint8LengthPrefixed(func(d *Builder) { - defer func() { - if recover() == nil { - t.Errorf("recover() = nil, want error; c.AddUint8() did not panic") - } + func() { + defer func() { + if recover() == nil { + t.Errorf("recover() = nil, want error; c.AddUint8() did not panic") + } + }() + c.AddUint8(2) // panics }() - c.AddUint8(2) // panics defer func() { if recover() == nil { @@ -351,6 +353,92 @@ func TestWriteWithPendingChild(t *testing.T) { }) } +func TestSetError(t *testing.T) { + const errorStr = "TestSetError" + var b Builder + b.SetError(errors.New(errorStr)) + + ret, err := b.Bytes() + if ret != nil { + t.Error("expected nil result") + } + if err == nil { + t.Fatal("unexpected nil error") + } + if s := err.Error(); s != errorStr { + t.Errorf("expected error %q, got %v", errorStr, s) + } +} + +func TestUnwrite(t *testing.T) { + var b Builder + b.AddBytes([]byte{1, 2, 3, 4, 5}) + b.Unwrite(2) + if err := builderBytesEq(&b, 1, 2, 3); err != nil { + t.Error(err) + } + + func() { + defer func() { + if recover() == nil { + t.Errorf("recover() = nil, want error; b.Unwrite() did not panic") + } + }() + b.Unwrite(4) // panics + }() + + b = Builder{} + b.AddBytes([]byte{1, 2, 3, 4, 5}) + b.AddUint8LengthPrefixed(func(b *Builder) { + b.AddBytes([]byte{1, 2, 3, 4, 5}) + + defer func() { + if recover() == nil { + t.Errorf("recover() = nil, want error; b.Unwrite() did not panic") + } + }() + b.Unwrite(6) // panics + }) + + b = Builder{} + b.AddBytes([]byte{1, 2, 3, 4, 5}) + b.AddUint8LengthPrefixed(func(c *Builder) { + defer func() { + if recover() == nil { + t.Errorf("recover() = nil, want error; b.Unwrite() did not panic") + } + }() + b.Unwrite(2) // panics (attempted unwrite while child is pending) + }) +} + +func TestFixedBuilderLengthPrefixed(t *testing.T) { + bufCap := 10 + inner := bytes.Repeat([]byte{0xff}, bufCap-2) + buf := make([]byte, 0, bufCap) + b := NewFixedBuilder(buf) + b.AddUint16LengthPrefixed(func(b *Builder) { + b.AddBytes(inner) + }) + if got := b.BytesOrPanic(); len(got) != bufCap { + t.Errorf("Expected output length to be %d, got %d", bufCap, len(got)) + } +} + +func TestFixedBuilderPanicReallocate(t *testing.T) { + defer func() { + recover() + }() + + b := NewFixedBuilder(make([]byte, 0, 10)) + b1 := NewFixedBuilder(make([]byte, 0, 10)) + b.AddUint16LengthPrefixed(func(b *Builder) { + *b = *b1 + }) + + t.Error("Builder did not panic") +} + // ASN.1 func TestASN1Int64(t *testing.T) { diff --git a/vendor/golang.org/x/crypto/hkdf/example_test.go b/vendor/golang.org/x/crypto/hkdf/example_test.go index df84395..e89c260 100644 --- a/vendor/golang.org/x/crypto/hkdf/example_test.go +++ b/vendor/golang.org/x/crypto/hkdf/example_test.go @@ -9,49 +9,44 @@ import ( "crypto/rand" "crypto/sha256" "fmt" - "golang.org/x/crypto/hkdf" "io" + + "golang.org/x/crypto/hkdf" ) -// Usage example that expands one master key into three other cryptographically -// secure keys. +// Usage example that expands one master secret into three other +// cryptographically secure keys. func Example_usage() { - // Underlying hash function to use + // Underlying hash function for HMAC. hash := sha256.New - // Cryptographically secure master key. - master := []byte{0x00, 0x01, 0x02, 0x03} // i.e. NOT this. + // Cryptographically secure master secret. + secret := []byte{0x00, 0x01, 0x02, 0x03} // i.e. NOT this. - // Non secret salt, optional (can be nil) - // Recommended: hash-length sized random + // Non-secret salt, optional (can be nil). + // Recommended: hash-length random value. salt := make([]byte, hash().Size()) - n, err := io.ReadFull(rand.Reader, salt) - if n != len(salt) || err != nil { - fmt.Println("error:", err) - return + if _, err := rand.Read(salt); err != nil { + panic(err) } - // Non secret context specific info, optional (can be nil). - // Note, independent from the master key. - info := []byte{0x03, 0x14, 0x15, 0x92, 0x65} + // Non-secret context info, optional (can be nil). + info := []byte("hkdf example") - // Create the key derivation function - hkdf := hkdf.New(hash, master, salt, info) + // Generate three 128-bit derived keys. + hkdf := hkdf.New(hash, secret, salt, info) - // Generate the required keys - keys := make([][]byte, 3) - for i := 0; i < len(keys); i++ { - keys[i] = make([]byte, 24) - n, err := io.ReadFull(hkdf, keys[i]) - if n != len(keys[i]) || err != nil { - fmt.Println("error:", err) - return + var keys [][]byte + for i := 0; i < 3; i++ { + key := make([]byte, 16) + if _, err := io.ReadFull(hkdf, key); err != nil { + panic(err) } + keys = append(keys, key) } - // Keys should contain 192 bit random keys - for i := 1; i <= len(keys); i++ { - fmt.Printf("Key #%d: %v\n", i, !bytes.Equal(keys[i-1], make([]byte, 24))) + for i := range keys { + fmt.Printf("Key #%d: %v\n", i+1, !bytes.Equal(keys[i], make([]byte, 16))) } // Output: diff --git a/vendor/golang.org/x/crypto/hkdf/hkdf.go b/vendor/golang.org/x/crypto/hkdf/hkdf.go index 5bc2463..dda3f14 100644 --- a/vendor/golang.org/x/crypto/hkdf/hkdf.go +++ b/vendor/golang.org/x/crypto/hkdf/hkdf.go @@ -8,8 +8,6 @@ // HKDF is a cryptographic key derivation function (KDF) with the goal of // expanding limited input keying material into one or more cryptographically // strong secret keys. -// -// RFC 5869: https://tools.ietf.org/html/rfc5869 package hkdf // import "golang.org/x/crypto/hkdf" import ( @@ -19,6 +17,21 @@ import ( "io" ) +// Extract generates a pseudorandom key for use with Expand from an input secret +// and an optional independent salt. +// +// Only use this function if you need to reuse the extracted key with multiple +// Expand invocations and different context values. Most common scenarios, +// including the generation of multiple keys, should use New instead. +func Extract(hash func() hash.Hash, secret, salt []byte) []byte { + if salt == nil { + salt = make([]byte, hash().Size()) + } + extractor := hmac.New(hash, salt) + extractor.Write(secret) + return extractor.Sum(nil) +} + type hkdf struct { expander hash.Hash size int @@ -26,22 +39,22 @@ type hkdf struct { info []byte counter byte - prev []byte - cache []byte + prev []byte + buf []byte } func (f *hkdf) Read(p []byte) (int, error) { // Check whether enough data can be generated need := len(p) - remains := len(f.cache) + int(255-f.counter+1)*f.size + remains := len(f.buf) + int(255-f.counter+1)*f.size if remains < need { return 0, errors.New("hkdf: entropy limit reached") } - // Read from the cache, if enough data is present - n := copy(p, f.cache) + // Read any leftover from the buffer + n := copy(p, f.buf) p = p[n:] - // Fill the buffer + // Fill the rest of the buffer for len(p) > 0 { f.expander.Reset() f.expander.Write(f.prev) @@ -51,25 +64,30 @@ func (f *hkdf) Read(p []byte) (int, error) { f.counter++ // Copy the new batch into p - f.cache = f.prev - n = copy(p, f.cache) + f.buf = f.prev + n = copy(p, f.buf) p = p[n:] } // Save leftovers for next run - f.cache = f.cache[n:] + f.buf = f.buf[n:] return need, nil } -// New returns a new HKDF using the given hash, the secret keying material to expand -// and optional salt and info fields. -func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader { - if salt == nil { - salt = make([]byte, hash().Size()) - } - extractor := hmac.New(hash, salt) - extractor.Write(secret) - prk := extractor.Sum(nil) - - return &hkdf{hmac.New(hash, prk), extractor.Size(), info, 1, nil, nil} +// Expand returns a Reader, from which keys can be read, using the given +// pseudorandom key and optional context info, skipping the extraction step. +// +// The pseudorandomKey should have been generated by Extract, or be a uniformly +// random or pseudorandom cryptographically strong key. See RFC 5869, Section +// 3.3. Most common scenarios will want to use New instead. +func Expand(hash func() hash.Hash, pseudorandomKey, info []byte) io.Reader { + expander := hmac.New(hash, pseudorandomKey) + return &hkdf{expander, expander.Size(), info, 1, nil, nil} +} + +// New returns a Reader, from which keys can be read, using the given hash, +// secret, salt and context info. Salt and info can be nil. +func New(hash func() hash.Hash, secret, salt, info []byte) io.Reader { + prk := Extract(hash, secret, salt) + return Expand(hash, prk, info) } diff --git a/vendor/golang.org/x/crypto/hkdf/hkdf_test.go b/vendor/golang.org/x/crypto/hkdf/hkdf_test.go index cee659b..ea57577 100644 --- a/vendor/golang.org/x/crypto/hkdf/hkdf_test.go +++ b/vendor/golang.org/x/crypto/hkdf/hkdf_test.go @@ -18,6 +18,7 @@ type hkdfTest struct { hash func() hash.Hash master []byte salt []byte + prk []byte info []byte out []byte } @@ -35,6 +36,12 @@ var hkdfTests = []hkdfTest{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, }, + []byte{ + 0x07, 0x77, 0x09, 0x36, 0x2c, 0x2e, 0x32, 0xdf, + 0x0d, 0xdc, 0x3f, 0x0d, 0xc4, 0x7b, 0xba, 0x63, + 0x90, 0xb6, 0xc7, 0x3b, 0xb5, 0x0f, 0x9c, 0x31, + 0x22, 0xec, 0x84, 0x4a, 0xd7, 0xc2, 0xb3, 0xe5, + }, []byte{ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, @@ -74,6 +81,12 @@ var hkdfTests = []hkdfTest{ 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, }, + []byte{ + 0x06, 0xa6, 0xb8, 0x8c, 0x58, 0x53, 0x36, 0x1a, + 0x06, 0x10, 0x4c, 0x9c, 0xeb, 0x35, 0xb4, 0x5c, + 0xef, 0x76, 0x00, 0x14, 0x90, 0x46, 0x71, 0x01, + 0x4a, 0x19, 0x3f, 0x40, 0xc1, 0x5f, 0xc2, 0x44, + }, []byte{ 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, @@ -108,6 +121,12 @@ var hkdfTests = []hkdfTest{ 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, }, []byte{}, + []byte{ + 0x19, 0xef, 0x24, 0xa3, 0x2c, 0x71, 0x7b, 0x16, + 0x7f, 0x33, 0xa9, 0x1d, 0x6f, 0x64, 0x8b, 0xdf, + 0x96, 0x59, 0x67, 0x76, 0xaf, 0xdb, 0x63, 0x77, + 0xac, 0x43, 0x4c, 0x1c, 0x29, 0x3c, 0xcb, 0x04, + }, []byte{}, []byte{ 0x8d, 0xa4, 0xe7, 0x75, 0xa5, 0x63, 0xc1, 0x8f, @@ -118,6 +137,30 @@ var hkdfTests = []hkdfTest{ 0x96, 0xc8, }, }, + { + sha256.New, + []byte{ + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, + }, + nil, + []byte{ + 0x19, 0xef, 0x24, 0xa3, 0x2c, 0x71, 0x7b, 0x16, + 0x7f, 0x33, 0xa9, 0x1d, 0x6f, 0x64, 0x8b, 0xdf, + 0x96, 0x59, 0x67, 0x76, 0xaf, 0xdb, 0x63, 0x77, + 0xac, 0x43, 0x4c, 0x1c, 0x29, 0x3c, 0xcb, 0x04, + }, + nil, + []byte{ + 0x8d, 0xa4, 0xe7, 0x75, 0xa5, 0x63, 0xc1, 0x8f, + 0x71, 0x5f, 0x80, 0x2a, 0x06, 0x3c, 0x5a, 0x31, + 0xb8, 0xa1, 0x1f, 0x5c, 0x5e, 0xe1, 0x87, 0x9e, + 0xc3, 0x45, 0x4e, 0x5f, 0x3c, 0x73, 0x8d, 0x2d, + 0x9d, 0x20, 0x13, 0x95, 0xfa, 0xa4, 0xb6, 0x1a, + 0x96, 0xc8, + }, + }, { sha1.New, []byte{ @@ -128,6 +171,11 @@ var hkdfTests = []hkdfTest{ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, }, + []byte{ + 0x9b, 0x6c, 0x18, 0xc4, 0x32, 0xa7, 0xbf, 0x8f, + 0x0e, 0x71, 0xc8, 0xeb, 0x88, 0xf4, 0xb3, 0x0b, + 0xaa, 0x2b, 0xa2, 0x43, + }, []byte{ 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, @@ -167,6 +215,11 @@ var hkdfTests = []hkdfTest{ 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, 0xac, 0xad, 0xae, 0xaf, }, + []byte{ + 0x8a, 0xda, 0xe0, 0x9a, 0x2a, 0x30, 0x70, 0x59, + 0x47, 0x8d, 0x30, 0x9b, 0x26, 0xc4, 0x11, 0x5a, + 0x22, 0x4c, 0xfa, 0xf6, + }, []byte{ 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, @@ -201,6 +254,11 @@ var hkdfTests = []hkdfTest{ 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, }, []byte{}, + []byte{ + 0xda, 0x8c, 0x8a, 0x73, 0xc7, 0xfa, 0x77, 0x28, + 0x8e, 0xc6, 0xf5, 0xe7, 0xc2, 0x97, 0x78, 0x6a, + 0xa0, 0xd3, 0x2d, 0x01, + }, []byte{}, []byte{ 0x0a, 0xc1, 0xaf, 0x70, 0x02, 0xb3, 0xd7, 0x61, @@ -219,7 +277,12 @@ var hkdfTests = []hkdfTest{ 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, }, nil, - []byte{}, + []byte{ + 0x2a, 0xdc, 0xca, 0xda, 0x18, 0x77, 0x9e, 0x7c, + 0x20, 0x77, 0xad, 0x2e, 0xb1, 0x9d, 0x3f, 0x3e, + 0x73, 0x13, 0x85, 0xdd, + }, + nil, []byte{ 0x2c, 0x91, 0x11, 0x72, 0x04, 0xd7, 0x45, 0xf3, 0x50, 0x0d, 0x63, 0x6a, 0x62, 0xf6, 0x4f, 0x0a, @@ -233,6 +296,11 @@ var hkdfTests = []hkdfTest{ func TestHKDF(t *testing.T) { for i, tt := range hkdfTests { + prk := Extract(tt.hash, tt.master, tt.salt) + if !bytes.Equal(prk, tt.prk) { + t.Errorf("test %d: incorrect PRK: have %v, need %v.", i, prk, tt.prk) + } + hkdf := New(tt.hash, tt.master, tt.salt, tt.info) out := make([]byte, len(tt.out)) @@ -244,6 +312,17 @@ func TestHKDF(t *testing.T) { if !bytes.Equal(out, tt.out) { t.Errorf("test %d: incorrect output: have %v, need %v.", i, out, tt.out) } + + hkdf = Expand(tt.hash, prk, tt.info) + + n, err = io.ReadFull(hkdf, out) + if n != len(tt.out) || err != nil { + t.Errorf("test %d: not enough output bytes from Expand: %d.", i, n) + } + + if !bytes.Equal(out, tt.out) { + t.Errorf("test %d: incorrect output from Expand: have %v, need %v.", i, out, tt.out) + } } } diff --git a/vendor/golang.org/x/crypto/nacl/sign/sign_test.go b/vendor/golang.org/x/crypto/nacl/sign/sign_test.go index 0a6439a..db26901 100644 --- a/vendor/golang.org/x/crypto/nacl/sign/sign_test.go +++ b/vendor/golang.org/x/crypto/nacl/sign/sign_test.go @@ -40,7 +40,7 @@ func TestOpen(t *testing.T) { if !bytes.Equal(message, testMessage) { t.Fatalf("message did not match, got\n%x\n, expected\n%x", message, testMessage) } - message, ok = Open(nil, testSignedMessage[1:], &testPublicKey) + _, ok = Open(nil, testSignedMessage[1:], &testPublicKey) if ok { t.Fatalf("invalid signed message successfully verified") } diff --git a/vendor/golang.org/x/crypto/ocsp/ocsp.go b/vendor/golang.org/x/crypto/ocsp/ocsp.go index 5edc9c9..d297ac9 100644 --- a/vendor/golang.org/x/crypto/ocsp/ocsp.go +++ b/vendor/golang.org/x/crypto/ocsp/ocsp.go @@ -63,7 +63,7 @@ func (r ResponseStatus) String() string { } // ResponseError is an error that may be returned by ParseResponse to indicate -// that the response itself is an error, not just that its indicating that a +// that the response itself is an error, not just that it's indicating that a // certificate is revoked, unknown, etc. type ResponseError struct { Status ResponseStatus @@ -487,6 +487,9 @@ func ParseResponseForCert(bytes []byte, cert, issuer *x509.Certificate) (*Respon if err != nil { return nil, err } + if len(rest) > 0 { + return nil, ParseError("trailing data in OCSP response") + } if n := len(basicResp.TBSResponseData.Responses); n == 0 || cert == nil && n > 1 { return nil, ParseError("OCSP response contains bad number of responses") diff --git a/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign.go b/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign.go index def4cab..a9437dc 100644 --- a/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign.go +++ b/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign.go @@ -13,6 +13,7 @@ import ( "bufio" "bytes" "crypto" + "fmt" "hash" "io" "net/textproto" @@ -177,8 +178,9 @@ func Decode(data []byte) (b *Block, rest []byte) { // message. type dashEscaper struct { buffered *bufio.Writer - h hash.Hash + hashers []hash.Hash // one per key in privateKeys hashType crypto.Hash + toHash io.Writer // writes to all the hashes in hashers atBeginningOfLine bool isFirstLine bool @@ -186,8 +188,8 @@ type dashEscaper struct { whitespace []byte byteBuf []byte // a one byte buffer to save allocations - privateKey *packet.PrivateKey - config *packet.Config + privateKeys []*packet.PrivateKey + config *packet.Config } func (d *dashEscaper) Write(data []byte) (n int, err error) { @@ -198,7 +200,7 @@ func (d *dashEscaper) Write(data []byte) (n int, err error) { // The final CRLF isn't included in the hash so we have to wait // until this point (the start of the next line) before writing it. if !d.isFirstLine { - d.h.Write(crlf) + d.toHash.Write(crlf) } d.isFirstLine = false } @@ -219,12 +221,12 @@ func (d *dashEscaper) Write(data []byte) (n int, err error) { if _, err = d.buffered.Write(dashEscape); err != nil { return } - d.h.Write(d.byteBuf) + d.toHash.Write(d.byteBuf) d.atBeginningOfLine = false } else if b == '\n' { // Nothing to do because we delay writing CRLF to the hash. } else { - d.h.Write(d.byteBuf) + d.toHash.Write(d.byteBuf) d.atBeginningOfLine = false } if err = d.buffered.WriteByte(b); err != nil { @@ -245,13 +247,13 @@ func (d *dashEscaper) Write(data []byte) (n int, err error) { // Any buffered whitespace wasn't at the end of the line so // we need to write it out. if len(d.whitespace) > 0 { - d.h.Write(d.whitespace) + d.toHash.Write(d.whitespace) if _, err = d.buffered.Write(d.whitespace); err != nil { return } d.whitespace = d.whitespace[:0] } - d.h.Write(d.byteBuf) + d.toHash.Write(d.byteBuf) if err = d.buffered.WriteByte(b); err != nil { return } @@ -269,25 +271,29 @@ func (d *dashEscaper) Close() (err error) { return } } - sig := new(packet.Signature) - sig.SigType = packet.SigTypeText - sig.PubKeyAlgo = d.privateKey.PubKeyAlgo - sig.Hash = d.hashType - sig.CreationTime = d.config.Now() - sig.IssuerKeyId = &d.privateKey.KeyId - - if err = sig.Sign(d.h, d.privateKey, d.config); err != nil { - return - } out, err := armor.Encode(d.buffered, "PGP SIGNATURE", nil) if err != nil { return } - if err = sig.Serialize(out); err != nil { - return + t := d.config.Now() + for i, k := range d.privateKeys { + sig := new(packet.Signature) + sig.SigType = packet.SigTypeText + sig.PubKeyAlgo = k.PubKeyAlgo + sig.Hash = d.hashType + sig.CreationTime = t + sig.IssuerKeyId = &k.KeyId + + if err = sig.Sign(d.hashers[i], k, d.config); err != nil { + return + } + if err = sig.Serialize(out); err != nil { + return + } } + if err = out.Close(); err != nil { return } @@ -300,8 +306,17 @@ func (d *dashEscaper) Close() (err error) { // Encode returns a WriteCloser which will clear-sign a message with privateKey // and write it to w. If config is nil, sensible defaults are used. func Encode(w io.Writer, privateKey *packet.PrivateKey, config *packet.Config) (plaintext io.WriteCloser, err error) { - if privateKey.Encrypted { - return nil, errors.InvalidArgumentError("signing key is encrypted") + return EncodeMulti(w, []*packet.PrivateKey{privateKey}, config) +} + +// EncodeMulti returns a WriteCloser which will clear-sign a message with all the +// private keys indicated and write it to w. If config is nil, sensible defaults +// are used. +func EncodeMulti(w io.Writer, privateKeys []*packet.PrivateKey, config *packet.Config) (plaintext io.WriteCloser, err error) { + for _, k := range privateKeys { + if k.Encrypted { + return nil, errors.InvalidArgumentError(fmt.Sprintf("signing key %s is encrypted", k.KeyIdString())) + } } hashType := config.Hash() @@ -313,7 +328,14 @@ func Encode(w io.Writer, privateKey *packet.PrivateKey, config *packet.Config) ( if !hashType.Available() { return nil, errors.UnsupportedError("unsupported hash type: " + strconv.Itoa(int(hashType))) } - h := hashType.New() + var hashers []hash.Hash + var ws []io.Writer + for range privateKeys { + h := hashType.New() + hashers = append(hashers, h) + ws = append(ws, h) + } + toHash := io.MultiWriter(ws...) buffered := bufio.NewWriter(w) // start has a \n at the beginning that we don't want here. @@ -338,16 +360,17 @@ func Encode(w io.Writer, privateKey *packet.PrivateKey, config *packet.Config) ( plaintext = &dashEscaper{ buffered: buffered, - h: h, + hashers: hashers, hashType: hashType, + toHash: toHash, atBeginningOfLine: true, isFirstLine: true, byteBuf: make([]byte, 1), - privateKey: privateKey, - config: config, + privateKeys: privateKeys, + config: config, } return diff --git a/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go b/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go index 2c09480..96f5d78 100644 --- a/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go +++ b/vendor/golang.org/x/crypto/openpgp/clearsign/clearsign_test.go @@ -6,8 +6,11 @@ package clearsign import ( "bytes" - "golang.org/x/crypto/openpgp" + "fmt" "testing" + + "golang.org/x/crypto/openpgp" + "golang.org/x/crypto/openpgp/packet" ) func testParse(t *testing.T, input []byte, expected, expectedPlaintext string) { @@ -125,6 +128,71 @@ func TestSigning(t *testing.T) { } } +// We use this to make test keys, so that they aren't all the same. +type quickRand byte + +func (qr *quickRand) Read(p []byte) (int, error) { + for i := range p { + p[i] = byte(*qr) + } + *qr++ + return len(p), nil +} + +func TestMultiSign(t *testing.T) { + zero := quickRand(0) + config := packet.Config{Rand: &zero} + + for nKeys := 0; nKeys < 4; nKeys++ { + nextTest: + for nExtra := 0; nExtra < 4; nExtra++ { + var signKeys []*packet.PrivateKey + var verifyKeys openpgp.EntityList + + desc := fmt.Sprintf("%d keys; %d of which will be used to verify", nKeys+nExtra, nKeys) + for i := 0; i < nKeys+nExtra; i++ { + e, err := openpgp.NewEntity("name", "comment", "email", &config) + if err != nil { + t.Errorf("cannot create key: %v", err) + continue nextTest + } + if i < nKeys { + verifyKeys = append(verifyKeys, e) + } + signKeys = append(signKeys, e.PrivateKey) + } + + input := []byte("this is random text\r\n4 17") + var output bytes.Buffer + w, err := EncodeMulti(&output, signKeys, nil) + if err != nil { + t.Errorf("EncodeMulti (%s) failed: %v", desc, err) + } + if _, err := w.Write(input); err != nil { + t.Errorf("Write(%q) to signer (%s) failed: %v", string(input), desc, err) + } + if err := w.Close(); err != nil { + t.Errorf("Close() of signer (%s) failed: %v", desc, err) + } + + block, _ := Decode(output.Bytes()) + if string(block.Bytes) != string(input) { + t.Errorf("Inline data didn't match original; got %q want %q", string(block.Bytes), string(input)) + } + _, err = openpgp.CheckDetachedSignature(verifyKeys, bytes.NewReader(block.Bytes), block.ArmoredSignature.Body) + if nKeys == 0 { + if err == nil { + t.Errorf("verifying inline (%s) succeeded; want failure", desc) + } + } else { + if err != nil { + t.Errorf("verifying inline (%s) failed (%v); want success", desc, err) + } + } + } + } +} + var clearsignInput = []byte(` ;lasjlkfdsa diff --git a/vendor/golang.org/x/crypto/openpgp/keys.go b/vendor/golang.org/x/crypto/openpgp/keys.go index a79a8c1..3e25186 100644 --- a/vendor/golang.org/x/crypto/openpgp/keys.go +++ b/vendor/golang.org/x/crypto/openpgp/keys.go @@ -333,7 +333,6 @@ func ReadEntity(packets *packet.Reader) (*Entity, error) { return nil, errors.StructuralError("primary key cannot be used for signatures") } - var current *Identity var revocations []*packet.Signature EachPacket: for { @@ -346,36 +345,8 @@ EachPacket: switch pkt := p.(type) { case *packet.UserId: - // Make a new Identity object, that we might wind up throwing away. - // We'll only add it if we get a valid self-signature over this - // userID. - current = new(Identity) - current.Name = pkt.Id - current.UserId = pkt - - for { - p, err = packets.Next() - if err == io.EOF { - break EachPacket - } else if err != nil { - return nil, err - } - - sig, ok := p.(*packet.Signature) - if !ok { - packets.Unread(p) - continue EachPacket - } - - if (sig.SigType == packet.SigTypePositiveCert || sig.SigType == packet.SigTypeGenericCert) && sig.IssuerKeyId != nil && *sig.IssuerKeyId == e.PrimaryKey.KeyId { - if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, e.PrimaryKey, sig); err != nil { - return nil, errors.StructuralError("user ID self-signature invalid: " + err.Error()) - } - current.SelfSignature = sig - e.Identities[pkt.Id] = current - } else { - current.Signatures = append(current.Signatures, sig) - } + if err := addUserID(e, packets, pkt); err != nil { + return nil, err } case *packet.Signature: if pkt.SigType == packet.SigTypeKeyRevocation { @@ -384,11 +355,9 @@ EachPacket: // TODO: RFC4880 5.2.1 permits signatures // directly on keys (eg. to bind additional // revocation keys). - } else if current == nil { - return nil, errors.StructuralError("signature packet found before user id packet") - } else { - current.Signatures = append(current.Signatures, pkt) } + // Else, ignoring the signature as it does not follow anything + // we would know to attach it to. case *packet.PrivateKey: if pkt.IsSubkey == false { packets.Unread(p) @@ -429,33 +398,105 @@ EachPacket: return e, nil } +func addUserID(e *Entity, packets *packet.Reader, pkt *packet.UserId) error { + // Make a new Identity object, that we might wind up throwing away. + // We'll only add it if we get a valid self-signature over this + // userID. + identity := new(Identity) + identity.Name = pkt.Id + identity.UserId = pkt + + for { + p, err := packets.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + sig, ok := p.(*packet.Signature) + if !ok { + packets.Unread(p) + break + } + + if (sig.SigType == packet.SigTypePositiveCert || sig.SigType == packet.SigTypeGenericCert) && sig.IssuerKeyId != nil && *sig.IssuerKeyId == e.PrimaryKey.KeyId { + if err = e.PrimaryKey.VerifyUserIdSignature(pkt.Id, e.PrimaryKey, sig); err != nil { + return errors.StructuralError("user ID self-signature invalid: " + err.Error()) + } + identity.SelfSignature = sig + e.Identities[pkt.Id] = identity + } else { + identity.Signatures = append(identity.Signatures, sig) + } + } + + return nil +} + func addSubkey(e *Entity, packets *packet.Reader, pub *packet.PublicKey, priv *packet.PrivateKey) error { var subKey Subkey subKey.PublicKey = pub subKey.PrivateKey = priv - p, err := packets.Next() - if err == io.EOF { - return io.ErrUnexpectedEOF + + for { + p, err := packets.Next() + if err == io.EOF { + break + } else if err != nil { + return errors.StructuralError("subkey signature invalid: " + err.Error()) + } + + sig, ok := p.(*packet.Signature) + if !ok { + packets.Unread(p) + break + } + + if sig.SigType != packet.SigTypeSubkeyBinding && sig.SigType != packet.SigTypeSubkeyRevocation { + return errors.StructuralError("subkey signature with wrong type") + } + + if err := e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, sig); err != nil { + return errors.StructuralError("subkey signature invalid: " + err.Error()) + } + + switch sig.SigType { + case packet.SigTypeSubkeyRevocation: + subKey.Sig = sig + case packet.SigTypeSubkeyBinding: + + if shouldReplaceSubkeySig(subKey.Sig, sig) { + subKey.Sig = sig + } + } } - if err != nil { - return errors.StructuralError("subkey signature invalid: " + err.Error()) - } - var ok bool - subKey.Sig, ok = p.(*packet.Signature) - if !ok { + + if subKey.Sig == nil { return errors.StructuralError("subkey packet not followed by signature") } - if subKey.Sig.SigType != packet.SigTypeSubkeyBinding && subKey.Sig.SigType != packet.SigTypeSubkeyRevocation { - return errors.StructuralError("subkey signature with wrong type") - } - err = e.PrimaryKey.VerifyKeySignature(subKey.PublicKey, subKey.Sig) - if err != nil { - return errors.StructuralError("subkey signature invalid: " + err.Error()) - } + e.Subkeys = append(e.Subkeys, subKey) + return nil } +func shouldReplaceSubkeySig(existingSig, potentialNewSig *packet.Signature) bool { + if potentialNewSig == nil { + return false + } + + if existingSig == nil { + return true + } + + if existingSig.SigType == packet.SigTypeSubkeyRevocation { + return false // never override a revocation signature + } + + return potentialNewSig.CreationTime.After(existingSig.CreationTime) +} + const defaultRSAKeyBits = 2048 // NewEntity returns an Entity that contains a fresh RSA/RSA keypair with a diff --git a/vendor/golang.org/x/crypto/openpgp/keys_data_test.go b/vendor/golang.org/x/crypto/openpgp/keys_data_test.go new file mode 100644 index 0000000..7779bd9 --- /dev/null +++ b/vendor/golang.org/x/crypto/openpgp/keys_data_test.go @@ -0,0 +1,200 @@ +package openpgp + +const expiringKeyHex = "988d0451d1ec5d010400ba3385721f2dc3f4ab096b2ee867ab77213f0a27a8538441c35d2fa225b08798a1439a66a5150e6bdc3f40f5d28d588c712394c632b6299f77db8c0d48d37903fb72ebd794d61be6aa774688839e5fdecfe06b2684cc115d240c98c66cb1ef22ae84e3aa0c2b0c28665c1e7d4d044e7f270706193f5223c8d44e0d70b7b8da830011010001b40f4578706972792074657374206b657988be041301020028050251d1ec5d021b03050900278d00060b090807030206150802090a0b0416020301021e01021780000a091072589ad75e237d8c033503fd10506d72837834eb7f994117740723adc39227104b0d326a1161871c0b415d25b4aedef946ca77ea4c05af9c22b32cf98be86ab890111fced1ee3f75e87b7cc3c00dc63bbc85dfab91c0dc2ad9de2c4d13a34659333a85c6acc1a669c5e1d6cecb0cf1e56c10e72d855ae177ddc9e766f9b2dda57ccbb75f57156438bbdb4e42b88d0451d1ec5d0104009c64906559866c5cb61578f5846a94fcee142a489c9b41e67b12bb54cfe86eb9bc8566460f9a720cb00d6526fbccfd4f552071a8e3f7744b1882d01036d811ee5a3fb91a1c568055758f43ba5d2c6a9676b012f3a1a89e47bbf624f1ad571b208f3cc6224eb378f1645dd3d47584463f9eadeacfd1ce6f813064fbfdcc4b5a53001101000188a504180102000f021b0c050251d1f06b050900093e89000a091072589ad75e237d8c20e00400ab8310a41461425b37889c4da28129b5fae6084fafbc0a47dd1adc74a264c6e9c9cc125f40462ee1433072a58384daef88c961c390ed06426a81b464a53194c4e291ddd7e2e2ba3efced01537d713bd111f48437bde2363446200995e8e0d4e528dda377fd1e8f8ede9c8e2198b393bd86852ce7457a7e3daf74d510461a5b77b88d0451d1ece8010400b3a519f83ab0010307e83bca895170acce8964a044190a2b368892f7a244758d9fc193482648acb1fb9780d28cc22d171931f38bb40279389fc9bf2110876d4f3db4fcfb13f22f7083877fe56592b3b65251312c36f83ffcb6d313c6a17f197dd471f0712aad15a8537b435a92471ba2e5b0c72a6c72536c3b567c558d7b6051001101000188a504180102000f021b0c050251d1f07b050900279091000a091072589ad75e237d8ce69e03fe286026afacf7c97ee20673864d4459a2240b5655219950643c7dba0ac384b1d4359c67805b21d98211f7b09c2a0ccf6410c8c04d4ff4a51293725d8d6570d9d8bb0e10c07d22357caeb49626df99c180be02d77d1fe8ed25e7a54481237646083a9f89a11566cd20b9e995b1487c5f9e02aeb434f3a1897cd416dd0a87861838da3e9e" +const subkeyUsageHex = "988d04533a52bc010400d26af43085558f65b9e7dbc90cb9238015259aed5e954637adcfa2181548b2d0b60c65f1f42ec5081cbf1bc0a8aa4900acfb77070837c58f26012fbce297d70afe96e759ad63531f0037538e70dbf8e384569b9720d99d8eb39d8d0a2947233ed242436cb6ac7dfe74123354b3d0119b5c235d3dd9c9d6c004f8ffaf67ad8583001101000188b7041f010200210502533b8552170c8001ce094aa433f7040bb2ddf0be3893cb843d0fe70c020700000a0910a42704b92866382aa98404009d63d916a27543da4221c60087c33f1c44bec9998c5438018ed370cca4962876c748e94b73eb39c58eb698063f3fd6346d58dd2a11c0247934c4a9d71f24754f7468f96fb24c3e791dd2392b62f626148ad724189498cbf993db2df7c0cdc2d677c35da0f16cb16c9ce7c33b4de65a4a91b1d21a130ae9cc26067718910ef8e2b417556d627261203c756d627261407379642e65642e61753e88b80413010200220502533a52bc021b03060b090807030206150802090a0b0416020301021e01021780000a0910a42704b92866382a47840400c0c2bd04f5fca586de408b395b3c280a278259c93eaaa8b79a53b97003f8ed502a8a00446dd9947fb462677e4fcac0dac2f0701847d15130aadb6cd9e0705ea0cf5f92f129136c7be21a718d46c8e641eb7f044f2adae573e11ae423a0a9ca51324f03a8a2f34b91fa40c3cc764bee4dccadedb54c768ba0469b683ea53f1c29b88d04533a52bc01040099c92a5d6f8b744224da27bc2369127c35269b58bec179de6bbc038f749344222f85a31933224f26b70243c4e4b2d242f0c4777eaef7b5502f9dad6d8bf3aaeb471210674b74de2d7078af497d55f5cdad97c7bedfbc1b41e8065a97c9c3d344b21fc81d27723af8e374bc595da26ea242dccb6ae497be26eea57e563ed517e90011010001889f0418010200090502533a52bc021b0c000a0910a42704b92866382afa1403ff70284c2de8a043ff51d8d29772602fa98009b7861c540535f874f2c230af8caf5638151a636b21f8255003997ccd29747fdd06777bb24f9593bd7d98a3e887689bf902f999915fcc94625ae487e5d13e6616f89090ebc4fdc7eb5cad8943e4056995bb61c6af37f8043016876a958ec7ebf39c43d20d53b7f546cfa83e8d2604b88d04533b8283010400c0b529316dbdf58b4c54461e7e669dc11c09eb7f73819f178ccd4177b9182b91d138605fcf1e463262fabefa73f94a52b5e15d1904635541c7ea540f07050ce0fb51b73e6f88644cec86e91107c957a114f69554548a85295d2b70bd0b203992f76eb5d493d86d9eabcaa7ef3fc7db7e458438db3fcdb0ca1cc97c638439a9170011010001889f0418010200090502533b8283021b0c000a0910a42704b92866382adc6d0400cfff6258485a21675adb7a811c3e19ebca18851533f75a7ba317950b9997fda8d1a4c8c76505c08c04b6c2cc31dc704d33da36a21273f2b388a1a706f7c3378b66d887197a525936ed9a69acb57fe7f718133da85ec742001c5d1864e9c6c8ea1b94f1c3759cebfd93b18606066c063a63be86085b7e37bdbc65f9a915bf084bb901a204533b85cd110400aed3d2c52af2b38b5b67904b0ef73d6dd7aef86adb770e2b153cd22489654dcc91730892087bb9856ae2d9f7ed1eb48f214243fe86bfe87b349ebd7c30e630e49c07b21fdabf78b7a95c8b7f969e97e3d33f2e074c63552ba64a2ded7badc05ce0ea2be6d53485f6900c7860c7aa76560376ce963d7271b9b54638a4028b573f00a0d8854bfcdb04986141568046202192263b9b67350400aaa1049dbc7943141ef590a70dcb028d730371d92ea4863de715f7f0f16d168bd3dc266c2450457d46dcbbf0b071547e5fbee7700a820c3750b236335d8d5848adb3c0da010e998908dfd93d961480084f3aea20b247034f8988eccb5546efaa35a92d0451df3aaf1aee5aa36a4c4d462c760ecd9cebcabfbe1412b1f21450f203fd126687cd486496e971a87fd9e1a8a765fe654baa219a6871ab97768596ab05c26c1aeea8f1a2c72395a58dbc12ef9640d2b95784e974a4d2d5a9b17c25fedacfe551bda52602de8f6d2e48443f5dd1a2a2a8e6a5e70ecdb88cd6e766ad9745c7ee91d78cc55c3d06536b49c3fee6c3d0b6ff0fb2bf13a314f57c953b8f4d93bf88e70418010200090502533b85cd021b0200520910a42704b92866382a47200419110200060502533b85cd000a091042ce2c64bc0ba99214b2009e26b26852c8b13b10c35768e40e78fbbb48bd084100a0c79d9ea0844fa5853dd3c85ff3ecae6f2c9dd6c557aa04008bbbc964cd65b9b8299d4ebf31f41cc7264b8cf33a00e82c5af022331fac79efc9563a822497ba012953cefe2629f1242fcdcb911dbb2315985bab060bfd58261ace3c654bdbbe2e8ed27a46e836490145c86dc7bae15c011f7e1ffc33730109b9338cd9f483e7cef3d2f396aab5bd80efb6646d7e778270ee99d934d187dd98" +const revokedKeyHex = "988d045331ce82010400c4fdf7b40a5477f206e6ee278eaef888ca73bf9128a9eef9f2f1ddb8b7b71a4c07cfa241f028a04edb405e4d916c61d6beabc333813dc7b484d2b3c52ee233c6a79b1eea4e9cc51596ba9cd5ac5aeb9df62d86ea051055b79d03f8a4fa9f38386f5bd17529138f3325d46801514ea9047977e0829ed728e68636802796801be10011010001889f04200102000905025331d0e3021d03000a0910a401d9f09a34f7c042aa040086631196405b7e6af71026b88e98012eab44aa9849f6ef3fa930c7c9f23deaedba9db1538830f8652fb7648ec3fcade8dbcbf9eaf428e83c6cbcc272201bfe2fbb90d41963397a7c0637a1a9d9448ce695d9790db2dc95433ad7be19eb3de72dacf1d6db82c3644c13eae2a3d072b99bb341debba012c5ce4006a7d34a1f4b94b444526567205265766f6b657220283c52656727732022424d204261726973746122204b657920262530305c303e5c29203c72656740626d626172697374612e636f2e61753e88b704130102002205025331ce82021b03060b090807030206150802090a0b0416020301021e01021780000a0910a401d9f09a34f7c0019c03f75edfbeb6a73e7225ad3cc52724e2872e04260d7daf0d693c170d8c4b243b8767bc7785763533febc62ec2600c30603c433c095453ede59ff2fcabeb84ce32e0ed9d5cf15ffcbc816202b64370d4d77c1e9077d74e94a16fb4fa2e5bec23a56d7a73cf275f91691ae1801a976fcde09e981a2f6327ac27ea1fecf3185df0d56889c04100102000605025331cfb5000a0910fe9645554e8266b64b4303fc084075396674fb6f778d302ac07cef6bc0b5d07b66b2004c44aef711cbac79617ef06d836b4957522d8772dd94bf41a2f4ac8b1ee6d70c57503f837445a74765a076d07b829b8111fc2a918423ddb817ead7ca2a613ef0bfb9c6b3562aec6c3cf3c75ef3031d81d95f6563e4cdcc9960bcb386c5d757b104fcca5fe11fc709df884604101102000605025331cfe7000a09107b15a67f0b3ddc0317f6009e360beea58f29c1d963a22b962b80788c3fa6c84e009d148cfde6b351469b8eae91187eff07ad9d08fcaab88d045331ce820104009f25e20a42b904f3fa555530fe5c46737cf7bd076c35a2a0d22b11f7e0b61a69320b768f4a80fe13980ce380d1cfc4a0cd8fbe2d2e2ef85416668b77208baa65bf973fe8e500e78cc310d7c8705cdb34328bf80e24f0385fce5845c33bc7943cf6b11b02348a23da0bf6428e57c05135f2dc6bd7c1ce325d666d5a5fd2fd5e410011010001889f04180102000905025331ce82021b0c000a0910a401d9f09a34f7c0418003fe34feafcbeaef348a800a0d908a7a6809cc7304017d820f70f0474d5e23cb17e38b67dc6dca282c6ca00961f4ec9edf2738d0f087b1d81e4871ef08e1798010863afb4eac4c44a376cb343be929c5be66a78cfd4456ae9ec6a99d97f4e1c3ff3583351db2147a65c0acef5c003fb544ab3a2e2dc4d43646f58b811a6c3a369d1f" +const revokedSubkeyHex = "988d04533121f6010400aefc803a3e4bb1a61c86e8a86d2726c6a43e0079e9f2713f1fa017e9854c83877f4aced8e331d675c67ea83ddab80aacbfa0b9040bb12d96f5a3d6be09455e2a76546cbd21677537db941cab710216b6d24ec277ee0bd65b910f416737ed120f6b93a9d3b306245c8cfd8394606fdb462e5cf43c551438d2864506c63367fc890011010001b41d416c696365203c616c69636540626d626172697374612e636f2e61753e88bb041301020025021b03060b090807030206150802090a0b0416020301021e01021780050253312798021901000a09104ef7e4beccde97f015a803ff5448437780f63263b0df8442a995e7f76c221351a51edd06f2063d8166cf3157aada4923dfc44aa0f2a6a4da5cf83b7fe722ba8ab416c976e77c6b5682e7f1069026673bd0de56ba06fd5d7a9f177607f277d9b55ff940a638c3e68525c67517e2b3d976899b93ca267f705b3e5efad7d61220e96b618a4497eab8d04403d23f8846041011020006050253312910000a09107b15a67f0b3ddc03d96e009f50b6365d86c4be5d5e9d0ea42d5e56f5794c617700a0ab274e19c2827780016d23417ce89e0a2c0d987d889c04100102000605025331cf7a000a0910a401d9f09a34f7c0ee970400aca292f213041c9f3b3fc49148cbda9d84afee6183c8dd6c5ff2600b29482db5fecd4303797be1ee6d544a20a858080fec43412061c9a71fae4039fd58013b4ae341273e6c66ad4c7cdd9e68245bedb260562e7b166f2461a1032f2b38c0e0e5715fb3d1656979e052b55ca827a76f872b78a9fdae64bc298170bfcebedc1271b41a416c696365203c616c696365407379646973702e6f722e61753e88b804130102002205025331278b021b03060b090807030206150802090a0b0416020301021e01021780000a09104ef7e4beccde97f06a7003fa03c3af68d272ebc1fa08aa72a03b02189c26496a2833d90450801c4e42c5b5f51ad96ce2d2c9cef4b7c02a6a2fcf1412d6a2d486098eb762f5010a201819c17fd2888aec8eda20c65a3b75744de7ee5cc8ac7bfc470cbe3cb982720405a27a3c6a8c229cfe36905f881b02ed5680f6a8f05866efb9d6c5844897e631deb949ca8846041011020006050253312910000a09107b15a67f0b3ddc0347bc009f7fa35db59147469eb6f2c5aaf6428accb138b22800a0caa2f5f0874bacc5909c652a57a31beda65eddd5889c04100102000605025331cf7a000a0910a401d9f09a34f7c0316403ff46f2a5c101256627f16384d34a38fb47a6c88ba60506843e532d91614339fccae5f884a5741e7582ffaf292ba38ee10a270a05f139bde3814b6a077e8cd2db0f105ebea2a83af70d385f13b507fac2ad93ff79d84950328bb86f3074745a8b7f9b64990fb142e2a12976e27e8d09a28dc5621f957ac49091116da410ac3cbde1b88d04533121f6010400cbd785b56905e4192e2fb62a720727d43c4fa487821203cf72138b884b78b701093243e1d8c92a0248a6c0203a5a88693da34af357499abacaf4b3309c640797d03093870a323b4b6f37865f6eaa2838148a67df4735d43a90ca87942554cdf1c4a751b1e75f9fd4ce4e97e278d6c1c7ed59d33441df7d084f3f02beb68896c70011010001889f0418010200090502533121f6021b0c000a09104ef7e4beccde97f0b98b03fc0a5ccf6a372995835a2f5da33b282a7d612c0ab2a97f59cf9fff73e9110981aac2858c41399afa29624a7fd8a0add11654e3d882c0fd199e161bdad65e5e2548f7b68a437ea64293db1246e3011cbb94dc1bcdeaf0f2539bd88ff16d95547144d97cead6a8c5927660a91e6db0d16eb36b7b49a3525b54d1644e65599b032b7eb901a204533127a0110400bd3edaa09eff9809c4edc2c2a0ebe52e53c50a19c1e49ab78e6167bf61473bb08f2050d78a5cbbc6ed66aff7b42cd503f16b4a0b99fa1609681fca9b7ce2bbb1a5b3864d6cdda4d7ef7849d156d534dea30fb0efb9e4cf8959a2b2ce623905882d5430b995a15c3b9fe92906086788b891002924f94abe139b42cbbfaaabe42f00a0b65dc1a1ad27d798adbcb5b5ad02d2688c89477b03ff4eebb6f7b15a73b96a96bed201c0e5e4ea27e4c6e2dd1005b94d4b90137a5b1cf5e01c6226c070c4cc999938101578877ee76d296b9aab8246d57049caacf489e80a3f40589cade790a020b1ac146d6f7a6241184b8c7fcde680eae3188f5dcbe846d7f7bdad34f6fcfca08413e19c1d5df83fc7c7c627d493492e009c2f52a80400a2fe82de87136fd2e8845888c4431b032ba29d9a29a804277e31002a8201fb8591a3e55c7a0d0881496caf8b9fb07544a5a4879291d0dc026a0ea9e5bd88eb4aa4947bbd694b25012e208a250d65ddc6f1eea59d3aed3b4ec15fcab85e2afaa23a40ab1ef9ce3e11e1bc1c34a0e758e7aa64deb8739276df0af7d4121f834a9b88e70418010200090502533127a0021b02005209104ef7e4beccde97f047200419110200060502533127a0000a0910dbce4ee19529437fe045009c0b32f5ead48ee8a7e98fac0dea3d3e6c0e2c552500a0ad71fadc5007cfaf842d9b7db3335a8cdad15d3d1a6404009b08e2c68fe8f3b45c1bb72a4b3278cdf3012aa0f229883ad74aa1f6000bb90b18301b2f85372ca5d6b9bf478d235b733b1b197d19ccca48e9daf8e890cb64546b4ce1b178faccfff07003c172a2d4f5ebaba9f57153955f3f61a9b80a4f5cb959908f8b211b03b7026a8a82fc612bfedd3794969bcf458c4ce92be215a1176ab88d045331d144010400a5063000c5aaf34953c1aa3bfc95045b3aab9882b9a8027fecfe2142dc6b47ba8aca667399990244d513dd0504716908c17d92c65e74219e004f7b83fc125e575dd58efec3ab6dd22e3580106998523dea42ec75bf9aa111734c82df54630bebdff20fe981cfc36c76f865eb1c2fb62c9e85bc3a6e5015a361a2eb1c8431578d0011010001889f04280102000905025331d433021d03000a09104ef7e4beccde97f02e5503ff5e0630d1b65291f4882b6d40a29da4616bb5088717d469fbcc3648b8276de04a04988b1f1b9f3e18f52265c1f8b6c85861691c1a6b8a3a25a1809a0b32ad330aec5667cb4262f4450649184e8113849b05e5ad06a316ea80c001e8e71838190339a6e48bbde30647bcf245134b9a97fa875c1d83a9862cae87ffd7e2c4ce3a1b89013d04180102000905025331d144021b0200a809104ef7e4beccde97f09d2004190102000605025331d144000a0910677815e371c2fd23522203fe22ab62b8e7a151383cea3edd3a12995693911426f8ccf125e1f6426388c0010f88d9ca7da2224aee8d1c12135998640c5e1813d55a93df472faae75bef858457248db41b4505827590aeccf6f9eb646da7f980655dd3050c6897feddddaca90676dee856d66db8923477d251712bb9b3186b4d0114daf7d6b59272b53218dd1da94a03ff64006fcbe71211e5daecd9961fba66cdb6de3f914882c58ba5beddeba7dcb950c1156d7fba18c19ea880dccc800eae335deec34e3b84ac75ffa24864f782f87815cda1c0f634b3dd2fa67cea30811d21723d21d9551fa12ccbcfa62b6d3a15d01307b99925707992556d50065505b090aadb8579083a20fe65bd2a270da9b011" + +const missingCrossSignatureKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- +Charset: UTF-8 + +mQENBFMYynYBCACVOZ3/e8Bm2b9KH9QyIlHGo/i1bnkpqsgXj8tpJ2MIUOnXMMAY +ztW7kKFLCmgVdLIC0vSoLA4yhaLcMojznh/2CcUglZeb6Ao8Gtelr//Rd5DRfPpG +zqcfUo+m+eO1co2Orabw0tZDfGpg5p3AYl0hmxhUyYSc/xUq93xL1UJzBFgYXY54 +QsM8dgeQgFseSk/YvdP5SMx1ev+eraUyiiUtWzWrWC1TdyRa5p4UZg6Rkoppf+WJ +QrW6BWrhAtqATHc8ozV7uJjeONjUEq24roRc/OFZdmQQGK6yrzKnnbA6MdHhqpdo +9kWDcXYb7pSE63Lc+OBa5X2GUVvXJLS/3nrtABEBAAG0F2ludmFsaWQtc2lnbmlu +Zy1zdWJrZXlziQEoBBMBAgASBQJTnKB5AhsBAgsHAhUIAh4BAAoJEO3UDQUIHpI/ +dN4H/idX4FQ1LIZCnpHS/oxoWQWfpRgdKAEM0qCqjMgiipJeEwSQbqjTCynuh5/R +JlODDz85ABR06aoF4l5ebGLQWFCYifPnJZ/Yf5OYcMGtb7dIbqxWVFL9iLMO/oDL +ioI3dotjPui5e+2hI9pVH1UHB/bZ/GvMGo6Zg0XxLPolKQODMVjpjLAQ0YJ3spew +RAmOGre6tIvbDsMBnm8qREt7a07cBJ6XK7xjxYaZHQBiHVxyEWDa6gyANONx8duW +/fhQ/zDTnyVM/ik6VO0Ty9BhPpcEYLFwh5c1ilFari1ta3e6qKo6ZGa9YMk/REhu +yBHd9nTkI+0CiQUmbckUiVjDKKe5AQ0EUxjKdgEIAJcXQeP+NmuciE99YcJoffxv +2gVLU4ZXBNHEaP0mgaJ1+tmMD089vUQAcyGRvw8jfsNsVZQIOAuRxY94aHQhIRHR +bUzBN28ofo/AJJtfx62C15xt6fDKRV6HXYqAiygrHIpEoRLyiN69iScUsjIJeyFL +C8wa72e8pSL6dkHoaV1N9ZH/xmrJ+k0vsgkQaAh9CzYufncDxcwkoP+aOlGtX1gP +WwWoIbz0JwLEMPHBWvDDXQcQPQTYQyj+LGC9U6f9VZHN25E94subM1MjuT9OhN9Y +MLfWaaIc5WyhLFyQKW2Upofn9wSFi8ubyBnv640Dfd0rVmaWv7LNTZpoZ/GbJAMA +EQEAAYkBHwQYAQIACQUCU5ygeQIbAgAKCRDt1A0FCB6SP0zCB/sEzaVR38vpx+OQ +MMynCBJrakiqDmUZv9xtplY7zsHSQjpd6xGflbU2n+iX99Q+nav0ETQZifNUEd4N +1ljDGQejcTyKD6Pkg6wBL3x9/RJye7Zszazm4+toJXZ8xJ3800+BtaPoI39akYJm ++ijzbskvN0v/j5GOFJwQO0pPRAFtdHqRs9Kf4YanxhedB4dIUblzlIJuKsxFit6N +lgGRblagG3Vv2eBszbxzPbJjHCgVLR3RmrVezKOsZjr/2i7X+xLWIR0uD3IN1qOW +CXQxLBizEEmSNVNxsp7KPGTLnqO3bPtqFirxS9PJLIMPTPLNBY7ZYuPNTMqVIUWF +4artDmrG +=7FfJ +-----END PGP PUBLIC KEY BLOCK-----` + +const invalidCrossSignatureKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFMYynYBCACVOZ3/e8Bm2b9KH9QyIlHGo/i1bnkpqsgXj8tpJ2MIUOnXMMAY +ztW7kKFLCmgVdLIC0vSoLA4yhaLcMojznh/2CcUglZeb6Ao8Gtelr//Rd5DRfPpG +zqcfUo+m+eO1co2Orabw0tZDfGpg5p3AYl0hmxhUyYSc/xUq93xL1UJzBFgYXY54 +QsM8dgeQgFseSk/YvdP5SMx1ev+eraUyiiUtWzWrWC1TdyRa5p4UZg6Rkoppf+WJ +QrW6BWrhAtqATHc8ozV7uJjeONjUEq24roRc/OFZdmQQGK6yrzKnnbA6MdHhqpdo +9kWDcXYb7pSE63Lc+OBa5X2GUVvXJLS/3nrtABEBAAG0F2ludmFsaWQtc2lnbmlu +Zy1zdWJrZXlziQEoBBMBAgASBQJTnKB5AhsBAgsHAhUIAh4BAAoJEO3UDQUIHpI/ +dN4H/idX4FQ1LIZCnpHS/oxoWQWfpRgdKAEM0qCqjMgiipJeEwSQbqjTCynuh5/R +JlODDz85ABR06aoF4l5ebGLQWFCYifPnJZ/Yf5OYcMGtb7dIbqxWVFL9iLMO/oDL +ioI3dotjPui5e+2hI9pVH1UHB/bZ/GvMGo6Zg0XxLPolKQODMVjpjLAQ0YJ3spew +RAmOGre6tIvbDsMBnm8qREt7a07cBJ6XK7xjxYaZHQBiHVxyEWDa6gyANONx8duW +/fhQ/zDTnyVM/ik6VO0Ty9BhPpcEYLFwh5c1ilFari1ta3e6qKo6ZGa9YMk/REhu +yBHd9nTkI+0CiQUmbckUiVjDKKe5AQ0EUxjKdgEIAIINDqlj7X6jYKc6DjwrOkjQ +UIRWbQQar0LwmNilehmt70g5DCL1SYm9q4LcgJJ2Nhxj0/5qqsYib50OSWMcKeEe +iRXpXzv1ObpcQtI5ithp0gR53YPXBib80t3bUzomQ5UyZqAAHzMp3BKC54/vUrSK +FeRaxDzNLrCeyI00+LHNUtwghAqHvdNcsIf8VRumK8oTm3RmDh0TyjASWYbrt9c8 +R1Um3zuoACOVy+mEIgIzsfHq0u7dwYwJB5+KeM7ZLx+HGIYdUYzHuUE1sLwVoELh ++SHIGHI1HDicOjzqgajShuIjj5hZTyQySVprrsLKiXS6NEwHAP20+XjayJ/R3tEA +EQEAAYkCPgQYAQIBKAUCU5ygeQIbAsBdIAQZAQIABgUCU5ygeQAKCRCpVlnFZmhO +52RJB/9uD1MSa0wjY6tHOIgquZcP3bHBvHmrHNMw9HR2wRCMO91ZkhrpdS3ZHtgb +u3/55etj0FdvDo1tb8P8FGSVtO5Vcwf5APM8sbbqoi8L951Q3i7qt847lfhu6sMl +w0LWFvPTOLHrliZHItPRjOltS1WAWfr2jUYhsU9ytaDAJmvf9DujxEOsN5G1YJep +54JCKVCkM/y585Zcnn+yxk/XwqoNQ0/iJUT9qRrZWvoeasxhl1PQcwihCwss44A+ +YXaAt3hbk+6LEQuZoYS73yR3WHj+42tfm7YxRGeubXfgCEz/brETEWXMh4pe0vCL +bfWrmfSPq2rDegYcAybxRQz0lF8PAAoJEO3UDQUIHpI/exkH/0vQfdHA8g/N4T6E +i6b1CUVBAkvtdJpCATZjWPhXmShOw62gkDw306vHPilL4SCvEEi4KzG72zkp6VsB +DSRcpxCwT4mHue+duiy53/aRMtSJ+vDfiV1Vhq+3sWAck/yUtfDU9/u4eFaiNok1 +8/Gd7reyuZt5CiJnpdPpjCwelK21l2w7sHAnJF55ITXdOxI8oG3BRKufz0z5lyDY +s2tXYmhhQIggdgelN8LbcMhWs/PBbtUr6uZlNJG2lW1yscD4aI529VjwJlCeo745 +U7pO4eF05VViUJ2mmfoivL3tkhoTUWhx8xs8xCUcCg8DoEoSIhxtOmoTPR22Z9BL +6LCg2mg= +=Dhm4 +-----END PGP PUBLIC KEY BLOCK-----` + +const goodCrossSignatureKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mI0EVUqeVwEEAMufHRrMPWK3gyvi0O0tABCs/oON9zV9KDZlr1a1M91ShCSFwCPo +7r80PxdWVWcj0V5h50/CJYtpN3eE/mUIgW2z1uDYQF1OzrQ8ubrksfsJvpAhENom +lTQEppv9mV8qhcM278teb7TX0pgrUHLYF5CfPdp1L957JLLXoQR/lwLVABEBAAG0 +E2dvb2Qtc2lnbmluZy1zdWJrZXmIuAQTAQIAIgUCVUqeVwIbAwYLCQgHAwIGFQgC +CQoLBBYCAwECHgECF4AACgkQNRjL95IRWP69XQQAlH6+eyXJN4DZTLX78KGjHrsw +6FCvxxClEPtPUjcJy/1KCRQmtLAt9PbbA78dvgzjDeZMZqRAwdjyJhjyg/fkU2OH +7wq4ktjUu+dLcOBb+BFMEY+YjKZhf6EJuVfxoTVr5f82XNPbYHfTho9/OABKH6kv +X70PaKZhbwnwij8Nts65AaIEVUqftREEAJ3WxZfqAX0bTDbQPf2CMT2IVMGDfhK7 +GyubOZgDFFjwUJQvHNvsrbeGLZ0xOBumLINyPO1amIfTgJNm1iiWFWfmnHReGcDl +y5mpYG60Mb79Whdcer7CMm3AqYh/dW4g6IB02NwZMKoUHo3PXmFLxMKXnWyJ0clw +R0LI/Qn509yXAKDh1SO20rqrBM+EAP2c5bfI98kyNwQAi3buu94qo3RR1ZbvfxgW +CKXDVm6N99jdZGNK7FbRifXqzJJDLcXZKLnstnC4Sd3uyfyf1uFhmDLIQRryn5m+ +LBYHfDBPN3kdm7bsZDDq9GbTHiFZUfm/tChVKXWxkhpAmHhU/tH6GGzNSMXuIWSO +aOz3Rqq0ED4NXyNKjdF9MiwD/i83S0ZBc0LmJYt4Z10jtH2B6tYdqnAK29uQaadx +yZCX2scE09UIm32/w7pV77CKr1Cp/4OzAXS1tmFzQ+bX7DR+Gl8t4wxr57VeEMvl +BGw4Vjh3X8//m3xynxycQU18Q1zJ6PkiMyPw2owZ/nss3hpSRKFJsxMLhW3fKmKr +Ey2KiOcEGAECAAkFAlVKn7UCGwIAUgkQNRjL95IRWP5HIAQZEQIABgUCVUqftQAK +CRD98VjDN10SqkWrAKDTpEY8D8HC02E/KVC5YUI01B30wgCgurpILm20kXEDCeHp +C5pygfXw1DJrhAP+NyPJ4um/bU1I+rXaHHJYroYJs8YSweiNcwiHDQn0Engh/mVZ +SqLHvbKh2dL/RXymC3+rjPvQf5cup9bPxNMa6WagdYBNAfzWGtkVISeaQW+cTEp/ +MtgVijRGXR/lGLGETPg2X3Afwn9N9bLMBkBprKgbBqU7lpaoPupxT61bL70= +=vtbN +-----END PGP PUBLIC KEY BLOCK-----` + +const revokedUserIDKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQENBFsgO5EBCADhREPmcjsPkXe1z7ctvyWL0S7oa9JaoGZ9oPDHFDlQxd0qlX2e +DZJZDg0qYvVixmaULIulApq1puEsaJCn3lHUbHlb4PYKwLEywYXM28JN91KtLsz/ +uaEX2KC5WqeP40utmzkNLq+oRX/xnRMgwbO7yUNVG2UlEa6eI+xOXO3YtLdmJMBW +ClQ066ZnOIzEo1JxnIwha1CDBMWLLfOLrg6l8InUqaXbtEBbnaIYO6fXVXELUjkx +nmk7t/QOk0tXCy8muH9UDqJkwDUESY2l79XwBAcx9riX8vY7vwC34pm22fAUVLCJ +x1SJx0J8bkeNp38jKM2Zd9SUQqSbfBopQ4pPABEBAAG0I0dvbGFuZyBHb3BoZXIg +PG5vLXJlcGx5QGdvbGFuZy5jb20+iQFUBBMBCgA+FiEE5Ik5JLcNx6l6rZfw1oFy +9I6cUoMFAlsgO5ECGwMFCQPCZwAFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQ +1oFy9I6cUoMIkwf8DNPeD23i4jRwd/pylbvxwZintZl1fSwTJW1xcOa1emXaEtX2 +depuqhP04fjlRQGfsYAQh7X9jOJxAHjTmhqFBi5sD7QvKU00cPFYbJ/JTx0B41bl +aXnSbGhRPh63QtEZL7ACAs+shwvvojJqysx7kyVRu0EW2wqjXdHwR/SJO6nhNBa2 +DXzSiOU/SUA42mmG+5kjF8Aabq9wPwT9wjraHShEweNerNMmOqJExBOy3yFeyDpa +XwEZFzBfOKoxFNkIaVf5GSdIUGhFECkGvBMB935khftmgR8APxdU4BE7XrXexFJU +8RCuPXonm4WQOwTWR0vQg64pb2WKAzZ8HhwTGbQiR29sYW5nIEdvcGhlciA8cmV2 +b2tlZEBnb2xhbmcuY29tPokBNgQwAQoAIBYhBOSJOSS3Dcepeq2X8NaBcvSOnFKD +BQJbIDv3Ah0AAAoJENaBcvSOnFKDfWMIAKhI/Tvu3h8fSUxp/gSAcduT6bC1JttG +0lYQ5ilKB/58lBUA5CO3ZrKDKlzW3M8VEcvohVaqeTMKeoQd5rCZq8KxHn/KvN6N +s85REfXfniCKfAbnGgVXX3kDmZ1g63pkxrFu0fDZjVDXC6vy+I0sGyI/Inro0Pzb +tvn0QCsxjapKK15BtmSrpgHgzVqVg0cUp8vqZeKFxarYbYB2idtGRci4b9tObOK0 +BSTVFy26+I/mrFGaPrySYiy2Kz5NMEcRhjmTxJ8jSwEr2O2sUR0yjbgUAXbTxDVE +/jg5fQZ1ACvBRQnB7LvMHcInbzjyeTM3FazkkSYQD6b97+dkWwb1iWG5AQ0EWyA7 +kQEIALkg04REDZo1JgdYV4x8HJKFS4xAYWbIva1ZPqvDNmZRUbQZR2+gpJGEwn7z +VofGvnOYiGW56AS5j31SFf5kro1+1bZQ5iOONBng08OOo58/l1hRseIIVGB5TGSa +PCdChKKHreJI6hS3mShxH6hdfFtiZuB45rwoaArMMsYcjaezLwKeLc396cpUwwcZ +snLUNd1Xu5EWEF2OdFkZ2a1qYdxBvAYdQf4+1Nr+NRIx1u1NS9c8jp3PuMOkrQEi +bNtc1v6v0Jy52mKLG4y7mC/erIkvkQBYJdxPaP7LZVaPYc3/xskcyijrJ/5ufoD8 +K71/ShtsZUXSQn9jlRaYR0EbojMAEQEAAYkBPAQYAQoAJhYhBOSJOSS3Dcepeq2X +8NaBcvSOnFKDBQJbIDuRAhsMBQkDwmcAAAoJENaBcvSOnFKDkFMIAIt64bVZ8x7+ +TitH1bR4pgcNkaKmgKoZz6FXu80+SnbuEt2NnDyf1cLOSimSTILpwLIuv9Uft5Pb +OraQbYt3xi9yrqdKqGLv80bxqK0NuryNkvh9yyx5WoG1iKqMj9/FjGghuPrRaT4l +QinNAghGVkEy1+aXGFrG2DsOC1FFI51CC2WVTzZ5RwR2GpiNRfESsU1rZAUqf/2V +yJl9bD5R4SUNy8oQmhOxi+gbhD4Ao34e4W0ilibslI/uawvCiOwlu5NGd8zv5n+U +heiQvzkApQup5c+BhH5zFDFdKJ2CBByxw9+7QjMFI/wgLixKuE0Ob2kAokXf7RlB +7qTZOahrETw= +=IKnw +-----END PGP PUBLIC KEY BLOCK-----` + +const keyWithSubKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- + +mI0EWyKwKQEEALwXhKBnyaaNFeK3ljfc/qn9X/QFw+28EUfgZPHjRmHubuXLE2uR +s3ZoSXY2z7Dkv+NyHYMt8p+X8q5fR7JvUjK2XbPyKoiJVnHINll83yl67DaWfKNL +EjNoO0kIfbXfCkZ7EG6DL+iKtuxniGTcnGT47e+HJSqb/STpLMnWwXjBABEBAAG0 +I0dvbGFuZyBHb3BoZXIgPG5vLXJlcGx5QGdvbGFuZy5jb20+iM4EEwEKADgWIQQ/ +lRafP/p9PytHbwxMvYJsOQdOOAUCWyKwKQIbAwULCQgHAwUVCgkICwUWAgMBAAIe +AQIXgAAKCRBMvYJsOQdOOOsFBAC62mXww8XuqvYLcVOvHkWLT6mhxrQOJXnlfpn7 +2uBV9CMhoG/Ycd43NONsJrB95Apr9TDIqWnVszNbqPCuBhZQSGLdbiDKjxnCWBk0 +69qv4RNtkpOhYB7jK4s8F5oQZqId6JasT/PmJTH92mhBYhhTQr0GYFuPX2UJdkw9 +Sn9C67iNBFsisDUBBAC3A+Yo9lgCnxi/pfskyLrweYif6kIXWLAtLTsM6g/6jt7b +wTrknuCPyTv0QKGXsAEe/cK/Xq3HvX9WfXPGIHc/X56ZIsHQ+RLowbZV/Lhok1IW +FAuQm8axr/by80cRwFnzhfPc/ukkAq2Qyj4hLsGblu6mxeAhzcp8aqmWOO2H9QAR +AQABiLYEKAEKACAWIQQ/lRafP/p9PytHbwxMvYJsOQdOOAUCWyK16gIdAAAKCRBM +vYJsOQdOOB1vA/4u4uLONsE+2GVOyBsHyy7uTdkuxaR9b54A/cz6jT/tzUbeIzgx +22neWhgvIEghnUZd0vEyK9k1wy5vbDlEo6nKzHso32N1QExGr5upRERAxweDxGOj +7luDwNypI7QcifE64lS/JmlnunwRCdRWMKc0Fp+7jtRc5mpwyHN/Suf5RokBagQY +AQoAIBYhBD+VFp8/+n0/K0dvDEy9gmw5B044BQJbIrA1AhsCAL8JEEy9gmw5B044 +tCAEGQEKAB0WIQSNdnkaWY6t62iX336UXbGvYdhXJwUCWyKwNQAKCRCUXbGvYdhX +JxJSA/9fCPHP6sUtGF1o3G1a3yvOUDGr1JWcct9U+QpbCt1mZoNopCNDDQAJvDWl +mvDgHfuogmgNJRjOMznvahbF+wpTXmB7LS0SK412gJzl1fFIpK4bgnhu0TwxNsO1 +8UkCZWqxRMgcNUn9z6XWONK8dgt5JNvHSHrwF4CxxwjL23AAtK+FA/UUoi3U4kbC +0XnSr1Sl+mrzQi1+H7xyMe7zjqe+gGANtskqexHzwWPUJCPZ5qpIa2l8ghiUim6b +4ymJ+N8/T8Yva1FaPEqfMzzqJr8McYFm0URioXJPvOAlRxdHPteZ0qUopt/Jawxl +Xt6B9h1YpeLoJwjwsvbi98UTRs0jXwoY +=3fWu +-----END PGP PUBLIC KEY BLOCK-----` + +const keyWithSubKeyAndBadSelfSigOrder = `-----BEGIN PGP PUBLIC KEY BLOCK----- + +mI0EWyLLDQEEAOqIOpJ/ha1OYAGduu9tS3rBz5vyjbNgJO4sFveEM0mgsHQ0X9/L +plonW+d0gRoO1dhJ8QICjDAc6+cna1DE3tEb5m6JtQ30teLZuqrR398Cf6w7NNVz +r3lrlmnH9JaKRuXl7tZciwyovneBfZVCdtsRZjaLI1uMQCz/BToiYe3DABEBAAG0 +I0dvbGFuZyBHb3BoZXIgPG5vLXJlcGx5QGdvbGFuZy5jb20+iM4EEwEKADgWIQRZ +sixZOfQcZdW0wUqmgmdsv1O9xgUCWyLLDQIbAwULCQgHAwUVCgkICwUWAgMBAAIe +AQIXgAAKCRCmgmdsv1O9xql2A/4pix98NxjhdsXtazA9agpAKeADf9tG4Za27Gj+ +3DCww/E4iP2X35jZimSm/30QRB6j08uGCqd9vXkkJxtOt63y/IpVOtWX6vMWSTUm +k8xKkaYMP0/IzKNJ1qC/qYEUYpwERBKg9Z+k99E2Ql4kRHdxXUHq6OzY79H18Y+s +GdeM/riNBFsiyxsBBAC54Pxg/8ZWaZX1phGdwfe5mek27SOYpC0AxIDCSOdMeQ6G +HPk38pywl1d+S+KmF/F4Tdi+kWro62O4eG2uc/T8JQuRDUhSjX0Qa51gPzJrUOVT +CFyUkiZ/3ZDhtXkgfuso8ua2ChBgR9Ngr4v43tSqa9y6AK7v0qjxD1x+xMrjXQAR +AQABiQFxBBgBCgAmAhsCFiEEWbIsWTn0HGXVtMFKpoJnbL9TvcYFAlsizTIFCQAN +MRcAv7QgBBkBCgAdFiEEJcoVUVJIk5RWj1c/o62jUpRPICQFAlsiyxsACgkQo62j +UpRPICQq5gQApoWIigZxXFoM0uw4uJBS5JFZtirTANvirZV5RhndwHeMN6JttaBS +YnjyA4+n1D+zB2VqliD2QrsX12KJN6rGOehCtEIClQ1Hodo9nC6kMzzAwW1O8bZs +nRJmXV+bsvD4sidLZLjdwOVa3Cxh6pvq4Uur6a7/UYx121hEY0Qx0s8JEKaCZ2y/ +U73GGi0D/i20VW8AWYAPACm2zMlzExKTOAV01YTQH/3vW0WLrOse53WcIVZga6es +HuO4So0SOEAvxKMe5HpRIu2dJxTvd99Bo9xk9xJU0AoFrO0vNCRnL+5y68xMlODK +lEw5/kl0jeaTBp6xX0HDQOEVOpPGUwWV4Ij2EnvfNDXaE1vK1kffiQFrBBgBCgAg +AhsCFiEEWbIsWTn0HGXVtMFKpoJnbL9TvcYFAlsi0AYAv7QgBBkBCgAdFiEEJcoV +UVJIk5RWj1c/o62jUpRPICQFAlsiyxsACgkQo62jUpRPICQq5gQApoWIigZxXFoM +0uw4uJBS5JFZtirTANvirZV5RhndwHeMN6JttaBSYnjyA4+n1D+zB2VqliD2QrsX +12KJN6rGOehCtEIClQ1Hodo9nC6kMzzAwW1O8bZsnRJmXV+bsvD4sidLZLjdwOVa +3Cxh6pvq4Uur6a7/UYx121hEY0Qx0s8JEKaCZ2y/U73GRl0EAJokkXmy4zKDHWWi +wvK9gi2gQgRkVnu2AiONxJb5vjeLhM/07BRmH6K1o+w3fOeEQp4FjXj1eQ5fPSM6 +Hhwx2CTl9SDnPSBMiKXsEFRkmwQ2AAsQZLmQZvKBkLZYeBiwf+IY621eYDhZfo+G +1dh1WoUCyREZsJQg2YoIpWIcvw+a +=bNRo +-----END PGP PUBLIC KEY BLOCK----- +` diff --git a/vendor/golang.org/x/crypto/openpgp/keys_test.go b/vendor/golang.org/x/crypto/openpgp/keys_test.go index d877589..0eb1a9e 100644 --- a/vendor/golang.org/x/crypto/openpgp/keys_test.go +++ b/vendor/golang.org/x/crypto/openpgp/keys_test.go @@ -180,6 +180,44 @@ func TestKeyRevocation(t *testing.T) { } } +func TestKeyWithRevokedSubKey(t *testing.T) { + // This key contains a revoked sub key: + // pub rsa1024/0x4CBD826C39074E38 2018-06-14 [SC] + // Key fingerprint = 3F95 169F 3FFA 7D3F 2B47 6F0C 4CBD 826C 3907 4E38 + // uid Golang Gopher + // sub rsa1024/0x945DB1AF61D85727 2018-06-14 [S] [revoked: 2018-06-14] + + keys, err := ReadArmoredKeyRing(bytes.NewBufferString(keyWithSubKey)) + if err != nil { + t.Fatal(err) + } + + if len(keys) != 1 { + t.Fatal("Failed to read key with a sub key") + } + + identity := keys[0].Identities["Golang Gopher "] + + // Test for an issue where Subkey Binding Signatures (RFC 4880 5.2.1) were added to the identity + // preceding the Subkey Packet if the Subkey Packet was followed by more than one signature. + // For example, the current key has the following layout: + // PUBKEY UID SELFSIG SUBKEY REV SELFSIG + // The last SELFSIG would be added to the UID's signatures. This is wrong. + if numIdentitySigs, numExpected := len(identity.Signatures), 0; numIdentitySigs != numExpected { + t.Fatalf("got %d identity signatures, expected %d", numIdentitySigs, numExpected) + } + + if numSubKeys, numExpected := len(keys[0].Subkeys), 1; numSubKeys != numExpected { + t.Fatalf("got %d subkeys, expected %d", numSubKeys, numExpected) + } + + subKey := keys[0].Subkeys[0] + if subKey.Sig == nil { + t.Fatalf("subkey signature is nil") + } + +} + func TestSubkeyRevocation(t *testing.T) { kring, err := ReadKeyRing(readerFromHex(revokedSubkeyHex)) if err != nil { @@ -216,6 +254,51 @@ func TestSubkeyRevocation(t *testing.T) { } } +func TestKeyWithSubKeyAndBadSelfSigOrder(t *testing.T) { + // This key was altered so that the self signatures following the + // subkey are in a sub-optimal order. + // + // Note: Should someone have to create a similar key again, look into + // gpgsplit, gpg --dearmor, and gpg --enarmor. + // + // The packet ordering is the following: + // PUBKEY UID UIDSELFSIG SUBKEY SELFSIG1 SELFSIG2 + // + // Where: + // SELFSIG1 expires on 2018-06-14 and was created first + // SELFSIG2 does not expire and was created after SELFSIG1 + // + // Test for RFC 4880 5.2.3.3: + // > An implementation that encounters multiple self-signatures on the + // > same object may resolve the ambiguity in any way it sees fit, but it + // > is RECOMMENDED that priority be given to the most recent self- + // > signature. + // + // This means that we should keep SELFSIG2. + + keys, err := ReadArmoredKeyRing(bytes.NewBufferString(keyWithSubKeyAndBadSelfSigOrder)) + if err != nil { + t.Fatal(err) + } + + if len(keys) != 1 { + t.Fatal("Failed to read key with a sub key and a bad selfsig packet order") + } + + key := keys[0] + + if numKeys, expected := len(key.Subkeys), 1; numKeys != expected { + t.Fatalf("Read %d subkeys, expected %d", numKeys, expected) + } + + subKey := key.Subkeys[0] + + if lifetime := subKey.Sig.KeyLifetimeSecs; lifetime != nil { + t.Errorf("The signature has a key lifetime (%d), but it should be nil", *lifetime) + } + +} + func TestKeyUsage(t *testing.T) { kring, err := ReadKeyRing(readerFromHex(subkeyUsageHex)) if err != nil { @@ -410,140 +493,3 @@ func TestNewEntityPublicSerialization(t *testing.T) { t.Fatal(err) } } - -const expiringKeyHex = "988d0451d1ec5d010400ba3385721f2dc3f4ab096b2ee867ab77213f0a27a8538441c35d2fa225b08798a1439a66a5150e6bdc3f40f5d28d588c712394c632b6299f77db8c0d48d37903fb72ebd794d61be6aa774688839e5fdecfe06b2684cc115d240c98c66cb1ef22ae84e3aa0c2b0c28665c1e7d4d044e7f270706193f5223c8d44e0d70b7b8da830011010001b40f4578706972792074657374206b657988be041301020028050251d1ec5d021b03050900278d00060b090807030206150802090a0b0416020301021e01021780000a091072589ad75e237d8c033503fd10506d72837834eb7f994117740723adc39227104b0d326a1161871c0b415d25b4aedef946ca77ea4c05af9c22b32cf98be86ab890111fced1ee3f75e87b7cc3c00dc63bbc85dfab91c0dc2ad9de2c4d13a34659333a85c6acc1a669c5e1d6cecb0cf1e56c10e72d855ae177ddc9e766f9b2dda57ccbb75f57156438bbdb4e42b88d0451d1ec5d0104009c64906559866c5cb61578f5846a94fcee142a489c9b41e67b12bb54cfe86eb9bc8566460f9a720cb00d6526fbccfd4f552071a8e3f7744b1882d01036d811ee5a3fb91a1c568055758f43ba5d2c6a9676b012f3a1a89e47bbf624f1ad571b208f3cc6224eb378f1645dd3d47584463f9eadeacfd1ce6f813064fbfdcc4b5a53001101000188a504180102000f021b0c050251d1f06b050900093e89000a091072589ad75e237d8c20e00400ab8310a41461425b37889c4da28129b5fae6084fafbc0a47dd1adc74a264c6e9c9cc125f40462ee1433072a58384daef88c961c390ed06426a81b464a53194c4e291ddd7e2e2ba3efced01537d713bd111f48437bde2363446200995e8e0d4e528dda377fd1e8f8ede9c8e2198b393bd86852ce7457a7e3daf74d510461a5b77b88d0451d1ece8010400b3a519f83ab0010307e83bca895170acce8964a044190a2b368892f7a244758d9fc193482648acb1fb9780d28cc22d171931f38bb40279389fc9bf2110876d4f3db4fcfb13f22f7083877fe56592b3b65251312c36f83ffcb6d313c6a17f197dd471f0712aad15a8537b435a92471ba2e5b0c72a6c72536c3b567c558d7b6051001101000188a504180102000f021b0c050251d1f07b050900279091000a091072589ad75e237d8ce69e03fe286026afacf7c97ee20673864d4459a2240b5655219950643c7dba0ac384b1d4359c67805b21d98211f7b09c2a0ccf6410c8c04d4ff4a51293725d8d6570d9d8bb0e10c07d22357caeb49626df99c180be02d77d1fe8ed25e7a54481237646083a9f89a11566cd20b9e995b1487c5f9e02aeb434f3a1897cd416dd0a87861838da3e9e" -const subkeyUsageHex = "988d04533a52bc010400d26af43085558f65b9e7dbc90cb9238015259aed5e954637adcfa2181548b2d0b60c65f1f42ec5081cbf1bc0a8aa4900acfb77070837c58f26012fbce297d70afe96e759ad63531f0037538e70dbf8e384569b9720d99d8eb39d8d0a2947233ed242436cb6ac7dfe74123354b3d0119b5c235d3dd9c9d6c004f8ffaf67ad8583001101000188b7041f010200210502533b8552170c8001ce094aa433f7040bb2ddf0be3893cb843d0fe70c020700000a0910a42704b92866382aa98404009d63d916a27543da4221c60087c33f1c44bec9998c5438018ed370cca4962876c748e94b73eb39c58eb698063f3fd6346d58dd2a11c0247934c4a9d71f24754f7468f96fb24c3e791dd2392b62f626148ad724189498cbf993db2df7c0cdc2d677c35da0f16cb16c9ce7c33b4de65a4a91b1d21a130ae9cc26067718910ef8e2b417556d627261203c756d627261407379642e65642e61753e88b80413010200220502533a52bc021b03060b090807030206150802090a0b0416020301021e01021780000a0910a42704b92866382a47840400c0c2bd04f5fca586de408b395b3c280a278259c93eaaa8b79a53b97003f8ed502a8a00446dd9947fb462677e4fcac0dac2f0701847d15130aadb6cd9e0705ea0cf5f92f129136c7be21a718d46c8e641eb7f044f2adae573e11ae423a0a9ca51324f03a8a2f34b91fa40c3cc764bee4dccadedb54c768ba0469b683ea53f1c29b88d04533a52bc01040099c92a5d6f8b744224da27bc2369127c35269b58bec179de6bbc038f749344222f85a31933224f26b70243c4e4b2d242f0c4777eaef7b5502f9dad6d8bf3aaeb471210674b74de2d7078af497d55f5cdad97c7bedfbc1b41e8065a97c9c3d344b21fc81d27723af8e374bc595da26ea242dccb6ae497be26eea57e563ed517e90011010001889f0418010200090502533a52bc021b0c000a0910a42704b92866382afa1403ff70284c2de8a043ff51d8d29772602fa98009b7861c540535f874f2c230af8caf5638151a636b21f8255003997ccd29747fdd06777bb24f9593bd7d98a3e887689bf902f999915fcc94625ae487e5d13e6616f89090ebc4fdc7eb5cad8943e4056995bb61c6af37f8043016876a958ec7ebf39c43d20d53b7f546cfa83e8d2604b88d04533b8283010400c0b529316dbdf58b4c54461e7e669dc11c09eb7f73819f178ccd4177b9182b91d138605fcf1e463262fabefa73f94a52b5e15d1904635541c7ea540f07050ce0fb51b73e6f88644cec86e91107c957a114f69554548a85295d2b70bd0b203992f76eb5d493d86d9eabcaa7ef3fc7db7e458438db3fcdb0ca1cc97c638439a9170011010001889f0418010200090502533b8283021b0c000a0910a42704b92866382adc6d0400cfff6258485a21675adb7a811c3e19ebca18851533f75a7ba317950b9997fda8d1a4c8c76505c08c04b6c2cc31dc704d33da36a21273f2b388a1a706f7c3378b66d887197a525936ed9a69acb57fe7f718133da85ec742001c5d1864e9c6c8ea1b94f1c3759cebfd93b18606066c063a63be86085b7e37bdbc65f9a915bf084bb901a204533b85cd110400aed3d2c52af2b38b5b67904b0ef73d6dd7aef86adb770e2b153cd22489654dcc91730892087bb9856ae2d9f7ed1eb48f214243fe86bfe87b349ebd7c30e630e49c07b21fdabf78b7a95c8b7f969e97e3d33f2e074c63552ba64a2ded7badc05ce0ea2be6d53485f6900c7860c7aa76560376ce963d7271b9b54638a4028b573f00a0d8854bfcdb04986141568046202192263b9b67350400aaa1049dbc7943141ef590a70dcb028d730371d92ea4863de715f7f0f16d168bd3dc266c2450457d46dcbbf0b071547e5fbee7700a820c3750b236335d8d5848adb3c0da010e998908dfd93d961480084f3aea20b247034f8988eccb5546efaa35a92d0451df3aaf1aee5aa36a4c4d462c760ecd9cebcabfbe1412b1f21450f203fd126687cd486496e971a87fd9e1a8a765fe654baa219a6871ab97768596ab05c26c1aeea8f1a2c72395a58dbc12ef9640d2b95784e974a4d2d5a9b17c25fedacfe551bda52602de8f6d2e48443f5dd1a2a2a8e6a5e70ecdb88cd6e766ad9745c7ee91d78cc55c3d06536b49c3fee6c3d0b6ff0fb2bf13a314f57c953b8f4d93bf88e70418010200090502533b85cd021b0200520910a42704b92866382a47200419110200060502533b85cd000a091042ce2c64bc0ba99214b2009e26b26852c8b13b10c35768e40e78fbbb48bd084100a0c79d9ea0844fa5853dd3c85ff3ecae6f2c9dd6c557aa04008bbbc964cd65b9b8299d4ebf31f41cc7264b8cf33a00e82c5af022331fac79efc9563a822497ba012953cefe2629f1242fcdcb911dbb2315985bab060bfd58261ace3c654bdbbe2e8ed27a46e836490145c86dc7bae15c011f7e1ffc33730109b9338cd9f483e7cef3d2f396aab5bd80efb6646d7e778270ee99d934d187dd98" -const revokedKeyHex = "988d045331ce82010400c4fdf7b40a5477f206e6ee278eaef888ca73bf9128a9eef9f2f1ddb8b7b71a4c07cfa241f028a04edb405e4d916c61d6beabc333813dc7b484d2b3c52ee233c6a79b1eea4e9cc51596ba9cd5ac5aeb9df62d86ea051055b79d03f8a4fa9f38386f5bd17529138f3325d46801514ea9047977e0829ed728e68636802796801be10011010001889f04200102000905025331d0e3021d03000a0910a401d9f09a34f7c042aa040086631196405b7e6af71026b88e98012eab44aa9849f6ef3fa930c7c9f23deaedba9db1538830f8652fb7648ec3fcade8dbcbf9eaf428e83c6cbcc272201bfe2fbb90d41963397a7c0637a1a9d9448ce695d9790db2dc95433ad7be19eb3de72dacf1d6db82c3644c13eae2a3d072b99bb341debba012c5ce4006a7d34a1f4b94b444526567205265766f6b657220283c52656727732022424d204261726973746122204b657920262530305c303e5c29203c72656740626d626172697374612e636f2e61753e88b704130102002205025331ce82021b03060b090807030206150802090a0b0416020301021e01021780000a0910a401d9f09a34f7c0019c03f75edfbeb6a73e7225ad3cc52724e2872e04260d7daf0d693c170d8c4b243b8767bc7785763533febc62ec2600c30603c433c095453ede59ff2fcabeb84ce32e0ed9d5cf15ffcbc816202b64370d4d77c1e9077d74e94a16fb4fa2e5bec23a56d7a73cf275f91691ae1801a976fcde09e981a2f6327ac27ea1fecf3185df0d56889c04100102000605025331cfb5000a0910fe9645554e8266b64b4303fc084075396674fb6f778d302ac07cef6bc0b5d07b66b2004c44aef711cbac79617ef06d836b4957522d8772dd94bf41a2f4ac8b1ee6d70c57503f837445a74765a076d07b829b8111fc2a918423ddb817ead7ca2a613ef0bfb9c6b3562aec6c3cf3c75ef3031d81d95f6563e4cdcc9960bcb386c5d757b104fcca5fe11fc709df884604101102000605025331cfe7000a09107b15a67f0b3ddc0317f6009e360beea58f29c1d963a22b962b80788c3fa6c84e009d148cfde6b351469b8eae91187eff07ad9d08fcaab88d045331ce820104009f25e20a42b904f3fa555530fe5c46737cf7bd076c35a2a0d22b11f7e0b61a69320b768f4a80fe13980ce380d1cfc4a0cd8fbe2d2e2ef85416668b77208baa65bf973fe8e500e78cc310d7c8705cdb34328bf80e24f0385fce5845c33bc7943cf6b11b02348a23da0bf6428e57c05135f2dc6bd7c1ce325d666d5a5fd2fd5e410011010001889f04180102000905025331ce82021b0c000a0910a401d9f09a34f7c0418003fe34feafcbeaef348a800a0d908a7a6809cc7304017d820f70f0474d5e23cb17e38b67dc6dca282c6ca00961f4ec9edf2738d0f087b1d81e4871ef08e1798010863afb4eac4c44a376cb343be929c5be66a78cfd4456ae9ec6a99d97f4e1c3ff3583351db2147a65c0acef5c003fb544ab3a2e2dc4d43646f58b811a6c3a369d1f" -const revokedSubkeyHex = "988d04533121f6010400aefc803a3e4bb1a61c86e8a86d2726c6a43e0079e9f2713f1fa017e9854c83877f4aced8e331d675c67ea83ddab80aacbfa0b9040bb12d96f5a3d6be09455e2a76546cbd21677537db941cab710216b6d24ec277ee0bd65b910f416737ed120f6b93a9d3b306245c8cfd8394606fdb462e5cf43c551438d2864506c63367fc890011010001b41d416c696365203c616c69636540626d626172697374612e636f2e61753e88bb041301020025021b03060b090807030206150802090a0b0416020301021e01021780050253312798021901000a09104ef7e4beccde97f015a803ff5448437780f63263b0df8442a995e7f76c221351a51edd06f2063d8166cf3157aada4923dfc44aa0f2a6a4da5cf83b7fe722ba8ab416c976e77c6b5682e7f1069026673bd0de56ba06fd5d7a9f177607f277d9b55ff940a638c3e68525c67517e2b3d976899b93ca267f705b3e5efad7d61220e96b618a4497eab8d04403d23f8846041011020006050253312910000a09107b15a67f0b3ddc03d96e009f50b6365d86c4be5d5e9d0ea42d5e56f5794c617700a0ab274e19c2827780016d23417ce89e0a2c0d987d889c04100102000605025331cf7a000a0910a401d9f09a34f7c0ee970400aca292f213041c9f3b3fc49148cbda9d84afee6183c8dd6c5ff2600b29482db5fecd4303797be1ee6d544a20a858080fec43412061c9a71fae4039fd58013b4ae341273e6c66ad4c7cdd9e68245bedb260562e7b166f2461a1032f2b38c0e0e5715fb3d1656979e052b55ca827a76f872b78a9fdae64bc298170bfcebedc1271b41a416c696365203c616c696365407379646973702e6f722e61753e88b804130102002205025331278b021b03060b090807030206150802090a0b0416020301021e01021780000a09104ef7e4beccde97f06a7003fa03c3af68d272ebc1fa08aa72a03b02189c26496a2833d90450801c4e42c5b5f51ad96ce2d2c9cef4b7c02a6a2fcf1412d6a2d486098eb762f5010a201819c17fd2888aec8eda20c65a3b75744de7ee5cc8ac7bfc470cbe3cb982720405a27a3c6a8c229cfe36905f881b02ed5680f6a8f05866efb9d6c5844897e631deb949ca8846041011020006050253312910000a09107b15a67f0b3ddc0347bc009f7fa35db59147469eb6f2c5aaf6428accb138b22800a0caa2f5f0874bacc5909c652a57a31beda65eddd5889c04100102000605025331cf7a000a0910a401d9f09a34f7c0316403ff46f2a5c101256627f16384d34a38fb47a6c88ba60506843e532d91614339fccae5f884a5741e7582ffaf292ba38ee10a270a05f139bde3814b6a077e8cd2db0f105ebea2a83af70d385f13b507fac2ad93ff79d84950328bb86f3074745a8b7f9b64990fb142e2a12976e27e8d09a28dc5621f957ac49091116da410ac3cbde1b88d04533121f6010400cbd785b56905e4192e2fb62a720727d43c4fa487821203cf72138b884b78b701093243e1d8c92a0248a6c0203a5a88693da34af357499abacaf4b3309c640797d03093870a323b4b6f37865f6eaa2838148a67df4735d43a90ca87942554cdf1c4a751b1e75f9fd4ce4e97e278d6c1c7ed59d33441df7d084f3f02beb68896c70011010001889f0418010200090502533121f6021b0c000a09104ef7e4beccde97f0b98b03fc0a5ccf6a372995835a2f5da33b282a7d612c0ab2a97f59cf9fff73e9110981aac2858c41399afa29624a7fd8a0add11654e3d882c0fd199e161bdad65e5e2548f7b68a437ea64293db1246e3011cbb94dc1bcdeaf0f2539bd88ff16d95547144d97cead6a8c5927660a91e6db0d16eb36b7b49a3525b54d1644e65599b032b7eb901a204533127a0110400bd3edaa09eff9809c4edc2c2a0ebe52e53c50a19c1e49ab78e6167bf61473bb08f2050d78a5cbbc6ed66aff7b42cd503f16b4a0b99fa1609681fca9b7ce2bbb1a5b3864d6cdda4d7ef7849d156d534dea30fb0efb9e4cf8959a2b2ce623905882d5430b995a15c3b9fe92906086788b891002924f94abe139b42cbbfaaabe42f00a0b65dc1a1ad27d798adbcb5b5ad02d2688c89477b03ff4eebb6f7b15a73b96a96bed201c0e5e4ea27e4c6e2dd1005b94d4b90137a5b1cf5e01c6226c070c4cc999938101578877ee76d296b9aab8246d57049caacf489e80a3f40589cade790a020b1ac146d6f7a6241184b8c7fcde680eae3188f5dcbe846d7f7bdad34f6fcfca08413e19c1d5df83fc7c7c627d493492e009c2f52a80400a2fe82de87136fd2e8845888c4431b032ba29d9a29a804277e31002a8201fb8591a3e55c7a0d0881496caf8b9fb07544a5a4879291d0dc026a0ea9e5bd88eb4aa4947bbd694b25012e208a250d65ddc6f1eea59d3aed3b4ec15fcab85e2afaa23a40ab1ef9ce3e11e1bc1c34a0e758e7aa64deb8739276df0af7d4121f834a9b88e70418010200090502533127a0021b02005209104ef7e4beccde97f047200419110200060502533127a0000a0910dbce4ee19529437fe045009c0b32f5ead48ee8a7e98fac0dea3d3e6c0e2c552500a0ad71fadc5007cfaf842d9b7db3335a8cdad15d3d1a6404009b08e2c68fe8f3b45c1bb72a4b3278cdf3012aa0f229883ad74aa1f6000bb90b18301b2f85372ca5d6b9bf478d235b733b1b197d19ccca48e9daf8e890cb64546b4ce1b178faccfff07003c172a2d4f5ebaba9f57153955f3f61a9b80a4f5cb959908f8b211b03b7026a8a82fc612bfedd3794969bcf458c4ce92be215a1176ab88d045331d144010400a5063000c5aaf34953c1aa3bfc95045b3aab9882b9a8027fecfe2142dc6b47ba8aca667399990244d513dd0504716908c17d92c65e74219e004f7b83fc125e575dd58efec3ab6dd22e3580106998523dea42ec75bf9aa111734c82df54630bebdff20fe981cfc36c76f865eb1c2fb62c9e85bc3a6e5015a361a2eb1c8431578d0011010001889f04280102000905025331d433021d03000a09104ef7e4beccde97f02e5503ff5e0630d1b65291f4882b6d40a29da4616bb5088717d469fbcc3648b8276de04a04988b1f1b9f3e18f52265c1f8b6c85861691c1a6b8a3a25a1809a0b32ad330aec5667cb4262f4450649184e8113849b05e5ad06a316ea80c001e8e71838190339a6e48bbde30647bcf245134b9a97fa875c1d83a9862cae87ffd7e2c4ce3a1b89013d04180102000905025331d144021b0200a809104ef7e4beccde97f09d2004190102000605025331d144000a0910677815e371c2fd23522203fe22ab62b8e7a151383cea3edd3a12995693911426f8ccf125e1f6426388c0010f88d9ca7da2224aee8d1c12135998640c5e1813d55a93df472faae75bef858457248db41b4505827590aeccf6f9eb646da7f980655dd3050c6897feddddaca90676dee856d66db8923477d251712bb9b3186b4d0114daf7d6b59272b53218dd1da94a03ff64006fcbe71211e5daecd9961fba66cdb6de3f914882c58ba5beddeba7dcb950c1156d7fba18c19ea880dccc800eae335deec34e3b84ac75ffa24864f782f87815cda1c0f634b3dd2fa67cea30811d21723d21d9551fa12ccbcfa62b6d3a15d01307b99925707992556d50065505b090aadb8579083a20fe65bd2a270da9b011" -const missingCrossSignatureKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- -Charset: UTF-8 - -mQENBFMYynYBCACVOZ3/e8Bm2b9KH9QyIlHGo/i1bnkpqsgXj8tpJ2MIUOnXMMAY -ztW7kKFLCmgVdLIC0vSoLA4yhaLcMojznh/2CcUglZeb6Ao8Gtelr//Rd5DRfPpG -zqcfUo+m+eO1co2Orabw0tZDfGpg5p3AYl0hmxhUyYSc/xUq93xL1UJzBFgYXY54 -QsM8dgeQgFseSk/YvdP5SMx1ev+eraUyiiUtWzWrWC1TdyRa5p4UZg6Rkoppf+WJ -QrW6BWrhAtqATHc8ozV7uJjeONjUEq24roRc/OFZdmQQGK6yrzKnnbA6MdHhqpdo -9kWDcXYb7pSE63Lc+OBa5X2GUVvXJLS/3nrtABEBAAG0F2ludmFsaWQtc2lnbmlu -Zy1zdWJrZXlziQEoBBMBAgASBQJTnKB5AhsBAgsHAhUIAh4BAAoJEO3UDQUIHpI/ -dN4H/idX4FQ1LIZCnpHS/oxoWQWfpRgdKAEM0qCqjMgiipJeEwSQbqjTCynuh5/R -JlODDz85ABR06aoF4l5ebGLQWFCYifPnJZ/Yf5OYcMGtb7dIbqxWVFL9iLMO/oDL -ioI3dotjPui5e+2hI9pVH1UHB/bZ/GvMGo6Zg0XxLPolKQODMVjpjLAQ0YJ3spew -RAmOGre6tIvbDsMBnm8qREt7a07cBJ6XK7xjxYaZHQBiHVxyEWDa6gyANONx8duW -/fhQ/zDTnyVM/ik6VO0Ty9BhPpcEYLFwh5c1ilFari1ta3e6qKo6ZGa9YMk/REhu -yBHd9nTkI+0CiQUmbckUiVjDKKe5AQ0EUxjKdgEIAJcXQeP+NmuciE99YcJoffxv -2gVLU4ZXBNHEaP0mgaJ1+tmMD089vUQAcyGRvw8jfsNsVZQIOAuRxY94aHQhIRHR -bUzBN28ofo/AJJtfx62C15xt6fDKRV6HXYqAiygrHIpEoRLyiN69iScUsjIJeyFL -C8wa72e8pSL6dkHoaV1N9ZH/xmrJ+k0vsgkQaAh9CzYufncDxcwkoP+aOlGtX1gP -WwWoIbz0JwLEMPHBWvDDXQcQPQTYQyj+LGC9U6f9VZHN25E94subM1MjuT9OhN9Y -MLfWaaIc5WyhLFyQKW2Upofn9wSFi8ubyBnv640Dfd0rVmaWv7LNTZpoZ/GbJAMA -EQEAAYkBHwQYAQIACQUCU5ygeQIbAgAKCRDt1A0FCB6SP0zCB/sEzaVR38vpx+OQ -MMynCBJrakiqDmUZv9xtplY7zsHSQjpd6xGflbU2n+iX99Q+nav0ETQZifNUEd4N -1ljDGQejcTyKD6Pkg6wBL3x9/RJye7Zszazm4+toJXZ8xJ3800+BtaPoI39akYJm -+ijzbskvN0v/j5GOFJwQO0pPRAFtdHqRs9Kf4YanxhedB4dIUblzlIJuKsxFit6N -lgGRblagG3Vv2eBszbxzPbJjHCgVLR3RmrVezKOsZjr/2i7X+xLWIR0uD3IN1qOW -CXQxLBizEEmSNVNxsp7KPGTLnqO3bPtqFirxS9PJLIMPTPLNBY7ZYuPNTMqVIUWF -4artDmrG -=7FfJ ------END PGP PUBLIC KEY BLOCK-----` - -const invalidCrossSignatureKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- - -mQENBFMYynYBCACVOZ3/e8Bm2b9KH9QyIlHGo/i1bnkpqsgXj8tpJ2MIUOnXMMAY -ztW7kKFLCmgVdLIC0vSoLA4yhaLcMojznh/2CcUglZeb6Ao8Gtelr//Rd5DRfPpG -zqcfUo+m+eO1co2Orabw0tZDfGpg5p3AYl0hmxhUyYSc/xUq93xL1UJzBFgYXY54 -QsM8dgeQgFseSk/YvdP5SMx1ev+eraUyiiUtWzWrWC1TdyRa5p4UZg6Rkoppf+WJ -QrW6BWrhAtqATHc8ozV7uJjeONjUEq24roRc/OFZdmQQGK6yrzKnnbA6MdHhqpdo -9kWDcXYb7pSE63Lc+OBa5X2GUVvXJLS/3nrtABEBAAG0F2ludmFsaWQtc2lnbmlu -Zy1zdWJrZXlziQEoBBMBAgASBQJTnKB5AhsBAgsHAhUIAh4BAAoJEO3UDQUIHpI/ -dN4H/idX4FQ1LIZCnpHS/oxoWQWfpRgdKAEM0qCqjMgiipJeEwSQbqjTCynuh5/R -JlODDz85ABR06aoF4l5ebGLQWFCYifPnJZ/Yf5OYcMGtb7dIbqxWVFL9iLMO/oDL -ioI3dotjPui5e+2hI9pVH1UHB/bZ/GvMGo6Zg0XxLPolKQODMVjpjLAQ0YJ3spew -RAmOGre6tIvbDsMBnm8qREt7a07cBJ6XK7xjxYaZHQBiHVxyEWDa6gyANONx8duW -/fhQ/zDTnyVM/ik6VO0Ty9BhPpcEYLFwh5c1ilFari1ta3e6qKo6ZGa9YMk/REhu -yBHd9nTkI+0CiQUmbckUiVjDKKe5AQ0EUxjKdgEIAIINDqlj7X6jYKc6DjwrOkjQ -UIRWbQQar0LwmNilehmt70g5DCL1SYm9q4LcgJJ2Nhxj0/5qqsYib50OSWMcKeEe -iRXpXzv1ObpcQtI5ithp0gR53YPXBib80t3bUzomQ5UyZqAAHzMp3BKC54/vUrSK -FeRaxDzNLrCeyI00+LHNUtwghAqHvdNcsIf8VRumK8oTm3RmDh0TyjASWYbrt9c8 -R1Um3zuoACOVy+mEIgIzsfHq0u7dwYwJB5+KeM7ZLx+HGIYdUYzHuUE1sLwVoELh -+SHIGHI1HDicOjzqgajShuIjj5hZTyQySVprrsLKiXS6NEwHAP20+XjayJ/R3tEA -EQEAAYkCPgQYAQIBKAUCU5ygeQIbAsBdIAQZAQIABgUCU5ygeQAKCRCpVlnFZmhO -52RJB/9uD1MSa0wjY6tHOIgquZcP3bHBvHmrHNMw9HR2wRCMO91ZkhrpdS3ZHtgb -u3/55etj0FdvDo1tb8P8FGSVtO5Vcwf5APM8sbbqoi8L951Q3i7qt847lfhu6sMl -w0LWFvPTOLHrliZHItPRjOltS1WAWfr2jUYhsU9ytaDAJmvf9DujxEOsN5G1YJep -54JCKVCkM/y585Zcnn+yxk/XwqoNQ0/iJUT9qRrZWvoeasxhl1PQcwihCwss44A+ -YXaAt3hbk+6LEQuZoYS73yR3WHj+42tfm7YxRGeubXfgCEz/brETEWXMh4pe0vCL -bfWrmfSPq2rDegYcAybxRQz0lF8PAAoJEO3UDQUIHpI/exkH/0vQfdHA8g/N4T6E -i6b1CUVBAkvtdJpCATZjWPhXmShOw62gkDw306vHPilL4SCvEEi4KzG72zkp6VsB -DSRcpxCwT4mHue+duiy53/aRMtSJ+vDfiV1Vhq+3sWAck/yUtfDU9/u4eFaiNok1 -8/Gd7reyuZt5CiJnpdPpjCwelK21l2w7sHAnJF55ITXdOxI8oG3BRKufz0z5lyDY -s2tXYmhhQIggdgelN8LbcMhWs/PBbtUr6uZlNJG2lW1yscD4aI529VjwJlCeo745 -U7pO4eF05VViUJ2mmfoivL3tkhoTUWhx8xs8xCUcCg8DoEoSIhxtOmoTPR22Z9BL -6LCg2mg= -=Dhm4 ------END PGP PUBLIC KEY BLOCK-----` - -const goodCrossSignatureKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- -Version: GnuPG v1 - -mI0EVUqeVwEEAMufHRrMPWK3gyvi0O0tABCs/oON9zV9KDZlr1a1M91ShCSFwCPo -7r80PxdWVWcj0V5h50/CJYtpN3eE/mUIgW2z1uDYQF1OzrQ8ubrksfsJvpAhENom -lTQEppv9mV8qhcM278teb7TX0pgrUHLYF5CfPdp1L957JLLXoQR/lwLVABEBAAG0 -E2dvb2Qtc2lnbmluZy1zdWJrZXmIuAQTAQIAIgUCVUqeVwIbAwYLCQgHAwIGFQgC -CQoLBBYCAwECHgECF4AACgkQNRjL95IRWP69XQQAlH6+eyXJN4DZTLX78KGjHrsw -6FCvxxClEPtPUjcJy/1KCRQmtLAt9PbbA78dvgzjDeZMZqRAwdjyJhjyg/fkU2OH -7wq4ktjUu+dLcOBb+BFMEY+YjKZhf6EJuVfxoTVr5f82XNPbYHfTho9/OABKH6kv -X70PaKZhbwnwij8Nts65AaIEVUqftREEAJ3WxZfqAX0bTDbQPf2CMT2IVMGDfhK7 -GyubOZgDFFjwUJQvHNvsrbeGLZ0xOBumLINyPO1amIfTgJNm1iiWFWfmnHReGcDl -y5mpYG60Mb79Whdcer7CMm3AqYh/dW4g6IB02NwZMKoUHo3PXmFLxMKXnWyJ0clw -R0LI/Qn509yXAKDh1SO20rqrBM+EAP2c5bfI98kyNwQAi3buu94qo3RR1ZbvfxgW -CKXDVm6N99jdZGNK7FbRifXqzJJDLcXZKLnstnC4Sd3uyfyf1uFhmDLIQRryn5m+ -LBYHfDBPN3kdm7bsZDDq9GbTHiFZUfm/tChVKXWxkhpAmHhU/tH6GGzNSMXuIWSO -aOz3Rqq0ED4NXyNKjdF9MiwD/i83S0ZBc0LmJYt4Z10jtH2B6tYdqnAK29uQaadx -yZCX2scE09UIm32/w7pV77CKr1Cp/4OzAXS1tmFzQ+bX7DR+Gl8t4wxr57VeEMvl -BGw4Vjh3X8//m3xynxycQU18Q1zJ6PkiMyPw2owZ/nss3hpSRKFJsxMLhW3fKmKr -Ey2KiOcEGAECAAkFAlVKn7UCGwIAUgkQNRjL95IRWP5HIAQZEQIABgUCVUqftQAK -CRD98VjDN10SqkWrAKDTpEY8D8HC02E/KVC5YUI01B30wgCgurpILm20kXEDCeHp -C5pygfXw1DJrhAP+NyPJ4um/bU1I+rXaHHJYroYJs8YSweiNcwiHDQn0Engh/mVZ -SqLHvbKh2dL/RXymC3+rjPvQf5cup9bPxNMa6WagdYBNAfzWGtkVISeaQW+cTEp/ -MtgVijRGXR/lGLGETPg2X3Afwn9N9bLMBkBprKgbBqU7lpaoPupxT61bL70= -=vtbN ------END PGP PUBLIC KEY BLOCK-----` - -const revokedUserIDKey = `-----BEGIN PGP PUBLIC KEY BLOCK----- - -mQENBFsgO5EBCADhREPmcjsPkXe1z7ctvyWL0S7oa9JaoGZ9oPDHFDlQxd0qlX2e -DZJZDg0qYvVixmaULIulApq1puEsaJCn3lHUbHlb4PYKwLEywYXM28JN91KtLsz/ -uaEX2KC5WqeP40utmzkNLq+oRX/xnRMgwbO7yUNVG2UlEa6eI+xOXO3YtLdmJMBW -ClQ066ZnOIzEo1JxnIwha1CDBMWLLfOLrg6l8InUqaXbtEBbnaIYO6fXVXELUjkx -nmk7t/QOk0tXCy8muH9UDqJkwDUESY2l79XwBAcx9riX8vY7vwC34pm22fAUVLCJ -x1SJx0J8bkeNp38jKM2Zd9SUQqSbfBopQ4pPABEBAAG0I0dvbGFuZyBHb3BoZXIg -PG5vLXJlcGx5QGdvbGFuZy5jb20+iQFUBBMBCgA+FiEE5Ik5JLcNx6l6rZfw1oFy -9I6cUoMFAlsgO5ECGwMFCQPCZwAFCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQ -1oFy9I6cUoMIkwf8DNPeD23i4jRwd/pylbvxwZintZl1fSwTJW1xcOa1emXaEtX2 -depuqhP04fjlRQGfsYAQh7X9jOJxAHjTmhqFBi5sD7QvKU00cPFYbJ/JTx0B41bl -aXnSbGhRPh63QtEZL7ACAs+shwvvojJqysx7kyVRu0EW2wqjXdHwR/SJO6nhNBa2 -DXzSiOU/SUA42mmG+5kjF8Aabq9wPwT9wjraHShEweNerNMmOqJExBOy3yFeyDpa -XwEZFzBfOKoxFNkIaVf5GSdIUGhFECkGvBMB935khftmgR8APxdU4BE7XrXexFJU -8RCuPXonm4WQOwTWR0vQg64pb2WKAzZ8HhwTGbQiR29sYW5nIEdvcGhlciA8cmV2 -b2tlZEBnb2xhbmcuY29tPokBNgQwAQoAIBYhBOSJOSS3Dcepeq2X8NaBcvSOnFKD -BQJbIDv3Ah0AAAoJENaBcvSOnFKDfWMIAKhI/Tvu3h8fSUxp/gSAcduT6bC1JttG -0lYQ5ilKB/58lBUA5CO3ZrKDKlzW3M8VEcvohVaqeTMKeoQd5rCZq8KxHn/KvN6N -s85REfXfniCKfAbnGgVXX3kDmZ1g63pkxrFu0fDZjVDXC6vy+I0sGyI/Inro0Pzb -tvn0QCsxjapKK15BtmSrpgHgzVqVg0cUp8vqZeKFxarYbYB2idtGRci4b9tObOK0 -BSTVFy26+I/mrFGaPrySYiy2Kz5NMEcRhjmTxJ8jSwEr2O2sUR0yjbgUAXbTxDVE -/jg5fQZ1ACvBRQnB7LvMHcInbzjyeTM3FazkkSYQD6b97+dkWwb1iWG5AQ0EWyA7 -kQEIALkg04REDZo1JgdYV4x8HJKFS4xAYWbIva1ZPqvDNmZRUbQZR2+gpJGEwn7z -VofGvnOYiGW56AS5j31SFf5kro1+1bZQ5iOONBng08OOo58/l1hRseIIVGB5TGSa -PCdChKKHreJI6hS3mShxH6hdfFtiZuB45rwoaArMMsYcjaezLwKeLc396cpUwwcZ -snLUNd1Xu5EWEF2OdFkZ2a1qYdxBvAYdQf4+1Nr+NRIx1u1NS9c8jp3PuMOkrQEi -bNtc1v6v0Jy52mKLG4y7mC/erIkvkQBYJdxPaP7LZVaPYc3/xskcyijrJ/5ufoD8 -K71/ShtsZUXSQn9jlRaYR0EbojMAEQEAAYkBPAQYAQoAJhYhBOSJOSS3Dcepeq2X -8NaBcvSOnFKDBQJbIDuRAhsMBQkDwmcAAAoJENaBcvSOnFKDkFMIAIt64bVZ8x7+ -TitH1bR4pgcNkaKmgKoZz6FXu80+SnbuEt2NnDyf1cLOSimSTILpwLIuv9Uft5Pb -OraQbYt3xi9yrqdKqGLv80bxqK0NuryNkvh9yyx5WoG1iKqMj9/FjGghuPrRaT4l -QinNAghGVkEy1+aXGFrG2DsOC1FFI51CC2WVTzZ5RwR2GpiNRfESsU1rZAUqf/2V -yJl9bD5R4SUNy8oQmhOxi+gbhD4Ao34e4W0ilibslI/uawvCiOwlu5NGd8zv5n+U -heiQvzkApQup5c+BhH5zFDFdKJ2CBByxw9+7QjMFI/wgLixKuE0Ob2kAokXf7RlB -7qTZOahrETw= -=IKnw ------END PGP PUBLIC KEY BLOCK-----` diff --git a/vendor/golang.org/x/crypto/openpgp/packet/opaque_test.go b/vendor/golang.org/x/crypto/openpgp/packet/opaque_test.go index f27bbfe..61159f4 100644 --- a/vendor/golang.org/x/crypto/openpgp/packet/opaque_test.go +++ b/vendor/golang.org/x/crypto/openpgp/packet/opaque_test.go @@ -32,7 +32,7 @@ func TestOpaqueParseReason(t *testing.T) { break } // try to parse opaque packet - p, err := op.Parse() + p, _ := op.Parse() switch pkt := p.(type) { case *UserId: uid = pkt diff --git a/vendor/golang.org/x/crypto/openpgp/packet/packet.go b/vendor/golang.org/x/crypto/openpgp/packet/packet.go index 625bb5a..5af64c5 100644 --- a/vendor/golang.org/x/crypto/openpgp/packet/packet.go +++ b/vendor/golang.org/x/crypto/openpgp/packet/packet.go @@ -404,14 +404,16 @@ const ( type PublicKeyAlgorithm uint8 const ( - PubKeyAlgoRSA PublicKeyAlgorithm = 1 - PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2 - PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3 - PubKeyAlgoElGamal PublicKeyAlgorithm = 16 - PubKeyAlgoDSA PublicKeyAlgorithm = 17 + PubKeyAlgoRSA PublicKeyAlgorithm = 1 + PubKeyAlgoElGamal PublicKeyAlgorithm = 16 + PubKeyAlgoDSA PublicKeyAlgorithm = 17 // RFC 6637, Section 5. PubKeyAlgoECDH PublicKeyAlgorithm = 18 PubKeyAlgoECDSA PublicKeyAlgorithm = 19 + + // Deprecated in RFC 4880, Section 13.5. Use key flags instead. + PubKeyAlgoRSAEncryptOnly PublicKeyAlgorithm = 2 + PubKeyAlgoRSASignOnly PublicKeyAlgorithm = 3 ) // CanEncrypt returns true if it's possible to encrypt a message to a public diff --git a/vendor/golang.org/x/crypto/openpgp/packet/private_key.go b/vendor/golang.org/x/crypto/openpgp/packet/private_key.go index 34734cc..bd31cce 100644 --- a/vendor/golang.org/x/crypto/openpgp/packet/private_key.go +++ b/vendor/golang.org/x/crypto/openpgp/packet/private_key.go @@ -64,14 +64,19 @@ func NewECDSAPrivateKey(currentTime time.Time, priv *ecdsa.PrivateKey) *PrivateK return pk } -// NewSignerPrivateKey creates a sign-only PrivateKey from a crypto.Signer that +// NewSignerPrivateKey creates a PrivateKey from a crypto.Signer that // implements RSA or ECDSA. func NewSignerPrivateKey(currentTime time.Time, signer crypto.Signer) *PrivateKey { pk := new(PrivateKey) + // In general, the public Keys should be used as pointers. We still + // type-switch on the values, for backwards-compatibility. switch pubkey := signer.Public().(type) { + case *rsa.PublicKey: + pk.PublicKey = *NewRSAPublicKey(currentTime, pubkey) case rsa.PublicKey: pk.PublicKey = *NewRSAPublicKey(currentTime, &pubkey) - pk.PubKeyAlgo = PubKeyAlgoRSASignOnly + case *ecdsa.PublicKey: + pk.PublicKey = *NewECDSAPublicKey(currentTime, pubkey) case ecdsa.PublicKey: pk.PublicKey = *NewECDSAPublicKey(currentTime, &pubkey) default: diff --git a/vendor/golang.org/x/crypto/openpgp/packet/private_key_test.go b/vendor/golang.org/x/crypto/openpgp/packet/private_key_test.go index ac651d9..cc08b48 100644 --- a/vendor/golang.org/x/crypto/openpgp/packet/private_key_test.go +++ b/vendor/golang.org/x/crypto/openpgp/packet/private_key_test.go @@ -14,7 +14,6 @@ import ( "crypto/x509" "encoding/hex" "hash" - "io" "testing" "time" ) @@ -162,15 +161,7 @@ func TestECDSAPrivateKey(t *testing.T) { } type rsaSigner struct { - priv *rsa.PrivateKey -} - -func (s *rsaSigner) Public() crypto.PublicKey { - return s.priv.PublicKey -} - -func (s *rsaSigner) Sign(rand io.Reader, msg []byte, opts crypto.SignerOpts) ([]byte, error) { - return s.priv.Sign(rand, msg, opts) + *rsa.PrivateKey } func TestRSASignerPrivateKey(t *testing.T) { @@ -181,12 +172,8 @@ func TestRSASignerPrivateKey(t *testing.T) { priv := NewSignerPrivateKey(time.Now(), &rsaSigner{rsaPriv}) - if priv.PubKeyAlgo != PubKeyAlgoRSASignOnly { - t.Fatal("NewSignerPrivateKey should have made a sign-only RSA private key") - } - sig := &Signature{ - PubKeyAlgo: PubKeyAlgoRSASignOnly, + PubKeyAlgo: PubKeyAlgoRSA, Hash: crypto.SHA256, } msg := []byte("Hello World!") @@ -208,15 +195,7 @@ func TestRSASignerPrivateKey(t *testing.T) { } type ecdsaSigner struct { - priv *ecdsa.PrivateKey -} - -func (s *ecdsaSigner) Public() crypto.PublicKey { - return s.priv.PublicKey -} - -func (s *ecdsaSigner) Sign(rand io.Reader, msg []byte, opts crypto.SignerOpts) ([]byte, error) { - return s.priv.Sign(rand, msg, opts) + *ecdsa.PrivateKey } func TestECDSASignerPrivateKey(t *testing.T) { diff --git a/vendor/golang.org/x/crypto/openpgp/packet/signature.go b/vendor/golang.org/x/crypto/openpgp/packet/signature.go index 6ce0cbe..b2a24a5 100644 --- a/vendor/golang.org/x/crypto/openpgp/packet/signature.go +++ b/vendor/golang.org/x/crypto/openpgp/packet/signature.go @@ -542,7 +542,7 @@ func (sig *Signature) Sign(h hash.Hash, priv *PrivateKey, config *Config) (err e r, s, err = ecdsa.Sign(config.Random(), pk, digest) } else { var b []byte - b, err = priv.PrivateKey.(crypto.Signer).Sign(config.Random(), digest, nil) + b, err = priv.PrivateKey.(crypto.Signer).Sign(config.Random(), digest, sig.Hash) if err == nil { r, s, err = unwrapECDSASig(b) } diff --git a/vendor/golang.org/x/crypto/openpgp/packet/userattribute.go b/vendor/golang.org/x/crypto/openpgp/packet/userattribute.go index 96a2b38..d19ffbc 100644 --- a/vendor/golang.org/x/crypto/openpgp/packet/userattribute.go +++ b/vendor/golang.org/x/crypto/openpgp/packet/userattribute.go @@ -80,7 +80,7 @@ func (uat *UserAttribute) Serialize(w io.Writer) (err error) { // ImageData returns zero or more byte slices, each containing // JPEG File Interchange Format (JFIF), for each photo in the -// the user attribute packet. +// user attribute packet. func (uat *UserAttribute) ImageData() (imageData [][]byte) { for _, sp := range uat.Contents { if sp.SubType == UserAttrImageSubpacket && len(sp.Contents) > 16 { diff --git a/vendor/golang.org/x/crypto/openpgp/write.go b/vendor/golang.org/x/crypto/openpgp/write.go index d6dede7..4ee7178 100644 --- a/vendor/golang.org/x/crypto/openpgp/write.go +++ b/vendor/golang.org/x/crypto/openpgp/write.go @@ -271,6 +271,7 @@ func Encrypt(ciphertext io.Writer, to []*Entity, signed *Entity, hints *FileHint // These are the possible hash functions that we'll use for the signature. candidateHashes := []uint8{ hashToHashId(crypto.SHA256), + hashToHashId(crypto.SHA384), hashToHashId(crypto.SHA512), hashToHashId(crypto.SHA1), hashToHashId(crypto.RIPEMD160), @@ -349,6 +350,7 @@ func Sign(output io.Writer, signed *Entity, hints *FileHints, config *packet.Con // These are the possible hash functions that we'll use for the signature. candidateHashes := []uint8{ hashToHashId(crypto.SHA256), + hashToHashId(crypto.SHA384), hashToHashId(crypto.SHA512), hashToHashId(crypto.SHA1), hashToHashId(crypto.RIPEMD160), diff --git a/vendor/golang.org/x/crypto/otr/otr.go b/vendor/golang.org/x/crypto/otr/otr.go index 173b753..1be103f 100644 --- a/vendor/golang.org/x/crypto/otr/otr.go +++ b/vendor/golang.org/x/crypto/otr/otr.go @@ -637,7 +637,7 @@ func (c *Conversation) serializeDHKey() []byte { } func (c *Conversation) processDHKey(in []byte) (isSame bool, err error) { - gy, in, ok := getMPI(in) + gy, _, ok := getMPI(in) if !ok { err = errors.New("otr: corrupt DH key message") return diff --git a/vendor/golang.org/x/crypto/pkcs12/pkcs12.go b/vendor/golang.org/x/crypto/pkcs12/pkcs12.go index eff9ad3..55f7691 100644 --- a/vendor/golang.org/x/crypto/pkcs12/pkcs12.go +++ b/vendor/golang.org/x/crypto/pkcs12/pkcs12.go @@ -7,6 +7,9 @@ // This implementation is distilled from https://tools.ietf.org/html/rfc7292 // and referenced documents. It is intended for decoding P12/PFX-stored // certificates and keys for use with the crypto/tls package. +// +// This package is frozen. If it's missing functionality you need, consider +// an alternative like software.sslmate.com/src/go-pkcs12. package pkcs12 import ( @@ -100,7 +103,7 @@ func unmarshal(in []byte, out interface{}) error { return nil } -// ConvertToPEM converts all "safe bags" contained in pfxData to PEM blocks. +// ToPEM converts all "safe bags" contained in pfxData to PEM blocks. func ToPEM(pfxData []byte, password string) ([]*pem.Block, error) { encodedPassword, err := bmpString(password) if err != nil { @@ -208,7 +211,7 @@ func convertAttribute(attribute *pkcs12Attribute) (key, value string, err error) // Decode extracts a certificate and private key from pfxData. This function // assumes that there is only one certificate and only one private key in the -// pfxData. +// pfxData; if there are more use ToPEM instead. func Decode(pfxData []byte, password string) (privateKey interface{}, certificate *x509.Certificate, err error) { encodedPassword, err := bmpString(password) if err != nil { diff --git a/vendor/golang.org/x/crypto/ripemd160/ripemd160.go b/vendor/golang.org/x/crypto/ripemd160/ripemd160.go index 6c6e842..fd97ba1 100644 --- a/vendor/golang.org/x/crypto/ripemd160/ripemd160.go +++ b/vendor/golang.org/x/crypto/ripemd160/ripemd160.go @@ -5,7 +5,7 @@ // Package ripemd160 implements the RIPEMD-160 hash algorithm. package ripemd160 // import "golang.org/x/crypto/ripemd160" -// RIPEMD-160 is designed by by Hans Dobbertin, Antoon Bosselaers, and Bart +// RIPEMD-160 is designed by Hans Dobbertin, Antoon Bosselaers, and Bart // Preneel with specifications available at: // http://homes.esat.kuleuven.be/~cosicart/pdf/AB-9601/AB-9601.pdf. diff --git a/vendor/golang.org/x/crypto/scrypt/scrypt.go b/vendor/golang.org/x/crypto/scrypt/scrypt.go index 9b25b5a..3362afd 100644 --- a/vendor/golang.org/x/crypto/scrypt/scrypt.go +++ b/vendor/golang.org/x/crypto/scrypt/scrypt.go @@ -29,7 +29,7 @@ func blockXOR(dst, src []uint32, n int) { } // salsaXOR applies Salsa20/8 to the XOR of 16 numbers from tmp and in, -// and puts the result into both both tmp and out. +// and puts the result into both tmp and out. func salsaXOR(tmp *[16]uint32, in, out []uint32) { w0 := tmp[0] ^ in[0] w1 := tmp[1] ^ in[1] diff --git a/vendor/golang.org/x/crypto/sha3/doc.go b/vendor/golang.org/x/crypto/sha3/doc.go index a0ee3ae..c2fef30 100644 --- a/vendor/golang.org/x/crypto/sha3/doc.go +++ b/vendor/golang.org/x/crypto/sha3/doc.go @@ -43,7 +43,7 @@ // is then "full" and the permutation is applied to "empty" it. This process is // repeated until all the input has been "absorbed". The input is then padded. // The digest is "squeezed" from the sponge in the same way, except that output -// output is copied out instead of input being XORed in. +// is copied out instead of input being XORed in. // // A sponge is parameterized by its generic security strength, which is equal // to half its capacity; capacity + rate is equal to the permutation's width. diff --git a/vendor/golang.org/x/crypto/sha3/hashes.go b/vendor/golang.org/x/crypto/sha3/hashes.go index 4fb38c0..0d8043f 100644 --- a/vendor/golang.org/x/crypto/sha3/hashes.go +++ b/vendor/golang.org/x/crypto/sha3/hashes.go @@ -58,6 +58,12 @@ func New512() hash.Hash { // that uses non-standard padding. All other users should use New256 instead. func NewLegacyKeccak256() hash.Hash { return &state{rate: 136, outputLen: 32, dsbyte: 0x01} } +// NewLegacyKeccak512 creates a new Keccak-512 hash. +// +// Only use this function if you require compatibility with an existing cryptosystem +// that uses non-standard padding. All other users should use New512 instead. +func NewLegacyKeccak512() hash.Hash { return &state{rate: 72, outputLen: 64, dsbyte: 0x01} } + // Sum224 returns the SHA3-224 digest of the data. func Sum224(data []byte) (digest [28]byte) { h := New224() diff --git a/vendor/golang.org/x/crypto/sha3/sha3_test.go b/vendor/golang.org/x/crypto/sha3/sha3_test.go index c1f6ca3..26d1549 100644 --- a/vendor/golang.org/x/crypto/sha3/sha3_test.go +++ b/vendor/golang.org/x/crypto/sha3/sha3_test.go @@ -44,6 +44,7 @@ var testDigests = map[string]func() hash.Hash{ "SHA3-384": New384, "SHA3-512": New512, "Keccak-256": NewLegacyKeccak256, + "Keccak-512": NewLegacyKeccak512, "SHAKE128": newHashShake128, "SHAKE256": newHashShake256, } @@ -137,6 +138,11 @@ func TestKeccak(t *testing.T) { []byte("abc"), "4e03657aea45a94fc7d47ba826c8d667c0d1e6e33a64a036ec44f58fa12d6c45", }, + { + NewLegacyKeccak512, + []byte("abc"), + "18587dc2ea106b9a1563e32b3312421ca164c7f1f07bc922a9c83d77cea3a1e5d0c69910739025372dc14ac9642629379540c17e2a65b19d77aa511a9d00bb96", + }, } for _, u := range tests { diff --git a/vendor/golang.org/x/crypto/ssh/agent/client.go b/vendor/golang.org/x/crypto/ssh/agent/client.go index b1808dd..51f7405 100644 --- a/vendor/golang.org/x/crypto/ssh/agent/client.go +++ b/vendor/golang.org/x/crypto/ssh/agent/client.go @@ -25,10 +25,22 @@ import ( "math/big" "sync" + "crypto" "golang.org/x/crypto/ed25519" "golang.org/x/crypto/ssh" ) +// SignatureFlags represent additional flags that can be passed to the signature +// requests an defined in [PROTOCOL.agent] section 4.5.1. +type SignatureFlags uint32 + +// SignatureFlag values as defined in [PROTOCOL.agent] section 5.3. +const ( + SignatureFlagReserved SignatureFlags = 1 << iota + SignatureFlagRsaSha256 + SignatureFlagRsaSha512 +) + // Agent represents the capabilities of an ssh-agent. type Agent interface { // List returns the identities known to the agent. @@ -57,6 +69,26 @@ type Agent interface { Signers() ([]ssh.Signer, error) } +type ExtendedAgent interface { + Agent + + // SignWithFlags signs like Sign, but allows for additional flags to be sent/received + SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) + + // Extension processes a custom extension request. Standard-compliant agents are not + // required to support any extensions, but this method allows agents to implement + // vendor-specific methods or add experimental features. See [PROTOCOL.agent] section 4.7. + // If agent extensions are unsupported entirely this method MUST return an + // ErrExtensionUnsupported error. Similarly, if just the specific extensionType in + // the request is unsupported by the agent then ErrExtensionUnsupported MUST be + // returned. + // + // In the case of success, since [PROTOCOL.agent] section 4.7 specifies that the contents + // of the response are unspecified (including the type of the message), the complete + // response will be returned as a []byte slice, including the "type" byte of the message. + Extension(extensionType string, contents []byte) ([]byte, error) +} + // ConstraintExtension describes an optional constraint defined by users. type ConstraintExtension struct { // ExtensionName consist of a UTF-8 string suffixed by the @@ -179,6 +211,23 @@ type constrainExtensionAgentMsg struct { Rest []byte `ssh:"rest"` } +// See [PROTOCOL.agent], section 4.7 +const agentExtension = 27 +const agentExtensionFailure = 28 + +// ErrExtensionUnsupported indicates that an extension defined in +// [PROTOCOL.agent] section 4.7 is unsupported by the agent. Specifically this +// error indicates that the agent returned a standard SSH_AGENT_FAILURE message +// as the result of a SSH_AGENTC_EXTENSION request. Note that the protocol +// specification (and therefore this error) does not distinguish between a +// specific extension being unsupported and extensions being unsupported entirely. +var ErrExtensionUnsupported = errors.New("agent: extension unsupported") + +type extensionAgentMsg struct { + ExtensionType string `sshtype:"27"` + Contents []byte +} + // Key represents a protocol 2 public key as defined in // [PROTOCOL.agent], section 2.5.2. type Key struct { @@ -260,7 +309,7 @@ type client struct { // NewClient returns an Agent that talks to an ssh-agent process over // the given connection. -func NewClient(rw io.ReadWriter) Agent { +func NewClient(rw io.ReadWriter) ExtendedAgent { return &client{conn: rw} } @@ -268,6 +317,21 @@ func NewClient(rw io.ReadWriter) Agent { // unmarshaled into reply and replyType is set to the first byte of // the reply, which contains the type of the message. func (c *client) call(req []byte) (reply interface{}, err error) { + buf, err := c.callRaw(req) + if err != nil { + return nil, err + } + reply, err = unmarshal(buf) + if err != nil { + return nil, clientErr(err) + } + return reply, nil +} + +// callRaw sends an RPC to the agent. On success, the raw +// bytes of the response are returned; no unmarshalling is +// performed on the response. +func (c *client) callRaw(req []byte) (reply []byte, err error) { c.mu.Lock() defer c.mu.Unlock() @@ -284,18 +348,14 @@ func (c *client) call(req []byte) (reply interface{}, err error) { } respSize := binary.BigEndian.Uint32(respSizeBuf[:]) if respSize > maxAgentResponseBytes { - return nil, clientErr(err) + return nil, clientErr(errors.New("response too large")) } buf := make([]byte, respSize) if _, err = io.ReadFull(c.conn, buf); err != nil { return nil, clientErr(err) } - reply, err = unmarshal(buf) - if err != nil { - return nil, clientErr(err) - } - return reply, err + return buf, nil } func (c *client) simpleCall(req []byte) error { @@ -369,9 +429,14 @@ func (c *client) List() ([]*Key, error) { // Sign has the agent sign the data using a protocol 2 key as defined // in [PROTOCOL.agent] section 2.6.2. func (c *client) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return c.SignWithFlags(key, data, 0) +} + +func (c *client) SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) { req := ssh.Marshal(signRequestAgentMsg{ KeyBlob: key.Marshal(), Data: data, + Flags: uint32(flags), }) msg, err := c.call(req) @@ -681,3 +746,44 @@ func (s *agentKeyringSigner) Sign(rand io.Reader, data []byte) (*ssh.Signature, // The agent has its own entropy source, so the rand argument is ignored. return s.agent.Sign(s.pub, data) } + +func (s *agentKeyringSigner) SignWithOpts(rand io.Reader, data []byte, opts crypto.SignerOpts) (*ssh.Signature, error) { + var flags SignatureFlags + if opts != nil { + switch opts.HashFunc() { + case crypto.SHA256: + flags = SignatureFlagRsaSha256 + case crypto.SHA512: + flags = SignatureFlagRsaSha512 + } + } + return s.agent.SignWithFlags(s.pub, data, flags) +} + +// Calls an extension method. It is up to the agent implementation as to whether or not +// any particular extension is supported and may always return an error. Because the +// type of the response is up to the implementation, this returns the bytes of the +// response and does not attempt any type of unmarshalling. +func (c *client) Extension(extensionType string, contents []byte) ([]byte, error) { + req := ssh.Marshal(extensionAgentMsg{ + ExtensionType: extensionType, + Contents: contents, + }) + buf, err := c.callRaw(req) + if err != nil { + return nil, err + } + if len(buf) == 0 { + return nil, errors.New("agent: failure; empty response") + } + // [PROTOCOL.agent] section 4.7 indicates that an SSH_AGENT_FAILURE message + // represents an agent that does not support the extension + if buf[0] == agentFailure { + return nil, ErrExtensionUnsupported + } + if buf[0] == agentExtensionFailure { + return nil, errors.New("agent: generic extension failure") + } + + return buf, nil +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/client_test.go b/vendor/golang.org/x/crypto/ssh/agent/client_test.go index 266fd6d..2f8abf1 100644 --- a/vendor/golang.org/x/crypto/ssh/agent/client_test.go +++ b/vendor/golang.org/x/crypto/ssh/agent/client_test.go @@ -20,7 +20,7 @@ import ( ) // startOpenSSHAgent executes ssh-agent, and returns an Agent interface to it. -func startOpenSSHAgent(t *testing.T) (client Agent, socket string, cleanup func()) { +func startOpenSSHAgent(t *testing.T) (client ExtendedAgent, socket string, cleanup func()) { if testing.Short() { // ssh-agent is not always available, and the key // types supported vary by platform. @@ -79,13 +79,12 @@ func startOpenSSHAgent(t *testing.T) (client Agent, socket string, cleanup func( } } -// startKeyringAgent uses Keyring to simulate a ssh-agent Server and returns a client. -func startKeyringAgent(t *testing.T) (client Agent, cleanup func()) { +func startAgent(t *testing.T, agent Agent) (client ExtendedAgent, cleanup func()) { c1, c2, err := netPipe() if err != nil { t.Fatalf("netPipe: %v", err) } - go ServeAgent(NewKeyring(), c2) + go ServeAgent(agent, c2) return NewClient(c1), func() { c1.Close() @@ -93,6 +92,11 @@ func startKeyringAgent(t *testing.T) (client Agent, cleanup func()) { } } +// startKeyringAgent uses Keyring to simulate a ssh-agent Server and returns a client. +func startKeyringAgent(t *testing.T) (client ExtendedAgent, cleanup func()) { + return startAgent(t, NewKeyring()) +} + func testOpenSSHAgent(t *testing.T, key interface{}, cert *ssh.Certificate, lifetimeSecs uint32) { agent, _, cleanup := startOpenSSHAgent(t) defer cleanup() @@ -107,7 +111,7 @@ func testKeyringAgent(t *testing.T, key interface{}, cert *ssh.Certificate, life testAgentInterface(t, agent, key, cert, lifetimeSecs) } -func testAgentInterface(t *testing.T, agent Agent, key interface{}, cert *ssh.Certificate, lifetimeSecs uint32) { +func testAgentInterface(t *testing.T, agent ExtendedAgent, key interface{}, cert *ssh.Certificate, lifetimeSecs uint32) { signer, err := ssh.NewSignerFromKey(key) if err != nil { t.Fatalf("NewSignerFromKey(%T): %v", key, err) @@ -159,6 +163,25 @@ func testAgentInterface(t *testing.T, agent Agent, key interface{}, cert *ssh.Ce t.Fatalf("Verify(%s): %v", pubKey.Type(), err) } + // For tests on RSA keys, try signing with SHA-256 and SHA-512 flags + if pubKey.Type() == "ssh-rsa" { + sshFlagTest := func(flag SignatureFlags, expectedSigFormat string) { + sig, err = agent.SignWithFlags(pubKey, data, flag) + if err != nil { + t.Fatalf("SignWithFlags(%s): %v", pubKey.Type(), err) + } + if sig.Format != expectedSigFormat { + t.Fatalf("Signature format didn't match expected value: %s != %s", sig.Format, expectedSigFormat) + } + if err := pubKey.Verify(data, sig); err != nil { + t.Fatalf("Verify(%s): %v", pubKey.Type(), err) + } + } + sshFlagTest(0, ssh.SigAlgoRSA) + sshFlagTest(SignatureFlagRsaSha256, ssh.SigAlgoRSASHA2256) + sshFlagTest(SignatureFlagRsaSha512, ssh.SigAlgoRSASHA2512) + } + // If the key has a lifetime, is it removed when it should be? if lifetimeSecs > 0 { time.Sleep(time.Second*time.Duration(lifetimeSecs) + 100*time.Millisecond) @@ -218,6 +241,35 @@ func netPipe() (net.Conn, net.Conn, error) { return c1, c2, nil } +func TestServerResponseTooLarge(t *testing.T) { + a, b, err := netPipe() + if err != nil { + t.Fatalf("netPipe: %v", err) + } + + defer a.Close() + defer b.Close() + + var response identitiesAnswerAgentMsg + response.NumKeys = 1 + response.Keys = make([]byte, maxAgentResponseBytes+1) + + agent := NewClient(a) + go func() { + n, _ := b.Write(ssh.Marshal(response)) + if n < 4 { + t.Fatalf("At least 4 bytes (the response size) should have been successfully written: %d < 4", n) + } + }() + _, err = agent.List() + if err == nil { + t.Fatal("Did not get error result") + } + if err.Error() != "agent: client error: response too large" { + t.Fatal("Did not get expected error result") + } +} + func TestAuth(t *testing.T) { agent, _, cleanup := startOpenSSHAgent(t) defer cleanup() @@ -377,3 +429,38 @@ func testAgentLifetime(t *testing.T, agent Agent) { t.Errorf("Want 0 keys, got %v", len(keys)) } } + +type keyringExtended struct { + *keyring +} + +func (r *keyringExtended) Extension(extensionType string, contents []byte) ([]byte, error) { + if extensionType != "my-extension@example.com" { + return []byte{agentExtensionFailure}, nil + } + return append([]byte{agentSuccess}, contents...), nil +} + +func TestAgentExtensions(t *testing.T) { + agent, _, cleanup := startOpenSSHAgent(t) + defer cleanup() + _, err := agent.Extension("my-extension@example.com", []byte{0x00, 0x01, 0x02}) + if err == nil { + t.Fatal("should have gotten agent extension failure") + } + + agent, cleanup = startAgent(t, &keyringExtended{}) + defer cleanup() + result, err := agent.Extension("my-extension@example.com", []byte{0x00, 0x01, 0x02}) + if err != nil { + t.Fatalf("agent extension failure: %v", err) + } + if len(result) != 4 || !bytes.Equal(result, []byte{agentSuccess, 0x00, 0x01, 0x02}) { + t.Fatalf("agent extension result invalid: %v", result) + } + + _, err = agent.Extension("bad-extension@example.com", []byte{0x00, 0x01, 0x02}) + if err == nil { + t.Fatal("should have gotten agent extension failure") + } +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/keyring.go b/vendor/golang.org/x/crypto/ssh/agent/keyring.go index 1a51632..c9d9794 100644 --- a/vendor/golang.org/x/crypto/ssh/agent/keyring.go +++ b/vendor/golang.org/x/crypto/ssh/agent/keyring.go @@ -182,6 +182,10 @@ func (r *keyring) Add(key AddedKey) error { // Sign returns a signature for the data. func (r *keyring) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return r.SignWithFlags(key, data, 0) +} + +func (r *keyring) SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) { r.mu.Lock() defer r.mu.Unlock() if r.locked { @@ -192,7 +196,24 @@ func (r *keyring) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { wanted := key.Marshal() for _, k := range r.keys { if bytes.Equal(k.signer.PublicKey().Marshal(), wanted) { - return k.signer.Sign(rand.Reader, data) + if flags == 0 { + return k.signer.Sign(rand.Reader, data) + } else { + if algorithmSigner, ok := k.signer.(ssh.AlgorithmSigner); !ok { + return nil, fmt.Errorf("agent: signature does not support non-default signature algorithm: %T", k.signer) + } else { + var algorithm string + switch flags { + case SignatureFlagRsaSha256: + algorithm = ssh.SigAlgoRSASHA2256 + case SignatureFlagRsaSha512: + algorithm = ssh.SigAlgoRSASHA2512 + default: + return nil, fmt.Errorf("agent: unsupported signature flags: %d", flags) + } + return algorithmSigner.SignWithAlgorithm(rand.Reader, data, algorithm) + } + } } } return nil, errors.New("not found") @@ -213,3 +234,8 @@ func (r *keyring) Signers() ([]ssh.Signer, error) { } return s, nil } + +// The keyring does not support any extensions +func (r *keyring) Extension(extensionType string, contents []byte) ([]byte, error) { + return nil, ErrExtensionUnsupported +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/server.go b/vendor/golang.org/x/crypto/ssh/agent/server.go index 2e4692c..a194976 100644 --- a/vendor/golang.org/x/crypto/ssh/agent/server.go +++ b/vendor/golang.org/x/crypto/ssh/agent/server.go @@ -128,7 +128,14 @@ func (s *server) processRequest(data []byte) (interface{}, error) { Blob: req.KeyBlob, } - sig, err := s.agent.Sign(k, req.Data) // TODO(hanwen): flags. + var sig *ssh.Signature + var err error + if extendedAgent, ok := s.agent.(ExtendedAgent); ok { + sig, err = extendedAgent.SignWithFlags(k, req.Data, SignatureFlags(req.Flags)) + } else { + sig, err = s.agent.Sign(k, req.Data) + } + if err != nil { return nil, err } @@ -150,6 +157,43 @@ func (s *server) processRequest(data []byte) (interface{}, error) { case agentAddIDConstrained, agentAddIdentity: return nil, s.insertIdentity(data) + + case agentExtension: + // Return a stub object where the whole contents of the response gets marshaled. + var responseStub struct { + Rest []byte `ssh:"rest"` + } + + if extendedAgent, ok := s.agent.(ExtendedAgent); !ok { + // If this agent doesn't implement extensions, [PROTOCOL.agent] section 4.7 + // requires that we return a standard SSH_AGENT_FAILURE message. + responseStub.Rest = []byte{agentFailure} + } else { + var req extensionAgentMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + res, err := extendedAgent.Extension(req.ExtensionType, req.Contents) + if err != nil { + // If agent extensions are unsupported, return a standard SSH_AGENT_FAILURE + // message as required by [PROTOCOL.agent] section 4.7. + if err == ErrExtensionUnsupported { + responseStub.Rest = []byte{agentFailure} + } else { + // As the result of any other error processing an extension request, + // [PROTOCOL.agent] section 4.7 requires that we return a + // SSH_AGENT_EXTENSION_FAILURE code. + responseStub.Rest = []byte{agentExtensionFailure} + } + } else { + if len(res) == 0 { + return nil, nil + } + responseStub.Rest = res + } + } + + return responseStub, nil } return nil, fmt.Errorf("unknown opcode %d", data[0]) diff --git a/vendor/golang.org/x/crypto/ssh/benchmark_test.go b/vendor/golang.org/x/crypto/ssh/benchmark_test.go index 20c3307..a13235d 100644 --- a/vendor/golang.org/x/crypto/ssh/benchmark_test.go +++ b/vendor/golang.org/x/crypto/ssh/benchmark_test.go @@ -93,6 +93,9 @@ func BenchmarkEndToEnd(b *testing.B) { b.Fatalf("Client: %v", err) } ch, incoming, err := newCh.Accept() + if err != nil { + b.Fatalf("Accept: %v", err) + } go DiscardRequests(incoming) for i := 0; i < b.N; i++ { if _, err := io.ReadFull(ch, output); err != nil { diff --git a/vendor/golang.org/x/crypto/ssh/certs.go b/vendor/golang.org/x/crypto/ssh/certs.go index 42106f3..00ed992 100644 --- a/vendor/golang.org/x/crypto/ssh/certs.go +++ b/vendor/golang.org/x/crypto/ssh/certs.go @@ -222,6 +222,11 @@ type openSSHCertSigner struct { signer Signer } +type algorithmOpenSSHCertSigner struct { + *openSSHCertSigner + algorithmSigner AlgorithmSigner +} + // NewCertSigner returns a Signer that signs with the given Certificate, whose // private key is held by signer. It returns an error if the public key in cert // doesn't match the key used by signer. @@ -230,7 +235,12 @@ func NewCertSigner(cert *Certificate, signer Signer) (Signer, error) { return nil, errors.New("ssh: signer and cert have different public key") } - return &openSSHCertSigner{cert, signer}, nil + if algorithmSigner, ok := signer.(AlgorithmSigner); ok { + return &algorithmOpenSSHCertSigner{ + &openSSHCertSigner{cert, signer}, algorithmSigner}, nil + } else { + return &openSSHCertSigner{cert, signer}, nil + } } func (s *openSSHCertSigner) Sign(rand io.Reader, data []byte) (*Signature, error) { @@ -241,6 +251,10 @@ func (s *openSSHCertSigner) PublicKey() PublicKey { return s.pub } +func (s *algorithmOpenSSHCertSigner) SignWithAlgorithm(rand io.Reader, data []byte, algorithm string) (*Signature, error) { + return s.algorithmSigner.SignWithAlgorithm(rand, data, algorithm) +} + const sourceAddressCriticalOption = "source-address" // CertChecker does the work of verifying a certificate. Its methods diff --git a/vendor/golang.org/x/crypto/ssh/client.go b/vendor/golang.org/x/crypto/ssh/client.go index ae6ca77..7b00bff 100644 --- a/vendor/golang.org/x/crypto/ssh/client.go +++ b/vendor/golang.org/x/crypto/ssh/client.go @@ -185,7 +185,7 @@ func Dial(network, addr string, config *ClientConfig) (*Client, error) { // keys. A HostKeyCallback must return nil if the host key is OK, or // an error to reject it. It receives the hostname as passed to Dial // or NewClientConn. The remote address is the RemoteAddr of the -// net.Conn underlying the the SSH connection. +// net.Conn underlying the SSH connection. type HostKeyCallback func(hostname string, remote net.Addr, key PublicKey) error // BannerCallback is the function type used for treat the banner sent by diff --git a/vendor/golang.org/x/crypto/ssh/client_auth_test.go b/vendor/golang.org/x/crypto/ssh/client_auth_test.go index 5fbb20d..026d137 100644 --- a/vendor/golang.org/x/crypto/ssh/client_auth_test.go +++ b/vendor/golang.org/x/crypto/ssh/client_auth_test.go @@ -9,6 +9,7 @@ import ( "crypto/rand" "errors" "fmt" + "io" "os" "strings" "testing" @@ -28,8 +29,14 @@ func (cr keyboardInteractive) Challenge(user string, instruction string, questio var clientPassword = "tiger" // tryAuth runs a handshake with a given config against an SSH server -// with config serverConfig +// with config serverConfig. Returns both client and server side errors. func tryAuth(t *testing.T, config *ClientConfig) error { + err, _ := tryAuthBothSides(t, config) + return err +} + +// tryAuthBothSides runs the handshake and returns the resulting errors from both sides of the connection. +func tryAuthBothSides(t *testing.T, config *ClientConfig) (clientError error, serverAuthErrors []error) { c1, c2, err := netPipe() if err != nil { t.Fatalf("netPipe: %v", err) @@ -79,9 +86,13 @@ func tryAuth(t *testing.T, config *ClientConfig) error { } serverConfig.AddHostKey(testSigners["rsa"]) + serverConfig.AuthLogCallback = func(conn ConnMetadata, method string, err error) { + serverAuthErrors = append(serverAuthErrors, err) + } + go newServer(c1, serverConfig) _, _, _, err = NewClientConn(c2, "", config) - return err + return err, serverAuthErrors } func TestClientAuthPublicKey(t *testing.T) { @@ -213,6 +224,45 @@ func TestAuthMethodRSAandDSA(t *testing.T) { } } +type invalidAlgSigner struct { + Signer +} + +func (s *invalidAlgSigner) Sign(rand io.Reader, data []byte) (*Signature, error) { + sig, err := s.Signer.Sign(rand, data) + if sig != nil { + sig.Format = "invalid" + } + return sig, err +} + +func TestMethodInvalidAlgorithm(t *testing.T) { + config := &ClientConfig{ + User: "testuser", + Auth: []AuthMethod{ + PublicKeys(&invalidAlgSigner{testSigners["rsa"]}), + }, + HostKeyCallback: InsecureIgnoreHostKey(), + } + + err, serverErrors := tryAuthBothSides(t, config) + if err == nil { + t.Fatalf("login succeeded") + } + + found := false + want := "algorithm \"invalid\"" + + var errStrings []string + for _, err := range serverErrors { + found = found || (err != nil && strings.Contains(err.Error(), want)) + errStrings = append(errStrings, err.Error()) + } + if !found { + t.Errorf("server got error %q, want substring %q", errStrings, want) + } +} + func TestClientHMAC(t *testing.T) { for _, mac := range supportedMACs { config := &ClientConfig{ diff --git a/vendor/golang.org/x/crypto/ssh/keys.go b/vendor/golang.org/x/crypto/ssh/keys.go index 34d9582..9698047 100644 --- a/vendor/golang.org/x/crypto/ssh/keys.go +++ b/vendor/golang.org/x/crypto/ssh/keys.go @@ -38,6 +38,16 @@ const ( KeyAlgoED25519 = "ssh-ed25519" ) +// These constants represent non-default signature algorithms that are supported +// as algorithm parameters to AlgorithmSigner.SignWithAlgorithm methods. See +// [PROTOCOL.agent] section 4.5.1 and +// https://tools.ietf.org/html/draft-ietf-curdle-rsa-sha2-10 +const ( + SigAlgoRSA = "ssh-rsa" + SigAlgoRSASHA2256 = "rsa-sha2-256" + SigAlgoRSASHA2512 = "rsa-sha2-512" +) + // parsePubKey parses a public key of the given algorithm. // Use ParsePublicKey for keys with prepended algorithm. func parsePubKey(in []byte, algo string) (pubKey PublicKey, rest []byte, err error) { @@ -301,6 +311,19 @@ type Signer interface { Sign(rand io.Reader, data []byte) (*Signature, error) } +// A AlgorithmSigner is a Signer that also supports specifying a specific +// algorithm to use for signing. +type AlgorithmSigner interface { + Signer + + // SignWithAlgorithm is like Signer.Sign, but allows specification of a + // non-default signing algorithm. See the SigAlgo* constants in this + // package for signature algorithms supported by this package. Callers may + // pass an empty string for the algorithm in which case the AlgorithmSigner + // will use its default algorithm. + SignWithAlgorithm(rand io.Reader, data []byte, algorithm string) (*Signature, error) +} + type rsaPublicKey rsa.PublicKey func (r *rsaPublicKey) Type() string { @@ -349,13 +372,21 @@ func (r *rsaPublicKey) Marshal() []byte { } func (r *rsaPublicKey) Verify(data []byte, sig *Signature) error { - if sig.Format != r.Type() { + var hash crypto.Hash + switch sig.Format { + case SigAlgoRSA: + hash = crypto.SHA1 + case SigAlgoRSASHA2256: + hash = crypto.SHA256 + case SigAlgoRSASHA2512: + hash = crypto.SHA512 + default: return fmt.Errorf("ssh: signature type %s for key type %s", sig.Format, r.Type()) } - h := crypto.SHA1.New() + h := hash.New() h.Write(data) digest := h.Sum(nil) - return rsa.VerifyPKCS1v15((*rsa.PublicKey)(r), crypto.SHA1, digest, sig.Blob) + return rsa.VerifyPKCS1v15((*rsa.PublicKey)(r), hash, digest, sig.Blob) } func (r *rsaPublicKey) CryptoPublicKey() crypto.PublicKey { @@ -459,6 +490,14 @@ func (k *dsaPrivateKey) PublicKey() PublicKey { } func (k *dsaPrivateKey) Sign(rand io.Reader, data []byte) (*Signature, error) { + return k.SignWithAlgorithm(rand, data, "") +} + +func (k *dsaPrivateKey) SignWithAlgorithm(rand io.Reader, data []byte, algorithm string) (*Signature, error) { + if algorithm != "" && algorithm != k.PublicKey().Type() { + return nil, fmt.Errorf("ssh: unsupported signature algorithm %s", algorithm) + } + h := crypto.SHA1.New() h.Write(data) digest := h.Sum(nil) @@ -691,16 +730,42 @@ func (s *wrappedSigner) PublicKey() PublicKey { } func (s *wrappedSigner) Sign(rand io.Reader, data []byte) (*Signature, error) { + return s.SignWithAlgorithm(rand, data, "") +} + +func (s *wrappedSigner) SignWithAlgorithm(rand io.Reader, data []byte, algorithm string) (*Signature, error) { var hashFunc crypto.Hash - switch key := s.pubKey.(type) { - case *rsaPublicKey, *dsaPublicKey: - hashFunc = crypto.SHA1 - case *ecdsaPublicKey: - hashFunc = ecHash(key.Curve) - case ed25519PublicKey: - default: - return nil, fmt.Errorf("ssh: unsupported key type %T", key) + if _, ok := s.pubKey.(*rsaPublicKey); ok { + // RSA keys support a few hash functions determined by the requested signature algorithm + switch algorithm { + case "", SigAlgoRSA: + algorithm = SigAlgoRSA + hashFunc = crypto.SHA1 + case SigAlgoRSASHA2256: + hashFunc = crypto.SHA256 + case SigAlgoRSASHA2512: + hashFunc = crypto.SHA512 + default: + return nil, fmt.Errorf("ssh: unsupported signature algorithm %s", algorithm) + } + } else { + // The only supported algorithm for all other key types is the same as the type of the key + if algorithm == "" { + algorithm = s.pubKey.Type() + } else if algorithm != s.pubKey.Type() { + return nil, fmt.Errorf("ssh: unsupported signature algorithm %s", algorithm) + } + + switch key := s.pubKey.(type) { + case *dsaPublicKey: + hashFunc = crypto.SHA1 + case *ecdsaPublicKey: + hashFunc = ecHash(key.Curve) + case ed25519PublicKey: + default: + return nil, fmt.Errorf("ssh: unsupported key type %T", key) + } } var digest []byte @@ -745,7 +810,7 @@ func (s *wrappedSigner) Sign(rand io.Reader, data []byte) (*Signature, error) { } return &Signature{ - Format: s.pubKey.Type(), + Format: algorithm, Blob: signature, }, nil } @@ -903,8 +968,8 @@ func ParseDSAPrivateKey(der []byte) (*dsa.PrivateKey, error) { // Implemented based on the documentation at // https://github.com/openssh/openssh-portable/blob/master/PROTOCOL.key func parseOpenSSHPrivateKey(key []byte) (crypto.PrivateKey, error) { - magic := append([]byte("openssh-key-v1"), 0) - if !bytes.Equal(magic, key[0:len(magic)]) { + const magic = "openssh-key-v1\x00" + if len(key) < len(magic) || string(key[:len(magic)]) != magic { return nil, errors.New("ssh: invalid openssh private key format") } remaining := key[len(magic):] diff --git a/vendor/golang.org/x/crypto/ssh/keys_test.go b/vendor/golang.org/x/crypto/ssh/keys_test.go index 9a90abc..3847b3b 100644 --- a/vendor/golang.org/x/crypto/ssh/keys_test.go +++ b/vendor/golang.org/x/crypto/ssh/keys_test.go @@ -13,7 +13,9 @@ import ( "crypto/rsa" "crypto/x509" "encoding/base64" + "encoding/pem" "fmt" + "io" "reflect" "strings" "testing" @@ -107,6 +109,49 @@ func TestKeySignVerify(t *testing.T) { } } +func TestKeySignWithAlgorithmVerify(t *testing.T) { + for _, priv := range testSigners { + if algorithmSigner, ok := priv.(AlgorithmSigner); !ok { + t.Errorf("Signers constructed by ssh package should always implement the AlgorithmSigner interface: %T", priv) + } else { + pub := priv.PublicKey() + data := []byte("sign me") + + signWithAlgTestCase := func(algorithm string, expectedAlg string) { + sig, err := algorithmSigner.SignWithAlgorithm(rand.Reader, data, algorithm) + if err != nil { + t.Fatalf("Sign(%T): %v", priv, err) + } + if sig.Format != expectedAlg { + t.Errorf("signature format did not match requested signature algorithm: %s != %s", sig.Format, expectedAlg) + } + + if err := pub.Verify(data, sig); err != nil { + t.Errorf("publicKey.Verify(%T): %v", priv, err) + } + sig.Blob[5]++ + if err := pub.Verify(data, sig); err == nil { + t.Errorf("publicKey.Verify on broken sig did not fail") + } + } + + // Using the empty string as the algorithm name should result in the same signature format as the algorithm-free Sign method. + defaultSig, err := priv.Sign(rand.Reader, data) + if err != nil { + t.Fatalf("Sign(%T): %v", priv, err) + } + signWithAlgTestCase("", defaultSig.Format) + + // RSA keys are the only ones which currently support more than one signing algorithm + if pub.Type() == KeyAlgoRSA { + for _, algorithm := range []string{SigAlgoRSA, SigAlgoRSASHA2256, SigAlgoRSASHA2512} { + signWithAlgTestCase(algorithm, algorithm) + } + } + } + } +} + func TestParseRSAPrivateKey(t *testing.T) { key := testPrivateKeys["rsa"] @@ -498,3 +543,32 @@ func TestFingerprintSHA256(t *testing.T) { t.Errorf("got fingerprint %q want %q", fingerprint, want) } } + +func TestInvalidKeys(t *testing.T) { + keyTypes := []string{ + "RSA PRIVATE KEY", + "PRIVATE KEY", + "EC PRIVATE KEY", + "DSA PRIVATE KEY", + "OPENSSH PRIVATE KEY", + } + + for _, keyType := range keyTypes { + for _, dataLen := range []int{0, 1, 2, 5, 10, 20} { + data := make([]byte, dataLen) + if _, err := io.ReadFull(rand.Reader, data); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + pem.Encode(&buf, &pem.Block{ + Type: keyType, + Bytes: data, + }) + + // This test is just to ensure that the function + // doesn't panic so the return value is ignored. + ParseRawPrivateKey(buf.Bytes()) + } + } +} diff --git a/vendor/golang.org/x/crypto/ssh/knownhosts/knownhosts.go b/vendor/golang.org/x/crypto/ssh/knownhosts/knownhosts.go index bc3db73..260cfe5 100644 --- a/vendor/golang.org/x/crypto/ssh/knownhosts/knownhosts.go +++ b/vendor/golang.org/x/crypto/ssh/knownhosts/knownhosts.go @@ -350,8 +350,8 @@ func (db *hostKeyDB) check(address string, remote net.Addr, remoteKey ssh.Public return db.checkAddr(hostToCheck, remoteKey) } -// checkAddrs checks if we can find the given public key for any of -// the given addresses. If we only find an entry for the IP address, +// checkAddr checks if we can find the given public key for the +// given address. If we only find an entry for the IP address, // or only the hostname, then this still succeeds. func (db *hostKeyDB) checkAddr(a addr, remoteKey ssh.PublicKey) error { // TODO(hanwen): are these the right semantics? What if there diff --git a/vendor/golang.org/x/crypto/ssh/mux_test.go b/vendor/golang.org/x/crypto/ssh/mux_test.go index d88b64e..94596ec 100644 --- a/vendor/golang.org/x/crypto/ssh/mux_test.go +++ b/vendor/golang.org/x/crypto/ssh/mux_test.go @@ -20,7 +20,7 @@ func muxPair() (*mux, *mux) { return s, c } -// Returns both ends of a channel, and the mux for the the 2nd +// Returns both ends of a channel, and the mux for the 2nd // channel. func channelPair(t *testing.T) (*channel, *channel, *mux) { c, s := muxPair() diff --git a/vendor/golang.org/x/crypto/ssh/server.go b/vendor/golang.org/x/crypto/ssh/server.go index d0f4825..e86e896 100644 --- a/vendor/golang.org/x/crypto/ssh/server.go +++ b/vendor/golang.org/x/crypto/ssh/server.go @@ -404,7 +404,7 @@ userAuthLoop: perms, authErr = config.PasswordCallback(s, password) case "keyboard-interactive": if config.KeyboardInteractiveCallback == nil { - authErr = errors.New("ssh: keyboard-interactive auth not configubred") + authErr = errors.New("ssh: keyboard-interactive auth not configured") break } @@ -484,6 +484,7 @@ userAuthLoop: // sig.Format. This is usually the same, but // for certs, the names differ. if !isAcceptableAlgo(sig.Format) { + authErr = fmt.Errorf("ssh: algorithm %q not accepted", sig.Format) break } signedData := buildDataSignedForAuth(sessionID, userAuthReq, algoBytes, pubKeyData) diff --git a/vendor/golang.org/x/crypto/ssh/terminal/terminal.go b/vendor/golang.org/x/crypto/ssh/terminal/terminal.go index 9a88759..9d666ff 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/terminal.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/terminal.go @@ -159,6 +159,10 @@ func bytesToKey(b []byte, pasteActive bool) (rune, []byte) { return keyClearScreen, b[1:] case 23: // ^W return keyDeleteWord, b[1:] + case 14: // ^N + return keyDown, b[1:] + case 16: // ^P + return keyUp, b[1:] } } diff --git a/vendor/golang.org/x/crypto/ssh/terminal/terminal_test.go b/vendor/golang.org/x/crypto/ssh/terminal/terminal_test.go index d9b77c1..3ae9116 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/terminal_test.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/terminal_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux,!appengine netbsd openbsd windows plan9 solaris +// +build aix darwin dragonfly freebsd linux,!appengine netbsd openbsd windows plan9 solaris package terminal @@ -91,6 +91,12 @@ var keyPressTests = []struct { { in: "\x1b[B\r", // down }, + { + in: "\016\r", // ^P + }, + { + in: "\014\r", // ^N + }, { in: "line\x1b[A\x1b[B\r", // up then down line: "line", diff --git a/vendor/golang.org/x/crypto/ssh/terminal/util.go b/vendor/golang.org/x/crypto/ssh/terminal/util.go index 731c89a..3911040 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/util.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/util.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build darwin dragonfly freebsd linux,!appengine netbsd openbsd +// +build aix darwin dragonfly freebsd linux,!appengine netbsd openbsd // Package terminal provides support functions for dealing with terminals, as // commonly found on UNIX systems. @@ -25,7 +25,7 @@ type State struct { termios unix.Termios } -// IsTerminal returns true if the given file descriptor is a terminal. +// IsTerminal returns whether the given file descriptor is a terminal. func IsTerminal(fd int) bool { _, err := unix.IoctlGetTermios(fd, ioctlReadTermios) return err == nil diff --git a/vendor/github.com/sirupsen/logrus/terminal_linux.go b/vendor/golang.org/x/crypto/ssh/terminal/util_aix.go similarity index 53% rename from vendor/github.com/sirupsen/logrus/terminal_linux.go rename to vendor/golang.org/x/crypto/ssh/terminal/util_aix.go index f29a009..dfcd627 100644 --- a/vendor/github.com/sirupsen/logrus/terminal_linux.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/util_aix.go @@ -1,14 +1,12 @@ -// Based on ssh/terminal: -// Copyright 2013 The Go Authors. All rights reserved. +// Copyright 2018 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !appengine,!gopherjs +// +build aix -package logrus +package terminal import "golang.org/x/sys/unix" const ioctlReadTermios = unix.TCGETS - -type Termios unix.Termios +const ioctlWriteTermios = unix.TCSETS diff --git a/vendor/golang.org/x/crypto/ssh/terminal/util_plan9.go b/vendor/golang.org/x/crypto/ssh/terminal/util_plan9.go index 799f049..9317ac7 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/util_plan9.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/util_plan9.go @@ -21,7 +21,7 @@ import ( type State struct{} -// IsTerminal returns true if the given file descriptor is a terminal. +// IsTerminal returns whether the given file descriptor is a terminal. func IsTerminal(fd int) bool { return false } diff --git a/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go b/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go index 9e41b9f..3d5f06a 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/util_solaris.go @@ -17,7 +17,7 @@ type State struct { termios unix.Termios } -// IsTerminal returns true if the given file descriptor is a terminal. +// IsTerminal returns whether the given file descriptor is a terminal. func IsTerminal(fd int) bool { _, err := unix.IoctlGetTermio(fd, unix.TCGETA) return err == nil diff --git a/vendor/golang.org/x/crypto/ssh/terminal/util_windows.go b/vendor/golang.org/x/crypto/ssh/terminal/util_windows.go index 8618955..6cb8a95 100644 --- a/vendor/golang.org/x/crypto/ssh/terminal/util_windows.go +++ b/vendor/golang.org/x/crypto/ssh/terminal/util_windows.go @@ -26,7 +26,7 @@ type State struct { mode uint32 } -// IsTerminal returns true if the given file descriptor is a terminal. +// IsTerminal returns whether the given file descriptor is a terminal. func IsTerminal(fd int) bool { var st uint32 err := windows.GetConsoleMode(windows.Handle(fd), &st) diff --git a/vendor/golang.org/x/crypto/ssh/testdata/keys.go b/vendor/golang.org/x/crypto/ssh/testdata/keys.go index bdaa9cb..bfae85f 100644 --- a/vendor/golang.org/x/crypto/ssh/testdata/keys.go +++ b/vendor/golang.org/x/crypto/ssh/testdata/keys.go @@ -155,7 +155,7 @@ var SSHCertificates = map[string][]byte{ // Generated by the following commands: // // 1. Assumes "rsa" key above in file named "rsa", write out the public key to "rsa.pub": - // ssh-keygen -y -f rsa > rsa.pu + // ssh-keygen -y -f rsa > rsa.pub // // 2. Assumes "ca" key above in file named "ca", sign a cert for "rsa.pub": // ssh-keygen -s ca -h -n host.example.com -V +500w -I host.example.com-key rsa.pub diff --git a/vendor/golang.org/x/net/bpf/constants.go b/vendor/golang.org/x/net/bpf/constants.go index b89ca35..12f3ee8 100644 --- a/vendor/golang.org/x/net/bpf/constants.go +++ b/vendor/golang.org/x/net/bpf/constants.go @@ -38,6 +38,7 @@ const ( type JumpTest uint16 // Supported operators for conditional jumps. +// K can be RegX for JumpIfX const ( // K == A JumpEqual JumpTest = iota @@ -134,12 +135,9 @@ const ( opMaskLoadDest = 0x01 opMaskLoadWidth = 0x18 opMaskLoadMode = 0xe0 - // opClsALU - opMaskOperandSrc = 0x08 - opMaskOperator = 0xf0 - // opClsJump - opMaskJumpConst = 0x0f - opMaskJumpCond = 0xf0 + // opClsALU & opClsJump + opMaskOperand = 0x08 + opMaskOperator = 0xf0 ) const ( @@ -192,15 +190,21 @@ const ( opLoadWidth1 ) -// Operator defined by ALUOp* +// Operand for ALU and Jump instructions +type opOperand uint16 +// Supported operand sources. const ( - opALUSrcConstant uint16 = iota << 3 - opALUSrcX + opOperandConstant opOperand = iota << 3 + opOperandX ) +// An jumpOp is a conditional jump condition. +type jumpOp uint16 + +// Supported jump conditions. const ( - opJumpAlways = iota << 4 + opJumpAlways jumpOp = iota << 4 opJumpEqual opJumpGT opJumpGE diff --git a/vendor/golang.org/x/net/bpf/instructions.go b/vendor/golang.org/x/net/bpf/instructions.go index f9dc0e8..3cffcaa 100644 --- a/vendor/golang.org/x/net/bpf/instructions.go +++ b/vendor/golang.org/x/net/bpf/instructions.go @@ -89,10 +89,14 @@ func (ri RawInstruction) Disassemble() Instruction { case opClsALU: switch op := ALUOp(ri.Op & opMaskOperator); op { case ALUOpAdd, ALUOpSub, ALUOpMul, ALUOpDiv, ALUOpOr, ALUOpAnd, ALUOpShiftLeft, ALUOpShiftRight, ALUOpMod, ALUOpXor: - if ri.Op&opMaskOperandSrc != 0 { + switch operand := opOperand(ri.Op & opMaskOperand); operand { + case opOperandX: return ALUOpX{Op: op} + case opOperandConstant: + return ALUOpConstant{Op: op, Val: ri.K} + default: + return ri } - return ALUOpConstant{Op: op, Val: ri.K} case aluOpNeg: return NegateA{} default: @@ -100,63 +104,18 @@ func (ri RawInstruction) Disassemble() Instruction { } case opClsJump: - if ri.Op&opMaskJumpConst != opClsJump { - return ri - } - switch ri.Op & opMaskJumpCond { + switch op := jumpOp(ri.Op & opMaskOperator); op { case opJumpAlways: return Jump{Skip: ri.K} - case opJumpEqual: - if ri.Jt == 0 { - return JumpIf{ - Cond: JumpNotEqual, - Val: ri.K, - SkipTrue: ri.Jf, - SkipFalse: 0, - } - } - return JumpIf{ - Cond: JumpEqual, - Val: ri.K, - SkipTrue: ri.Jt, - SkipFalse: ri.Jf, - } - case opJumpGT: - if ri.Jt == 0 { - return JumpIf{ - Cond: JumpLessOrEqual, - Val: ri.K, - SkipTrue: ri.Jf, - SkipFalse: 0, - } - } - return JumpIf{ - Cond: JumpGreaterThan, - Val: ri.K, - SkipTrue: ri.Jt, - SkipFalse: ri.Jf, - } - case opJumpGE: - if ri.Jt == 0 { - return JumpIf{ - Cond: JumpLessThan, - Val: ri.K, - SkipTrue: ri.Jf, - SkipFalse: 0, - } - } - return JumpIf{ - Cond: JumpGreaterOrEqual, - Val: ri.K, - SkipTrue: ri.Jt, - SkipFalse: ri.Jf, - } - case opJumpSet: - return JumpIf{ - Cond: JumpBitsSet, - Val: ri.K, - SkipTrue: ri.Jt, - SkipFalse: ri.Jf, + case opJumpEqual, opJumpGT, opJumpGE, opJumpSet: + cond, skipTrue, skipFalse := jumpOpToTest(op, ri.Jt, ri.Jf) + switch operand := opOperand(ri.Op & opMaskOperand); operand { + case opOperandX: + return JumpIfX{Cond: cond, SkipTrue: skipTrue, SkipFalse: skipFalse} + case opOperandConstant: + return JumpIf{Cond: cond, Val: ri.K, SkipTrue: skipTrue, SkipFalse: skipFalse} + default: + return ri } default: return ri @@ -187,6 +146,41 @@ func (ri RawInstruction) Disassemble() Instruction { } } +func jumpOpToTest(op jumpOp, skipTrue uint8, skipFalse uint8) (JumpTest, uint8, uint8) { + var test JumpTest + + // Decode "fake" jump conditions that don't appear in machine code + // Ensures the Assemble -> Disassemble stage recreates the same instructions + // See https://github.com/golang/go/issues/18470 + if skipTrue == 0 { + switch op { + case opJumpEqual: + test = JumpNotEqual + case opJumpGT: + test = JumpLessOrEqual + case opJumpGE: + test = JumpLessThan + case opJumpSet: + test = JumpBitsNotSet + } + + return test, skipFalse, 0 + } + + switch op { + case opJumpEqual: + test = JumpEqual + case opJumpGT: + test = JumpGreaterThan + case opJumpGE: + test = JumpGreaterOrEqual + case opJumpSet: + test = JumpBitsSet + } + + return test, skipTrue, skipFalse +} + // LoadConstant loads Val into register Dst. type LoadConstant struct { Dst Register @@ -413,7 +407,7 @@ type ALUOpConstant struct { // Assemble implements the Instruction Assemble method. func (a ALUOpConstant) Assemble() (RawInstruction, error) { return RawInstruction{ - Op: opClsALU | opALUSrcConstant | uint16(a.Op), + Op: opClsALU | uint16(opOperandConstant) | uint16(a.Op), K: a.Val, }, nil } @@ -454,7 +448,7 @@ type ALUOpX struct { // Assemble implements the Instruction Assemble method. func (a ALUOpX) Assemble() (RawInstruction, error) { return RawInstruction{ - Op: opClsALU | opALUSrcX | uint16(a.Op), + Op: opClsALU | uint16(opOperandX) | uint16(a.Op), }, nil } @@ -509,7 +503,7 @@ type Jump struct { // Assemble implements the Instruction Assemble method. func (a Jump) Assemble() (RawInstruction, error) { return RawInstruction{ - Op: opClsJump | opJumpAlways, + Op: opClsJump | uint16(opJumpAlways), K: a.Skip, }, nil } @@ -530,11 +524,39 @@ type JumpIf struct { // Assemble implements the Instruction Assemble method. func (a JumpIf) Assemble() (RawInstruction, error) { + return jumpToRaw(a.Cond, opOperandConstant, a.Val, a.SkipTrue, a.SkipFalse) +} + +// String returns the instruction in assembler notation. +func (a JumpIf) String() string { + return jumpToString(a.Cond, fmt.Sprintf("#%d", a.Val), a.SkipTrue, a.SkipFalse) +} + +// JumpIfX skips the following Skip instructions in the program if A +// X is true. +type JumpIfX struct { + Cond JumpTest + SkipTrue uint8 + SkipFalse uint8 +} + +// Assemble implements the Instruction Assemble method. +func (a JumpIfX) Assemble() (RawInstruction, error) { + return jumpToRaw(a.Cond, opOperandX, 0, a.SkipTrue, a.SkipFalse) +} + +// String returns the instruction in assembler notation. +func (a JumpIfX) String() string { + return jumpToString(a.Cond, "x", a.SkipTrue, a.SkipFalse) +} + +// jumpToRaw assembles a jump instruction into a RawInstruction +func jumpToRaw(test JumpTest, operand opOperand, k uint32, skipTrue, skipFalse uint8) (RawInstruction, error) { var ( - cond uint16 + cond jumpOp flip bool ) - switch a.Cond { + switch test { case JumpEqual: cond = opJumpEqual case JumpNotEqual: @@ -552,63 +574,63 @@ func (a JumpIf) Assemble() (RawInstruction, error) { case JumpBitsNotSet: cond, flip = opJumpSet, true default: - return RawInstruction{}, fmt.Errorf("unknown JumpTest %v", a.Cond) + return RawInstruction{}, fmt.Errorf("unknown JumpTest %v", test) } - jt, jf := a.SkipTrue, a.SkipFalse + jt, jf := skipTrue, skipFalse if flip { jt, jf = jf, jt } return RawInstruction{ - Op: opClsJump | cond, + Op: opClsJump | uint16(cond) | uint16(operand), Jt: jt, Jf: jf, - K: a.Val, + K: k, }, nil } -// String returns the instruction in assembler notation. -func (a JumpIf) String() string { - switch a.Cond { +// jumpToString converts a jump instruction to assembler notation +func jumpToString(cond JumpTest, operand string, skipTrue, skipFalse uint8) string { + switch cond { // K == A case JumpEqual: - return conditionalJump(a, "jeq", "jneq") + return conditionalJump(operand, skipTrue, skipFalse, "jeq", "jneq") // K != A case JumpNotEqual: - return fmt.Sprintf("jneq #%d,%d", a.Val, a.SkipTrue) + return fmt.Sprintf("jneq %s,%d", operand, skipTrue) // K > A case JumpGreaterThan: - return conditionalJump(a, "jgt", "jle") + return conditionalJump(operand, skipTrue, skipFalse, "jgt", "jle") // K < A case JumpLessThan: - return fmt.Sprintf("jlt #%d,%d", a.Val, a.SkipTrue) + return fmt.Sprintf("jlt %s,%d", operand, skipTrue) // K >= A case JumpGreaterOrEqual: - return conditionalJump(a, "jge", "jlt") + return conditionalJump(operand, skipTrue, skipFalse, "jge", "jlt") // K <= A case JumpLessOrEqual: - return fmt.Sprintf("jle #%d,%d", a.Val, a.SkipTrue) + return fmt.Sprintf("jle %s,%d", operand, skipTrue) // K & A != 0 case JumpBitsSet: - if a.SkipFalse > 0 { - return fmt.Sprintf("jset #%d,%d,%d", a.Val, a.SkipTrue, a.SkipFalse) + if skipFalse > 0 { + return fmt.Sprintf("jset %s,%d,%d", operand, skipTrue, skipFalse) } - return fmt.Sprintf("jset #%d,%d", a.Val, a.SkipTrue) + return fmt.Sprintf("jset %s,%d", operand, skipTrue) // K & A == 0, there is no assembler instruction for JumpBitNotSet, use JumpBitSet and invert skips case JumpBitsNotSet: - return JumpIf{Cond: JumpBitsSet, SkipTrue: a.SkipFalse, SkipFalse: a.SkipTrue, Val: a.Val}.String() + return jumpToString(JumpBitsSet, operand, skipFalse, skipTrue) default: - return fmt.Sprintf("unknown instruction: %#v", a) + return fmt.Sprintf("unknown JumpTest %#v", cond) } } -func conditionalJump(inst JumpIf, positiveJump, negativeJump string) string { - if inst.SkipTrue > 0 { - if inst.SkipFalse > 0 { - return fmt.Sprintf("%s #%d,%d,%d", positiveJump, inst.Val, inst.SkipTrue, inst.SkipFalse) +func conditionalJump(operand string, skipTrue, skipFalse uint8, positiveJump, negativeJump string) string { + if skipTrue > 0 { + if skipFalse > 0 { + return fmt.Sprintf("%s %s,%d,%d", positiveJump, operand, skipTrue, skipFalse) } - return fmt.Sprintf("%s #%d,%d", positiveJump, inst.Val, inst.SkipTrue) + return fmt.Sprintf("%s %s,%d", positiveJump, operand, skipTrue) } - return fmt.Sprintf("%s #%d,%d", negativeJump, inst.Val, inst.SkipFalse) + return fmt.Sprintf("%s %s,%d", negativeJump, operand, skipFalse) } // RetA exits the BPF program, returning the value of register A. diff --git a/vendor/golang.org/x/net/bpf/instructions_test.go b/vendor/golang.org/x/net/bpf/instructions_test.go index dde474a..69b25c5 100644 --- a/vendor/golang.org/x/net/bpf/instructions_test.go +++ b/vendor/golang.org/x/net/bpf/instructions_test.go @@ -64,14 +64,22 @@ var allInstructions = []Instruction{ NegateA{}, - Jump{Skip: 10}, - JumpIf{Cond: JumpEqual, Val: 42, SkipTrue: 8, SkipFalse: 9}, - JumpIf{Cond: JumpNotEqual, Val: 42, SkipTrue: 8}, - JumpIf{Cond: JumpLessThan, Val: 42, SkipTrue: 7}, - JumpIf{Cond: JumpLessOrEqual, Val: 42, SkipTrue: 6}, - JumpIf{Cond: JumpGreaterThan, Val: 42, SkipTrue: 4, SkipFalse: 5}, - JumpIf{Cond: JumpGreaterOrEqual, Val: 42, SkipTrue: 3, SkipFalse: 4}, - JumpIf{Cond: JumpBitsSet, Val: 42, SkipTrue: 2, SkipFalse: 3}, + Jump{Skip: 17}, + JumpIf{Cond: JumpEqual, Val: 42, SkipTrue: 15, SkipFalse: 16}, + JumpIf{Cond: JumpNotEqual, Val: 42, SkipTrue: 15}, + JumpIf{Cond: JumpLessThan, Val: 42, SkipTrue: 14}, + JumpIf{Cond: JumpLessOrEqual, Val: 42, SkipTrue: 13}, + JumpIf{Cond: JumpGreaterThan, Val: 42, SkipTrue: 11, SkipFalse: 12}, + JumpIf{Cond: JumpGreaterOrEqual, Val: 42, SkipTrue: 10, SkipFalse: 11}, + JumpIf{Cond: JumpBitsSet, Val: 42, SkipTrue: 9, SkipFalse: 10}, + + JumpIfX{Cond: JumpEqual, SkipTrue: 8, SkipFalse: 9}, + JumpIfX{Cond: JumpNotEqual, SkipTrue: 8}, + JumpIfX{Cond: JumpLessThan, SkipTrue: 7}, + JumpIfX{Cond: JumpLessOrEqual, SkipTrue: 6}, + JumpIfX{Cond: JumpGreaterThan, SkipTrue: 4, SkipFalse: 5}, + JumpIfX{Cond: JumpGreaterOrEqual, SkipTrue: 3, SkipFalse: 4}, + JumpIfX{Cond: JumpBitsSet, SkipTrue: 2, SkipFalse: 3}, TAX{}, TXA{}, @@ -487,7 +495,67 @@ func TestString(t *testing.T) { }, { instruction: JumpIf{Cond: 0xffff, Val: 42, SkipTrue: 1, SkipFalse: 2}, - assembler: "unknown instruction: bpf.JumpIf{Cond:0xffff, Val:0x2a, SkipTrue:0x1, SkipFalse:0x2}", + assembler: "unknown JumpTest 0xffff", + }, + { + instruction: JumpIfX{Cond: JumpEqual, SkipTrue: 8, SkipFalse: 9}, + assembler: "jeq x,8,9", + }, + { + instruction: JumpIfX{Cond: JumpEqual, SkipTrue: 8}, + assembler: "jeq x,8", + }, + { + instruction: JumpIfX{Cond: JumpEqual, SkipFalse: 8}, + assembler: "jneq x,8", + }, + { + instruction: JumpIfX{Cond: JumpNotEqual, SkipTrue: 8}, + assembler: "jneq x,8", + }, + { + instruction: JumpIfX{Cond: JumpLessThan, SkipTrue: 7}, + assembler: "jlt x,7", + }, + { + instruction: JumpIfX{Cond: JumpLessOrEqual, SkipTrue: 6}, + assembler: "jle x,6", + }, + { + instruction: JumpIfX{Cond: JumpGreaterThan, SkipTrue: 4, SkipFalse: 5}, + assembler: "jgt x,4,5", + }, + { + instruction: JumpIfX{Cond: JumpGreaterThan, SkipTrue: 4}, + assembler: "jgt x,4", + }, + { + instruction: JumpIfX{Cond: JumpGreaterOrEqual, SkipTrue: 3, SkipFalse: 4}, + assembler: "jge x,3,4", + }, + { + instruction: JumpIfX{Cond: JumpGreaterOrEqual, SkipTrue: 3}, + assembler: "jge x,3", + }, + { + instruction: JumpIfX{Cond: JumpBitsSet, SkipTrue: 2, SkipFalse: 3}, + assembler: "jset x,2,3", + }, + { + instruction: JumpIfX{Cond: JumpBitsSet, SkipTrue: 2}, + assembler: "jset x,2", + }, + { + instruction: JumpIfX{Cond: JumpBitsNotSet, SkipTrue: 2, SkipFalse: 3}, + assembler: "jset x,3,2", + }, + { + instruction: JumpIfX{Cond: JumpBitsNotSet, SkipTrue: 2}, + assembler: "jset x,0,2", + }, + { + instruction: JumpIfX{Cond: 0xffff, SkipTrue: 1, SkipFalse: 2}, + assembler: "unknown JumpTest 0xffff", }, { instruction: TAX{}, diff --git a/vendor/golang.org/x/net/bpf/testdata/all_instructions.bpf b/vendor/golang.org/x/net/bpf/testdata/all_instructions.bpf index f871440..929eb25 100644 --- a/vendor/golang.org/x/net/bpf/testdata/all_instructions.bpf +++ b/vendor/golang.org/x/net/bpf/testdata/all_instructions.bpf @@ -1 +1 @@ -50,0 0 0 42,1 0 0 42,96 0 0 3,97 0 0 3,48 0 0 42,40 0 0 42,32 0 0 42,80 0 0 42,72 0 0 42,64 0 0 42,177 0 0 42,128 0 0 0,32 0 0 4294963200,32 0 0 4294963204,32 0 0 4294963256,2 0 0 3,3 0 0 3,4 0 0 42,20 0 0 42,36 0 0 42,52 0 0 42,68 0 0 42,84 0 0 42,100 0 0 42,116 0 0 42,148 0 0 42,164 0 0 42,12 0 0 0,28 0 0 0,44 0 0 0,60 0 0 0,76 0 0 0,92 0 0 0,108 0 0 0,124 0 0 0,156 0 0 0,172 0 0 0,132 0 0 0,5 0 0 10,21 8 9 42,21 0 8 42,53 0 7 42,37 0 6 42,37 4 5 42,53 3 4 42,69 2 3 42,7 0 0 0,135 0 0 0,22 0 0 0,6 0 0 0, +57,0 0 0 42,1 0 0 42,96 0 0 3,97 0 0 3,48 0 0 42,40 0 0 42,32 0 0 42,80 0 0 42,72 0 0 42,64 0 0 42,177 0 0 42,128 0 0 0,32 0 0 4294963200,32 0 0 4294963204,32 0 0 4294963256,2 0 0 3,3 0 0 3,4 0 0 42,20 0 0 42,36 0 0 42,52 0 0 42,68 0 0 42,84 0 0 42,100 0 0 42,116 0 0 42,148 0 0 42,164 0 0 42,12 0 0 0,28 0 0 0,44 0 0 0,60 0 0 0,76 0 0 0,92 0 0 0,108 0 0 0,124 0 0 0,156 0 0 0,172 0 0 0,132 0 0 0,5 0 0 17,21 15 16 42,21 0 15 42,53 0 14 42,37 0 13 42,37 11 12 42,53 10 11 42,69 9 10 42,29 8 9 0,29 0 8 0,61 0 7 0,45 0 6 0,45 4 5 0,61 3 4 0,77 2 3 0,7 0 0 0,135 0 0 0,22 0 0 0,6 0 0 42, diff --git a/vendor/golang.org/x/net/bpf/testdata/all_instructions.txt b/vendor/golang.org/x/net/bpf/testdata/all_instructions.txt index 3045501..8e4d758 100644 --- a/vendor/golang.org/x/net/bpf/testdata/all_instructions.txt +++ b/vendor/golang.org/x/net/bpf/testdata/all_instructions.txt @@ -1,6 +1,6 @@ # This filter is compiled to all_instructions.bpf by the `bpf_asm` # tool, which can be found in the linux kernel source tree under -# tools/net. +# tools/bpf. # Load immediate ld #42 @@ -60,7 +60,7 @@ xor x # !A neg -# Jumps +# Jump A constant ja end jeq #42,prev,end jne #42,end @@ -70,6 +70,15 @@ jgt #42,prev,end jge #42,prev,end jset #42,prev,end +# Jump A X +jeq x,prev,end +jne x,end +jlt x,end +jle x,end +jgt x,prev,end +jge x,prev,end +jset x,prev,end + # Register transfers tax txa diff --git a/vendor/golang.org/x/net/bpf/vm.go b/vendor/golang.org/x/net/bpf/vm.go index 4c656f1..73f57f1 100644 --- a/vendor/golang.org/x/net/bpf/vm.go +++ b/vendor/golang.org/x/net/bpf/vm.go @@ -35,6 +35,13 @@ func NewVM(filter []Instruction) (*VM, error) { if check <= int(ins.SkipFalse) { return nil, fmt.Errorf("cannot jump %d instructions in false case; jumping past program bounds", ins.SkipFalse) } + case JumpIfX: + if check <= int(ins.SkipTrue) { + return nil, fmt.Errorf("cannot jump %d instructions in true case; jumping past program bounds", ins.SkipTrue) + } + if check <= int(ins.SkipFalse) { + return nil, fmt.Errorf("cannot jump %d instructions in false case; jumping past program bounds", ins.SkipFalse) + } // Check for division or modulus by zero case ALUOpConstant: if ins.Val != 0 { @@ -109,6 +116,9 @@ func (v *VM) Run(in []byte) (int, error) { case JumpIf: jump := jumpIf(ins, regA) i += jump + case JumpIfX: + jump := jumpIfX(ins, regA, regX) + i += jump case LoadAbsolute: regA, ok = loadAbsolute(ins, in) case LoadConstant: diff --git a/vendor/golang.org/x/net/bpf/vm_instructions.go b/vendor/golang.org/x/net/bpf/vm_instructions.go index 516f946..f0d2e55 100644 --- a/vendor/golang.org/x/net/bpf/vm_instructions.go +++ b/vendor/golang.org/x/net/bpf/vm_instructions.go @@ -55,34 +55,41 @@ func aluOpCommon(op ALUOp, regA uint32, value uint32) uint32 { } } -func jumpIf(ins JumpIf, value uint32) int { - var ok bool - inV := uint32(ins.Val) +func jumpIf(ins JumpIf, regA uint32) int { + return jumpIfCommon(ins.Cond, ins.SkipTrue, ins.SkipFalse, regA, ins.Val) +} - switch ins.Cond { +func jumpIfX(ins JumpIfX, regA uint32, regX uint32) int { + return jumpIfCommon(ins.Cond, ins.SkipTrue, ins.SkipFalse, regA, regX) +} + +func jumpIfCommon(cond JumpTest, skipTrue, skipFalse uint8, regA uint32, value uint32) int { + var ok bool + + switch cond { case JumpEqual: - ok = value == inV + ok = regA == value case JumpNotEqual: - ok = value != inV + ok = regA != value case JumpGreaterThan: - ok = value > inV + ok = regA > value case JumpLessThan: - ok = value < inV + ok = regA < value case JumpGreaterOrEqual: - ok = value >= inV + ok = regA >= value case JumpLessOrEqual: - ok = value <= inV + ok = regA <= value case JumpBitsSet: - ok = (value & inV) != 0 + ok = (regA & value) != 0 case JumpBitsNotSet: - ok = (value & inV) == 0 + ok = (regA & value) == 0 } if ok { - return int(ins.SkipTrue) + return int(skipTrue) } - return int(ins.SkipFalse) + return int(skipFalse) } func loadAbsolute(ins LoadAbsolute, in []byte) (uint32, bool) { diff --git a/vendor/golang.org/x/net/bpf/vm_jump_test.go b/vendor/golang.org/x/net/bpf/vm_jump_test.go index e0a3a98..77a2d47 100644 --- a/vendor/golang.org/x/net/bpf/vm_jump_test.go +++ b/vendor/golang.org/x/net/bpf/vm_jump_test.go @@ -83,6 +83,32 @@ func TestVMJumpIfFalseOutOfProgram(t *testing.T) { } } +func TestVMJumpIfXTrueOutOfProgram(t *testing.T) { + _, _, err := testVM(t, []bpf.Instruction{ + bpf.JumpIfX{ + Cond: bpf.JumpEqual, + SkipTrue: 2, + }, + bpf.RetA{}, + }) + if errStr(err) != "cannot jump 2 instructions in true case; jumping past program bounds" { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestVMJumpIfXFalseOutOfProgram(t *testing.T) { + _, _, err := testVM(t, []bpf.Instruction{ + bpf.JumpIfX{ + Cond: bpf.JumpEqual, + SkipFalse: 3, + }, + bpf.RetA{}, + }) + if errStr(err) != "cannot jump 3 instructions in false case; jumping past program bounds" { + t.Fatalf("unexpected error: %v", err) + } +} + func TestVMJumpIfEqual(t *testing.T) { vm, done, err := testVM(t, []bpf.Instruction{ bpf.LoadAbsolute{ @@ -378,3 +404,323 @@ func TestVMJumpIfBitsNotSet(t *testing.T) { want, got) } } + +func TestVMJumpIfXEqual(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 1, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 1, + }, + bpf.JumpIfX{ + Cond: bpf.JumpEqual, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 9, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 1, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 1, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXNotEqual(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 1, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 1, + }, + bpf.JumpIfX{ + Cond: bpf.JumpNotEqual, + SkipFalse: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 9, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 1, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 1, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXGreaterThan(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 4, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 0x00010202, + }, + bpf.JumpIfX{ + Cond: bpf.JumpGreaterThan, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 12, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 4, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXLessThan(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 4, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 0xff010203, + }, + bpf.JumpIfX{ + Cond: bpf.JumpLessThan, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 12, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 4, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXGreaterOrEqual(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 4, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 0x00010203, + }, + bpf.JumpIfX{ + Cond: bpf.JumpGreaterOrEqual, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 12, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 4, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXLessOrEqual(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 4, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 0xff010203, + }, + bpf.JumpIfX{ + Cond: bpf.JumpLessOrEqual, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 12, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0, 1, 2, 3, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 4, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXBitsSet(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 2, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 0x1122, + }, + bpf.JumpIfX{ + Cond: bpf.JumpBitsSet, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 10, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0x01, 0x02, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 2, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} + +func TestVMJumpIfXBitsNotSet(t *testing.T) { + vm, done, err := testVM(t, []bpf.Instruction{ + bpf.LoadAbsolute{ + Off: 8, + Size: 2, + }, + bpf.LoadConstant{ + Dst: bpf.RegX, + Val: 0x1221, + }, + bpf.JumpIfX{ + Cond: bpf.JumpBitsNotSet, + SkipTrue: 1, + }, + bpf.RetConstant{ + Val: 0, + }, + bpf.RetConstant{ + Val: 10, + }, + }) + if err != nil { + t.Fatalf("failed to load BPF program: %v", err) + } + defer done() + + out, err := vm.Run([]byte{ + 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, + 0x01, 0x02, + }) + if err != nil { + t.Fatalf("unexpected error while running program: %v", err) + } + if want, got := 2, out; want != got { + t.Fatalf("unexpected number of output bytes:\n- want: %d\n- got: %d", + want, got) + } +} diff --git a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go index 606cf1f..37dc0cf 100644 --- a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go +++ b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go @@ -2,18 +2,15 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build go1.7 - // Package ctxhttp provides helper functions for performing context-aware HTTP requests. package ctxhttp // import "golang.org/x/net/context/ctxhttp" import ( + "context" "io" "net/http" "net/url" "strings" - - "golang.org/x/net/context" ) // Do sends an HTTP request with the provided http.Client and returns diff --git a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_17_test.go b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_17_test.go deleted file mode 100644 index 72411b1..0000000 --- a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_17_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !plan9,go1.7 - -package ctxhttp - -import ( - "io" - "net/http" - "net/http/httptest" - "testing" - - "context" -) - -func TestGo17Context(t *testing.T) { - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - io.WriteString(w, "ok") - })) - defer ts.Close() - ctx := context.Background() - resp, err := Get(ctx, http.DefaultClient, ts.URL) - if resp == nil || err != nil { - t.Fatalf("error received from client: %v %v", err, resp) - } - resp.Body.Close() -} diff --git a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17.go b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17.go deleted file mode 100644 index 926870c..0000000 --- a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !go1.7 - -package ctxhttp // import "golang.org/x/net/context/ctxhttp" - -import ( - "io" - "net/http" - "net/url" - "strings" - - "golang.org/x/net/context" -) - -func nop() {} - -var ( - testHookContextDoneBeforeHeaders = nop - testHookDoReturned = nop - testHookDidBodyClose = nop -) - -// Do sends an HTTP request with the provided http.Client and returns an HTTP response. -// If the client is nil, http.DefaultClient is used. -// If the context is canceled or times out, ctx.Err() will be returned. -func Do(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) { - if client == nil { - client = http.DefaultClient - } - - // TODO(djd): Respect any existing value of req.Cancel. - cancel := make(chan struct{}) - req.Cancel = cancel - - type responseAndError struct { - resp *http.Response - err error - } - result := make(chan responseAndError, 1) - - // Make local copies of test hooks closed over by goroutines below. - // Prevents data races in tests. - testHookDoReturned := testHookDoReturned - testHookDidBodyClose := testHookDidBodyClose - - go func() { - resp, err := client.Do(req) - testHookDoReturned() - result <- responseAndError{resp, err} - }() - - var resp *http.Response - - select { - case <-ctx.Done(): - testHookContextDoneBeforeHeaders() - close(cancel) - // Clean up after the goroutine calling client.Do: - go func() { - if r := <-result; r.resp != nil { - testHookDidBodyClose() - r.resp.Body.Close() - } - }() - return nil, ctx.Err() - case r := <-result: - var err error - resp, err = r.resp, r.err - if err != nil { - return resp, err - } - } - - c := make(chan struct{}) - go func() { - select { - case <-ctx.Done(): - close(cancel) - case <-c: - // The response's Body is closed. - } - }() - resp.Body = ¬ifyingReader{resp.Body, c} - - return resp, nil -} - -// Get issues a GET request via the Do function. -func Get(ctx context.Context, client *http.Client, url string) (*http.Response, error) { - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - return Do(ctx, client, req) -} - -// Head issues a HEAD request via the Do function. -func Head(ctx context.Context, client *http.Client, url string) (*http.Response, error) { - req, err := http.NewRequest("HEAD", url, nil) - if err != nil { - return nil, err - } - return Do(ctx, client, req) -} - -// Post issues a POST request via the Do function. -func Post(ctx context.Context, client *http.Client, url string, bodyType string, body io.Reader) (*http.Response, error) { - req, err := http.NewRequest("POST", url, body) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", bodyType) - return Do(ctx, client, req) -} - -// PostForm issues a POST request via the Do function. -func PostForm(ctx context.Context, client *http.Client, url string, data url.Values) (*http.Response, error) { - return Post(ctx, client, url, "application/x-www-form-urlencoded", strings.NewReader(data.Encode())) -} - -// notifyingReader is an io.ReadCloser that closes the notify channel after -// Close is called or a Read fails on the underlying ReadCloser. -type notifyingReader struct { - io.ReadCloser - notify chan<- struct{} -} - -func (r *notifyingReader) Read(p []byte) (int, error) { - n, err := r.ReadCloser.Read(p) - if err != nil && r.notify != nil { - close(r.notify) - r.notify = nil - } - return n, err -} - -func (r *notifyingReader) Close() error { - err := r.ReadCloser.Close() - if r.notify != nil { - close(r.notify) - r.notify = nil - } - return err -} diff --git a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17_test.go b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17_test.go deleted file mode 100644 index 9159cf0..0000000 --- a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_pre17_test.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !plan9,!go1.7 - -package ctxhttp - -import ( - "net" - "net/http" - "net/http/httptest" - "sync" - "testing" - "time" - - "golang.org/x/net/context" -) - -// golang.org/issue/14065 -func TestClosesResponseBodyOnCancel(t *testing.T) { - defer func() { testHookContextDoneBeforeHeaders = nop }() - defer func() { testHookDoReturned = nop }() - defer func() { testHookDidBodyClose = nop }() - - ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) - defer ts.Close() - - ctx, cancel := context.WithCancel(context.Background()) - - // closed when Do enters select case <-ctx.Done() - enteredDonePath := make(chan struct{}) - - testHookContextDoneBeforeHeaders = func() { - close(enteredDonePath) - } - - testHookDoReturned = func() { - // We now have the result (the Flush'd headers) at least, - // so we can cancel the request. - cancel() - - // But block the client.Do goroutine from sending - // until Do enters into the <-ctx.Done() path, since - // otherwise if both channels are readable, select - // picks a random one. - <-enteredDonePath - } - - sawBodyClose := make(chan struct{}) - testHookDidBodyClose = func() { close(sawBodyClose) } - - tr := &http.Transport{} - defer tr.CloseIdleConnections() - c := &http.Client{Transport: tr} - req, _ := http.NewRequest("GET", ts.URL, nil) - _, doErr := Do(ctx, c, req) - - select { - case <-sawBodyClose: - case <-time.After(5 * time.Second): - t.Fatal("timeout waiting for body to close") - } - - if doErr != ctx.Err() { - t.Errorf("Do error = %v; want %v", doErr, ctx.Err()) - } -} - -type noteCloseConn struct { - net.Conn - onceClose sync.Once - closefn func() -} - -func (c *noteCloseConn) Close() error { - c.onceClose.Do(c.closefn) - return c.Conn.Close() -} diff --git a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_test.go b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_test.go index 1e41551..c4339c4 100644 --- a/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_test.go +++ b/vendor/golang.org/x/net/context/ctxhttp/ctxhttp_test.go @@ -7,16 +7,28 @@ package ctxhttp import ( + "context" "io" "io/ioutil" "net/http" "net/http/httptest" "testing" "time" - - "golang.org/x/net/context" ) +func TestGo17Context(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + io.WriteString(w, "ok") + })) + defer ts.Close() + ctx := context.Background() + resp, err := Get(ctx, http.DefaultClient, ts.URL) + if resp == nil || err != nil { + t.Fatalf("error received from client: %v %v", err, resp) + } + resp.Body.Close() +} + const ( requestDuration = 100 * time.Millisecond requestBody = "ok" diff --git a/vendor/golang.org/x/net/html/const.go b/vendor/golang.org/x/net/html/const.go index 5eb7c5a..a3a918f 100644 --- a/vendor/golang.org/x/net/html/const.go +++ b/vendor/golang.org/x/net/html/const.go @@ -97,8 +97,16 @@ func isSpecialElement(element *Node) bool { switch element.Namespace { case "", "html": return isSpecialElementMap[element.Data] + case "math": + switch element.Data { + case "mi", "mo", "mn", "ms", "mtext", "annotation-xml": + return true + } case "svg": - return element.Data == "foreignObject" + switch element.Data { + case "foreignObject", "desc", "title": + return true + } } return false } diff --git a/vendor/golang.org/x/net/html/node.go b/vendor/golang.org/x/net/html/node.go index 2c1cade..633ee15 100644 --- a/vendor/golang.org/x/net/html/node.go +++ b/vendor/golang.org/x/net/html/node.go @@ -177,7 +177,7 @@ func (s *nodeStack) index(n *Node) int { // contains returns whether a is within s. func (s *nodeStack) contains(a atom.Atom) bool { for _, n := range *s { - if n.DataAtom == a { + if n.DataAtom == a && n.Namespace == "" { return true } } diff --git a/vendor/golang.org/x/net/html/parse.go b/vendor/golang.org/x/net/html/parse.go index 7e539b1..488e8d3 100644 --- a/vendor/golang.org/x/net/html/parse.go +++ b/vendor/golang.org/x/net/html/parse.go @@ -209,27 +209,6 @@ loop: p.oe = p.oe[:i+1] } -// generateAllImpliedEndTags pops nodes off the stack of open elements as long as -// the top node has a tag name of caption, colgroup, dd, div, dt, li, optgroup, option, p, rb, -// rp, rt, rtc, span, tbody, td, tfoot, th, thead or tr. -func (p *parser) generateAllImpliedEndTags() { - var i int - for i = len(p.oe) - 1; i >= 0; i-- { - n := p.oe[i] - if n.Type == ElementNode { - switch n.DataAtom { - // TODO: remove this divergence from the HTML5 spec - case a.Caption, a.Colgroup, a.Dd, a.Div, a.Dt, a.Li, a.Optgroup, a.Option, a.P, a.Rb, - a.Rp, a.Rt, a.Rtc, a.Span, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: - continue - } - } - break - } - - p.oe = p.oe[:i+1] -} - // addChild adds a child node n to the top element, and pushes n onto the stack // of open elements if it is an element node. func (p *parser) addChild(n *Node) { @@ -276,7 +255,7 @@ func (p *parser) fosterParent(n *Node) { } } - if template != nil && (table == nil || j < i) { + if template != nil && (table == nil || j > i) { template.AppendChild(n) return } @@ -491,6 +470,10 @@ func (p *parser) resetInsertionMode() { case a.Table: p.im = inTableIM case a.Template: + // TODO: remove this divergence from the HTML5 spec. + if n.Namespace != "" { + continue + } p.im = p.templateStack.top() case a.Head: // TODO: remove this divergence from the HTML5 spec. @@ -679,11 +662,16 @@ func inHeadIM(p *parser) bool { if !p.oe.contains(a.Template) { return true } - p.generateAllImpliedEndTags() - if n := p.oe.top(); n.DataAtom != a.Template { - return true + // TODO: remove this divergence from the HTML5 spec. + // + // See https://bugs.chromium.org/p/chromium/issues/detail?id=829668 + p.generateImpliedEndTags() + for i := len(p.oe) - 1; i >= 0; i-- { + if n := p.oe[i]; n.Namespace == "" && n.DataAtom == a.Template { + p.oe = p.oe[:i] + break + } } - p.popUntil(defaultScope, a.Template) p.clearActiveFormattingElements() p.templateStack.pop() p.resetInsertionMode() @@ -1000,6 +988,14 @@ func inBodyIM(p *parser) bool { p.acknowledgeSelfClosingTag() p.popUntil(buttonScope, a.P) p.parseImpliedToken(StartTagToken, a.Form, a.Form.String()) + if p.form == nil { + // NOTE: The 'isindex' element has been removed, + // and the 'template' element has not been designed to be + // collaborative with the index element. + // + // Ignore the token. + return true + } if action != "" { p.form.Attr = []Attribute{{Key: "action", Val: action}} } @@ -1074,13 +1070,7 @@ func inBodyIM(p *parser) bool { p.acknowledgeSelfClosingTag() } return true - case a.Frame: - // TODO: remove this divergence from the HTML5 spec. - if p.oe.contains(a.Template) { - p.addElement() - return true - } - case a.Caption, a.Col, a.Colgroup, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: + case a.Caption, a.Col, a.Colgroup, a.Frame, a.Head, a.Tbody, a.Td, a.Tfoot, a.Th, a.Thead, a.Tr: // Ignore the token. default: p.reconstructActiveFormattingElements() @@ -1274,12 +1264,6 @@ func (p *parser) inBodyEndTagFormatting(tagAtom a.Atom) { switch commonAncestor.DataAtom { case a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr: p.fosterParent(lastNode) - case a.Template: - // TODO: remove namespace checking - if commonAncestor.Namespace == "html" { - commonAncestor = commonAncestor.LastChild - } - fallthrough default: commonAncestor.AppendChild(lastNode) } @@ -1351,9 +1335,6 @@ func textIM(p *parser) bool { // Section 12.2.6.4.9. func inTableIM(p *parser) bool { switch p.tok.Type { - case ErrorToken: - // Stop parsing. - return true case TextToken: p.tok.Data = strings.Replace(p.tok.Data, "\x00", "", -1) switch p.oe.top().DataAtom { @@ -1448,6 +1429,8 @@ func inTableIM(p *parser) bool { case DoctypeToken: // Ignore the token. return true + case ErrorToken: + return inBodyIM(p) } p.fosterParenting = true @@ -1550,6 +1533,8 @@ func inColumnGroupIM(p *parser) bool { case a.Template: return inHeadIM(p) } + case ErrorToken: + return inBodyIM(p) } if p.oe.top().DataAtom != a.Colgroup { return true @@ -1714,9 +1699,6 @@ func inCellIM(p *parser) bool { // Section 12.2.6.4.16. func inSelectIM(p *parser) bool { switch p.tok.Type { - case ErrorToken: - // Stop parsing. - return true case TextToken: p.addText(strings.Replace(p.tok.Data, "\x00", "", -1)) case StartTagToken: @@ -1737,8 +1719,12 @@ func inSelectIM(p *parser) bool { } p.addElement() case a.Select: - p.tok.Type = EndTagToken - return false + if p.popUntil(selectScope, a.Select) { + p.resetInsertionMode() + } else { + // Ignore the token. + return true + } case a.Input, a.Keygen, a.Textarea: if p.elementInScope(selectScope, a.Select) { p.parseImpliedToken(EndTagToken, a.Select, a.Select.String()) @@ -1768,6 +1754,9 @@ func inSelectIM(p *parser) bool { case a.Select: if p.popUntil(selectScope, a.Select) { p.resetInsertionMode() + } else { + // Ignore the token. + return true } case a.Template: return inHeadIM(p) @@ -1780,6 +1769,8 @@ func inSelectIM(p *parser) bool { case DoctypeToken: // Ignore the token. return true + case ErrorToken: + return inBodyIM(p) } return true @@ -1791,13 +1782,22 @@ func inSelectInTableIM(p *parser) bool { case StartTagToken, EndTagToken: switch p.tok.DataAtom { case a.Caption, a.Table, a.Tbody, a.Tfoot, a.Thead, a.Tr, a.Td, a.Th: - if p.tok.Type == StartTagToken || p.elementInScope(tableScope, p.tok.DataAtom) { - p.parseImpliedToken(EndTagToken, a.Select, a.Select.String()) - return false - } else { + if p.tok.Type == EndTagToken && !p.elementInScope(tableScope, p.tok.DataAtom) { // Ignore the token. return true } + // This is like p.popUntil(selectScope, a.Select), but it also + // matches , not just brx
    aoe`: true, `