Compare commits

...
77 Commits
Author SHA1 Message Date
Joona Hoikkala 5708096bbe Merge branch 'master' of github.com:joohoi/acme-dns 2019-02-04 00:22:02 +02:00
Joona HoikkalaandGitHub 09dc25d336 Update vendored dependencies (#147)
* Prepare readme for release

* Update vendored packages

* New version of dns dependency handles rejecting UPDATEs under the hood

* Go 1.11 required
2019-02-04 00:21:42 +02:00
Joona Hoikkala 7b5973669f Merge branch 'master' of github.com:joohoi/acme-dns 2019-02-03 20:32:24 +02:00
Joona HoikkalaandGitHub a88ee29755 Prepare readme for release (#146) 2019-02-03 20:32:02 +02:00
Joona Hoikkala 277fa48405 Prepare readme for release 2019-02-03 20:21:03 +02:00
Joona HoikkalaandGitHub de7fe3cb1d Fix responses to be more standards compliant (#145)
* Handle OPT questions (EDNS)

* Handle authoritative bit, and append SOA for authoritative NXDOMAIN responses

* Changelog entry

* Fix linter errors
2019-02-03 20:19:15 +02:00
Joona HoikkalaandGitHub 7a2f9f06b1 Refactoring DNS server part for safer paraller execution (#144)
* Refactoring DNS server part for safer paraller execution and better data structures

* Fix linter issues
2019-02-03 17:23:04 +02:00
Joona HoikkalaandGitHub d695f72963 Fix the default configuration SQLite db path (#143) 2019-02-01 09:55:26 +02:00
Marvin BlumandJoona Hoikkala aff13a02fb Added http health check endpoint. (#137)
* Added http health check endpoint.

* Fixed performing POST on GET endpoint.

* Explicitly return http status 200 in health check endpoint.

* Updated changelog.
2019-01-25 19:22:53 +02:00
Joona HoikkalaandGitHub 7fbb5261c8 Fix test not to assume preset UseHeaders value (#139) 2019-01-25 18:36:29 +02:00
Philip VieiraandJoona Hoikkala 2da94f1462 Clarify how to test that the DNS Lookup is working (#133)
Reasoning: Unless you specify that you're looking for a TXT record, you will not get the answer from the DNS server
2019-01-20 18:23:07 +02:00
Christopher CookandJoona Hoikkala ea4d125663 Add Certify The Web to clients list (#131)
Certify now supports acme-dns in the latest beta.
https://github.com/webprofusion/certify/issues/391
2018-12-18 11:05:01 +02:00
Joona HoikkalaandGitHub f64de0353d Fix Docker instructions and add option to bind both UDP and TCP DNS listeners (#130) 2018-12-13 12:19:10 +02:00
Joona HoikkalaandGitHub 20411b650f Update changelog and prepare for release v0.6 (#125) 2018-10-31 15:13:29 +02:00
Joona HoikkalaandGitHub 3bb130d055 Refuse dynamic updates properly (#124) 2018-10-31 15:01:42 +02:00
Joona HoikkalaandGitHub 1d85bf75b0 Added Traefik and cert-manager to client list (#123) 2018-10-31 01:13:10 +02:00
Joona HoikkalaandGitHub c2c5c5cd70 Better error handling in goroutines (#122)
* More robust goroutine error handling using channels

* Fix tests and make startup log msg saner

* Clarification to README and config file
2018-10-31 00:54:51 +02:00
golint fixerandJoona Hoikkala a09073da12 Fix golint import path (#121) 2018-10-29 10:29:34 +02:00
Julien SchmidtandJoona Hoikkala 4bb5e265ab README: use absolute path for sqlite DB in example config (#116) 2018-09-28 11:40:50 +03:00
Julien SchmidtandJoona Hoikkala b452d504c8 README: fix NS address in test command (#115) 2018-09-28 11:14:31 +03:00
Yannik SembritzkiandJoona Hoikkala f76790426d fix inconsistency between README and actual config (#114)
* fix inconsistency between README and actual config

* clarify example config in sync w/ config.cfg
2018-09-28 00:46:07 +03:00
Yannik SembritzkiandJoona Hoikkala f650e47fe5 fix README inconsistencies (#113) 2018-09-28 00:33:51 +03:00
Yannik SembritzkiandJoona Hoikkala dc0dd43017 simplify example dns config (#106) 2018-09-27 23:58:37 +03:00
Julien SchmidtandJoona Hoikkala c827ee4801 README: fix section of HTTP header vars in example config (#112) 2018-09-27 23:51:31 +03:00
Stefan BethkeandJoona Hoikkala db2a6bc288 Cmd line flag -c for config location (#108)
Add command line parsing and a flag `-c` to specify where the config
file should be loaded from.
2018-09-21 13:38:23 +03:00
Joona HoikkalaandGitHub d66ccffaf6 Sign releases (#105) 2018-08-12 21:04:07 +03:00
Joona HoikkalaandGitHub eeff02e63b Changelog entries for v0.5 (#104) 2018-08-12 20:51:04 +03:00
Joona HoikkalaandGitHub 90ae6ee268 Default value for added configuration option to keep backwards compatibility with old config (#103) 2018-08-12 20:19:58 +03:00
Joona HoikkalaandGitHub 0fc5a8e848 Use umask 0077 across the process in order to have the created files readable only by the acme-dns user (#102) 2018-08-12 20:06:54 +03:00
Joona HoikkalaandGitHub ec013c0f25 Better error messages for missing DB config values (#101)
* Better error messages for missing DB config values

* Make linter happy
2018-08-12 19:48:39 +03:00
Joona HoikkalaandGitHub 856cc05881 Added supplementary error checking (#99)
* Added supplementary errorichecking

* After running util.go through gofmt

* Updated main and util

* Minor updates to main and util

* Slight refactoring

* Add tests
2018-08-12 18:49:17 +03:00
Joona HoikkalaandGitHub 75d4a30c1f Update dependencies and replace uuid library (#100) 2018-08-10 16:51:32 +03:00
Daniel McCarneyandJoona Hoikkala 8aa869b2f8 README: Add Lego to supported clients (#94)
* README: Add Lego to supported clients

* README: List clients in alpha sort
2018-07-10 00:28:53 +03:00
Daniel McCarneyandJoona Hoikkala 52e977ce1d README: Add goacmedns, a Go acme-dns client library to README. (#92) 2018-07-01 22:54:37 +03:00
Josh SorefandJoona Hoikkala 0bbbf5ed39 Minor (#90)
* add periods to sentences

* markdown

* reword

* reword: the item _acme-challenge should be a cname, not a cname to _acme-challenge
2018-06-05 11:50:42 +03:00
Josh SorefandJoona Hoikkala fbd5c65a6a Spelling (#89)
* spelling: the challenging

* spelling: effects

* spelling: received

* spelling: vagrantfile
2018-06-05 11:42:16 +03:00
Gabe CookandJoona Hoikkala 4646e7f7e5 Add systemd service information (#83)
* Add systemd service information

* Requested changes for systemd service information
2018-05-27 00:08:55 +03:00
Andrew MeyerandJoona Hoikkala 3006cb712b Make ACME cache directory location configurable (#81)
* Remove trailing whitespace from README and config

* Make ACME cache directory location configurable
2018-05-14 13:42:39 +03:00
Ryan BolgerandJoona Hoikkala fde566fe67 Added Posh-ACME to client list (#78)
* Added Posh-ACME to client list

* alphabetical order as requested
2018-05-02 23:15:50 +03:00
Joona HoikkalaandGitHub a17792d33c Merge pull request #75 from joohoi/golintfix-revert
Revert "Disable golint until golang/go#25048 is resolved."
2018-04-25 11:50:32 +03:00
Joona Hoikkala d1af1d029f Revert "Disable golint until golang/go#25048 is resolved."
This reverts commit 1c918fcaa4.
2018-04-25 11:44:34 +03:00
Joona HoikkalaandGitHub a5b417901c Merge pull request #73 from joohoi/add-sewer
README: Add Sewer client
2018-04-24 15:00:30 +03:00
Joona Hoikkala 065083781e Merge remote-tracking branch 'origin/master' into add-sewer 2018-04-24 14:55:32 +03:00
Joona HoikkalaandGitHub 0af5dc2cd9 Merge pull request #74 from joohoi/golintfix
Fix golint dependency issue
2018-04-24 14:55:05 +03:00
Joona Hoikkala 1c918fcaa4 Disable golint until golang/go#25048 is resolved. 2018-04-24 14:49:20 +03:00
Joona Hoikkala 299c520c2b Add Sewer client 2018-04-24 14:17:59 +03:00
Joona HoikkalaandGitHub 3343d943d6 Merge pull request #68 from joohoi/update-clients
README: add acme.sh and change Certbot hook URL
2018-04-08 12:55:59 +03:00
Joona Hoikkala 7744357b61 README: add acme.sh and change Certbot hook URL 2018-04-08 12:51:06 +03:00
Joona HoikkalaandGitHub 14f552e970 Merge pull request #66 from cpu/cpu-typo-fix
db_test: fix 'recieve' typo
2018-04-08 12:41:20 +03:00
Joona HoikkalaandGitHub f2d1fc692b Merge branch 'master' into cpu-typo-fix 2018-04-08 12:37:14 +03:00
Joona HoikkalaandGitHub dd12fa7c9b Merge pull request #64 from Daniel15/patch-2
Make installation instructions more comprehensive
2018-04-08 12:34:00 +03:00
Joona HoikkalaandGitHub 0991b3e3c9 Merge branch 'master' into patch-2 2018-04-08 12:24:24 +03:00
Daniel d18cd65374 db_test: fix 'recieve' typo 2018-03-26 17:22:23 -04:00
Daniel Lo NigroandGitHub 8eed75b495 Explain CNAMEs in DNS records section 2018-03-22 20:37:07 -07:00
Daniel Lo NigroandGitHub 46b9c45e1b Remove unintended newline 2018-03-22 20:33:30 -07:00
Daniel Lo NigroandGitHub ed82e15349 Refactor readme changes
- Move DNS config to separate "DNS Records" section
- Move testing to separate "Testing It Out" section
2018-03-22 20:32:56 -07:00
Joona HoikkalaandGitHub 7b2203ccca Merge pull request #65 from joohoi/clients
README: Add link to a client hook written in Go
2018-03-20 15:01:51 +02:00
Joona Hoikkala 945013579f README: Add link to a client hook written in Go 2018-03-20 14:56:35 +02:00
Daniel Lo NigroandGitHub 3ee50d0c20 Make installation instructions more comprehensive 2018-03-18 23:47:45 -07:00
Joona HoikkalaandGitHub f2fe21934c Merge pull request #60 from joohoi/link_clientlib
README: Add link to the client library
2018-03-18 17:13:23 +02:00
Joona Hoikkala 07aa5e0043 README: Add link to the client library 2018-03-18 16:02:25 +02:00
Joona HoikkalaandGitHub 01f010e35e README: link to client implementation (#55) 2018-03-15 13:40:45 +02:00
Joona HoikkalaandGitHub cd3a587e43 Release v0.4 (#53) 2018-03-15 00:47:36 +02:00
Joona HoikkalaandGitHub 439da9c09f Properly parse r.RemoteAddr (#50)
* Properly parse r.RemoteAddr

* Add tests, and fix net.ParseCIDR issues with IPv6 addresses enclosed in brackets
2018-03-15 00:23:55 +02:00
Joona HoikkalaandGitHub 5c2e60a828 Add configuration option to disable registration endpoint (#51) 2018-03-14 23:35:39 +02:00
Joona HoikkalaandGitHub d542ee03b5 Merge pull request #42 from cpu/cpu-differentiate-errors
API: Differentiate bad TXT update error.
2018-03-13 11:32:40 +02:00
Joona HoikkalaandGitHub 92f8cc2802 Merge branch 'master' into cpu-differentiate-errors 2018-03-13 10:55:02 +02:00
Joona HoikkalaandGitHub 978ac5d62b Merge pull request #49 from Yannik/patch-1
return cname if requested RR was not found and cname exists (fixes #48)
2018-03-13 10:42:44 +02:00
Yannik Sembritzki 4d214d7f52 return cname if requested RR was not found and cname exists 2018-03-03 11:24:32 +01:00
Daniel McCarneyandGitHub f463d07d0b Merge branch 'master' into cpu-differentiate-errors 2018-03-01 09:54:41 -05:00
Joona HoikkalaandGitHub 9a908d7d6b Log IP address that we're matching against allowFrom values stored in the DB (#46)
* Add logging for IP matching

* Fix typo
2018-03-01 16:53:38 +02:00
Daniel McCarneyandJoona Hoikkala 11c852ee91 README: simplify installation instructions with go get. (#41)
* README: simplify installation instructions with `go get`.

Prior to this commit the installation instructions in the README had you
use `git clone` and then `go build`. This can be collapsed into one step
using a more idiomatic `go get` command. This commit updates the README
accordingly.

* Also mention supported config file locations

* Fix go get URL
2018-02-28 20:38:44 +02:00
Joona HoikkalaandGitHub 32608e9f47 Merge pull request #44 from cpu/cpu-add-api-ip-example-config
README: Add example 'ip' in example API config.
2018-02-28 20:24:08 +02:00
Daniel 77b5fda6fb README: Add example 'ip' in example API config.
`acme-dns` supports binding the API to a specific interface instead of
all interfaces by providing an `ip` address in the `[api]` configuration
section. Prior to this commit the `ip` field wasn't shown in the example
configuration in the README. This commit adds an example showing how to
configure `acme-dns` to listen only on `127.0.0.1` to the example config
and describes what the default value (`""`) does.
2018-02-27 21:45:13 -05:00
Daniel efdd560ee4 API: Differentiate bad TXT update error.
Previous to this commit, if the update message had a valid subdomain but
an invalid TXT the error returned was for a bad subdomain. This can
confuse developers who were POSTing junk TXT records to test acme-dns
:-)

This commit adjusts the `webUpdatePost` error handling such that
`!validSubdomain(input)` and `!validTXT(input)` give distinct errors.

The `!validSubdomain` case should never happen in `webUpdatePost`
because `auth.go`'s `Auth` function already vets the post data
subdomain but I retained the error handling code just in case.

Unit tests for an update with an invalid subdomain and an update with an
invalid TXT are included.
2018-02-27 19:26:15 -05:00
Joona HoikkalaandGitHub 830cceb62c Release v0.3.2 (#40) 2018-02-01 16:10:24 +02:00
Leon KyneurandJoona Hoikkala 9f896759f0 reinstall ca-certificates (#39) 2018-02-01 16:02:06 +02:00
2172 changed files with 191944 additions and 59272 deletions
+1
View File
@@ -3,3 +3,4 @@ acme-dns.db
acme-dns.log
.vagrant
coverage.out
.idea/
+4
View File
@@ -15,3 +15,7 @@ archive:
- README.md
- Dockerfile
- config.cfg
- acme-dns.service
sign:
artifacts: checksum
+2 -2
View File
@@ -1,10 +1,10 @@
language: go
go:
- 1.9
- 1.11
env:
- "PATH=/home/travis/gopath/bin:$PATH"
before_install:
- go get github.com/golang/lint/golint
- go get golang.org/x/lint/golint
- go get github.com/mattn/goveralls
script:
- go vet
+3 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.9.2-alpine AS builder
FROM golang:1.11.4-alpine3.8 AS builder
LABEL maintainer="joona@kuori.org"
RUN apk add --update gcc musl-dev git
@@ -14,7 +14,9 @@ COPY --from=builder /go/src/github.com/joohoi/acme-dns .
RUN mkdir -p /etc/acme-dns
RUN mkdir -p /var/lib/acme-dns
RUN rm -rf ./config.cfg
RUN apk --no-cache add ca-certificates && update-ca-certificates
VOLUME ["/etc/acme-dns", "/var/lib/acme-dns"]
ENTRYPOINT ["./acme-dns"]
EXPOSE 53 80 443
EXPOSE 53/udp
Generated
+211 -69
View File
@@ -2,224 +2,366 @@
[[projects]]
digest = "1:e4b30804a381d7603b8a344009987c1ba351c26043501b23b8c7ce21f0b67474"
name = "github.com/BurntSushi/toml"
packages = ["."]
revision = "b26d9c308763d68093482582cea63d69be07a0f0"
version = "v0.3.0"
pruneopts = ""
revision = "3012a1dbe2e4bd1391d42b32f0577cb7bbc7f005"
version = "v0.3.1"
[[projects]]
digest = "1:9ceecb4271682fd824475d451b6abf02b99ad04e72f8de3408a9d8b7fd15b933"
name = "github.com/ajg/form"
packages = ["."]
pruneopts = ""
revision = "cc2954064ec9ea8d93917f0f87456e11d7b881ad"
version = "v1.5"
[[projects]]
digest = "1:0deddd908b6b4b768cfc272c16ee61e7088a60f7fe2f06c547bd3d8e1f8b8e77"
name = "github.com/davecgh/go-spew"
packages = ["spew"]
revision = "346938d642f2ec3594ed81d874461961cd0faa76"
pruneopts = ""
revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73"
version = "v1.1.1"
[[projects]]
branch = "master"
digest = "1:358f9731027e87f8ae3a7e52f0473031be47d1e041295b3c0afc84dc779c0666"
name = "github.com/erikstmartin/go-testdb"
packages = ["."]
pruneopts = ""
revision = "8d10e4a1bae52cd8b81ffdec3445890d6dccab3d"
[[projects]]
digest = "1:2defd14040b1ca18840524e9d0f9cfab0ea8d2f2d396b343d221f8bc8e21ab9d"
name = "github.com/fatih/structs"
packages = ["."]
pruneopts = ""
revision = "4966fc68f5b7593aafa6cbbba2d65ec6e1416047"
version = "v1.1.0"
[[projects]]
branch = "master"
name = "github.com/erikstmartin/go-testdb"
packages = ["."]
revision = "8d10e4a1bae52cd8b81ffdec3445890d6dccab3d"
[[projects]]
name = "github.com/fatih/structs"
packages = ["."]
revision = "a720dfa8df582c51dee1b36feabb906bde1588bd"
version = "v1.0"
[[projects]]
branch = "master"
digest = "1:7a98bc5cfa8e0c82156a33edb7ed582a9e700af089994e1b89bdb8517cd2db58"
name = "github.com/gavv/httpexpect"
packages = ["."]
revision = "c44a6d7bb636b17e880a53998a7f7061a56ffacb"
pruneopts = ""
revision = "bdde308713130a703436e014ff782958c251d20a"
[[projects]]
branch = "master"
digest = "1:44e52e928b91a686e2b9518f7eea82ba729d227d94b69574416bbe2d9d418b33"
name = "github.com/gavv/monotime"
packages = ["."]
pruneopts = ""
revision = "6f8212e8d10df7383609d3c377ca08884d8f3ec0"
[[projects]]
branch = "master"
digest = "1:cea4aa2038169ee558bf507d5ea02c94ca85bcca28a4c7bb99fd59b31e43a686"
name = "github.com/google/go-querystring"
packages = ["query"]
revision = "53e6ce116135b80d037921a7fdd5138cf32d7a8a"
pruneopts = ""
revision = "44c6ddd0a2342c386950e880b658017258da92fc"
version = "v1.0.0"
[[projects]]
digest = "1:a25a2c5ae694b01713fb6cd03c3b1ac1ccc1902b9f0a922680a88ec254f968e1"
name = "github.com/google/uuid"
packages = ["."]
pruneopts = ""
revision = "9b3b1e0f5f99ae461456d768e7d301a7acdaa2d8"
version = "v1.1.0"
[[projects]]
digest = "1:788735d9cba7f51b7cc86e6c7ba7b40b0b7c7b374bf3a0b5fa7c929fa2af2da8"
name = "github.com/imkira/go-interpol"
packages = ["."]
pruneopts = ""
revision = "5accad8134979a6ac504d456a6c7f1c53da237ca"
version = "v1.1.0"
[[projects]]
digest = "1:deeb6536ebbcb66ab11213ecfaac9daadfcd004dd057446a4980ffd6c15c36e2"
name = "github.com/julienschmidt/httprouter"
packages = ["."]
revision = "8c199fb6259ffc1af525cc3ad52ee60ba8359669"
version = "v1.1"
pruneopts = ""
revision = "348b672cd90d8190f8240323e372ecd1e66b59dc"
version = "v1.2.0"
[[projects]]
digest = "1:064eb4d2277fb6a23944380644268b99acc0ae0576c72d0b6d7288d3c2fe4a1e"
name = "github.com/klauspost/compress"
packages = ["flate","gzip","zlib"]
revision = "6c8db69c4b49dd4df1fff66996cf556176d0b9bf"
version = "v1.2.1"
packages = [
"flate",
"gzip",
"zlib",
]
pruneopts = ""
revision = "30be6041bed523c18e269a700ebd9c2ea9328574"
version = "v1.4.1"
[[projects]]
digest = "1:198af4ad42013a3779477b597458f860a6a99e57e1674fa4f61eb04b8b5b768f"
name = "github.com/klauspost/cpuid"
packages = ["."]
revision = "ae7887de9fa5d2db4eaa8174a7eff2c1ac00f2da"
version = "v1.1"
pruneopts = ""
revision = "e7e905edc00ea8827e58662220139109efea09db"
version = "v1.2.0"
[[projects]]
name = "github.com/klauspost/crc32"
digest = "1:6a874e3ddfb9db2b42bd8c85b6875407c702fa868eed20634ff489bc896ccfd3"
name = "github.com/konsorten/go-windows-terminal-sequences"
packages = ["."]
revision = "cb6bfca970f6908083f26f39a79009d608efd5cd"
version = "v1.1"
pruneopts = ""
revision = "5c8c8bd35d3832f5d134ae1e1e375b69a4d25242"
version = "v1.0.1"
[[projects]]
branch = "master"
digest = "1:bc36cd980d0069137800b505af4937c6109f4dc7cefe39d7c2efc7c8d51528d6"
name = "github.com/lib/pq"
packages = [".","oid"]
revision = "27ea5d92de30060e7121ddd543fe14e9a327e0cc"
packages = [
".",
"oid",
]
pruneopts = ""
revision = "9eb73efc1fcc404148b56765b0d3f61d9a5ef8ee"
[[projects]]
digest = "1:8bbdb2b3dce59271877770d6fe7dcbb8362438fa7d2e1e1f688e4bf2aac72706"
name = "github.com/mattn/go-sqlite3"
packages = ["."]
revision = "6c771bb9887719704b210e87e934f08be014bdb1"
version = "v1.6.0"
pruneopts = ""
revision = "c7c4067b79cc51e6dfdcef5c702e74b1e0fa7c75"
version = "v1.10.0"
[[projects]]
digest = "1:103c1001b79009329824bc60f93f8f7150bf1ad31f18d29a476deb8a2de0f002"
name = "github.com/miekg/dns"
packages = ["."]
revision = "5ec25f2a5044291b6c8abf43ed8a201da241e69e"
version = "v1.0.3"
pruneopts = ""
revision = "8fc2e5773bbd308ca2fcc962fd8d25c1bd0f6743"
version = "v1.1.4"
[[projects]]
branch = "master"
digest = "1:fe67641b990bdc1802f8a1e462a4924210a8762a8a17b72e09656049c906b871"
name = "github.com/moul/http2curl"
packages = ["."]
pruneopts = ""
revision = "9ac6cf4d929b2fa8fd2d2e6dec5bb0feb4f4911d"
version = "v1.0.0"
[[projects]]
digest = "1:256484dbbcd271f9ecebc6795b2df8cad4c458dd0f5fd82a8c2fa0c29f233411"
name = "github.com/pmezard/go-difflib"
packages = ["difflib"]
pruneopts = ""
revision = "792786c7400a136282c1664665ae0a8db921c6c2"
version = "v1.0.0"
[[projects]]
digest = "1:5f47c69f85311c4dc292be6cc995a0a3fe8337a6ce38ef4f71e5b7efd5ad42e0"
name = "github.com/rs/cors"
packages = ["."]
revision = "7af7a1e09ba336d2ea14b1ce73bf693c6837dbf6"
version = "v1.2"
pruneopts = ""
revision = "9a47f48565a795472d43519dd49aac781f3034fb"
version = "v1.6.0"
[[projects]]
name = "github.com/satori/go.uuid"
packages = ["."]
revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3"
version = "v1.2.0"
[[projects]]
branch = "master"
digest = "1:3962f553b77bf6c03fc07cd687a22dd3b00fe11aa14d31194f5505f5bb65cdc8"
name = "github.com/sergi/go-diff"
packages = ["diffmatchpatch"]
pruneopts = ""
revision = "1744e2970ca51c86172c8190fadad617561ed6e7"
version = "v1.0.0"
[[projects]]
digest = "1:9a3c631555e0351fdc4e696577bb63afd90c399d782a8462dba9d100d7021db3"
name = "github.com/sirupsen/logrus"
packages = [".","hooks/test"]
revision = "d682213848ed68c0a260ca37d6dd5ace8423f5ba"
version = "v1.0.4"
packages = [
".",
"hooks/test",
]
pruneopts = ""
revision = "e1e72e9de974bd926e5c56f83753fba2df402ce5"
version = "v1.3.0"
[[projects]]
digest = "1:381bcbeb112a51493d9d998bbba207a529c73dbb49b3fd789e48c63fac1f192c"
name = "github.com/stretchr/testify"
packages = ["assert","require"]
revision = "b91bfb9ebec76498946beb6af7c0230c7cc7ba6c"
version = "v1.2.0"
packages = [
"assert",
"require",
]
pruneopts = ""
revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053"
version = "v1.3.0"
[[projects]]
branch = "master"
digest = "1:857a9ecd5cb13379ecc8f798f6e6b6b574c98b9355657d91e068275f1120aaf7"
name = "github.com/valyala/bytebufferpool"
packages = ["."]
pruneopts = ""
revision = "e746df99fe4a3986f4d4f79e13c1e0117ce9c2f7"
version = "v1.0.0"
[[projects]]
branch = "master"
digest = "1:5838606075eb2e11d5b99d4526e36863a9315f006d322abfbf5a7fff73250271"
name = "github.com/valyala/fasthttp"
packages = [".","fasthttputil","stackless"]
revision = "e5f51c11919d4f66400334047b897ef0a94c6f3c"
packages = [
".",
"fasthttputil",
"stackless",
]
pruneopts = ""
revision = "aaec9b0fe2d01bd3ff761345ba47bb94a9b0dc59"
[[projects]]
branch = "master"
digest = "1:e6338f2518362ff701a556bd76afd90a2168d1c658ec5d1ea1e9c5ef30a7d157"
name = "github.com/xeipuuv/gojsonpointer"
packages = ["."]
revision = "6fe8760cad3569743d51ddbb243b26f8456742dc"
pruneopts = ""
revision = "4e3ac2762d5f479393488629ee9370b50873b3a6"
[[projects]]
branch = "master"
digest = "1:604f98a38394d2805a78c462396a4992b93fdd5b7306130add330f1a99ac6b0a"
name = "github.com/xeipuuv/gojsonreference"
packages = ["."]
revision = "e02fc20de94c78484cd5ffb007f8af96be030a45"
pruneopts = ""
revision = "bd5ef7bd5415a7ac448318e64f11a24cd21e594b"
[[projects]]
branch = "master"
digest = "1:c2c74271189fe48e23c25216570008b7ecc5782688d6efd81325886ac6f62ad0"
name = "github.com/xeipuuv/gojsonschema"
packages = ["."]
revision = "511d08a359d14c0dd9c4302af52ee9abb6f93c2a"
pruneopts = ""
revision = "f971f3cd73b2899de6923801c147f075263e0c50"
version = "v1.1.0"
[[projects]]
branch = "master"
digest = "1:81daf39130b8efb47ab2b841ee42adedeaaf20ab3675236f577722ae78d37728"
name = "github.com/yalp/jsonpath"
packages = ["."]
revision = "31a79c7593bb93eb10b163650d4a3e6ca190e4dc"
pruneopts = ""
revision = "5cc68e5049a040829faef3a44c00ec4332f6dec7"
[[projects]]
digest = "1:529ed3f98838f69e13761788d0cc71b44e130058fab13bae2ce09f7a176bced4"
name = "github.com/yudai/gojsondiff"
packages = [".","formatter"]
packages = [
".",
"formatter",
]
pruneopts = ""
revision = "7b1b7adf999dab73a6eb02669c3d82dbb27a3dd6"
version = "1.0.0"
[[projects]]
branch = "master"
digest = "1:9857bb2293f372b2181004d8b62179bbdb4ab0982ec6f762abe6cf2bfedaff85"
name = "github.com/yudai/golcs"
packages = ["."]
pruneopts = ""
revision = "ecda9a501e8220fae3b4b600c3db4b0ba22cfc68"
[[projects]]
branch = "master"
digest = "1:cbc1ebc01ec2ceb2c3cc7b9e33357dbc3eff173335f02d9f01603f14102602a7"
name = "golang.org/x/crypto"
packages = ["acme","acme/autocert","bcrypt","blowfish","ed25519","ed25519/internal/edwards25519","ssh/terminal"]
revision = "a6600008915114d9c087fad9f03d75087b1a74df"
packages = [
"acme",
"acme/autocert",
"bcrypt",
"blowfish",
"ed25519",
"ed25519/internal/edwards25519",
"ssh/terminal",
]
pruneopts = ""
revision = "b8fe1690c61389d7d2a8074a507d1d40c5d30448"
[[projects]]
branch = "master"
digest = "1:1e9704e5379e68ac473c28aeb3f7e7cd4036ae8a246bf0285b5ebdbb8e0cfacf"
name = "golang.org/x/net"
packages = ["bpf","idna","internal/iana","internal/socket","ipv4","ipv6","publicsuffix"]
revision = "5ccada7d0a7ba9aeb5d3aca8d3501b4c2a509fec"
packages = [
"bpf",
"idna",
"internal/iana",
"internal/socket",
"ipv4",
"ipv6",
"publicsuffix",
]
pruneopts = ""
revision = "d26f9f9a57f3fab6a695bec0d84433c2c50f8bbf"
[[projects]]
branch = "master"
digest = "1:4d6f20f5bc52af99569b1be5ea55d4e22646d5e022b92f2d3d75347628614734"
name = "golang.org/x/sys"
packages = ["unix","windows"]
revision = "af50095a40f9041b3b38960738837185c26e9419"
packages = [
"unix",
"windows",
]
pruneopts = ""
revision = "7ae0202eb74c2b534255c71b5a15fa4115aabbcc"
[[projects]]
branch = "master"
digest = "1:5acd3512b047305d49e8763eef7ba423901e85d5dd2fd1e71778a0ea8de10bd4"
name = "golang.org/x/text"
packages = ["collate","collate/build","internal/colltab","internal/gen","internal/tag","internal/triegen","internal/ucd","language","secure/bidirule","transform","unicode/bidi","unicode/cldr","unicode/norm","unicode/rangetable"]
revision = "e19ae1496984b1c655b8044a65c0300a3c878dd3"
packages = [
"collate",
"collate/build",
"internal/colltab",
"internal/gen",
"internal/tag",
"internal/triegen",
"internal/ucd",
"language",
"secure/bidirule",
"transform",
"unicode/bidi",
"unicode/cldr",
"unicode/norm",
"unicode/rangetable",
]
pruneopts = ""
revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0"
version = "v0.3.0"
[[projects]]
digest = "1:4014584c076f25aaf35d9de36c79ae2d208bba32c780f405e3395dad79292e22"
name = "gopkg.in/DATA-DOG/go-sqlmock.v1"
packages = ["."]
pruneopts = ""
revision = "d76b18b42f285b792bf985118980ce9eacea9d10"
version = "v1.3.0"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "991ecc43a6e9bcfe3c39169d7509ee821076b0b75bbf7cb38ec06db3041cd009"
input-imports = [
"github.com/BurntSushi/toml",
"github.com/erikstmartin/go-testdb",
"github.com/gavv/httpexpect",
"github.com/google/uuid",
"github.com/julienschmidt/httprouter",
"github.com/lib/pq",
"github.com/mattn/go-sqlite3",
"github.com/miekg/dns",
"github.com/rs/cors",
"github.com/sirupsen/logrus",
"github.com/sirupsen/logrus/hooks/test",
"github.com/valyala/fasthttp",
"golang.org/x/crypto/acme/autocert",
"golang.org/x/crypto/bcrypt",
"gopkg.in/DATA-DOG/go-sqlmock.v1",
]
solver-name = "gps-cdcl"
solver-version = 1
-4
View File
@@ -59,10 +59,6 @@ required = ["github.com/valyala/fasthttp"]
name = "github.com/rs/cors"
version = "1.2.0"
[[constraint]]
name = "github.com/satori/go.uuid"
version = "1.2.0"
[[constraint]]
name = "github.com/sirupsen/logrus"
version = "1.0.4"
+175 -41
View File
@@ -8,9 +8,9 @@ A simplified DNS server with a RESTful HTTP API to provide a simple way to autom
Many DNS servers do not provide an API to enable automation for the ACME DNS challenges. Those which do, give the keys way too much power.
Leaving the keys laying around your random boxes is too often a requirement to have a meaningful process automation.
Acme-dns provides a simple API exclusively for TXT record updates and should be used with ACME magic "\_acme-challenge" - subdomain CNAME records. This way, in the unfortunate exposure of API keys, the effetcs are limited to the subdomain TXT record in question.
Acme-dns provides a simple API exclusively for TXT record updates and should be used with ACME magic "\_acme-challenge" - subdomain CNAME records. This way, in the unfortunate exposure of API keys, the effects are limited to the subdomain TXT record in question.
So basically it boils down to **accessibility** and **security**
So basically it boils down to **accessibility** and **security**.
## Features
- Simplified DNS server, serving your ACME DNS challenges (TXT)
@@ -22,13 +22,16 @@ So basically it boils down to **accessibility** and **security**
- Simple deployment (it's Go after all)
## Usage
A Certbot authentication hook for acme-dns is available at: [https://github.com/joohoi/acme-dns-certbot](https://github.com/joohoi/acme-dns-certbot).
[![asciicast](https://asciinema.org/a/94903.png)](https://asciinema.org/a/94903)
Using acme-dns is a three-step process (provided you already have the self-hosted server set up):
- Get credentials and unique subdomain (simple POST request to eg. https://auth.acme-dns.io/register)
- Create a (ACME magic) CNAME record to your existing zone, pointing to the subdomain you got from the registration. (eg. `_acme-challenge.domainiwantcertfor.tld. CNAME a097455b-52cc-4569-90c8-7a4b97c6eba8.auth.example.org` )
- Use your credentials to POST a new DNS challenge values to an acme-dns server for the CA to validate them off of.
- Use your credentials to POST new DNS challenge values to an acme-dns server for the CA to validate from.
- Crontab and forget.
## API
@@ -37,9 +40,9 @@ Using acme-dns is a three-step process (provided you already have the self-hoste
The method returns a new unique subdomain and credentials needed to update your record.
Fulldomain is where you can point your own `_acme-challenge` subdomain CNAME record to.
With the credentials, you can update the TXT response in the service to match the challenge token, later referred as \_\_\_validation\_token\_recieved\_from\_the\_ca\_\_\_, given out by the Certificate Authority.
With the credentials, you can update the TXT response in the service to match the challenge token, later referred as \_\_\_validation\_token\_received\_from\_the\_ca\_\_\_, given out by the Certificate Authority.
**Optional:**: You can POST JSON data to limit the /update requests to predefined source networks using CIDR notation.
**Optional:**: You can POST JSON data to limit the `/update` requests to predefined source networks using CIDR notation.
```POST /register```
@@ -79,14 +82,14 @@ The method allows you to update the TXT answer contents of your unique subdomain
#### Required headers
| Header name | Description | Example |
| ------------- |--------------------------------------------|-------------------------------------------------------|
| X-Api-User | UUIDv4 username recieved from registration | `X-Api-User: c36f50e8-4632-44f0-83fe-e070fef28a10` |
| X-Api-Key | Password recieved from registration | `X-Api-Key: htB9mR9DYgcu9bX_afHF62erXaH2TS7bg9KW3F7Z` |
| X-Api-User | UUIDv4 username received from registration | `X-Api-User: c36f50e8-4632-44f0-83fe-e070fef28a10` |
| X-Api-Key | Password received from registration | `X-Api-Key: htB9mR9DYgcu9bX_afHF62erXaH2TS7bg9KW3F7Z` |
#### Example input
```json
{
"subdomain": "8e5700ea-a4bf-41c7-8a77-e990661dcc6a",
"txt": "___validation_token_recieved_from_the_ca___"
"txt": "___validation_token_received_from_the_ca___"
}
```
@@ -95,83 +98,161 @@ The method allows you to update the TXT answer contents of your unique subdomain
```Status: 200 OK```
```json
{
"txt": "___validation_token_recieved_from_the_ca___"
"txt": "___validation_token_received_from_the_ca___"
}
```
### Health check endpoint
The method can be used to check readiness and/or liveness of the server. It will return status code 200 on success or won't be reachable.
```GET /health```
#### Example using a Kubernetes deployment
```
# ...
readinessProbe:
httpGet:
path: /health
port: 80
periodSeconds: 2
initialDelaySeconds: 2
failureThreshold: 3
successThreshold: 1
livenessProbe:
# same as for readinessProbe...
```
## Self-hosted
You are encouraged to run your own acme-dns instance, because you are effectively authorizing the acme-dns server to act on your behalf in providing the answer to challengeing CA, making the instance able to request (and get issued) a TLS certificate for the domain that has CNAME pointing to it.
You are encouraged to run your own acme-dns instance, because you are effectively authorizing the acme-dns server to act on your behalf in providing the answer to the challenging CA, making the instance able to request (and get issued) a TLS certificate for the domain that has CNAME pointing to it.
Check out how in the INSTALL section.
See the INSTALL section for information on how to do this.
## Installation
1) Install [Go 1.9 or newer](https://golang.org/doc/install)
1) Install [Go 1.11 or newer](https://golang.org/doc/install).
2) Clone this repo: `git clone https://github.com/joohoi/acme-dns $GOPATH/src/acme-dns`
2) Install acme-dns: `go get github.com/joohoi/acme-dns/...`. This will install acme-dns to `~/go/bin/acme-dns`.
3) Build ACME-DNS: `go build`
3) Edit config.cfg to suit your needs (see [configuration](#configuration)). `acme-dns` will read the configuration file from `/etc/acme-dns/config.cfg` or `./config.cfg`, or a location specified with the `-c` flag.
4) Edit config.cfg to suit your needs (see [configuration](#configuration))
4) If your system has systemd, you can optionally install acme-dns as a service so that it will start on boot and be tracked by systemd. This also allows us to add the `CAP_NET_BIND_SERVICE` capability so that acme-dns can be run by a user other than root.
5) Run acme-dns. Please note that acme-dns needs to open a privileged port (53, domain), so it needs to be run with elevated privileges.
1) Make sure that you have moved the configuration file to `/etc/acme-dns/config.cfg` so that acme-dns can access it globally.
## Using Docker
2) Move the acme-dns executable from `~/go/bin/acme-dns` to `/usr/local/bin/acme-dns` (Any location will work, just be sure to change `acme-dns.service` to match).
1) Pull the latest acme-dns Docker image: `docker pull joohoi/acme-dns`
3) Create a minimal acme-dns user: `sudo adduser --system --gecos "acme-dns Service" --disabled-password --group --home /var/lib/acme-dns acme-dns`.
4) Move the systemd service unit from `acme-dns.service` to `/etc/systemd/system/acme-dns.service`.
5) Reload systemd units: `sudo systemctl daemon-reload`.
6) Enable acme-dns on boot: `sudo systemctl enable acme-dns.service`.
7) Run acme-dns: `sudo systemctl start acme-dns.service`.
5) If you did not install the systemd service, run `acme-dns`. Please note that acme-dns needs to open a privileged port (53, domain), so it needs to be run with elevated privileges.
### Using Docker
1) Pull the latest acme-dns Docker image: `docker pull joohoi/acme-dns`.
2) Create directories: `config` for the configuration file, and `data` for the sqlite3 database.
3) Copy [configuration template](https://raw.githubusercontent.com/joohoi/acme-dns/master/config.cfg) to `config/config.cfg`
3) Copy [configuration template](https://raw.githubusercontent.com/joohoi/acme-dns/master/config.cfg) to `config/config.cfg`.
4) Modify the config.cfg to suit your needs.
4) Modify the `config.cfg` to suit your needs.
5) Run Docker, this example expects that you have `port = "80"` in your config.cfg:
5) Run Docker, this example expects that you have `port = "80"` in your `config.cfg`:
```
docker run --rm --name acmedns \
-p 53:53 \
-p 53:53/udp \
-p 80:80 \
-v /path/to/your/config:/etc/acme-dns:ro \
-v /path/to/your/data:/var/lib/acme-dns \
-d joohoi/acme-dns
```
## Docker Compose
### Docker Compose
1) Create directories: `config` for the configuration file, and `data` for the sqlite3 database.
2) Copy [configuration template](https://raw.githubusercontent.com/joohoi/acme-dns/master/config.cfg) to `config/config.cfg`
2) Copy [configuration template](https://raw.githubusercontent.com/joohoi/acme-dns/master/config.cfg) to `config/config.cfg`.
3) Copy [docker-compose.yml from the project](https://raw.githubusercontent.com/joohoi/acme-dns/master/docker-compose.yml), or create your own.
4) Edit the `config/config.cfg` and `docker-compose.yml` to suit your needs, and run `docker-compose up -d`
4) Edit the `config/config.cfg` and `docker-compose.yml` to suit your needs, and run `docker-compose up -d`.
## DNS Records
Note: In this documentation:
- `auth.example.org` is the hostname of the acme-dns server
- acme-dns will serve `*.auth.example.org` records
- `198.51.100.1` is the **public** IP address of the system running acme-dns
These values should be changed based on your environment.
You will need to add some DNS records on your domain's regular DNS server:
- `NS` record for `auth.example.org` pointing to `auth.example.org` (this means, that `auth.example.org` is responsible for any `*.auth.example.org` records)
- `A` record for `auth.example.org` pointing to `198.51.100.1`
- If using IPv6, an `AAAA` record pointing to the IPv6 address.
- Each domain you will be authenticating will need a `_acme-challenge` `CNAME` subdomain added. The [client](README.md#clients) you use will explain how to do this.
## Testing It Out
You may want to test that acme-dns is working before using it for real queries.
1) Confirm that DNS lookups for the acme-dns subdomain works as expected: `dig auth.example.org`.
2) Call the `/register` API endpoint to register a test domain:
```
$ curl -X POST http://auth.example.org/register
{"username":"eabcdb41-d89f-4580-826f-3e62e9755ef2","password":"pbAXVjlIOE01xbut7YnAbkhMQIkcwoHO0ek2j4Q0","fulldomain":"d420c923-bbd7-4056-ab64-c3ca54c9b3cf.auth.example.org","subdomain":"d420c923-bbd7-4056-ab64-c3ca54c9b3cf","allowfrom":[]}
```
3) Call the `/update` API endpoint to set a test TXT record. Pass the `username`, `password` and `subdomain` received from the `register` call performed above:
```
$ curl -X POST \
-H "X-Api-User: eabcdb41-d89f-4580-826f-3e62e9755ef2" \
-H "X-Api-Key: pbAXVjlIOE01xbut7YnAbkhMQIkcwoHO0ek2j4Q0" \
-d '{"subdomain": "d420c923-bbd7-4056-ab64-c3ca54c9b3cf", "txt": "___validation_token_received_from_the_ca___"}' \
http://auth.example.org/update
```
Note: The `txt` field must be exactly 43 characters long, otherwise acme-dns will reject it
4) Perform a DNS lookup to the test subdomain to confirm the updated TXT record is being served:
```
$ dig -t txt @auth.example.org d420c923-bbd7-4056-ab64-c3ca54c9b3cf.auth.example.org
```
## Configuration
```bash
[general]
# dns interface
# DNS interface. Note that systemd-resolved may reserve port 53 on 127.0.0.53
# In this case acme-dns will error out and you will need to define the listening interface
# for example: listen = "127.0.0.1:53"
listen = ":53"
# protocol, "udp", "udp4", "udp6" or "tcp", "tcp4", "tcp6"
protocol = "udp"
# protocol, "both", "both4", "both6", "udp", "udp4", "udp6" or "tcp", "tcp4", "tcp6"
protocol = "both"
# domain name to serve the requests off of
domain = "auth.example.org"
# zone name server
nsname = "ns1.auth.example.org"
nsname = "auth.example.org"
# admin email address, where @ is substituted with .
nsadmin = "admin.example.org"
# predefined records served in addition to the TXT
records = [
# default A
"auth.example.org. A 192.168.1.100",
# A
"ns1.auth.example.org. A 192.168.1.100",
"ns2.auth.example.org. A 192.168.1.100",
# NS
"auth.example.org. NS ns1.auth.example.org.",
"auth.example.org. NS ns2.auth.example.org.",
# domain pointing to the public IP of your acme-dns server
"auth.example.org. A 198.51.100.1",
# specify that auth.example.org will resolve any *.auth.example.org records
"auth.example.org. NS auth.example.org.",
]
# debug messages from CORS etc
debug = false
@@ -180,14 +261,18 @@ debug = false
# Database engine to use, sqlite3 or postgres
engine = "sqlite3"
# Connection string, filename for sqlite3 and postgres://$username:$password@$host/$db_name for postgres
connection = "acme-dns.db"
connection = "/var/lib/acme-dns/acme-dns.db"
# connection = "postgres://user:password@localhost/acmedns_db"
[api]
# domain name to listen requests for, mandatory if using tls = "letsencrypt"
api_domain = ""
# disable registration endpoint
disable_registration = false
# autocert HTTP port, eg. 80 for answering Let's Encrypt HTTP-01 challenges. Mandatory if using tls = "letsencrypt".
autocert_port = "80"
# listen ip, default "" listens on all interfaces/addresses
ip = "127.0.0.1"
# listen port, eg. 443 for default HTTPS
port = "8080"
# possible values: "letsencrypt", "cert", "none"
@@ -195,10 +280,16 @@ tls = "none"
# only used if tls = "cert"
tls_cert_privkey = "/etc/tls/example.org/privkey.pem"
tls_cert_fullchain = "/etc/tls/example.org/fullchain.pem"
# only used if tls = "letsencrypt"
acme_cache_dir = "api-certs"
# CORS AllowOrigins, wildcards can be used
corsorigins = [
"*"
]
# use HTTP header to get the client ip
use_header = false
# header name to pull the ip address / list of ip addresses from
header_name = "X-Forwarded-For"
[logconfig]
# logging level: "error", "warning", "info" or "debug"
@@ -209,13 +300,56 @@ logtype = "stdout"
# logfile = "./acme-dns.log"
# format, either "json" or "text"
logformat = "text"
# use HTTP header to get the client ip
use_header = false
# header name to pull the ip address / list of ip addresses from
header_name = "X-Forwarded-For"
```
## Clients
- acme.sh: [https://github.com/Neilpang/acme.sh](https://github.com/Neilpang/acme.sh)
- Certify The Web: [https://github.com/webprofusion/certify](https://github.com/webprofusion/certify)
- cert-manager: [https://github.com/jetstack/cert-manager](https://github.com/jetstack/cert-manager)
- Lego: [https://github.com/xenolf/lego](https://github.com/xenolf/lego)
- Posh-ACME: [https://github.com/rmbolger/Posh-ACME](https://github.com/rmbolger/Posh-ACME)
- Sewer: [https://github.com/komuw/sewer](https://github.com/komuw/sewer)
- Traefik: [https://github.com/containous/traefik](https://github.com/containous/traefik)
### Authentication hooks
- Certbot authentication hook in Python: [https://github.com/joohoi/acme-dns-certbot-joohoi](https://github.com/joohoi/acme-dns-certbot-joohoi)
- Certbot authentication hook in Go: [https://github.com/koesie10/acme-dns-certbot-hook](https://github.com/koesie10/acme-dns-certbot-hook)
### Libraries
- Generic client library in Python ([PyPI](https://pypi.python.org/pypi/pyacmedns/)): [https://github.com/joohoi/pyacmedns](https://github.com/joohoi/pyacmedns)
- Generic client library in Go: [https://github.com/cpu/goacmedns](https://github.com/cpu/goacmedns)
## Changelog
- v0.7
- New
- Added an endpoint to perform health checks
- Changed
- A new protocol selection for DNS server "both", that binds both - UDP and TCP ports.
- Refactored DNS server internals.
- Handle some aspects of DNS spec better.
- v0.6
- New
- Command line flag `-c` to specify location of config file.
- Proper refusal of dynamic update requests.
- Release signing
- Changed
- Better error messages for goroutines
- v0.5
- New
- Configurable certificate cache directory
- Changed
- Process wide umask to ensure created files are only readable by the user running acme-dns
- Replaced package that handles UUIDs because of a flaw in the original package
- Updated dependencies
- Better error messages
- v0.4 Clear error messages for bad TXT record content, proper handling of static CNAME records, fixed IP address parsing from the request, added option to disable registration endpoint in the configuration.
- v0.3.2 Dockerfile was fixed for users using autocert feature
- v0.3.1 Added goreleaser for distributing binary builds of the releases
- v0.3 Changed autocert to use HTTP-01 challenges, as TLS-SNI is disabled by Let's Encrypt
- v0.2 Now powered by httprouter, support wildcard certificates, Docker images
- v0.1 Initial release
Vendored
+1 -1
View File
@@ -1,7 +1,7 @@
# -*- mode: ruby -*-
# vi: set ft=ruby :
# Vagratnfile for running integration tests with PostgreSQL
# Vagrantfile for running integration tests with PostgreSQL
VAGRANTFILE_API_VERSION = "2"
+13
View File
@@ -0,0 +1,13 @@
[Unit]
Description=Limited DNS server with RESTful HTTP API to handle ACME DNS challenges easily and securely
After=network.target
[Service]
User=acme-dns
Group=acme-dns
AmbientCapabilities=CAP_NET_BIND_SERVICE
ExecStart=/usr/local/bin/acme-dns
Restart=on-failure
[Install]
WantedBy=multi-user.target
+7 -5
View File
@@ -4,7 +4,8 @@ import (
"encoding/json"
"net"
"github.com/satori/go.uuid"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
)
// ACMETxt is the default structure for the user controlled record
@@ -32,9 +33,9 @@ func (c *cidrslice) JSON() string {
func (c *cidrslice) ValidEntries() []string {
valid := []string{}
for _, v := range *c {
_, _, err := net.ParseCIDR(v)
_, _, err := net.ParseCIDR(sanitizeIPv6addr(v))
if err == nil {
valid = append(valid, v)
valid = append(valid, sanitizeIPv6addr(v))
}
}
return valid
@@ -47,6 +48,7 @@ func (a ACMETxt) allowedFrom(ip string) bool {
if len(a.AllowFrom.ValidEntries()) == 0 {
return true
}
log.WithFields(log.Fields{"ip": remoteIP}).Debug("Checking if update is permitted from IP")
for _, v := range a.AllowFrom.ValidEntries() {
_, vnet, _ := net.ParseCIDR(v)
if vnet.Contains(remoteIP) {
@@ -74,8 +76,8 @@ func (a ACMETxt) allowedFromList(ips []string) bool {
func newACMETxt() ACMETxt {
var a = ACMETxt{}
password := generatePassword(40)
a.Username = uuid.NewV4()
a.Username = uuid.New()
a.Password = password
a.Subdomain = uuid.NewV4().String()
a.Subdomain = uuid.New().String()
return a
}
+17 -5
View File
@@ -66,7 +66,18 @@ func webUpdatePost(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
if !ok {
log.WithFields(log.Fields{"error": "context"}).Error("Context error")
}
if validSubdomain(a.Subdomain) && validTXT(a.Value) {
// NOTE: An invalid subdomain should not happen - the auth handler should
// reject POSTs with an invalid subdomain before this handler. Reject any
// invalid subdomains anyway as a matter of caution.
if !validSubdomain(a.Subdomain) {
log.WithFields(log.Fields{"error": "subdomain", "subdomain": a.Subdomain, "txt": a.Value}).Debug("Bad update data")
updStatus = http.StatusBadRequest
upd = jsonError("bad_subdomain")
} else if !validTXT(a.Value) {
log.WithFields(log.Fields{"error": "txt", "subdomain": a.Subdomain, "txt": a.Value}).Debug("Bad update data")
updStatus = http.StatusBadRequest
upd = jsonError("bad_txt")
} else if validSubdomain(a.Subdomain) && validTXT(a.Value) {
err := DB.Update(a)
if err != nil {
log.WithFields(log.Fields{"error": err.Error()}).Debug("Error while trying to update record")
@@ -77,12 +88,13 @@ func webUpdatePost(w http.ResponseWriter, r *http.Request, _ httprouter.Params)
updStatus = http.StatusOK
upd = []byte("{\"txt\": \"" + a.Value + "\"}")
}
} else {
log.WithFields(log.Fields{"error": "subdomain", "subdomain": a.Subdomain, "txt": a.Value}).Debug("Bad update data")
updStatus = http.StatusBadRequest
upd = jsonError("bad_subdomain")
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(updStatus)
w.Write(upd)
}
// Endpoint used to check the readiness and/or liveness (health) of the server.
func healthCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
w.WriteHeader(http.StatusOK)
}
+71 -2
View File
@@ -9,9 +9,9 @@ import (
"testing"
"github.com/gavv/httpexpect"
"github.com/google/uuid"
"github.com/julienschmidt/httprouter"
"github.com/rs/cors"
"github.com/satori/go.uuid"
"gopkg.in/DATA-DOG/go-sqlmock.v1"
)
@@ -25,7 +25,7 @@ func noAuth(update httprouter.Handle) httprouter.Handle {
dec := json.NewDecoder(r.Body)
_ = dec.Decode(&postData)
// Set user info to the decoded ACMETxt object
postData.Username, _ = uuid.FromString(uname)
postData.Username, _ = uuid.Parse(uname)
postData.Password = passwd
// Set the ACMETxt struct to context to pull in from update function
ctx := r.Context()
@@ -71,6 +71,7 @@ func setupRouter(debug bool, noauth bool) http.Handler {
Debug: Config.General.Debug,
})
api.POST("/register", webRegisterPost)
api.GET("/health", healthCheck)
if noauth {
api.POST("/update", noAuth(webUpdatePost))
} else {
@@ -159,6 +160,66 @@ func TestApiRegisterWithMockDB(t *testing.T) {
DB.SetBackend(oldDb)
}
func TestApiUpdateWithInvalidSubdomain(t *testing.T) {
validTxtData := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
updateJSON := map[string]interface{}{
"subdomain": "",
"txt": ""}
router := setupRouter(false, false)
server := httptest.NewServer(router)
defer server.Close()
e := getExpect(t, server)
newUser, err := DB.Register(cidrslice{})
if err != nil {
t.Errorf("Could not create new user, got error [%v]", err)
}
// Invalid subdomain data
updateJSON["subdomain"] = "example.com"
updateJSON["txt"] = validTxtData
e.POST("/update").
WithJSON(updateJSON).
WithHeader("X-Api-User", newUser.Username.String()).
WithHeader("X-Api-Key", newUser.Password).
Expect().
Status(http.StatusUnauthorized).
JSON().Object().
ContainsKey("error").
NotContainsKey("txt").
ValueEqual("error", "forbidden")
}
func TestApiUpdateWithInvalidTxt(t *testing.T) {
invalidTXTData := "idk m8 bbl lmao"
updateJSON := map[string]interface{}{
"subdomain": "",
"txt": ""}
router := setupRouter(false, false)
server := httptest.NewServer(router)
defer server.Close()
e := getExpect(t, server)
newUser, err := DB.Register(cidrslice{})
if err != nil {
t.Errorf("Could not create new user, got error [%v]", err)
}
updateJSON["subdomain"] = newUser.Subdomain
// Invalid txt data
updateJSON["txt"] = invalidTXTData
e.POST("/update").
WithJSON(updateJSON).
WithHeader("X-Api-User", newUser.Username.String()).
WithHeader("X-Api-Key", newUser.Password).
Expect().
Status(http.StatusBadRequest).
JSON().Object().
ContainsKey("error").
NotContainsKey("txt").
ValueEqual("error", "bad_txt")
}
func TestApiUpdateWithoutCredentials(t *testing.T) {
router := setupRouter(false, false)
server := httptest.NewServer(router)
@@ -346,3 +407,11 @@ func TestApiManyUpdateWithIpCheckHeaders(t *testing.T) {
}
Config.API.UseHeader = false
}
func TestApiHealthCheck(t *testing.T) {
router := setupRouter(false, false)
server := httptest.NewServer(router)
defer server.Close()
e := getExpect(t, server)
e.GET("/health").Expect().Status(http.StatusOK)
}
+7 -1
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"github.com/julienschmidt/httprouter"
@@ -83,5 +84,10 @@ func updateAllowedFromIP(r *http.Request, user ACMETxt) bool {
ips := getIPListFromHeader(r.Header.Get(Config.API.HeaderName))
return user.allowedFromList(ips)
}
return user.allowedFrom(r.RemoteAddr)
host, _, err := net.SplitHostPort(r.RemoteAddr)
if err != nil {
log.WithFields(log.Fields{"error": err.Error(), "remoteaddr": r.RemoteAddr}).Error("Error while parsing remote address")
host = ""
}
return user.allowedFrom(host)
}
+34
View File
@@ -0,0 +1,34 @@
package main
import (
"net/http"
"testing"
)
func TestUpdateAllowedFromIP(t *testing.T) {
Config.API.UseHeader = false
userWithAllow := newACMETxt()
userWithAllow.AllowFrom = cidrslice{"192.168.1.2/32", "[::1]/128"}
userWithoutAllow := newACMETxt()
for i, test := range []struct {
remoteaddr string
expected bool
}{
{"192.168.1.2:1234", true},
{"192.168.1.1:1234", false},
{"invalid", false},
{"[::1]:4567", true},
} {
newreq, _ := http.NewRequest("GET", "/whatever", nil)
newreq.RemoteAddr = test.remoteaddr
ret := updateAllowedFromIP(newreq, userWithAllow)
if test.expected != ret {
t.Errorf("Test %d: Unexpected result for user with allowForm set", i)
}
if !updateAllowedFromIP(newreq, userWithoutAllow) {
t.Errorf("Test %d: Unexpected result for user without allowForm set", i)
}
}
}
+15 -13
View File
@@ -1,24 +1,22 @@
[general]
# dns interface
listen = ":53"
# protocol, "udp", "udp4", "udp6" or "tcp", "tcp4", "tcp6"
protocol = "udp"
# DNS interface. Note that systemd-resolved may reserve port 53 on 127.0.0.53
# In this case acme-dns will error out and you will need to define the listening interface
# for example: listen = "127.0.0.1:53"
listen = "127.0.0.1:53"
# protocol, "both", "both4", "both6", "udp", "udp4", "udp6" or "tcp", "tcp4", "tcp6"
protocol = "both"
# domain name to serve the requests off of
domain = "auth.example.org"
# zone name server
nsname = "ns1.auth.example.org"
nsname = "auth.example.org"
# admin email address, where @ is substituted with .
nsadmin = "admin.example.org"
# predefined records served in addition to the TXT
records = [
# default A
"auth.example.org. A 192.168.1.100",
# A
"ns1.auth.example.org. A 192.168.1.100",
"ns2.auth.example.org. A 192.168.1.100",
# NS
"auth.example.org. NS ns1.auth.example.org.",
"auth.example.org. NS ns2.auth.example.org.",
# domain pointing to the public IP of your acme-dns server
"auth.example.org. A 198.51.100.1",
# specify that auth.example.org will resolve any *.auth.example.org records
"auth.example.org. NS auth.example.org.",
]
# debug messages from CORS etc
debug = false
@@ -36,6 +34,8 @@ connection = "/var/lib/acme-dns/acme-dns.db"
api_domain = ""
# listen ip eg. 127.0.0.1
ip = "0.0.0.0"
# disable registration endpoint
disable_registration = false
# autocert HTTP port, eg. 80 for answering Let's Encrypt HTTP-01 challenges. Mandatory if using tls = "letsencrypt".
autocert_port = "80"
# listen port, eg. 443 for default HTTPS
@@ -45,6 +45,8 @@ tls = "none"
# only used if tls = "cert"
tls_cert_privkey = "/etc/tls/example.org/privkey.pem"
tls_cert_fullchain = "/etc/tls/example.org/fullchain.pem"
# only used if tls = "letsencrypt"
acme_cache_dir = "api-certs"
# CORS AllowOrigins, wildcards can be used
corsorigins = [
"*"
+1 -1
View File
@@ -9,9 +9,9 @@ import (
"strconv"
"time"
"github.com/google/uuid"
_ "github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
"github.com/satori/go.uuid"
log "github.com/sirupsen/logrus"
"golang.org/x/crypto/bcrypt"
)
+2 -2
View File
@@ -67,10 +67,10 @@ func TestRegisterMany(t *testing.T) {
t.Errorf("Test %d: Got error when fetching username: [%v]", i, err)
}
if len(user.AllowFrom) != len(test.output) {
t.Errorf("Test %d: Expected to recieve struct with [%d] entries in AllowFrom, but got [%d] records", i, len(test.output), len(user.AllowFrom))
t.Errorf("Test %d: Expected to receive struct with [%d] entries in AllowFrom, but got [%d] records", i, len(test.output), len(user.AllowFrom))
}
if len(res.AllowFrom) != len(test.output) {
t.Errorf("Test %d: Expected to recieve struct with [%d] entries in AllowFrom, but got [%d] records", i, len(test.output), len(res.AllowFrom))
t.Errorf("Test %d: Expected to receive struct with [%d] entries in AllowFrom, but got [%d] records", i, len(test.output), len(res.AllowFrom))
}
}
+164 -73
View File
@@ -8,25 +8,182 @@ import (
"time"
)
func readQuery(m *dns.Msg) {
// Records is a slice of ResourceRecords
type Records struct {
Records []dns.RR
}
// DNSServer is the main struct for acme-dns DNS server
type DNSServer struct {
DB database
Server *dns.Server
SOA dns.RR
Domains map[string]Records
}
// NewDNSServer parses the DNS records from config and returns a new DNSServer struct
func NewDNSServer(db database, addr string, proto string) *DNSServer {
var server DNSServer
server.Server = &dns.Server{Addr: addr, Net: proto}
server.DB = db
server.Domains = make(map[string]Records)
return &server
}
// Start starts the DNSServer
func (d *DNSServer) Start(errorChannel chan error) {
// DNS server part
dns.HandleFunc(".", d.handleRequest)
log.WithFields(log.Fields{"addr": d.Server.Addr, "proto": d.Server.Net}).Info("Listening DNS")
err := d.Server.ListenAndServe()
if err != nil {
errorChannel <- err
}
}
// ParseRecords parses a slice of DNS record string
func (d *DNSServer) ParseRecords(config DNSConfig) {
for _, v := range config.General.StaticRecords {
rr, err := dns.NewRR(strings.ToLower(v))
if err != nil {
log.WithFields(log.Fields{"error": err.Error(), "rr": v}).Warning("Could not parse RR from config")
continue
}
// Add parsed RR
d.appendRR(rr)
}
// Create serial
serial := time.Now().Format("2006010215")
// Add SOA
SOAstring := fmt.Sprintf("%s. SOA %s. %s. %s 28800 7200 604800 86400", strings.ToLower(config.General.Domain), strings.ToLower(config.General.Nsname), strings.ToLower(config.General.Nsadmin), serial)
soarr, err := dns.NewRR(SOAstring)
if err != nil {
log.WithFields(log.Fields{"error": err.Error(), "soa": SOAstring}).Error("Error while adding SOA record")
} else {
d.appendRR(soarr)
d.SOA = soarr
}
}
func (d *DNSServer) appendRR(rr dns.RR) {
addDomain := rr.Header().Name
_, ok := d.Domains[addDomain]
if !ok {
d.Domains[addDomain] = Records{[]dns.RR{rr}}
} else {
drecs := d.Domains[addDomain]
drecs.Records = append(drecs.Records, rr)
d.Domains[addDomain] = drecs
}
log.WithFields(log.Fields{"recordtype": dns.TypeToString[rr.Header().Rrtype], "domain": addDomain}).Debug("Adding new record to domain")
}
func (d *DNSServer) handleRequest(w dns.ResponseWriter, r *dns.Msg) {
m := new(dns.Msg)
m.SetReply(r)
if r.Opcode == dns.OpcodeQuery {
d.readQuery(m)
}
w.WriteMsg(m)
}
func (d *DNSServer) readQuery(m *dns.Msg) {
var authoritative = false
for _, que := range m.Question {
if rr, rc, err := answer(que); err == nil {
if rr, rc, auth, err := d.answer(que); err == nil {
if auth {
authoritative = auth
}
m.MsgHdr.Rcode = rc
for _, r := range rr {
m.Answer = append(m.Answer, r)
}
}
}
m.MsgHdr.Authoritative = authoritative
if authoritative {
if m.MsgHdr.Rcode == dns.RcodeNameError {
m.Answer = append(m.Answer, d.SOA)
}
}
}
func answerTXT(q dns.Question) ([]dns.RR, int, error) {
func (d *DNSServer) getRecord(q dns.Question) ([]dns.RR, error) {
var rr []dns.RR
var cnames []dns.RR
domain, ok := d.Domains[q.Name]
if !ok {
return rr, fmt.Errorf("No records for domain %s", q.Name)
}
for _, ri := range domain.Records {
if ri.Header().Rrtype == q.Qtype {
rr = append(rr, ri)
}
if ri.Header().Rrtype == dns.TypeCNAME {
cnames = append(cnames, ri)
}
}
if len(rr) == 0 {
return cnames, nil
}
return rr, nil
}
// answeringForDomain checks if we have any records for a domain
func (d *DNSServer) answeringForDomain(name string) bool {
_, ok := d.Domains[name]
return ok
}
func (d *DNSServer) isAuthoritative(q dns.Question) bool {
if d.answeringForDomain(q.Name) {
return true
}
domainParts := strings.Split(q.Name, ".")
for i := range domainParts {
if d.answeringForDomain(strings.Join(domainParts[i:], ".")) {
return true
}
}
return false
}
func (d *DNSServer) answer(q dns.Question) ([]dns.RR, int, bool, error) {
var rcode int
var authoritative = d.isAuthoritative(q)
if !d.answeringForDomain(q.Name) {
rcode = dns.RcodeNameError
}
r, _ := d.getRecord(q)
if q.Qtype == dns.TypeTXT {
txtRRs, err := d.answerTXT(q)
if err == nil {
for _, txtRR := range txtRRs {
r = append(r, txtRR)
}
}
}
if len(r) > 0 {
// Make sure that we return NOERROR if there were dynamic records for the domain
rcode = dns.RcodeSuccess
}
// Handle EDNS (no support at the moment)
if q.Qtype == dns.TypeOPT {
return []dns.RR{}, dns.RcodeFormatError, authoritative, nil
}
log.WithFields(log.Fields{"qtype": dns.TypeToString[q.Qtype], "domain": q.Name, "rcode": dns.RcodeToString[rcode]}).Debug("Answering question for domain")
return r, rcode, authoritative, nil
}
func (d *DNSServer) answerTXT(q dns.Question) ([]dns.RR, error) {
var ra []dns.RR
rcode := dns.RcodeNameError
subdomain := sanitizeDomainQuestion(q.Name)
atxt, err := DB.GetTXTForDomain(subdomain)
atxt, err := d.DB.GetTXTForDomain(subdomain)
if err != nil {
log.WithFields(log.Fields{"error": err.Error()}).Debug("Error while trying to get record")
return ra, dns.RcodeNameError, err
return ra, err
}
for _, v := range atxt {
if len(v) > 0 {
@@ -34,73 +191,7 @@ func answerTXT(q dns.Question) ([]dns.RR, int, error) {
r.Hdr = dns.RR_Header{Name: q.Name, Rrtype: dns.TypeTXT, Class: dns.ClassINET, Ttl: 1}
r.Txt = append(r.Txt, v)
ra = append(ra, r)
rcode = dns.RcodeSuccess
}
}
log.WithFields(log.Fields{"domain": q.Name}).Info("Answering TXT question for domain")
return ra, rcode, nil
}
func answer(q dns.Question) ([]dns.RR, int, error) {
if q.Qtype == dns.TypeTXT {
return answerTXT(q)
}
var r []dns.RR
var rcode = dns.RcodeSuccess
var domain = strings.ToLower(q.Name)
var rtype = q.Qtype
r, ok := RR.Records[rtype][domain]
if !ok {
rcode = dns.RcodeNameError
}
log.WithFields(log.Fields{"qtype": dns.TypeToString[rtype], "domain": domain, "rcode": dns.RcodeToString[rcode]}).Debug("Answering question for domain")
return r, rcode, nil
}
func handleRequest(w dns.ResponseWriter, r *dns.Msg) {
m := new(dns.Msg)
m.SetReply(r)
if r.Opcode == dns.OpcodeQuery {
readQuery(m)
}
w.WriteMsg(m)
}
// Parse config records
func (r *Records) Parse(config general) {
rrmap := make(map[uint16]map[string][]dns.RR)
for _, v := range config.StaticRecords {
rr, err := dns.NewRR(strings.ToLower(v))
if err != nil {
log.WithFields(log.Fields{"error": err.Error(), "rr": v}).Warning("Could not parse RR from config")
continue
}
// Add parsed RR to the list
rrmap = appendRR(rrmap, rr)
}
// Create serial
serial := time.Now().Format("2006010215")
// Add SOA
SOAstring := fmt.Sprintf("%s. SOA %s. %s. %s 28800 7200 604800 86400", strings.ToLower(config.Domain), strings.ToLower(config.Nsname), strings.ToLower(config.Nsadmin), serial)
soarr, err := dns.NewRR(SOAstring)
if err != nil {
log.WithFields(log.Fields{"error": err.Error(), "soa": SOAstring}).Error("Error while adding SOA record")
} else {
rrmap = appendRR(rrmap, soarr)
}
r.Records = rrmap
}
func appendRR(rrmap map[uint16]map[string][]dns.RR, rr dns.RR) map[uint16]map[string][]dns.RR {
_, ok := rrmap[rr.Header().Rrtype]
if !ok {
newrr := make(map[string][]dns.RR)
rrmap[rr.Header().Rrtype] = newrr
}
rrmap[rr.Header().Rrtype][rr.Header().Name] = append(rrmap[rr.Header().Rrtype][rr.Header().Name], rr)
log.WithFields(log.Fields{"recordtype": dns.TypeToString[rr.Header().Rrtype], "domain": rr.Header().Name}).Debug("Adding new record type to domain")
return rrmap
return ra, nil
}
+81 -56
View File
@@ -5,10 +5,10 @@ import (
"database/sql/driver"
"errors"
"fmt"
"testing"
"github.com/erikstmartin/go-testdb"
"github.com/miekg/dns"
"strings"
"testing"
)
var resolv resolver
@@ -18,20 +18,20 @@ type resolver struct {
server string
}
func (r *resolver) lookup(host string, qtype uint16) ([]dns.RR, error) {
func (r *resolver) lookup(host string, qtype uint16) (*dns.Msg, error) {
msg := new(dns.Msg)
msg.Id = dns.Id()
msg.Question = make([]dns.Question, 1)
msg.Question[0] = dns.Question{Name: dns.Fqdn(host), Qtype: qtype, Qclass: dns.ClassINET}
in, err := dns.Exchange(msg, r.server)
if err != nil {
return []dns.RR{}, fmt.Errorf("Error querying the server [%v]", err)
return in, fmt.Errorf("Error querying the server [%v]", err)
}
if in != nil && in.Rcode != dns.RcodeSuccess {
return []dns.RR{}, fmt.Errorf("Received error from the server [%s]", dns.RcodeToString[in.Rcode])
return in, fmt.Errorf("Received error from the server [%s]", dns.RcodeToString[in.Rcode])
}
return in.Answer, nil
return in, nil
}
func hasExpectedTXTAnswer(answer []dns.RR, cmpTXT string) error {
@@ -51,25 +51,6 @@ func hasExpectedTXTAnswer(answer []dns.RR, cmpTXT string) error {
return errors.New("Expected answer not found")
}
func findRecordFromMemory(rrstr string, host string, qtype uint16) error {
var errmsg = "No record found"
arr, _ := dns.NewRR(strings.ToLower(rrstr))
if arrQt, ok := RR.Records[qtype]; ok {
if arrHst, ok := arrQt[host]; ok {
for _, v := range arrHst {
if arr.String() == v.String() {
return nil
}
}
} else {
errmsg = "No records for domain"
}
} else {
errmsg = "No records for this type in DB"
}
return errors.New(errmsg)
}
func TestQuestionDBError(t *testing.T) {
testdb.SetQueryWithArgsFunc(func(query string, args []driver.Value) (result driver.Rows, err error) {
columns := []string{"Username", "Password", "Subdomain", "Value", "LastActive"}
@@ -88,44 +69,36 @@ func TestQuestionDBError(t *testing.T) {
defer DB.SetBackend(oldDb)
q := dns.Question{Name: dns.Fqdn("whatever.tld"), Qtype: dns.TypeTXT, Qclass: dns.ClassINET}
_, rcode, err := answerTXT(q)
_, err = dnsserver.answerTXT(q)
if err == nil {
t.Errorf("Expected error but got none")
}
if rcode != dns.RcodeNameError {
t.Errorf("Expected [%s] rcode, but got [%s]", dns.RcodeToString[dns.RcodeNameError], dns.RcodeToString[rcode])
}
}
func TestParse(t *testing.T) {
var testcfg = general{
Domain: ")",
Nsname: "ns1.auth.example.org",
Nsadmin: "admin.example.org",
StaticRecords: []string{},
Debug: false,
var testcfg = DNSConfig{
General: general{
Domain: ")",
Nsname: "ns1.auth.example.org",
Nsadmin: "admin.example.org",
StaticRecords: []string{},
Debug: false,
},
}
var testRR Records
testRR.Parse(testcfg)
dnsserver.ParseRecords(testcfg)
if !loggerHasEntryWithMessage("Error while adding SOA record") {
t.Errorf("Expected SOA parsing to return error, but did not find one")
}
}
func TestResolveA(t *testing.T) {
resolv := resolver{server: "0.0.0.0:15353"}
resolv := resolver{server: "127.0.0.1:15353"}
answer, err := resolv.lookup("auth.example.org", dns.TypeA)
if err != nil {
t.Errorf("%v", err)
}
if len(answer) > 0 {
err = findRecordFromMemory(answer[0].String(), "auth.example.org.", dns.TypeA)
if err != nil {
t.Errorf("Answer [%s] did not match the expected, got error: [%s], debug: [%q]", answer[0].String(), err, RR.Records)
}
} else {
if len(answer.Answer) == 0 {
t.Error("No answer for DNS query")
}
@@ -135,8 +108,58 @@ func TestResolveA(t *testing.T) {
}
}
func TestEDNS(t *testing.T) {
resolv := resolver{server: "127.0.0.1:15353"}
answer, _ := resolv.lookup("auth.example.org", dns.TypeOPT)
if answer.Rcode != dns.RcodeFormatError {
t.Errorf("Was expecing FORMERR rcode for OPT query, but got [%s] instead.", dns.RcodeToString[answer.Rcode])
}
}
func TestResolveCNAME(t *testing.T) {
resolv := resolver{server: "127.0.0.1:15353"}
expected := "cn.example.org. 3600 IN CNAME something.example.org."
answer, err := resolv.lookup("cn.example.org", dns.TypeCNAME)
if err != nil {
t.Errorf("Got unexpected error: %s", err)
}
if len(answer.Answer) != 1 {
t.Errorf("Expected exactly 1 RR in answer, but got %d instead.", len(answer.Answer))
}
if answer.Answer[0].Header().Rrtype != dns.TypeCNAME {
t.Errorf("Expected a CNAME answer, but got [%s] instead.", dns.TypeToString[answer.Answer[0].Header().Rrtype])
}
if answer.Answer[0].String() != expected {
t.Errorf("Expected CNAME answer [%s] but got [%s] instead.", expected, answer.Answer[0].String())
}
}
func TestAuthoritative(t *testing.T) {
resolv := resolver{server: "127.0.0.1:15353"}
answer, _ := resolv.lookup("nonexistent.auth.example.org", dns.TypeA)
if answer.Rcode != dns.RcodeNameError {
t.Errorf("Was expecing NXDOMAIN rcode, but got [%s] instead.", dns.RcodeToString[answer.Rcode])
}
if len(answer.Answer) != 1 {
t.Errorf("Was expecting exactly one answer (SOA) for invalid subdomain, but got %d", len(answer.Answer))
}
if answer.Answer[0].Header().Rrtype != dns.TypeSOA {
t.Errorf("Was expecting SOA record as answer for NXDOMAIN but got [%s]", dns.TypeToString[answer.Answer[0].Header().Rrtype])
}
if !answer.MsgHdr.Authoritative {
t.Errorf("Was expecting authoritative bit to be set")
}
nanswer, _ := resolv.lookup("nonexsitent.nonauth.tld", dns.TypeA)
if len(nanswer.Answer) > 0 {
t.Errorf("Didn't expect answers for non authotitative domain query")
}
if nanswer.MsgHdr.Authoritative {
t.Errorf("Authoritative bit should not be set for non-authoritative domain.")
}
}
func TestResolveTXT(t *testing.T) {
resolv := resolver{server: "0.0.0.0:15353"}
resolv := resolver{server: "127.0.0.1:15353"}
validTXT := "______________valid_response_______________"
atxt, err := DB.Register(cidrslice{})
@@ -172,18 +195,20 @@ func TestResolveTXT(t *testing.T) {
}
}
if len(answer) > 0 {
if !test.getAnswer {
if len(answer.Answer) > 0 {
if !test.getAnswer && answer.Answer[0].Header().Rrtype != dns.TypeSOA {
t.Errorf("Test %d: Expected no answer, but got: [%q]", i, answer)
}
err = hasExpectedTXTAnswer(answer, test.expTXT)
if err != nil {
if test.validAnswer {
t.Errorf("Test %d: %v", i, err)
}
} else {
if !test.validAnswer {
t.Errorf("Test %d: Answer was not expected to be valid, answer [%q], compared to [%s]", i, answer, test.expTXT)
if test.getAnswer {
err = hasExpectedTXTAnswer(answer.Answer, test.expTXT)
if err != nil {
if test.validAnswer {
t.Errorf("Test %d: %v", i, err)
}
} else {
if !test.validAnswer {
t.Errorf("Test %d: Answer was not expected to be valid, answer [%q], compared to [%s]", i, answer, test.expTXT)
}
}
}
} else {
+1
View File
@@ -8,6 +8,7 @@ services:
ports:
- "443:443"
- "53:53"
- "53:53/udp"
- "80:80"
volumes:
- ./config:/etc/acme-dns:ro
+68 -19
View File
@@ -4,9 +4,12 @@ package main
import (
"crypto/tls"
"flag"
stdlog "log"
"net/http"
"os"
"strings"
"syscall"
"github.com/julienschmidt/httprouter"
"github.com/rs/cors"
@@ -15,24 +18,32 @@ import (
)
func main() {
// Created files are not world writable
syscall.Umask(0077)
configPtr := flag.String("c", "/etc/acme-dns/config.cfg", "config file location")
flag.Parse()
// Read global config
if fileExists("/etc/acme-dns/config.cfg") {
Config = readConfig("/etc/acme-dns/config.cfg")
log.WithFields(log.Fields{"file": "/etc/acme-dns/config.cfg"}).Info("Using config file")
} else {
var err error
if fileIsAccessible(*configPtr) {
log.WithFields(log.Fields{"file": *configPtr}).Info("Using config file")
Config, err = readConfig(*configPtr)
} else if fileIsAccessible("./config.cfg") {
log.WithFields(log.Fields{"file": "./config.cfg"}).Info("Using config file")
Config = readConfig("config.cfg")
Config, err = readConfig("./config.cfg")
} else {
log.Errorf("Configuration file not found.")
os.Exit(1)
}
if err != nil {
log.Errorf("Encountered an error while trying to read configuration file: %s", err)
os.Exit(1)
}
setupLogging(Config.Logconfig.Format, Config.Logconfig.Level)
// Read the default records in
RR.Parse(Config.General)
// Open database
newDB := new(acmedb)
err := newDB.Init(Config.Database.Engine, Config.Database.Connection)
err = newDB.Init(Config.Database.Engine, Config.Database.Connection)
if err != nil {
log.Errorf("Could not open database [%v]", err)
os.Exit(1)
@@ -42,16 +53,48 @@ func main() {
DB = newDB
defer DB.Close()
// Error channel for servers
errChan := make(chan error, 1)
// DNS server
startDNS(Config.General.Listen, Config.General.Proto)
if strings.HasPrefix(Config.General.Proto, "both") {
// Handle the case where DNS server should be started for both udp and tcp
udpProto := "udp"
tcpProto := "tcp"
if strings.HasSuffix(Config.General.Proto, "4") {
udpProto += "4"
tcpProto += "4"
} else if strings.HasSuffix(Config.General.Proto, "6") {
udpProto += "6"
tcpProto += "6"
}
dnsServerUDP := NewDNSServer(DB, Config.General.Listen, udpProto)
dnsServerUDP.ParseRecords(Config)
dnsServerTCP := NewDNSServer(DB, Config.General.Listen, tcpProto)
// No need to parse records from config again
dnsServerTCP.Domains = dnsServerUDP.Domains
go dnsServerUDP.Start(errChan)
go dnsServerTCP.Start(errChan)
} else {
dnsServer := NewDNSServer(DB, Config.General.Listen, Config.General.Proto)
dnsServer.ParseRecords(Config)
go dnsServer.Start(errChan)
}
// HTTP API
startHTTPAPI()
go startHTTPAPI(errChan)
// block waiting for error
select {
case err = <-errChan:
if err != nil {
log.Fatal(err)
}
}
log.Debugf("Shutting down...")
}
func startHTTPAPI() {
func startHTTPAPI(errChan chan error) {
// Setup http logger
logger := log.New()
logwriter := logger.Writer()
@@ -67,19 +110,22 @@ func startHTTPAPI() {
// Logwriter for saner log output
c.Log = stdlog.New(logwriter, "", 0)
}
api.POST("/register", webRegisterPost)
if !Config.API.DisableRegistration {
api.POST("/register", webRegisterPost)
}
api.POST("/update", Auth(webUpdatePost))
api.GET("/health", healthCheck)
host := Config.API.IP + ":" + Config.API.Port
cfg := &tls.Config{
MinVersion: tls.VersionTLS12,
}
var err error
switch Config.API.TLS {
case "letsencrypt":
m := autocert.Manager{
Cache: autocert.DirCache("api-certs"),
Cache: autocert.DirCache(Config.API.ACMECacheDir),
Prompt: autocert.AcceptTOS,
HostPolicy: autocert.HostWhitelist(Config.API.Domain),
}
@@ -94,7 +140,7 @@ func startHTTPAPI() {
ErrorLog: stdlog.New(logwriter, "", 0),
}
log.WithFields(log.Fields{"host": host, "domain": Config.API.Domain}).Info("Listening HTTPS, using certificate from autocert")
log.Fatal(srv.ListenAndServeTLS("", ""))
err = srv.ListenAndServeTLS("", "")
case "cert":
srv := &http.Server{
Addr: host,
@@ -103,9 +149,12 @@ func startHTTPAPI() {
ErrorLog: stdlog.New(logwriter, "", 0),
}
log.WithFields(log.Fields{"host": host}).Info("Listening HTTPS")
log.Fatal(srv.ListenAndServeTLS(Config.API.TLSCertFullchain, Config.API.TLSCertPrivkey))
err = srv.ListenAndServeTLS(Config.API.TLSCertFullchain, Config.API.TLSCertPrivkey)
default:
log.WithFields(log.Fields{"host": host}).Info("Listening HTTP")
log.Fatal(http.ListenAndServe(host, c.Handler(api)))
err = http.ListenAndServe(host, c.Handler(api))
}
if err != nil {
errChan <- err
}
}
+17 -3
View File
@@ -7,10 +7,12 @@ import (
logrustest "github.com/sirupsen/logrus/hooks/test"
"io/ioutil"
"os"
"sync"
"testing"
)
var loghook = new(logrustest.Hook)
var dnsserver *DNSServer
var (
postgres = flag.Bool("postgres", false, "run integration tests against PostgreSQL")
@@ -19,6 +21,7 @@ var (
var records = []string{
"auth.example.org. A 192.168.1.100",
"ns1.auth.example.org. A 192.168.1.101",
"cn.example.org CNAME something.example.org.",
"!''b', unparseable ",
"ns2.auth.example.org. A 192.168.1.102",
}
@@ -26,7 +29,6 @@ var records = []string{
func TestMain(m *testing.M) {
setupTestLogger()
setupConfig()
RR.Parse(Config.General)
flag.Parse()
newDb := new(acmedb)
@@ -42,9 +44,19 @@ func TestMain(m *testing.M) {
_ = newDb.Init("sqlite3", ":memory:")
}
DB = newDb
server := startDNS("0.0.0.0:15353", "udp")
dnsserver = NewDNSServer(DB, Config.General.Listen, Config.General.Proto)
dnsserver.ParseRecords(Config)
// Make sure that we're not creating a race condition in tests
var wg sync.WaitGroup
wg.Add(1)
dnsserver.Server.NotifyStartedFunc = func() {
wg.Done()
}
go dnsserver.Start(make(chan error, 1))
wg.Wait()
exitval := m.Run()
server.Shutdown()
dnsserver.Server.Shutdown()
DB.Close()
os.Exit(exitval)
}
@@ -57,6 +69,8 @@ func setupConfig() {
var generalcfg = general{
Domain: "auth.example.org",
Listen: "127.0.0.1:15353",
Proto: "udp",
Nsname: "ns1.auth.example.org",
Nsadmin: "admin.example.org",
StaticRecords: records,
+14 -20
View File
@@ -2,9 +2,9 @@ package main
import (
"database/sql"
"github.com/miekg/dns"
"github.com/satori/go.uuid"
"sync"
"github.com/google/uuid"
)
// Config is global configuration struct
@@ -13,14 +13,6 @@ var Config DNSConfig
// DB is used to access the database functions in acme-dns
var DB database
// RR holds the static DNS records
var RR Records
// Records is for static records
type Records struct {
Records map[uint16]map[string][]dns.RR
}
// DNSConfig holds the config structure
type DNSConfig struct {
General general
@@ -50,16 +42,18 @@ type dbsettings struct {
// API config
type httpapi struct {
Domain string `toml:"api_domain"`
IP string
AutocertPort string `toml:"autocert_port"`
Port string `toml:"port"`
TLS string
TLSCertPrivkey string `toml:"tls_cert_privkey"`
TLSCertFullchain string `toml:"tls_cert_fullchain"`
CorsOrigins []string
UseHeader bool `toml:"use_header"`
HeaderName string `toml:"header_name"`
Domain string `toml:"api_domain"`
IP string
DisableRegistration bool `toml:"disable_registration"`
AutocertPort string `toml:"autocert_port"`
Port string `toml:"port"`
TLS string
TLSCertPrivkey string `toml:"tls_cert_privkey"`
TLSCertFullchain string `toml:"tls_cert_fullchain"`
ACMECacheDir string `toml:"acme_cache_dir"`
CorsOrigins []string
UseHeader bool `toml:"use_header"`
HeaderName string `toml:"header_name"`
}
// Logging config
+37 -14
View File
@@ -2,6 +2,7 @@ package main
import (
"crypto/rand"
"errors"
"fmt"
"math/big"
"os"
@@ -9,7 +10,6 @@ import (
"strings"
"github.com/BurntSushi/toml"
"github.com/miekg/dns"
log "github.com/sirupsen/logrus"
)
@@ -17,19 +17,44 @@ func jsonError(message string) []byte {
return []byte(fmt.Sprintf("{\"error\": \"%s\"}", message))
}
func fileExists(fname string) bool {
func fileIsAccessible(fname string) bool {
_, err := os.Stat(fname)
if err != nil {
return false
}
f, err := os.Open(fname)
if err != nil {
return false
}
f.Close()
return true
}
func readConfig(fname string) DNSConfig {
func readConfig(fname string) (DNSConfig, error) {
var conf DNSConfig
// Practically never errors
_, _ = toml.DecodeFile(fname, &conf)
return conf
_, err := toml.DecodeFile(fname, &conf)
if err != nil {
// Return with config file parsing errors from toml package
return conf, err
}
return prepareConfig(conf)
}
// prepareConfig checks that mandatory values exist, and can be used to set default values in the future
func prepareConfig(conf DNSConfig) (DNSConfig, error) {
if conf.Database.Engine == "" {
return conf, errors.New("missing database configuration option \"engine\"")
}
if conf.Database.Connection == "" {
return conf, errors.New("missing database configuration option \"connection\"")
}
// Default values for options added to config to keep backwards compatibility with old config
if conf.API.ACMECacheDir == "" {
conf.API.ACMECacheDir = "api-certs"
}
return conf, nil
}
func sanitizeString(s string) string {
@@ -38,6 +63,12 @@ func sanitizeString(s string) string {
return re.ReplaceAllString(s, "")
}
func sanitizeIPv6addr(s string) string {
// Remove brackets from IPv6 addresses, net.ParseCIDR needs this
re, _ := regexp.Compile("[\\[\\]]+")
return re.ReplaceAllString(s, "")
}
func generatePassword(length int) string {
ret := make([]byte, length)
const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890-_"
@@ -76,14 +107,6 @@ func setupLogging(format string, level string) {
// TODO: file logging
}
func startDNS(listen string, proto string) *dns.Server {
// DNS server part
dns.HandleFunc(".", handleRequest)
server := &dns.Server{Addr: listen, Net: proto}
go server.ListenAndServe()
return server
}
func getIPListFromHeader(header string) []string {
iplist := []string{}
for _, v := range strings.Split(header, ",") {
+56 -2
View File
@@ -1,10 +1,12 @@
package main
import (
log "github.com/sirupsen/logrus"
"io/ioutil"
"os"
"syscall"
"testing"
log "github.com/sirupsen/logrus"
)
func TestSetupLogging(t *testing.T) {
@@ -62,7 +64,7 @@ func TestReadConfig(t *testing.T) {
if err := tmpfile.Close(); err != nil {
t.Error("Could not close temporary file")
}
ret := readConfig(tmpfile.Name())
ret, _ := readConfig(tmpfile.Name())
if ret.General.Listen != test.output.General.Listen {
t.Errorf("Test %d: Expected listen value %s, but got %s", i, test.output.General.Listen, ret.General.Listen)
}
@@ -95,3 +97,55 @@ func TestGetIPListFromHeader(t *testing.T) {
}
}
}
func TestFileCheckPermissionDenied(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "acmedns")
if err != nil {
t.Error("Could not create temporary file")
}
defer os.Remove(tmpfile.Name())
syscall.Chmod(tmpfile.Name(), 0000)
if fileIsAccessible(tmpfile.Name()) {
t.Errorf("File should not be accessible")
}
syscall.Chmod(tmpfile.Name(), 0644)
}
func TestFileCheckNotExists(t *testing.T) {
if fileIsAccessible("/path/that/does/not/exist") {
t.Errorf("File should not be accessible")
}
}
func TestFileCheckOK(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "acmedns")
if err != nil {
t.Error("Could not create temporary file")
}
defer os.Remove(tmpfile.Name())
if !fileIsAccessible(tmpfile.Name()) {
t.Errorf("File should be accessible")
}
}
func TestPrepareConfig(t *testing.T) {
for i, test := range []struct {
input DNSConfig
shoulderror bool
}{
{DNSConfig{Database: dbsettings{Engine: "whatever", Connection: "whatever_too"}}, false},
{DNSConfig{Database: dbsettings{Engine: "", Connection: "whatever_too"}}, true},
{DNSConfig{Database: dbsettings{Engine: "whatever", Connection: ""}}, true},
} {
_, err := prepareConfig(test.input)
if test.shoulderror {
if err == nil {
t.Errorf("Test %d: Expected error with prepareConfig input data [%v]", i, test.input)
}
} else {
if err != nil {
t.Errorf("Test %d: Expected no error with prepareConfig input data [%v]", i, test.input)
}
}
}
}
+3 -3
View File
@@ -3,12 +3,12 @@ package main
import (
"unicode/utf8"
"github.com/satori/go.uuid"
"github.com/google/uuid"
"golang.org/x/crypto/bcrypt"
)
func getValidUsername(u string) (uuid.UUID, error) {
uname, err := uuid.FromString(u)
uname, err := uuid.Parse(u)
if err != nil {
return uuid.UUID{}, err
}
@@ -25,7 +25,7 @@ func validKey(k string) bool {
}
func validSubdomain(s string) bool {
_, err := uuid.FromString(s)
_, err := uuid.Parse(s)
if err == nil {
return true
}
+3 -2
View File
@@ -1,12 +1,13 @@
package main
import (
"github.com/satori/go.uuid"
"testing"
"github.com/google/uuid"
)
func TestGetValidUsername(t *testing.T) {
v1, _ := uuid.FromString("a097455b-52cc-4569-90c8-7a4b97c6eba8")
v1, _ := uuid.Parse("a097455b-52cc-4569-90c8-7a4b97c6eba8")
for i, test := range []struct {
uname string
output uuid.UUID
+17 -10
View File
@@ -1,14 +1,21 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
The MIT License (MIT)
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Copyright (c) 2013 TOML authors
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+17 -10
View File
@@ -1,14 +1,21 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
The MIT License (MIT)
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Copyright (c) 2013 TOML authors
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+17 -10
View File
@@ -1,14 +1,21 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
The MIT License (MIT)
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Copyright (c) 2013 TOML authors
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+17 -10
View File
@@ -1,14 +1,21 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
The MIT License (MIT)
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Copyright (c) 2013 TOML authors
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
+14
View File
@@ -16,6 +16,8 @@ age = 250
andrew = "gallant"
kait = "brady"
now = 1987-07-05T05:45:00Z
nowEast = 2017-06-22T16:15:21+08:00
nowWest = 2017-06-22T02:14:36-06:00
yesOrNo = true
pi = 3.14
colors = [
@@ -38,6 +40,8 @@ cauchy = "cat 2"
Pi float64
YesOrNo bool
Now time.Time
NowEast time.Time
NowWest time.Time
Andrew string
Kait string
My map[string]cats
@@ -53,11 +57,21 @@ cauchy = "cat 2"
if err != nil {
panic(err)
}
nowEast, err := time.Parse("2006-01-02T15:04:05-07:00", "2017-06-22T16:15:21+08:00")
if err != nil {
panic(err)
}
nowWest, err := time.Parse("2006-01-02T15:04:05-07:00", "2017-06-22T02:14:36-06:00")
if err != nil {
panic(err)
}
var answer = simple{
Age: 250,
Andrew: "gallant",
Kait: "brady",
Now: now,
NowEast: nowEast,
NowWest: nowWest,
YesOrNo: true,
Pi: 3.14,
Colors: [][]string{
+1 -1
View File
@@ -775,7 +775,7 @@ func lexDatetime(lx *lexer) stateFn {
return lexDatetime
}
switch r {
case '-', 'T', ':', '.', 'Z':
case '-', 'T', ':', '.', 'Z', '+':
return lexDatetime
}
+21 -7
View File
@@ -1,14 +1,28 @@
language: go
go_import_path: github.com/davecgh/go-spew
go:
- 1.5.4
- 1.6.3
- 1.7
- 1.6.x
- 1.7.x
- 1.8.x
- 1.9.x
- 1.10.x
- tip
sudo: false
install:
- go get -v golang.org/x/tools/cmd/cover
- go get -v github.com/alecthomas/gometalinter
- gometalinter --install
script:
- go test -v -tags=safe ./spew
- go test -v -tags=testcgo ./spew -covermode=count -coverprofile=profile.cov
- export PATH=$PATH:$HOME/gopath/bin
- export GORACE="halt_on_error=1"
- test -z "$(gometalinter --disable-all
--enable=gofmt
--enable=golint
--enable=vet
--enable=gosimple
--enable=unconvert
--deadline=4m ./spew | tee /dev/stderr)"
- go test -v -race -tags safe ./spew
- go test -v -race -tags testcgo ./spew -covermode=atomic -coverprofile=profile.cov
after_success:
- go get -v github.com/mattn/goveralls
- export PATH=$PATH:$HOME/gopath/bin
- goveralls -coverprofile=profile.cov -service=travis-ci
+1 -1
View File
@@ -2,7 +2,7 @@ ISC License
Copyright (c) 2012-2016 Dave Collins <dave@davec.name>
Permission to use, copy, modify, and distribute this software for any
Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.
+4 -8
View File
@@ -1,12 +1,9 @@
go-spew
=======
[![Build Status](https://img.shields.io/travis/davecgh/go-spew.svg)]
(https://travis-ci.org/davecgh/go-spew) [![ISC License]
(http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org) [![Coverage Status]
(https://img.shields.io/coveralls/davecgh/go-spew.svg)]
(https://coveralls.io/r/davecgh/go-spew?branch=master)
[![Build Status](https://img.shields.io/travis/davecgh/go-spew.svg)](https://travis-ci.org/davecgh/go-spew)
[![ISC License](http://img.shields.io/badge/license-ISC-blue.svg)](http://copyfree.org)
[![Coverage Status](https://img.shields.io/coveralls/davecgh/go-spew.svg)](https://coveralls.io/r/davecgh/go-spew?branch=master)
Go-spew implements a deep pretty printer for Go data structures to aid in
debugging. A comprehensive suite of tests with 100% test coverage is provided
@@ -21,8 +18,7 @@ post about it
## Documentation
[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)]
(http://godoc.org/github.com/davecgh/go-spew/spew)
[![GoDoc](https://img.shields.io/badge/godoc-reference-blue.svg)](http://godoc.org/github.com/davecgh/go-spew/spew)
Full `go doc` style documentation for the project can be viewed online without
installing this package by using the excellent GoDoc site here:
+90 -97
View File
@@ -16,7 +16,9 @@
// when the code is not running on Google App Engine, compiled by GopherJS, and
// "-tags safe" is not added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build !js,!appengine,!safe,!disableunsafe
// Go versions prior to 1.4 are disabled because they use a different layout
// for interfaces which make the implementation of unsafeReflectValue more complex.
// +build !js,!appengine,!safe,!disableunsafe,go1.4
package spew
@@ -34,80 +36,49 @@ const (
ptrSize = unsafe.Sizeof((*byte)(nil))
)
var (
// offsetPtr, offsetScalar, and offsetFlag are the offsets for the
// internal reflect.Value fields. These values are valid before golang
// commit ecccf07e7f9d which changed the format. The are also valid
// after commit 82f48826c6c7 which changed the format again to mirror
// the original format. Code in the init function updates these offsets
// as necessary.
offsetPtr = uintptr(ptrSize)
offsetScalar = uintptr(0)
offsetFlag = uintptr(ptrSize * 2)
type flag uintptr
// flagKindWidth and flagKindShift indicate various bits that the
// reflect package uses internally to track kind information.
//
// flagRO indicates whether or not the value field of a reflect.Value is
// read-only.
//
// flagIndir indicates whether the value field of a reflect.Value is
// the actual data or a pointer to the data.
//
// These values are valid before golang commit 90a7c3c86944 which
// changed their positions. Code in the init function updates these
// flags as necessary.
flagKindWidth = uintptr(5)
flagKindShift = uintptr(flagKindWidth - 1)
flagRO = uintptr(1 << 0)
flagIndir = uintptr(1 << 1)
var (
// flagRO indicates whether the value field of a reflect.Value
// is read-only.
flagRO flag
// flagAddr indicates whether the address of the reflect.Value's
// value may be taken.
flagAddr flag
)
func init() {
// Older versions of reflect.Value stored small integers directly in the
// ptr field (which is named val in the older versions). Versions
// between commits ecccf07e7f9d and 82f48826c6c7 added a new field named
// scalar for this purpose which unfortunately came before the flag
// field, so the offset of the flag field is different for those
// versions.
//
// This code constructs a new reflect.Value from a known small integer
// and checks if the size of the reflect.Value struct indicates it has
// the scalar field. When it does, the offsets are updated accordingly.
vv := reflect.ValueOf(0xf00)
if unsafe.Sizeof(vv) == (ptrSize * 4) {
offsetScalar = ptrSize * 2
offsetFlag = ptrSize * 3
}
// flagKindMask holds the bits that make up the kind
// part of the flags field. In all the supported versions,
// it is in the lower 5 bits.
const flagKindMask = flag(0x1f)
// Commit 90a7c3c86944 changed the flag positions such that the low
// order bits are the kind. This code extracts the kind from the flags
// field and ensures it's the correct type. When it's not, the flag
// order has been changed to the newer format, so the flags are updated
// accordingly.
upf := unsafe.Pointer(uintptr(unsafe.Pointer(&vv)) + offsetFlag)
upfv := *(*uintptr)(upf)
flagKindMask := uintptr((1<<flagKindWidth - 1) << flagKindShift)
if (upfv&flagKindMask)>>flagKindShift != uintptr(reflect.Int) {
flagKindShift = 0
flagRO = 1 << 5
flagIndir = 1 << 6
// Different versions of Go have used different
// bit layouts for the flags type. This table
// records the known combinations.
var okFlags = []struct {
ro, addr flag
}{{
// From Go 1.4 to 1.5
ro: 1 << 5,
addr: 1 << 7,
}, {
// Up to Go tip.
ro: 1<<5 | 1<<6,
addr: 1 << 8,
}}
// Commit adf9b30e5594 modified the flags to separate the
// flagRO flag into two bits which specifies whether or not the
// field is embedded. This causes flagIndir to move over a bit
// and means that flagRO is the combination of either of the
// original flagRO bit and the new bit.
//
// This code detects the change by extracting what used to be
// the indirect bit to ensure it's set. When it's not, the flag
// order has been changed to the newer format, so the flags are
// updated accordingly.
if upfv&flagIndir == 0 {
flagRO = 3 << 5
flagIndir = 1 << 7
}
var flagValOffset = func() uintptr {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
return field.Offset
}()
// flagField returns a pointer to the flag field of a reflect.Value.
func flagField(v *reflect.Value) *flag {
return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset))
}
// unsafeReflectValue converts the passed reflect.Value into a one that bypasses
@@ -119,34 +90,56 @@ func init() {
// This allows us to check for implementations of the Stringer and error
// interfaces to be used for pretty printing ordinarily unaddressable and
// inaccessible values such as unexported struct fields.
func unsafeReflectValue(v reflect.Value) (rv reflect.Value) {
indirects := 1
vt := v.Type()
upv := unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetPtr)
rvf := *(*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(&v)) + offsetFlag))
if rvf&flagIndir != 0 {
vt = reflect.PtrTo(v.Type())
indirects++
} else if offsetScalar != 0 {
// The value is in the scalar field when it's not one of the
// reference types.
switch vt.Kind() {
case reflect.Uintptr:
case reflect.Chan:
case reflect.Func:
case reflect.Map:
case reflect.Ptr:
case reflect.UnsafePointer:
default:
upv = unsafe.Pointer(uintptr(unsafe.Pointer(&v)) +
offsetScalar)
func unsafeReflectValue(v reflect.Value) reflect.Value {
if !v.IsValid() || (v.CanInterface() && v.CanAddr()) {
return v
}
flagFieldPtr := flagField(&v)
*flagFieldPtr &^= flagRO
*flagFieldPtr |= flagAddr
return v
}
// Sanity checks against future reflect package changes
// to the type or semantics of the Value.flag field.
func init() {
field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag")
if !ok {
panic("reflect.Value has no flag field")
}
if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() {
panic("reflect.Value flag field has changed kind")
}
type t0 int
var t struct {
A t0
// t0 will have flagEmbedRO set.
t0
// a will have flagStickyRO set
a t0
}
vA := reflect.ValueOf(t).FieldByName("A")
va := reflect.ValueOf(t).FieldByName("a")
vt0 := reflect.ValueOf(t).FieldByName("t0")
// Infer flagRO from the difference between the flags
// for the (otherwise identical) fields in t.
flagPublic := *flagField(&vA)
flagWithRO := *flagField(&va) | *flagField(&vt0)
flagRO = flagPublic ^ flagWithRO
// Infer flagAddr from the difference between a value
// taken from a pointer and not.
vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A")
flagNoPtr := *flagField(&vA)
flagPtr := *flagField(&vPtrA)
flagAddr = flagNoPtr ^ flagPtr
// Check that the inferred flags tally with one of the known versions.
for _, f := range okFlags {
if flagRO == f.ro && flagAddr == f.addr {
return
}
}
pv := reflect.NewAt(vt, upv)
rv = pv
for i := 0; i < indirects; i++ {
rv = rv.Elem()
}
return rv
panic("reflect.Value read-only flag has changed semantics")
}
+1 -1
View File
@@ -16,7 +16,7 @@
// when the code is running on Google App Engine, compiled by GopherJS, or
// "-tags safe" is added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build js appengine safe disableunsafe
// +build js appengine safe disableunsafe !go1.4
package spew
+1 -1
View File
@@ -180,7 +180,7 @@ func printComplex(w io.Writer, c complex128, floatPrecision int) {
w.Write(closeParenBytes)
}
// printHexPtr outputs a uintptr formatted as hexidecimal with a leading '0x'
// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x'
// prefix to Writer w.
func printHexPtr(w io.Writer, p uintptr) {
// Null pointer.
+5 -5
View File
@@ -35,16 +35,16 @@ var (
// cCharRE is a regular expression that matches a cgo char.
// It is used to detect character arrays to hexdump them.
cCharRE = regexp.MustCompile("^.*\\._Ctype_char$")
cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`)
// cUnsignedCharRE is a regular expression that matches a cgo unsigned
// char. It is used to detect unsigned character arrays to hexdump
// them.
cUnsignedCharRE = regexp.MustCompile("^.*\\._Ctype_unsignedchar$")
cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`)
// cUint8tCharRE is a regular expression that matches a cgo uint8_t.
// It is used to detect uint8_t arrays to hexdump them.
cUint8tCharRE = regexp.MustCompile("^.*\\._Ctype_uint8_t$")
cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`)
)
// dumpState contains information about the state of a dump operation.
@@ -143,10 +143,10 @@ func (d *dumpState) dumpPtr(v reflect.Value) {
// Display dereferenced value.
d.w.Write(openParenBytes)
switch {
case nilFound == true:
case nilFound:
d.w.Write(nilAngleBytes)
case cycleFound == true:
case cycleFound:
d.w.Write(circularBytes)
default:
+1 -1
View File
@@ -768,7 +768,7 @@ func addUintptrDumpTests() {
func addUnsafePointerDumpTests() {
// Null pointer.
v := unsafe.Pointer(uintptr(0))
v := unsafe.Pointer(nil)
nv := (*unsafe.Pointer)(nil)
pv := &v
vAddr := fmt.Sprintf("%p", pv)
+4 -2
View File
@@ -82,18 +82,20 @@ func addCgoDumpTests() {
v5Len := fmt.Sprintf("%d", v5l)
v5Cap := fmt.Sprintf("%d", v5c)
v5t := "[6]testdata._Ctype_uint8_t"
v5t2 := "[6]testdata._Ctype_uchar"
v5s := "(len=" + v5Len + " cap=" + v5Cap + ") " +
"{\n 00000000 74 65 73 74 35 00 " +
" |test5.|\n}"
addDumpTest(v5, "("+v5t+") "+v5s+"\n")
addDumpTest(v5, "("+v5t+") "+v5s+"\n", "("+v5t2+") "+v5s+"\n")
// C typedefed unsigned char array.
v6, v6l, v6c := testdata.GetCgoTypdefedUnsignedCharArray()
v6Len := fmt.Sprintf("%d", v6l)
v6Cap := fmt.Sprintf("%d", v6c)
v6t := "[6]testdata._Ctype_custom_uchar_t"
v6t2 := "[6]testdata._Ctype_uchar"
v6s := "(len=" + v6Len + " cap=" + v6Cap + ") " +
"{\n 00000000 74 65 73 74 36 00 " +
" |test6.|\n}"
addDumpTest(v6, "("+v6t+") "+v6s+"\n")
addDumpTest(v6, "("+v6t+") "+v6s+"\n", "("+v6t2+") "+v6s+"\n")
}
+2 -2
View File
@@ -182,10 +182,10 @@ func (f *formatState) formatPtr(v reflect.Value) {
// Display dereferenced value.
switch {
case nilFound == true:
case nilFound:
f.fs.Write(nilAngleBytes)
case cycleFound == true:
case cycleFound:
f.fs.Write(circularShortBytes)
default:
+3 -3
View File
@@ -1083,7 +1083,7 @@ func addUintptrFormatterTests() {
func addUnsafePointerFormatterTests() {
// Null pointer.
v := unsafe.Pointer(uintptr(0))
v := unsafe.Pointer(nil)
nv := (*unsafe.Pointer)(nil)
pv := &v
vAddr := fmt.Sprintf("%p", pv)
@@ -1536,14 +1536,14 @@ func TestPrintSortedKeys(t *testing.T) {
t.Errorf("Sorted keys mismatch 3:\n %v %v", s, expected)
}
s = cfg.Sprint(map[testStruct]int{testStruct{1}: 1, testStruct{3}: 3, testStruct{2}: 2})
s = cfg.Sprint(map[testStruct]int{{1}: 1, {3}: 3, {2}: 2})
expected = "map[ts.1:1 ts.2:2 ts.3:3]"
if s != expected {
t.Errorf("Sorted keys mismatch 4:\n %v %v", s, expected)
}
if !spew.UnsafeDisabled {
s = cfg.Sprint(map[testStructP]int{testStructP{1}: 1, testStructP{3}: 3, testStructP{2}: 2})
s = cfg.Sprint(map[testStructP]int{{1}: 1, {3}: 3, {2}: 2})
expected = "map[ts.1:1 ts.2:2 ts.3:3]"
if s != expected {
t.Errorf("Sorted keys mismatch 5:\n %v %v", s, expected)
+1 -4
View File
@@ -36,10 +36,7 @@ type dummyFmtState struct {
}
func (dfs *dummyFmtState) Flag(f int) bool {
if f == int('+') {
return true
}
return false
return f == int('+')
}
func (dfs *dummyFmtState) Precision() (int, bool) {
+5 -6
View File
@@ -16,7 +16,7 @@
// when the code is not running on Google App Engine, compiled by GopherJS, and
// "-tags safe" is not added to the go build command line. The "disableunsafe"
// tag is deprecated and thus should not be used.
// +build !js,!appengine,!safe,!disableunsafe
// +build !js,!appengine,!safe,!disableunsafe,go1.4
/*
This test file is part of the spew package rather than than the spew_test
@@ -30,7 +30,6 @@ import (
"bytes"
"reflect"
"testing"
"unsafe"
)
// changeKind uses unsafe to intentionally change the kind of a reflect.Value to
@@ -38,13 +37,13 @@ import (
// fallback code which punts to the standard fmt library for new types that
// might get added to the language.
func changeKind(v *reflect.Value, readOnly bool) {
rvf := (*uintptr)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + offsetFlag))
*rvf = *rvf | ((1<<flagKindWidth - 1) << flagKindShift)
flags := flagField(v)
if readOnly {
*rvf |= flagRO
*flags |= flagRO
} else {
*rvf &= ^uintptr(flagRO)
*flags &^= flagRO
}
*flags |= flagKindMask
}
// TestAddedReflectValue tests functionaly of the dump and formatter code which
+2
View File
@@ -1,6 +1,8 @@
language: go
go:
- 1.7.x
- 1.8.x
- 1.9.x
- tip
sudo: false
before_install:
+2 -2
View File
@@ -81,8 +81,8 @@ n := s.Names() // Get a []string
f := s.Field(name) // Get a *Field based on the given field name
f, ok := s.FieldOk(name) // Get a *Field based on the given field name
n := s.Name() // Get the struct name
h := s.HasZero() // Check if any field is initialized
z := s.IsZero() // Check if all fields are initialized
h := s.HasZero() // Check if any field is uninitialized
z := s.IsZero() // Check if all fields are uninitialized
```
### Field methods
+1 -1
View File
@@ -135,7 +135,7 @@ func TestField_Set(t *testing.T) {
func TestField_NotSettable(t *testing.T) {
a := map[int]Baz{
4: Baz{
4: {
A: "value",
},
}
+2 -4
View File
@@ -203,9 +203,7 @@ func (s *Struct) Values() []interface{} {
if IsStruct(val.Interface()) && !tagOpts.Has("omitnested") {
// look out for embedded structs, and convert them to a
// []interface{} to be added to the final values slice
for _, embeddedVal := range Values(val.Interface()) {
t = append(t, embeddedVal)
}
t = append(t, Values(val.Interface())...)
} else {
t = append(t, val.Interface())
}
@@ -573,7 +571,7 @@ func (s *Struct) nested(val reflect.Value) interface{} {
break
}
slices := make([]interface{}, val.Len(), val.Len())
slices := make([]interface{}, val.Len())
for x := 0; x < val.Len(); x++ {
slices[x] = s.nested(val.Index(x))
}
+1 -1
View File
@@ -301,7 +301,7 @@ func ExampleIsZero() {
Enabled bool
}
// Nothing is initalized
// Nothing is initialized
a := &Server{}
isZeroA := IsZero(a)
+9 -9
View File
@@ -380,7 +380,7 @@ func TestMap_NestedMapWithSliceIntValues(t *testing.T) {
b := &B{
Foo: map[string][]int{
"example_key": []int{80},
"example_key": {80},
},
}
@@ -399,7 +399,7 @@ func TestMap_NestedMapWithSliceIntValues(t *testing.T) {
foo := in["Foo"].(map[string][]int)
if name := foo["example_key"]; name[0] != 80 {
t.Errorf("Map nested struct's name field should give example, got: %s", name)
t.Errorf("Map nested struct's name field should give example, got: %v", name)
}
}
@@ -418,7 +418,7 @@ func TestMap_NestedMapWithSliceStructValues(t *testing.T) {
b := &B{
Foo: map[string][]address{
"example_key": []address{
"example_key": {
{Country: "Turkey"},
},
},
@@ -463,8 +463,8 @@ func TestMap_NestedSliceWithStructValues(t *testing.T) {
p := person{
Name: "test",
Addresses: []address{
address{Country: "England"},
address{Country: "Italy"},
{Country: "England"},
{Country: "Italy"},
},
}
mp := Map(p)
@@ -492,8 +492,8 @@ func TestMap_NestedSliceWithPointerOfStructValues(t *testing.T) {
p := person{
Name: "test",
Addresses: []*address{
&address{Country: "England"},
&address{Country: "Italy"},
{Country: "England"},
{Country: "Italy"},
},
}
mp := Map(p)
@@ -1430,14 +1430,14 @@ func TestPointer2Pointer(t *testing.T) {
func TestMap_InterfaceTypeWithMapValue(t *testing.T) {
type A struct {
Name string `structs:"name"`
Ip string `structs:"ip"`
IP string `structs:"ip"`
Query string `structs:"query"`
Payload interface{} `structs:"payload"`
}
a := A{
Name: "test",
Ip: "127.0.0.1",
IP: "127.0.0.1",
Query: "",
Payload: map[string]string{"test_param": "test_param"},
}
+1 -1
View File
@@ -5,7 +5,7 @@ import "strings"
// tagOptions contains a slice of tag options
type tagOptions []string
// Has returns true if the given optiton is available in tagOptions
// Has returns true if the given option is available in tagOptions
func (t tagOptions) Has(opt string) bool {
for _, tagOpt := range t {
if tagOpt == opt {
+1 -3
View File
@@ -5,14 +5,12 @@
"gotype",
"goimports",
"golint",
"gosimple",
"gofmt",
"vet",
"lll",
"staticcheck",
"megacheck",
"ineffassign",
"deadcode",
"unused",
"unconvert",
"misspell",
"test",
+1 -2
View File
@@ -2,8 +2,7 @@ language: go
sudo: false
go:
- 1.7
- 1.8
- 1.10.x
before_install:
- go get golang.org/x/tools/cmd/cover
+201
View File
@@ -411,6 +411,207 @@ func TestValuePathError(t *testing.T) {
}
}
// based on github.com/yalp/jsonpath
func TestValuePathExpressions(t *testing.T) {
data := map[string]interface{}{
"A": []interface{}{
"string",
23.3,
3.0,
true,
false,
nil,
},
"B": "value",
"C": 3.14,
"D": map[string]interface{}{
"C": 3.1415,
"V": []interface{}{
"string2a",
"string2b",
map[string]interface{}{
"C": 3.141592,
},
},
},
"E": map[string]interface{}{
"A": []interface{}{"string3"},
"D": map[string]interface{}{
"V": map[string]interface{}{
"C": 3.14159265,
},
},
},
"F": map[string]interface{}{
"V": []interface{}{
"string4a",
"string4b",
map[string]interface{}{
"CC": 3.1415926535,
},
map[string]interface{}{
"CC": "hello",
},
[]interface{}{
"string5a",
"string5b",
},
[]interface{}{
"string6a",
"string6b",
},
},
},
}
reporter := newMockReporter(t)
runTests := func(tests map[string]interface{}) {
value := NewValue(reporter, data)
value.chain.assertOK(t)
for path, expected := range tests {
actual := value.Path(path)
actual.chain.assertOK(t)
assert.Equal(t, expected, actual.Raw())
}
}
t.Run("pick", func(t *testing.T) {
runTests(map[string]interface{}{
"$": data,
"$.A[0]": "string",
`$["A"][0]`: "string",
"$.A": []interface{}{"string", 23.3, 3.0, true, false, nil},
"$.A[*]": []interface{}{"string", 23.3, 3.0, true, false, nil},
"$.A.*": []interface{}{"string", 23.3, 3.0, true, false, nil},
"$.A.*.a": []interface{}{},
})
})
t.Run("slice", func(t *testing.T) {
runTests(map[string]interface{}{
"$.A[1,4,2]": []interface{}{23.3, false, 3.0},
`$["B","C"]`: []interface{}{"value", 3.14},
`$["C","B"]`: []interface{}{3.14, "value"},
"$.A[1:4]": []interface{}{23.3, 3.0, true},
"$.A[::2]": []interface{}{"string", 3.0, false},
"$.A[-2:]": []interface{}{false, nil},
"$.A[:-1]": []interface{}{"string", 23.3, 3.0, true, false},
"$.A[::-1]": []interface{}{nil, false, true, 3.0, 23.3, "string"},
"$.F.V[4:5][0,1]": []interface{}{"string5a", "string5b"},
"$.F.V[4:6][1]": []interface{}{"string5b", "string6b"},
"$.F.V[4:6][0,1]": []interface{}{"string5a", "string5b", "string6a", "string6b"},
"$.F.V[4,5][0:2]": []interface{}{"string5a", "string5b", "string6a", "string6b"},
"$.F.V[4:6]": []interface{}{
[]interface{}{
"string5a",
"string5b",
},
[]interface{}{
"string6a",
"string6b",
},
},
})
})
t.Run("quote", func(t *testing.T) {
runTests(map[string]interface{}{
`$[A][0]`: "string",
`$["A"][0]`: "string",
`$[B,C]`: []interface{}{"value", 3.14},
`$["B","C"]`: []interface{}{"value", 3.14},
})
})
t.Run("search", func(t *testing.T) {
runTests(map[string]interface{}{
"$..C": []interface{}{3.14, 3.1415, 3.141592, 3.14159265},
`$..["C"]`: []interface{}{3.14, 3.1415, 3.141592, 3.14159265},
"$.D.V..C": []interface{}{3.141592},
"$.D.V.*.C": []interface{}{3.141592},
"$.D.V..*.C": []interface{}{3.141592},
"$.D.*..C": []interface{}{3.141592},
"$.*.V..C": []interface{}{3.141592},
"$.*.D.V.C": []interface{}{3.14159265},
"$.*.D..C": []interface{}{3.14159265},
"$.*.D.V..*": []interface{}{3.14159265},
"$..D..V..C": []interface{}{3.141592, 3.14159265},
"$.*.*.*.C": []interface{}{3.141592, 3.14159265},
"$..V..C": []interface{}{3.141592, 3.14159265},
"$.D.V..*": []interface{}{
"string2a",
"string2b",
map[string]interface{}{
"C": 3.141592,
},
3.141592,
},
"$..A": []interface{}{
[]interface{}{"string", 23.3, 3.0, true, false, nil},
[]interface{}{"string3"},
},
"$..A..*": []interface{}{"string", 23.3, 3.0, true, false, nil, "string3"},
"$.A..*": []interface{}{"string", 23.3, 3.0, true, false, nil},
"$.A.*": []interface{}{"string", 23.3, 3.0, true, false, nil},
"$..A[0,1]": []interface{}{"string", 23.3},
"$..A[0]": []interface{}{"string", "string3"},
"$.*.V[0]": []interface{}{"string2a", "string4a"},
"$.*.V[1]": []interface{}{"string2b", "string4b"},
"$.*.V[0,1]": []interface{}{"string2a", "string2b", "string4a", "string4b"},
"$.*.V[0:2]": []interface{}{"string2a", "string2b", "string4a", "string4b"},
"$.*.V[2].C": []interface{}{3.141592},
"$..V[2].C": []interface{}{3.141592},
"$..V[*].C": []interface{}{3.141592},
"$.*.V[2].*": []interface{}{3.141592, 3.1415926535},
"$.*.V[2:3].*": []interface{}{3.141592, 3.1415926535},
"$.*.V[2:4].*": []interface{}{3.141592, 3.1415926535, "hello"},
"$..V[2,3].CC": []interface{}{3.1415926535, "hello"},
"$..V[2:4].CC": []interface{}{3.1415926535, "hello"},
"$..V[*].*": []interface{}{
3.141592,
3.1415926535,
"hello",
"string5a",
"string5b",
"string6a",
"string6b",
},
"$..[0]": []interface{}{
"string",
"string2a",
"string3",
"string4a",
"string5a",
"string6a",
},
"$..ZZ": []interface{}{},
})
})
}
func TestValuePathIntFloat(t *testing.T) {
reporter := newMockReporter(t)
data := map[string]interface{}{
"A": 123,
"B": 123.0,
}
value := NewValue(reporter, data)
value.chain.assertOK(t)
a := value.Path(`$["A"]`)
a.chain.assertOK(t)
assert.Equal(t, 123.0, a.Raw())
b := value.Path(`$["B"]`)
b.chain.assertOK(t)
assert.Equal(t, 123.0, b.Raw())
}
func TestValueSchema(t *testing.T) {
reporter := newMockReporter(t)
+1
View File
@@ -0,0 +1 @@
module github.com/google/go-querystring
+9
View File
@@ -0,0 +1,9 @@
language: go
go:
- 1.4.3
- 1.5.3
- tip
script:
- go test -v ./...
+10
View File
@@ -0,0 +1,10 @@
# How to contribute
We definitely welcome patches and contribution to this project!
### Legal requirements
In order to protect both you and ourselves, you will need to sign the
[Contributor License Agreement](https://cla.developers.google.com/clas).
You may have already signed it for other Google projects.
+9
View File
@@ -0,0 +1,9 @@
Paul Borman <borman@google.com>
bmatsuo
shawnps
theory
jboverfelt
dsymonds
cd1
wallclockbuilder
dansouza
@@ -1,5 +1,4 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Copyright (c) 2015 Klaus Post
Copyright (c) 2009,2014 Google Inc. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
+19
View File
@@ -0,0 +1,19 @@
# uuid ![build status](https://travis-ci.org/google/uuid.svg?branch=master)
The uuid package generates and inspects UUIDs based on
[RFC 4122](http://tools.ietf.org/html/rfc4122)
and DCE 1.1: Authentication and Security Services.
This package is based on the github.com/pborman/uuid package (previously named
code.google.com/p/go-uuid). It differs from these earlier packages in that
a UUID is a 16 byte array rather than a byte slice. One loss due to this
change is the ability to represent an invalid UUID (vs a NIL UUID).
###### Install
`go get github.com/google/uuid`
###### Documentation
[![GoDoc](https://godoc.org/github.com/google/uuid?status.svg)](http://godoc.org/github.com/google/uuid)
Full `go doc` style documentation for the package can be viewed online without
installing this package by using the GoDoc site here:
http://godoc.org/github.com/google/uuid
+80
View File
@@ -0,0 +1,80 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/binary"
"fmt"
"os"
)
// A Domain represents a Version 2 domain
type Domain byte
// Domain constants for DCE Security (Version 2) UUIDs.
const (
Person = Domain(0)
Group = Domain(1)
Org = Domain(2)
)
// NewDCESecurity returns a DCE Security (Version 2) UUID.
//
// The domain should be one of Person, Group or Org.
// On a POSIX system the id should be the users UID for the Person
// domain and the users GID for the Group. The meaning of id for
// the domain Org or on non-POSIX systems is site defined.
//
// For a given domain/id pair the same token may be returned for up to
// 7 minutes and 10 seconds.
func NewDCESecurity(domain Domain, id uint32) (UUID, error) {
uuid, err := NewUUID()
if err == nil {
uuid[6] = (uuid[6] & 0x0f) | 0x20 // Version 2
uuid[9] = byte(domain)
binary.BigEndian.PutUint32(uuid[0:], id)
}
return uuid, err
}
// NewDCEPerson returns a DCE Security (Version 2) UUID in the person
// domain with the id returned by os.Getuid.
//
// NewDCESecurity(Person, uint32(os.Getuid()))
func NewDCEPerson() (UUID, error) {
return NewDCESecurity(Person, uint32(os.Getuid()))
}
// NewDCEGroup returns a DCE Security (Version 2) UUID in the group
// domain with the id returned by os.Getgid.
//
// NewDCESecurity(Group, uint32(os.Getgid()))
func NewDCEGroup() (UUID, error) {
return NewDCESecurity(Group, uint32(os.Getgid()))
}
// Domain returns the domain for a Version 2 UUID. Domains are only defined
// for Version 2 UUIDs.
func (uuid UUID) Domain() Domain {
return Domain(uuid[9])
}
// ID returns the id for a Version 2 UUID. IDs are only defined for Version 2
// UUIDs.
func (uuid UUID) ID() uint32 {
return binary.BigEndian.Uint32(uuid[0:4])
}
func (d Domain) String() string {
switch d {
case Person:
return "Person"
case Group:
return "Group"
case Org:
return "Org"
}
return fmt.Sprintf("Domain%d", int(d))
}
+12
View File
@@ -0,0 +1,12 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package uuid generates and inspects UUIDs.
//
// UUIDs are based on RFC 4122 and DCE 1.1: Authentication and Security
// Services.
//
// A UUID is a 16 byte (128 bit) array. UUIDs may be used as keys to
// maps or compared directly.
package uuid
+1
View File
@@ -0,0 +1 @@
module github.com/google/uuid
+53
View File
@@ -0,0 +1,53 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"crypto/md5"
"crypto/sha1"
"hash"
)
// Well known namespace IDs and UUIDs
var (
NameSpaceDNS = Must(Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8"))
NameSpaceURL = Must(Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8"))
NameSpaceOID = Must(Parse("6ba7b812-9dad-11d1-80b4-00c04fd430c8"))
NameSpaceX500 = Must(Parse("6ba7b814-9dad-11d1-80b4-00c04fd430c8"))
Nil UUID // empty UUID, all zeros
)
// NewHash returns a new UUID derived from the hash of space concatenated with
// data generated by h. The hash should be at least 16 byte in length. The
// first 16 bytes of the hash are used to form the UUID. The version of the
// UUID will be the lower 4 bits of version. NewHash is used to implement
// NewMD5 and NewSHA1.
func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID {
h.Reset()
h.Write(space[:])
h.Write(data)
s := h.Sum(nil)
var uuid UUID
copy(uuid[:], s)
uuid[6] = (uuid[6] & 0x0f) | uint8((version&0xf)<<4)
uuid[8] = (uuid[8] & 0x3f) | 0x80 // RFC 4122 variant
return uuid
}
// NewMD5 returns a new MD5 (Version 3) UUID based on the
// supplied name space and data. It is the same as calling:
//
// NewHash(md5.New(), space, data, 3)
func NewMD5(space UUID, data []byte) UUID {
return NewHash(md5.New(), space, data, 3)
}
// NewSHA1 returns a new SHA1 (Version 5) UUID based on the
// supplied name space and data. It is the same as calling:
//
// NewHash(sha1.New(), space, data, 5)
func NewSHA1(space UUID, data []byte) UUID {
return NewHash(sha1.New(), space, data, 5)
}
+62
View File
@@ -0,0 +1,62 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/json"
"reflect"
"testing"
)
var testUUID = Must(Parse("f47ac10b-58cc-0372-8567-0e02b2c3d479"))
func TestJSON(t *testing.T) {
type S struct {
ID1 UUID
ID2 UUID
}
s1 := S{ID1: testUUID}
data, err := json.Marshal(&s1)
if err != nil {
t.Fatal(err)
}
var s2 S
if err := json.Unmarshal(data, &s2); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(&s1, &s2) {
t.Errorf("got %#v, want %#v", s2, s1)
}
}
func BenchmarkUUID_MarshalJSON(b *testing.B) {
x := &struct {
UUID UUID `json:"uuid"`
}{}
var err error
x.UUID, err = Parse("f47ac10b-58cc-0372-8567-0e02b2c3d479")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
js, err := json.Marshal(x)
if err != nil {
b.Fatalf("marshal json: %#v (%v)", js, err)
}
}
}
func BenchmarkUUID_UnmarshalJSON(b *testing.B) {
js := []byte(`{"uuid":"f47ac10b-58cc-0372-8567-0e02b2c3d479"}`)
var x *struct {
UUID UUID `json:"uuid"`
}
for i := 0; i < b.N; i++ {
err := json.Unmarshal(js, &x)
if err != nil {
b.Fatalf("marshal json: %#v (%v)", js, err)
}
}
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import "fmt"
// MarshalText implements encoding.TextMarshaler.
func (uuid UUID) MarshalText() ([]byte, error) {
var js [36]byte
encodeHex(js[:], uuid)
return js[:], nil
}
// UnmarshalText implements encoding.TextUnmarshaler.
func (uuid *UUID) UnmarshalText(data []byte) error {
id, err := ParseBytes(data)
if err == nil {
*uuid = id
}
return err
}
// MarshalBinary implements encoding.BinaryMarshaler.
func (uuid UUID) MarshalBinary() ([]byte, error) {
return uuid[:], nil
}
// UnmarshalBinary implements encoding.BinaryUnmarshaler.
func (uuid *UUID) UnmarshalBinary(data []byte) error {
if len(data) != 16 {
return fmt.Errorf("invalid UUID (got %d bytes)", len(data))
}
copy(uuid[:], data)
return nil
}
+89
View File
@@ -0,0 +1,89 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"sync"
)
var (
nodeMu sync.Mutex
ifname string // name of interface being used
nodeID [6]byte // hardware for version 1 UUIDs
zeroID [6]byte // nodeID with only 0's
)
// NodeInterface returns the name of the interface from which the NodeID was
// derived. The interface "user" is returned if the NodeID was set by
// SetNodeID.
func NodeInterface() string {
defer nodeMu.Unlock()
nodeMu.Lock()
return ifname
}
// SetNodeInterface selects the hardware address to be used for Version 1 UUIDs.
// If name is "" then the first usable interface found will be used or a random
// Node ID will be generated. If a named interface cannot be found then false
// is returned.
//
// SetNodeInterface never fails when name is "".
func SetNodeInterface(name string) bool {
defer nodeMu.Unlock()
nodeMu.Lock()
return setNodeInterface(name)
}
func setNodeInterface(name string) bool {
iname, addr := getHardwareInterface(name) // null implementation for js
if iname != "" && addr != nil {
ifname = iname
copy(nodeID[:], addr)
return true
}
// We found no interfaces with a valid hardware address. If name
// does not specify a specific interface generate a random Node ID
// (section 4.1.6)
if name == "" {
randomBits(nodeID[:])
return true
}
return false
}
// NodeID returns a slice of a copy of the current Node ID, setting the Node ID
// if not already set.
func NodeID() []byte {
defer nodeMu.Unlock()
nodeMu.Lock()
if nodeID == zeroID {
setNodeInterface("")
}
nid := nodeID
return nid[:]
}
// SetNodeID sets the Node ID to be used for Version 1 UUIDs. The first 6 bytes
// of id are used. If id is less than 6 bytes then false is returned and the
// Node ID is not set.
func SetNodeID(id []byte) bool {
if len(id) < 6 {
return false
}
defer nodeMu.Unlock()
nodeMu.Lock()
copy(nodeID[:], id)
ifname = "user"
return true
}
// NodeID returns the 6 byte node id encoded in uuid. It returns nil if uuid is
// not valid. The NodeID is only well defined for version 1 and 2 UUIDs.
func (uuid UUID) NodeID() []byte {
var node [6]byte
copy(node[:], uuid[10:])
return node[:]
}
+12
View File
@@ -0,0 +1,12 @@
// Copyright 2017 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build js
package uuid
// getHardwareInterface returns nil values for the JS version of the code.
// This remvoves the "net" dependency, because it is not used in the browser.
// Using the "net" library inflates the size of the transpiled JS code by 673k bytes.
func getHardwareInterface(name string) (string, []byte) { return "", nil }
+33
View File
@@ -0,0 +1,33 @@
// Copyright 2017 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build !js
package uuid
import "net"
var interfaces []net.Interface // cached list of interfaces
// getHardwareInterface returns the name and hardware address of interface name.
// If name is "" then the name and hardware address of one of the system's
// interfaces is returned. If no interfaces are found (name does not exist or
// there are no interfaces) then "", nil is returned.
//
// Only addresses of at least 6 bytes are returned.
func getHardwareInterface(name string) (string, []byte) {
if interfaces == nil {
var err error
interfaces, err = net.Interfaces()
if err != nil {
return "", nil
}
}
for _, ifs := range interfaces {
if len(ifs.HardwareAddr) >= 6 && (name == "" || name == ifs.Name) {
return ifs.Name, ifs.HardwareAddr
}
}
return "", nil
}
+66
View File
@@ -0,0 +1,66 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"flag"
"runtime"
"testing"
"time"
)
// This test is only run when --regressions is passed on the go test line.
var regressions = flag.Bool("regressions", false, "run uuid regression tests")
// TestClockSeqRace tests for a particular race condition of returning two
// identical Version1 UUIDs. The duration of 1 minute was chosen as the race
// condition, before being fixed, nearly always occurred in under 30 seconds.
func TestClockSeqRace(t *testing.T) {
if !*regressions {
t.Skip("skipping regression tests")
}
duration := time.Minute
done := make(chan struct{})
defer close(done)
ch := make(chan UUID, 10000)
ncpu := runtime.NumCPU()
switch ncpu {
case 0, 1:
// We can't run the test effectively.
t.Skip("skipping race test, only one CPU detected")
return
default:
runtime.GOMAXPROCS(ncpu)
}
for i := 0; i < ncpu; i++ {
go func() {
for {
select {
case <-done:
return
case ch <- Must(NewUUID()):
}
}
}()
}
uuids := make(map[string]bool)
cnt := 0
start := time.Now()
for u := range ch {
s := u.String()
if uuids[s] {
t.Errorf("duplicate uuid after %d in %v: %s", cnt, time.Since(start), s)
return
}
uuids[s] = true
if time.Since(start) > duration {
return
}
cnt++
}
}
+59
View File
@@ -0,0 +1,59 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"database/sql/driver"
"fmt"
)
// Scan implements sql.Scanner so UUIDs can be read from databases transparently
// Currently, database types that map to string and []byte are supported. Please
// consult database-specific driver documentation for matching types.
func (uuid *UUID) Scan(src interface{}) error {
switch src := src.(type) {
case nil:
return nil
case string:
// if an empty UUID comes from a table, we return a null UUID
if src == "" {
return nil
}
// see Parse for required string format
u, err := Parse(src)
if err != nil {
return fmt.Errorf("Scan: %v", err)
}
*uuid = u
case []byte:
// if an empty UUID comes from a table, we return a null UUID
if len(src) == 0 {
return nil
}
// assumes a simple slice of bytes if 16 bytes
// otherwise attempts to parse
if len(src) != 16 {
return uuid.Scan(string(src))
}
copy((*uuid)[:], src)
default:
return fmt.Errorf("Scan: unable to scan type %T into UUID", src)
}
return nil
}
// Value implements sql.Valuer so that UUIDs can be written to databases
// transparently. Currently, UUIDs map to strings. Please consult
// database-specific driver documentation for matching types.
func (uuid UUID) Value() (driver.Value, error) {
return uuid.String(), nil
}
+113
View File
@@ -0,0 +1,113 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"strings"
"testing"
)
func TestScan(t *testing.T) {
stringTest := "f47ac10b-58cc-0372-8567-0e02b2c3d479"
badTypeTest := 6
invalidTest := "f47ac10b-58cc-0372-8567-0e02b2c3d4"
byteTest := make([]byte, 16)
byteTestUUID := Must(Parse(stringTest))
copy(byteTest, byteTestUUID[:])
// sunny day tests
var uuid UUID
err := (&uuid).Scan(stringTest)
if err != nil {
t.Fatal(err)
}
err = (&uuid).Scan([]byte(stringTest))
if err != nil {
t.Fatal(err)
}
err = (&uuid).Scan(byteTest)
if err != nil {
t.Fatal(err)
}
// bad type tests
err = (&uuid).Scan(badTypeTest)
if err == nil {
t.Error("int correctly parsed and shouldn't have")
}
if !strings.Contains(err.Error(), "unable to scan type") {
t.Error("attempting to parse an int returned an incorrect error message")
}
// invalid/incomplete uuids
err = (&uuid).Scan(invalidTest)
if err == nil {
t.Error("invalid uuid was parsed without error")
}
if !strings.Contains(err.Error(), "invalid UUID") {
t.Error("attempting to parse an invalid UUID returned an incorrect error message")
}
err = (&uuid).Scan(byteTest[:len(byteTest)-2])
if err == nil {
t.Error("invalid byte uuid was parsed without error")
}
if !strings.Contains(err.Error(), "invalid UUID") {
t.Error("attempting to parse an invalid byte UUID returned an incorrect error message")
}
// empty tests
uuid = UUID{}
var emptySlice []byte
err = (&uuid).Scan(emptySlice)
if err != nil {
t.Fatal(err)
}
for _, v := range uuid {
if v != 0 {
t.Error("UUID was not nil after scanning empty byte slice")
}
}
uuid = UUID{}
var emptyString string
err = (&uuid).Scan(emptyString)
if err != nil {
t.Fatal(err)
}
for _, v := range uuid {
if v != 0 {
t.Error("UUID was not nil after scanning empty byte slice")
}
}
uuid = UUID{}
err = (&uuid).Scan(nil)
if err != nil {
t.Fatal(err)
}
for _, v := range uuid {
if v != 0 {
t.Error("UUID was not nil after scanning nil")
}
}
}
func TestValue(t *testing.T) {
stringTest := "f47ac10b-58cc-0372-8567-0e02b2c3d479"
uuid := Must(Parse(stringTest))
val, _ := uuid.Value()
if val != stringTest {
t.Error("Value() did not return expected string")
}
}
+123
View File
@@ -0,0 +1,123 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/binary"
"sync"
"time"
)
// A Time represents a time as the number of 100's of nanoseconds since 15 Oct
// 1582.
type Time int64
const (
lillian = 2299160 // Julian day of 15 Oct 1582
unix = 2440587 // Julian day of 1 Jan 1970
epoch = unix - lillian // Days between epochs
g1582 = epoch * 86400 // seconds between epochs
g1582ns100 = g1582 * 10000000 // 100s of a nanoseconds between epochs
)
var (
timeMu sync.Mutex
lasttime uint64 // last time we returned
clockSeq uint16 // clock sequence for this run
timeNow = time.Now // for testing
)
// UnixTime converts t the number of seconds and nanoseconds using the Unix
// epoch of 1 Jan 1970.
func (t Time) UnixTime() (sec, nsec int64) {
sec = int64(t - g1582ns100)
nsec = (sec % 10000000) * 100
sec /= 10000000
return sec, nsec
}
// GetTime returns the current Time (100s of nanoseconds since 15 Oct 1582) and
// clock sequence as well as adjusting the clock sequence as needed. An error
// is returned if the current time cannot be determined.
func GetTime() (Time, uint16, error) {
defer timeMu.Unlock()
timeMu.Lock()
return getTime()
}
func getTime() (Time, uint16, error) {
t := timeNow()
// If we don't have a clock sequence already, set one.
if clockSeq == 0 {
setClockSequence(-1)
}
now := uint64(t.UnixNano()/100) + g1582ns100
// If time has gone backwards with this clock sequence then we
// increment the clock sequence
if now <= lasttime {
clockSeq = ((clockSeq + 1) & 0x3fff) | 0x8000
}
lasttime = now
return Time(now), clockSeq, nil
}
// ClockSequence returns the current clock sequence, generating one if not
// already set. The clock sequence is only used for Version 1 UUIDs.
//
// The uuid package does not use global static storage for the clock sequence or
// the last time a UUID was generated. Unless SetClockSequence is used, a new
// random clock sequence is generated the first time a clock sequence is
// requested by ClockSequence, GetTime, or NewUUID. (section 4.2.1.1)
func ClockSequence() int {
defer timeMu.Unlock()
timeMu.Lock()
return clockSequence()
}
func clockSequence() int {
if clockSeq == 0 {
setClockSequence(-1)
}
return int(clockSeq & 0x3fff)
}
// SetClockSequence sets the clock sequence to the lower 14 bits of seq. Setting to
// -1 causes a new sequence to be generated.
func SetClockSequence(seq int) {
defer timeMu.Unlock()
timeMu.Lock()
setClockSequence(seq)
}
func setClockSequence(seq int) {
if seq == -1 {
var b [2]byte
randomBits(b[:]) // clock sequence
seq = int(b[0])<<8 | int(b[1])
}
oldSeq := clockSeq
clockSeq = uint16(seq&0x3fff) | 0x8000 // Set our variant
if oldSeq != clockSeq {
lasttime = 0
}
}
// Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in
// uuid. The time is only defined for version 1 and 2 UUIDs.
func (uuid UUID) Time() Time {
time := int64(binary.BigEndian.Uint32(uuid[0:4]))
time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32
time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48
return Time(time)
}
// ClockSequence returns the clock sequence encoded in uuid.
// The clock sequence is only well defined for version 1 and 2 UUIDs.
func (uuid UUID) ClockSequence() int {
return int(binary.BigEndian.Uint16(uuid[8:10])) & 0x3fff
}
+43
View File
@@ -0,0 +1,43 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"io"
)
// randomBits completely fills slice b with random data.
func randomBits(b []byte) {
if _, err := io.ReadFull(rander, b); err != nil {
panic(err.Error()) // rand should never fail
}
}
// xvalues returns the value of a byte as a hexadecimal digit or 255.
var xvalues = [256]byte{
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 10, 11, 12, 13, 14, 15, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255,
}
// xtob converts hex characters x1 and x2 into a byte.
func xtob(x1, x2 byte) (byte, bool) {
b1 := xvalues[x1]
b2 := xvalues[x2]
return (b1 << 4) | b2, b1 != 255 && b2 != 255
}
+245
View File
@@ -0,0 +1,245 @@
// Copyright 2018 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"bytes"
"crypto/rand"
"encoding/hex"
"errors"
"fmt"
"io"
"strings"
)
// A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC
// 4122.
type UUID [16]byte
// A Version represents a UUID's version.
type Version byte
// A Variant represents a UUID's variant.
type Variant byte
// Constants returned by Variant.
const (
Invalid = Variant(iota) // Invalid UUID
RFC4122 // The variant specified in RFC4122
Reserved // Reserved, NCS backward compatibility.
Microsoft // Reserved, Microsoft Corporation backward compatibility.
Future // Reserved for future definition.
)
var rander = rand.Reader // random function
// Parse decodes s into a UUID or returns an error. Both the standard UUID
// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and
// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the
// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex
// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.
func Parse(s string) (UUID, error) {
var uuid UUID
switch len(s) {
// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
case 36:
// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
case 36 + 9:
if strings.ToLower(s[:9]) != "urn:uuid:" {
return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9])
}
s = s[9:]
// {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}
case 36 + 2:
s = s[1:]
// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
case 32:
var ok bool
for i := range uuid {
uuid[i], ok = xtob(s[i*2], s[i*2+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
}
return uuid, nil
default:
return uuid, fmt.Errorf("invalid UUID length: %d", len(s))
}
// s is now at least 36 bytes long
// it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' {
return uuid, errors.New("invalid UUID format")
}
for i, x := range [16]int{
0, 2, 4, 6,
9, 11,
14, 16,
19, 21,
24, 26, 28, 30, 32, 34} {
v, ok := xtob(s[x], s[x+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
uuid[i] = v
}
return uuid, nil
}
// ParseBytes is like Parse, except it parses a byte slice instead of a string.
func ParseBytes(b []byte) (UUID, error) {
var uuid UUID
switch len(b) {
case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) {
return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9])
}
b = b[9:]
case 36 + 2: // {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}
b = b[1:]
case 32: // xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
var ok bool
for i := 0; i < 32; i += 2 {
uuid[i/2], ok = xtob(b[i], b[i+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
}
return uuid, nil
default:
return uuid, fmt.Errorf("invalid UUID length: %d", len(b))
}
// s is now at least 36 bytes long
// it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
if b[8] != '-' || b[13] != '-' || b[18] != '-' || b[23] != '-' {
return uuid, errors.New("invalid UUID format")
}
for i, x := range [16]int{
0, 2, 4, 6,
9, 11,
14, 16,
19, 21,
24, 26, 28, 30, 32, 34} {
v, ok := xtob(b[x], b[x+1])
if !ok {
return uuid, errors.New("invalid UUID format")
}
uuid[i] = v
}
return uuid, nil
}
// MustParse is like Parse but panics if the string cannot be parsed.
// It simplifies safe initialization of global variables holding compiled UUIDs.
func MustParse(s string) UUID {
uuid, err := Parse(s)
if err != nil {
panic(`uuid: Parse(` + s + `): ` + err.Error())
}
return uuid
}
// FromBytes creates a new UUID from a byte slice. Returns an error if the slice
// does not have a length of 16. The bytes are copied from the slice.
func FromBytes(b []byte) (uuid UUID, err error) {
err = uuid.UnmarshalBinary(b)
return uuid, err
}
// Must returns uuid if err is nil and panics otherwise.
func Must(uuid UUID, err error) UUID {
if err != nil {
panic(err)
}
return uuid
}
// String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
// , or "" if uuid is invalid.
func (uuid UUID) String() string {
var buf [36]byte
encodeHex(buf[:], uuid)
return string(buf[:])
}
// URN returns the RFC 2141 URN form of uuid,
// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx, or "" if uuid is invalid.
func (uuid UUID) URN() string {
var buf [36 + 9]byte
copy(buf[:], "urn:uuid:")
encodeHex(buf[9:], uuid)
return string(buf[:])
}
func encodeHex(dst []byte, uuid UUID) {
hex.Encode(dst, uuid[:4])
dst[8] = '-'
hex.Encode(dst[9:13], uuid[4:6])
dst[13] = '-'
hex.Encode(dst[14:18], uuid[6:8])
dst[18] = '-'
hex.Encode(dst[19:23], uuid[8:10])
dst[23] = '-'
hex.Encode(dst[24:], uuid[10:])
}
// Variant returns the variant encoded in uuid.
func (uuid UUID) Variant() Variant {
switch {
case (uuid[8] & 0xc0) == 0x80:
return RFC4122
case (uuid[8] & 0xe0) == 0xc0:
return Microsoft
case (uuid[8] & 0xe0) == 0xe0:
return Future
default:
return Reserved
}
}
// Version returns the version of uuid.
func (uuid UUID) Version() Version {
return Version(uuid[6] >> 4)
}
func (v Version) String() string {
if v > 15 {
return fmt.Sprintf("BAD_VERSION_%d", v)
}
return fmt.Sprintf("VERSION_%d", v)
}
func (v Variant) String() string {
switch v {
case RFC4122:
return "RFC4122"
case Reserved:
return "Reserved"
case Microsoft:
return "Microsoft"
case Future:
return "Future"
case Invalid:
return "Invalid"
}
return fmt.Sprintf("BadVariant%d", int(v))
}
// SetRand sets the random number generator to r, which implements io.Reader.
// If r.Read returns an error when the package requests random data then
// a panic will be issued.
//
// Calling SetRand with nil sets the random number generator to the default
// generator.
func SetRand(r io.Reader) {
if r == nil {
rander = rand.Reader
return
}
rander = r
}
+559
View File
@@ -0,0 +1,559 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"bytes"
"fmt"
"os"
"runtime"
"strings"
"testing"
"time"
"unsafe"
)
type test struct {
in string
version Version
variant Variant
isuuid bool
}
var tests = []test{
{"f47ac10b-58cc-0372-8567-0e02b2c3d479", 0, RFC4122, true},
{"f47ac10b-58cc-1372-8567-0e02b2c3d479", 1, RFC4122, true},
{"f47ac10b-58cc-2372-8567-0e02b2c3d479", 2, RFC4122, true},
{"f47ac10b-58cc-3372-8567-0e02b2c3d479", 3, RFC4122, true},
{"f47ac10b-58cc-4372-8567-0e02b2c3d479", 4, RFC4122, true},
{"f47ac10b-58cc-5372-8567-0e02b2c3d479", 5, RFC4122, true},
{"f47ac10b-58cc-6372-8567-0e02b2c3d479", 6, RFC4122, true},
{"f47ac10b-58cc-7372-8567-0e02b2c3d479", 7, RFC4122, true},
{"f47ac10b-58cc-8372-8567-0e02b2c3d479", 8, RFC4122, true},
{"f47ac10b-58cc-9372-8567-0e02b2c3d479", 9, RFC4122, true},
{"f47ac10b-58cc-a372-8567-0e02b2c3d479", 10, RFC4122, true},
{"f47ac10b-58cc-b372-8567-0e02b2c3d479", 11, RFC4122, true},
{"f47ac10b-58cc-c372-8567-0e02b2c3d479", 12, RFC4122, true},
{"f47ac10b-58cc-d372-8567-0e02b2c3d479", 13, RFC4122, true},
{"f47ac10b-58cc-e372-8567-0e02b2c3d479", 14, RFC4122, true},
{"f47ac10b-58cc-f372-8567-0e02b2c3d479", 15, RFC4122, true},
{"urn:uuid:f47ac10b-58cc-4372-0567-0e02b2c3d479", 4, Reserved, true},
{"URN:UUID:f47ac10b-58cc-4372-0567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-0567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-1567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-2567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-3567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-4567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-5567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-6567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-7567-0e02b2c3d479", 4, Reserved, true},
{"f47ac10b-58cc-4372-8567-0e02b2c3d479", 4, RFC4122, true},
{"f47ac10b-58cc-4372-9567-0e02b2c3d479", 4, RFC4122, true},
{"f47ac10b-58cc-4372-a567-0e02b2c3d479", 4, RFC4122, true},
{"f47ac10b-58cc-4372-b567-0e02b2c3d479", 4, RFC4122, true},
{"f47ac10b-58cc-4372-c567-0e02b2c3d479", 4, Microsoft, true},
{"f47ac10b-58cc-4372-d567-0e02b2c3d479", 4, Microsoft, true},
{"f47ac10b-58cc-4372-e567-0e02b2c3d479", 4, Future, true},
{"f47ac10b-58cc-4372-f567-0e02b2c3d479", 4, Future, true},
{"f47ac10b158cc-5372-a567-0e02b2c3d479", 0, Invalid, false},
{"f47ac10b-58cc25372-a567-0e02b2c3d479", 0, Invalid, false},
{"f47ac10b-58cc-53723a567-0e02b2c3d479", 0, Invalid, false},
{"f47ac10b-58cc-5372-a56740e02b2c3d479", 0, Invalid, false},
{"f47ac10b-58cc-5372-a567-0e02-2c3d479", 0, Invalid, false},
{"g47ac10b-58cc-4372-a567-0e02b2c3d479", 0, Invalid, false},
{"{f47ac10b-58cc-0372-8567-0e02b2c3d479}", 0, RFC4122, true},
{"{f47ac10b-58cc-0372-8567-0e02b2c3d479", 0, Invalid, false},
{"f47ac10b-58cc-0372-8567-0e02b2c3d479}", 0, Invalid, false},
{"f47ac10b58cc037285670e02b2c3d479", 0, RFC4122, true},
{"f47ac10b58cc037285670e02b2c3d4790", 0, Invalid, false},
{"f47ac10b58cc037285670e02b2c3d47", 0, Invalid, false},
}
var constants = []struct {
c interface{}
name string
}{
{Person, "Person"},
{Group, "Group"},
{Org, "Org"},
{Invalid, "Invalid"},
{RFC4122, "RFC4122"},
{Reserved, "Reserved"},
{Microsoft, "Microsoft"},
{Future, "Future"},
{Domain(17), "Domain17"},
{Variant(42), "BadVariant42"},
}
func testTest(t *testing.T, in string, tt test) {
uuid, err := Parse(in)
if ok := (err == nil); ok != tt.isuuid {
t.Errorf("Parse(%s) got %v expected %v\b", in, ok, tt.isuuid)
}
if err != nil {
return
}
if v := uuid.Variant(); v != tt.variant {
t.Errorf("Variant(%s) got %d expected %d\b", in, v, tt.variant)
}
if v := uuid.Version(); v != tt.version {
t.Errorf("Version(%s) got %d expected %d\b", in, v, tt.version)
}
}
func testBytes(t *testing.T, in []byte, tt test) {
uuid, err := ParseBytes(in)
if ok := (err == nil); ok != tt.isuuid {
t.Errorf("ParseBytes(%s) got %v expected %v\b", in, ok, tt.isuuid)
}
if err != nil {
return
}
suuid, _ := Parse(string(in))
if uuid != suuid {
t.Errorf("ParseBytes(%s) got %v expected %v\b", in, uuid, suuid)
}
}
func TestUUID(t *testing.T) {
for _, tt := range tests {
testTest(t, tt.in, tt)
testTest(t, strings.ToUpper(tt.in), tt)
testBytes(t, []byte(tt.in), tt)
}
}
func TestFromBytes(t *testing.T) {
b := []byte{
0x7d, 0x44, 0x48, 0x40,
0x9d, 0xc0,
0x11, 0xd1,
0xb2, 0x45,
0x5f, 0xfd, 0xce, 0x74, 0xfa, 0xd2,
}
uuid, err := FromBytes(b)
if err != nil {
t.Fatalf("%s", err)
}
for i := 0; i < len(uuid); i++ {
if b[i] != uuid[i] {
t.Fatalf("FromBytes() got %v expected %v\b", uuid[:], b)
}
}
}
func TestConstants(t *testing.T) {
for x, tt := range constants {
v, ok := tt.c.(fmt.Stringer)
if !ok {
t.Errorf("%x: %v: not a stringer", x, v)
} else if s := v.String(); s != tt.name {
v, _ := tt.c.(int)
t.Errorf("%x: Constant %T:%d gives %q, expected %q", x, tt.c, v, s, tt.name)
}
}
}
func TestRandomUUID(t *testing.T) {
m := make(map[string]bool)
for x := 1; x < 32; x++ {
uuid := New()
s := uuid.String()
if m[s] {
t.Errorf("NewRandom returned duplicated UUID %s", s)
}
m[s] = true
if v := uuid.Version(); v != 4 {
t.Errorf("Random UUID of version %s", v)
}
if uuid.Variant() != RFC4122 {
t.Errorf("Random UUID is variant %d", uuid.Variant())
}
}
}
func TestNew(t *testing.T) {
m := make(map[UUID]bool)
for x := 1; x < 32; x++ {
s := New()
if m[s] {
t.Errorf("New returned duplicated UUID %s", s)
}
m[s] = true
uuid, err := Parse(s.String())
if err != nil {
t.Errorf("New.String() returned %q which does not decode", s)
continue
}
if v := uuid.Version(); v != 4 {
t.Errorf("Random UUID of version %s", v)
}
if uuid.Variant() != RFC4122 {
t.Errorf("Random UUID is variant %d", uuid.Variant())
}
}
}
func TestClockSeq(t *testing.T) {
// Fake time.Now for this test to return a monotonically advancing time; restore it at end.
defer func(orig func() time.Time) { timeNow = orig }(timeNow)
monTime := time.Now()
timeNow = func() time.Time {
monTime = monTime.Add(1 * time.Second)
return monTime
}
SetClockSequence(-1)
uuid1, err := NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
uuid2, err := NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
if s1, s2 := uuid1.ClockSequence(), uuid2.ClockSequence(); s1 != s2 {
t.Errorf("clock sequence %d != %d", s1, s2)
}
SetClockSequence(-1)
uuid2, err = NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
// Just on the very off chance we generated the same sequence
// two times we try again.
if uuid1.ClockSequence() == uuid2.ClockSequence() {
SetClockSequence(-1)
uuid2, err = NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
}
if s1, s2 := uuid1.ClockSequence(), uuid2.ClockSequence(); s1 == s2 {
t.Errorf("Duplicate clock sequence %d", s1)
}
SetClockSequence(0x1234)
uuid1, err = NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
if seq := uuid1.ClockSequence(); seq != 0x1234 {
t.Errorf("%s: expected seq 0x1234 got 0x%04x", uuid1, seq)
}
}
func TestCoding(t *testing.T) {
text := "7d444840-9dc0-11d1-b245-5ffdce74fad2"
urn := "urn:uuid:7d444840-9dc0-11d1-b245-5ffdce74fad2"
data := UUID{
0x7d, 0x44, 0x48, 0x40,
0x9d, 0xc0,
0x11, 0xd1,
0xb2, 0x45,
0x5f, 0xfd, 0xce, 0x74, 0xfa, 0xd2,
}
if v := data.String(); v != text {
t.Errorf("%x: encoded to %s, expected %s", data, v, text)
}
if v := data.URN(); v != urn {
t.Errorf("%x: urn is %s, expected %s", data, v, urn)
}
uuid, err := Parse(text)
if err != nil {
t.Errorf("Parse returned unexpected error %v", err)
}
if data != uuid {
t.Errorf("%s: decoded to %s, expected %s", text, uuid, data)
}
}
func TestVersion1(t *testing.T) {
uuid1, err := NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
uuid2, err := NewUUID()
if err != nil {
t.Fatalf("could not create UUID: %v", err)
}
if uuid1 == uuid2 {
t.Errorf("%s:duplicate uuid", uuid1)
}
if v := uuid1.Version(); v != 1 {
t.Errorf("%s: version %s expected 1", uuid1, v)
}
if v := uuid2.Version(); v != 1 {
t.Errorf("%s: version %s expected 1", uuid2, v)
}
n1 := uuid1.NodeID()
n2 := uuid2.NodeID()
if !bytes.Equal(n1, n2) {
t.Errorf("Different nodes %x != %x", n1, n2)
}
t1 := uuid1.Time()
t2 := uuid2.Time()
q1 := uuid1.ClockSequence()
q2 := uuid2.ClockSequence()
switch {
case t1 == t2 && q1 == q2:
t.Error("time stopped")
case t1 > t2 && q1 == q2:
t.Error("time reversed")
case t1 < t2 && q1 != q2:
t.Error("clock sequence changed unexpectedly")
}
}
func TestNode(t *testing.T) {
// This test is mostly to make sure we don't leave nodeMu locked.
ifname = ""
if ni := NodeInterface(); ni != "" {
t.Errorf("NodeInterface got %q, want %q", ni, "")
}
if SetNodeInterface("xyzzy") {
t.Error("SetNodeInterface succeeded on a bad interface name")
}
if !SetNodeInterface("") {
t.Error("SetNodeInterface failed")
}
if runtime.GOARCH != "js" {
if ni := NodeInterface(); ni == "" {
t.Error("NodeInterface returned an empty string")
}
}
ni := NodeID()
if len(ni) != 6 {
t.Errorf("ni got %d bytes, want 6", len(ni))
}
hasData := false
for _, b := range ni {
if b != 0 {
hasData = true
}
}
if !hasData {
t.Error("nodeid is all zeros")
}
id := []byte{1, 2, 3, 4, 5, 6, 7, 8}
SetNodeID(id)
ni = NodeID()
if !bytes.Equal(ni, id[:6]) {
t.Errorf("got nodeid %v, want %v", ni, id[:6])
}
if ni := NodeInterface(); ni != "user" {
t.Errorf("got interface %q, want %q", ni, "user")
}
}
func TestNodeAndTime(t *testing.T) {
// Time is February 5, 1998 12:30:23.136364800 AM GMT
uuid, err := Parse("7d444840-9dc0-11d1-b245-5ffdce74fad2")
if err != nil {
t.Fatalf("Parser returned unexpected error %v", err)
}
node := []byte{0x5f, 0xfd, 0xce, 0x74, 0xfa, 0xd2}
ts := uuid.Time()
c := time.Unix(ts.UnixTime())
want := time.Date(1998, 2, 5, 0, 30, 23, 136364800, time.UTC)
if !c.Equal(want) {
t.Errorf("Got time %v, want %v", c, want)
}
if !bytes.Equal(node, uuid.NodeID()) {
t.Errorf("Expected node %v got %v", node, uuid.NodeID())
}
}
func TestMD5(t *testing.T) {
uuid := NewMD5(NameSpaceDNS, []byte("python.org")).String()
want := "6fa459ea-ee8a-3ca4-894e-db77e160355e"
if uuid != want {
t.Errorf("MD5: got %q expected %q", uuid, want)
}
}
func TestSHA1(t *testing.T) {
uuid := NewSHA1(NameSpaceDNS, []byte("python.org")).String()
want := "886313e1-3b8a-5372-9b90-0c9aee199e5d"
if uuid != want {
t.Errorf("SHA1: got %q expected %q", uuid, want)
}
}
func TestNodeID(t *testing.T) {
nid := []byte{1, 2, 3, 4, 5, 6}
SetNodeInterface("")
s := NodeInterface()
if runtime.GOARCH != "js" {
if s == "" || s == "user" {
t.Errorf("NodeInterface %q after SetInterface", s)
}
}
node1 := NodeID()
if node1 == nil {
t.Error("NodeID nil after SetNodeInterface", s)
}
SetNodeID(nid)
s = NodeInterface()
if s != "user" {
t.Errorf("Expected NodeInterface %q got %q", "user", s)
}
node2 := NodeID()
if node2 == nil {
t.Error("NodeID nil after SetNodeID", s)
}
if bytes.Equal(node1, node2) {
t.Error("NodeID not changed after SetNodeID", s)
} else if !bytes.Equal(nid, node2) {
t.Errorf("NodeID is %x, expected %x", node2, nid)
}
}
func testDCE(t *testing.T, name string, uuid UUID, err error, domain Domain, id uint32) {
if err != nil {
t.Errorf("%s failed: %v", name, err)
return
}
if v := uuid.Version(); v != 2 {
t.Errorf("%s: %s: expected version 2, got %s", name, uuid, v)
return
}
if v := uuid.Domain(); v != domain {
t.Errorf("%s: %s: expected domain %d, got %d", name, uuid, domain, v)
}
if v := uuid.ID(); v != id {
t.Errorf("%s: %s: expected id %d, got %d", name, uuid, id, v)
}
}
func TestDCE(t *testing.T) {
uuid, err := NewDCESecurity(42, 12345678)
testDCE(t, "NewDCESecurity", uuid, err, 42, 12345678)
uuid, err = NewDCEPerson()
testDCE(t, "NewDCEPerson", uuid, err, Person, uint32(os.Getuid()))
uuid, err = NewDCEGroup()
testDCE(t, "NewDCEGroup", uuid, err, Group, uint32(os.Getgid()))
}
type badRand struct{}
func (r badRand) Read(buf []byte) (int, error) {
for i := range buf {
buf[i] = byte(i)
}
return len(buf), nil
}
func TestBadRand(t *testing.T) {
SetRand(badRand{})
uuid1 := New()
uuid2 := New()
if uuid1 != uuid2 {
t.Errorf("expected duplicates, got %q and %q", uuid1, uuid2)
}
SetRand(nil)
uuid1 = New()
uuid2 = New()
if uuid1 == uuid2 {
t.Errorf("unexpected duplicates, got %q", uuid1)
}
}
var asString = "f47ac10b-58cc-0372-8567-0e02b2c3d479"
var asBytes = []byte(asString)
func BenchmarkParse(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := Parse(asString)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkParseBytes(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := ParseBytes(asBytes)
if err != nil {
b.Fatal(err)
}
}
}
// parseBytesUnsafe is to benchmark using unsafe.
func parseBytesUnsafe(b []byte) (UUID, error) {
return Parse(*(*string)(unsafe.Pointer(&b)))
}
func BenchmarkParseBytesUnsafe(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := parseBytesUnsafe(asBytes)
if err != nil {
b.Fatal(err)
}
}
}
// parseBytesCopy is to benchmark not using unsafe.
func parseBytesCopy(b []byte) (UUID, error) {
return Parse(string(b))
}
func BenchmarkParseBytesCopy(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := parseBytesCopy(asBytes)
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkNew(b *testing.B) {
for i := 0; i < b.N; i++ {
New()
}
}
func BenchmarkUUID_String(b *testing.B) {
uuid, err := Parse("f47ac10b-58cc-0372-8567-0e02b2c3d479")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
if uuid.String() == "" {
b.Fatal("invalid uuid")
}
}
}
func BenchmarkUUID_URN(b *testing.B) {
uuid, err := Parse("f47ac10b-58cc-0372-8567-0e02b2c3d479")
if err != nil {
b.Fatal(err)
}
for i := 0; i < b.N; i++ {
if uuid.URN() == "" {
b.Fatal("invalid uuid")
}
}
}
+44
View File
@@ -0,0 +1,44 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import (
"encoding/binary"
)
// NewUUID returns a Version 1 UUID based on the current NodeID and clock
// sequence, and the current time. If the NodeID has not been set by SetNodeID
// or SetNodeInterface then it will be set automatically. If the NodeID cannot
// be set NewUUID returns nil. If clock sequence has not been set by
// SetClockSequence then it will be set automatically. If GetTime fails to
// return the current NewUUID returns nil and an error.
//
// In most cases, New should be used.
func NewUUID() (UUID, error) {
nodeMu.Lock()
if nodeID == zeroID {
setNodeInterface("")
}
nodeMu.Unlock()
var uuid UUID
now, seq, err := GetTime()
if err != nil {
return uuid, err
}
timeLow := uint32(now & 0xffffffff)
timeMid := uint16((now >> 32) & 0xffff)
timeHi := uint16((now >> 48) & 0x0fff)
timeHi |= 0x1000 // Version 1
binary.BigEndian.PutUint32(uuid[0:], timeLow)
binary.BigEndian.PutUint16(uuid[4:], timeMid)
binary.BigEndian.PutUint16(uuid[6:], timeHi)
binary.BigEndian.PutUint16(uuid[8:], seq)
copy(uuid[10:], nodeID[:])
return uuid, nil
}
+38
View File
@@ -0,0 +1,38 @@
// Copyright 2016 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package uuid
import "io"
// New creates a new random UUID or panics. New is equivalent to
// the expression
//
// uuid.Must(uuid.NewRandom())
func New() UUID {
return Must(NewRandom())
}
// NewRandom returns a Random (Version 4) UUID.
//
// The strength of the UUIDs is based on the strength of the crypto/rand
// package.
//
// A note about uniqueness derived from the UUID Wikipedia entry:
//
// Randomly generated UUIDs have 122 random bits. One's annual risk of being
// hit by a meteorite is estimated to be one chance in 17 billion, that
// means the probability is about 0.00000000006 (6 × 10−11),
// equivalent to the odds of creating a few tens of trillions of UUIDs in a
// year and having one duplicate.
func NewRandom() (UUID, error) {
var uuid UUID
_, err := io.ReadFull(rander, uuid[:])
if err != nil {
return Nil, err
}
uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4
uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10
return uuid, nil
}
+14 -4
View File
@@ -1,8 +1,18 @@
sudo: false
language: go
go:
- 1.1
- 1.2
- 1.3
- 1.4
- 1.7
- 1.8
- 1.9
- "1.10"
- tip
before_install:
- go get golang.org/x/tools/cmd/cover
- go get github.com/mattn/goveralls
- go get github.com/golang/lint/golint
script:
- go test -v -covermode=count -coverprofile=coverage.out
- go vet ./...
- test -z "$(gofmt -d -s . | tee /dev/stderr)"
- test -z "$(golint ./... | tee /dev/stderr)"
- $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci
+76 -133
View File
@@ -1,58 +1,37 @@
# HttpRouter [![Build Status](https://travis-ci.org/julienschmidt/httprouter.png?branch=master)](https://travis-ci.org/julienschmidt/httprouter) [![Coverage](http://gocover.io/_badge/github.com/julienschmidt/httprouter?0)](http://gocover.io/github.com/julienschmidt/httprouter) [![GoDoc](http://godoc.org/github.com/julienschmidt/httprouter?status.png)](http://godoc.org/github.com/julienschmidt/httprouter)
# HttpRouter [![Build Status](https://travis-ci.org/julienschmidt/httprouter.svg?branch=master)](https://travis-ci.org/julienschmidt/httprouter) [![Coverage Status](https://coveralls.io/repos/github/julienschmidt/httprouter/badge.svg?branch=master)](https://coveralls.io/github/julienschmidt/httprouter?branch=master) [![GoDoc](https://godoc.org/github.com/julienschmidt/httprouter?status.svg)](http://godoc.org/github.com/julienschmidt/httprouter)
HttpRouter is a lightweight high performance HTTP request router
(also called *multiplexer* or just *mux* for short) for [Go](http://golang.org/).
HttpRouter is a lightweight high performance HTTP request router (also called *multiplexer* or just *mux* for short) for [Go](https://golang.org/).
In contrast to the [default mux](http://golang.org/pkg/net/http/#ServeMux) of Go's net/http package, this router supports
variables in the routing pattern and matches against the request method.
It also scales better.
In contrast to the [default mux](https://golang.org/pkg/net/http/#ServeMux) of Go's `net/http` package, this router supports variables in the routing pattern and matches against the request method. It also scales better.
The router is optimized for high performance and a small memory footprint.
It scales well even with very long paths and a large number of routes.
A compressing dynamic trie (radix tree) structure is used for efficient matching.
The router is optimized for high performance and a small memory footprint. It scales well even with very long paths and a large number of routes. A compressing dynamic trie (radix tree) structure is used for efficient matching.
## Features
**Only explicit matches:** With other routers, like [http.ServeMux](http://golang.org/pkg/net/http/#ServeMux),
a requested URL path could match multiple patterns. Therefore they have some
awkward pattern priority rules, like *longest match* or *first registered,
first matched*. By design of this router, a request can only match exactly one
or no route. As a result, there are also no unintended matches, which makes it
great for SEO and improves the user experience.
**Stop caring about trailing slashes:** Choose the URL style you like, the
router automatically redirects the client if a trailing slash is missing or if
there is one extra. Of course it only does so, if the new path has a handler.
If you don't like it, you can [turn off this behavior](http://godoc.org/github.com/julienschmidt/httprouter#Router.RedirectTrailingSlash).
**Only explicit matches:** With other routers, like [`http.ServeMux`](https://golang.org/pkg/net/http/#ServeMux), a requested URL path could match multiple patterns. Therefore they have some awkward pattern priority rules, like *longest match* or *first registered, first matched*. By design of this router, a request can only match exactly one or no route. As a result, there are also no unintended matches, which makes it great for SEO and improves the user experience.
**Path auto-correction:** Besides detecting the missing or additional trailing
slash at no extra cost, the router can also fix wrong cases and remove
superfluous path elements (like `../` or `//`).
Is [CAPTAIN CAPS LOCK](http://www.urbandictionary.com/define.php?term=Captain+Caps+Lock) one of your users?
HttpRouter can help him by making a case-insensitive look-up and redirecting him
to the correct URL.
**Stop caring about trailing slashes:** Choose the URL style you like, the router automatically redirects the client if a trailing slash is missing or if there is one extra. Of course it only does so, if the new path has a handler. If you don't like it, you can [turn off this behavior](https://godoc.org/github.com/julienschmidt/httprouter#Router.RedirectTrailingSlash).
**Parameters in your routing pattern:** Stop parsing the requested URL path,
just give the path segment a name and the router delivers the dynamic value to
you. Because of the design of the router, path parameters are very cheap.
**Path auto-correction:** Besides detecting the missing or additional trailing slash at no extra cost, the router can also fix wrong cases and remove superfluous path elements (like `../` or `//`). Is [CAPTAIN CAPS LOCK](http://www.urbandictionary.com/define.php?term=Captain+Caps+Lock) one of your users? HttpRouter can help him by making a case-insensitive look-up and redirecting him to the correct URL.
**Zero Garbage:** The matching and dispatching process generates zero bytes of
garbage. In fact, the only heap allocations that are made, is by building the
slice of the key-value pairs for path parameters. If the request path contains
no parameters, not a single heap allocation is necessary.
**Parameters in your routing pattern:** Stop parsing the requested URL path, just give the path segment a name and the router delivers the dynamic value to you. Because of the design of the router, path parameters are very cheap.
**Best Performance:** [Benchmarks speak for themselves](https://github.com/julienschmidt/go-http-routing-benchmark).
See below for technical details of the implementation.
**Zero Garbage:** The matching and dispatching process generates zero bytes of garbage. The only heap allocations that are made are building the slice of the key-value pairs for path parameters, and building new context and request objects (the latter only in the standard `Handler`/`HandlerFunc` api). In the 3-argument API, if the request path contains no parameters not a single heap allocation is necessary.
**No more server crashes:** You can set a [Panic handler](http://godoc.org/github.com/julienschmidt/httprouter#Router.PanicHandler) to deal with panics
occurring during handling a HTTP request. The router then recovers and lets the
PanicHandler log what happened and deliver a nice error page.
**Best Performance:** [Benchmarks speak for themselves](https://github.com/julienschmidt/go-http-routing-benchmark). See below for technical details of the implementation.
Of course you can also set **custom [NotFound](http://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) and [MethodNotAllowed](http://godoc.org/github.com/julienschmidt/httprouter#Router.MethodNotAllowed) handlers** and [**serve static files**](http://godoc.org/github.com/julienschmidt/httprouter#Router.ServeFiles).
**No more server crashes:** You can set a [Panic handler](https://godoc.org/github.com/julienschmidt/httprouter#Router.PanicHandler) to deal with panics occurring during handling a HTTP request. The router then recovers and lets the `PanicHandler` log what happened and deliver a nice error page.
**Perfect for APIs:** The router design encourages to build sensible, hierarchical RESTful APIs. Moreover it has builtin native support for [OPTIONS requests](http://zacstewart.com/2012/04/14/http-options-method.html) and `405 Method Not Allowed` replies.
Of course you can also set **custom [`NotFound`](https://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) and [`MethodNotAllowed`](https://godoc.org/github.com/julienschmidt/httprouter#Router.MethodNotAllowed) handlers** and [**serve static files**](https://godoc.org/github.com/julienschmidt/httprouter#Router.ServeFiles).
## Usage
This is just a quick introduction, view the [GoDoc](http://godoc.org/github.com/julienschmidt/httprouter) for details.
Let's start with a trivial example:
```go
package main
@@ -81,12 +60,11 @@ func main() {
```
### Named parameters
As you can see, `:name` is a *named parameter*.
The values are accessible via `httprouter.Params`, which is just a slice of `httprouter.Param`s.
You can get the value of a parameter either by its index in the slice, or by using the `ByName(name)` method:
`:name` can be retrived by `ByName("name")`.
As you can see, `:name` is a *named parameter*. The values are accessible via `httprouter.Params`, which is just a slice of `httprouter.Param`s. You can get the value of a parameter either by its index in the slice, or by using the `ByName(name)` method: `:name` can be retrived by `ByName("name")`.
Named parameters only match a single path segment:
```
Pattern: /user/:user
@@ -99,9 +77,9 @@ Pattern: /user/:user
**Note:** Since this router has only explicit matches, you can not register static routes and parameters for the same path segment. For example you can not register the patterns `/user/new` and `/user/:user` for the same request method at the same time. The routing of different request methods is independent from each other.
### Catch-All parameters
The second type are *catch-all* parameters and have the form `*name`.
Like the name suggests, they match everything.
Therefore they must always be at the **end** of the pattern:
The second type are *catch-all* parameters and have the form `*name`. Like the name suggests, they match everything. Therefore they must always be at the **end** of the pattern:
```
Pattern: /src/*filepath
@@ -111,11 +89,8 @@ Pattern: /src/*filepath
```
## How does it work?
The router relies on a tree structure which makes heavy use of *common prefixes*,
it is basically a *compact* [*prefix tree*](http://en.wikipedia.org/wiki/Trie)
(or just [*Radix tree*](http://en.wikipedia.org/wiki/Radix_tree)).
Nodes with a common prefix also share a common parent. Here is a short example
what the routing tree for the `GET` request method could look like:
The router relies on a tree structure which makes heavy use of *common prefixes*, it is basically a *compact* [*prefix tree*](https://en.wikipedia.org/wiki/Trie) (or just [*Radix tree*](https://en.wikipedia.org/wiki/Radix_tree)). Nodes with a common prefix also share a common parent. Here is a short example what the routing tree for the `GET` request method could look like:
```
Priority Path Handle
@@ -130,33 +105,15 @@ Priority Path Handle
1 | └team\ *<7>
1 └contact\ *<8>
```
Every `*<num>` represents the memory address of a handler function (a pointer).
If you follow a path trough the tree from the root to the leaf, you get the
complete route path, e.g `\blog\:post\`, where `:post` is just a placeholder
([*parameter*](#named-parameters)) for an actual post name. Unlike hash-maps, a
tree structure also allows us to use dynamic parts like the `:post` parameter,
since we actually match against the routing patterns instead of just comparing
hashes. [As benchmarks show](https://github.com/julienschmidt/go-http-routing-benchmark),
this works very well and efficient.
Since URL paths have a hierarchical structure and make use only of a limited set
of characters (byte values), it is very likely that there are a lot of common
prefixes. This allows us to easily reduce the routing into ever smaller problems.
Moreover the router manages a separate tree for every request method.
For one thing it is more space efficient than holding a method->handle map in
every single node, for another thing is also allows us to greatly reduce the
routing problem before even starting the look-up in the prefix-tree.
Every `*<num>` represents the memory address of a handler function (a pointer). If you follow a path trough the tree from the root to the leaf, you get the complete route path, e.g `\blog\:post\`, where `:post` is just a placeholder ([*parameter*](#named-parameters)) for an actual post name. Unlike hash-maps, a tree structure also allows us to use dynamic parts like the `:post` parameter, since we actually match against the routing patterns instead of just comparing hashes. [As benchmarks show](https://github.com/julienschmidt/go-http-routing-benchmark), this works very well and efficient.
For even better scalability, the child nodes on each tree level are ordered by
priority, where the priority is just the number of handles registered in sub
nodes (children, grandchildren, and so on..).
This helps in two ways:
Since URL paths have a hierarchical structure and make use only of a limited set of characters (byte values), it is very likely that there are a lot of common prefixes. This allows us to easily reduce the routing into ever smaller problems. Moreover the router manages a separate tree for every request method. For one thing it is more space efficient than holding a method->handle map in every single node, it also allows us to greatly reduce the routing problem before even starting the look-up in the prefix-tree.
1. Nodes which are part of the most routing paths are evaluated first. This
helps to make as much routes as possible to be reachable as fast as possible.
2. It is some sort of cost compensation. The longest reachable path (highest
cost) can always be evaluated first. The following scheme visualizes the tree
structure. Nodes are evaluated from top to bottom and from left to right.
For even better scalability, the child nodes on each tree level are ordered by priority, where the priority is just the number of handles registered in sub nodes (children, grandchildren, and so on..). This helps in two ways:
1. Nodes which are part of the most routing paths are evaluated first. This helps to make as much routes as possible to be reachable as fast as possible.
2. It is some sort of cost compensation. The longest reachable path (highest cost) can always be evaluated first. The following scheme visualizes the tree structure. Nodes are evaluated from top to bottom and from left to right.
```
├------------
@@ -168,47 +125,38 @@ structure. Nodes are evaluated from top to bottom and from left to right.
└-
```
## Why doesn't this work with `http.Handler`?
## Why doesn't this work with http.Handler?
**It does!** The router itself implements the http.Handler interface.
Moreover the router provides convenient [adapters for http.Handler](http://godoc.org/github.com/julienschmidt/httprouter#Router.Handler)s and [http.HandlerFunc](http://godoc.org/github.com/julienschmidt/httprouter#Router.HandlerFunc)s
which allows them to be used as a [httprouter.Handle](http://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) when registering a route.
The only disadvantage is, that no parameter values can be retrieved when a
http.Handler or http.HandlerFunc is used, since there is no efficient way to
pass the values with the existing function parameters.
Therefore [httprouter.Handle](http://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) has a third function parameter.
**It does!** The router itself implements the `http.Handler` interface. Moreover the router provides convenient [adapters for `http.Handler`](https://godoc.org/github.com/julienschmidt/httprouter#Router.Handler)s and [`http.HandlerFunc`](https://godoc.org/github.com/julienschmidt/httprouter#Router.HandlerFunc)s which allows them to be used as a [`httprouter.Handle`](https://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) when registering a route. The only disadvantage is, that no parameter values can be retrieved when a `http.Handler` or `http.HandlerFunc` is used, since there is no efficient way to pass the values with the existing function parameters. Therefore [`httprouter.Handle`](https://godoc.org/github.com/julienschmidt/httprouter#Router.Handle) has a third function parameter.
Just try it out for yourself, the usage of HttpRouter is very straightforward. The package is compact and minimalistic, but also probably one of the easiest routers to set up.
## Where can I find Middleware *X*?
This package just provides a very efficient request router with a few extra
features. The router is just a [http.Handler](http://golang.org/pkg/net/http/#Handler),
you can chain any http.Handler compatible middleware before the router,
for example the [Gorilla handlers](http://www.gorillatoolkit.org/pkg/handlers).
Or you could [just write your own](http://justinas.org/writing-http-middleware-in-go/),
it's very easy!
This package just provides a very efficient request router with a few extra features. The router is just a [`http.Handler`](https://golang.org/pkg/net/http/#Handler), you can chain any http.Handler compatible middleware before the router, for example the [Gorilla handlers](http://www.gorillatoolkit.org/pkg/handlers). Or you could [just write your own](https://justinas.org/writing-http-middleware-in-go/), it's very easy!
Alternatively, you could try [a web framework based on HttpRouter](#web-frameworks-based-on-httprouter).
### Multi-domain / Sub-domains
Here is a quick example: Does your server serve multiple domains / hosts?
You want to use sub-domains?
Define a router per host!
```go
// We need an object that implements the http.Handler interface.
// Therefore we need a type for which we implement the ServeHTTP method.
// We just use a map here, in which we map host names (with port) to http.Handlers
type HostSwitch map[string]http.Handler
// Implement the ServerHTTP method on our new type
// Implement the ServeHTTP method on our new type
func (hs HostSwitch) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Check if a http.Handler is registered for the given host.
// If yes, use it to handle the request.
if handler := hs[r.Host]; handler != nil {
handler.ServeHTTP(w, r)
} else {
// Handle host names for wich no handler is registered
// Handle host names for which no handler is registered
http.Error(w, "Forbidden", 403) // Or Redirect?
}
}
@@ -230,94 +178,89 @@ func main() {
```
### Basic Authentication
Another quick example: Basic Authentification (RFC 2617) for handles:
Another quick example: Basic Authentication (RFC 2617) for handles:
```go
package main
import (
"bytes"
"encoding/base64"
"fmt"
"github.com/julienschmidt/httprouter"
"net/http"
"log"
"strings"
"fmt"
"log"
"net/http"
"github.com/julienschmidt/httprouter"
)
func BasicAuth(h httprouter.Handle, user, pass []byte) httprouter.Handle {
func BasicAuth(h httprouter.Handle, requiredUser, requiredPassword string) httprouter.Handle {
return func(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
const basicAuthPrefix string = "Basic "
// Get the Basic Authentication credentials
auth := r.Header.Get("Authorization")
if strings.HasPrefix(auth, basicAuthPrefix) {
// Check credentials
payload, err := base64.StdEncoding.DecodeString(auth[len(basicAuthPrefix):])
if err == nil {
pair := bytes.SplitN(payload, []byte(":"), 2)
if len(pair) == 2 &&
bytes.Equal(pair[0], user) &&
bytes.Equal(pair[1], pass) {
user, password, hasAuth := r.BasicAuth()
// Delegate request to the given handle
h(w, r, ps)
return
}
}
if hasAuth && user == requiredUser && password == requiredPassword {
// Delegate request to the given handle
h(w, r, ps)
} else {
// Request Basic Authentication otherwise
w.Header().Set("WWW-Authenticate", "Basic realm=Restricted")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
// Request Basic Authentication otherwise
w.Header().Set("WWW-Authenticate", "Basic realm=Restricted")
http.Error(w, http.StatusText(http.StatusUnauthorized), http.StatusUnauthorized)
}
}
func Index(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
fmt.Fprint(w, "Not protected!\n")
fmt.Fprint(w, "Not protected!\n")
}
func Protected(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
fmt.Fprint(w, "Protected!\n")
fmt.Fprint(w, "Protected!\n")
}
func main() {
user := []byte("gordon")
pass := []byte("secret!")
user := "gordon"
pass := "secret!"
router := httprouter.New()
router.GET("/", Index)
router.GET("/protected/", BasicAuth(Protected, user, pass))
router := httprouter.New()
router.GET("/", Index)
router.GET("/protected/", BasicAuth(Protected, user, pass))
log.Fatal(http.ListenAndServe(":8080", router))
log.Fatal(http.ListenAndServe(":8080", router))
}
```
## Chaining with the NotFound handler
**NOTE: It might be required to set [Router.HandleMethodNotAllowed](http://godoc.org/github.com/julienschmidt/httprouter#Router.HandleMethodNotAllowed) to `false` to avoid problems.**
**NOTE: It might be required to set [`Router.HandleMethodNotAllowed`](https://godoc.org/github.com/julienschmidt/httprouter#Router.HandleMethodNotAllowed) to `false` to avoid problems.**
You can use another [http.HandlerFunc](http://golang.org/pkg/net/http/#HandlerFunc), for example another router, to handle requests which could not be matched by this router by using the [Router.NotFound](http://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) handler. This allows chaining.
You can use another [`http.Handler`](https://golang.org/pkg/net/http/#Handler), for example another router, to handle requests which could not be matched by this router by using the [`Router.NotFound`](https://godoc.org/github.com/julienschmidt/httprouter#Router.NotFound) handler. This allows chaining.
### Static files
The `NotFound` handler can for example be used to serve static files from the root path `/` (like an index.html file along with other assets):
The `NotFound` handler can for example be used to serve static files from the root path `/` (like an `index.html` file along with other assets):
```go
// Serve static files from the ./public directory
router.NotFound = http.FileServer(http.Dir("public")).ServeHTTP
router.NotFound = http.FileServer(http.Dir("public"))
```
But this approach sidesteps the strict core rules of this router to avoid routing problems. A cleaner approach is to use a distinct sub-path for serving files, like `/static/*filepath` or `/files/*filepath`.
## Web Frameworks based on HttpRouter
If the HttpRouter is a bit too minimalistic for you, you might try one of the following more high-level 3rd-party web frameworks building upon the HttpRouter package:
* [Ace](https://github.com/plimble/ace): Blazing fast Go Web Framework
* [api2go](https://github.com/univedo/api2go): A JSON API Implementation for Go
* [api2go](https://github.com/manyminds/api2go): A JSON API Implementation for Go
* [Gin](https://github.com/gin-gonic/gin): Features a martini-like API with much better performance
* [Goat](https://github.com/bahlo/goat): A minimalistic REST API server in Go
* [goMiddlewareChain](https://github.com/TobiEiss/goMiddlewareChain): An express.js-like-middleware-chain
* [Hikaru](https://github.com/najeira/hikaru): Supports standalone and Google AppEngine
* [Hitch](https://github.com/nbio/hitch): Hitch ties httprouter, [httpcontext](https://github.com/nbio/httpcontext), and middleware up in a bow
* [httpway](https://github.com/corneldamian/httpway): Simple middleware extension with context for httprouter and a server with gracefully shutdown support
* [kami](https://github.com/guregu/kami): A tiny web framework using x/net/context
* [Medeina](https://github.com/imdario/medeina): Inspired by Ruby's Roda and Cuba
* [Neko](https://github.com/rocwong/neko): A lightweight web application framework for Golang
* [River](https://github.com/abiosoft/river): River is a simple and lightweight REST server
* [Roxanna](https://github.com/iamthemuffinman/Roxanna): An amalgamation of httprouter, better logging, and hot reload
* [siesta](https://github.com/VividCortex/siesta): Composable HTTP handlers with contexts
* [xmux](https://github.com/rs/xmux): xmux is a httprouter fork on top of xhandler (net/context aware)
+38
View File
@@ -0,0 +1,38 @@
// +build go1.7
package httprouter
import (
"context"
"net/http"
)
type paramsKey struct{}
// ParamsKey is the request context key under which URL params are stored.
//
// This is only present from go 1.7.
var ParamsKey = paramsKey{}
// Handler is an adapter which allows the usage of an http.Handler as a
// request handle. With go 1.7+, the Params will be available in the
// request context under ParamsKey.
func (r *Router) Handler(method, path string, handler http.Handler) {
r.Handle(method, path,
func(w http.ResponseWriter, req *http.Request, p Params) {
ctx := req.Context()
ctx = context.WithValue(ctx, ParamsKey, p)
req = req.WithContext(ctx)
handler.ServeHTTP(w, req)
},
)
}
// ParamsFromContext pulls the URL parameters from a request context,
// or returns nil if none are present.
//
// This is only present from go 1.7.
func ParamsFromContext(ctx context.Context) Params {
p, _ := ctx.Value(ParamsKey).(Params)
return p
}
+16
View File
@@ -0,0 +1,16 @@
// +build !go1.7
package httprouter
import "net/http"
// Handler is an adapter which allows the usage of an http.Handler as a
// request handle. With go 1.7+, the Params will be available in the
// request context under ParamsKey.
func (r *Router) Handler(method, path string, handler http.Handler) {
r.Handle(method, path,
func(w http.ResponseWriter, req *http.Request, _ Params) {
handler.ServeHTTP(w, req)
},
)
}
+3 -3
View File
@@ -41,7 +41,7 @@ func CleanPath(p string) string {
buf[0] = '/'
}
trailing := n > 2 && p[n-1] == '/'
trailing := n > 1 && p[n-1] == '/'
// A bit more clunky without a 'lazybuf' like the path package, but the loop
// gets completely inlined (bufApp). So in contrast to the path package this
@@ -59,11 +59,11 @@ func CleanPath(p string) string {
case p[r] == '.' && p[r+1] == '/':
// . element
r++
r += 2
case p[r] == '.' && p[r+1] == '.' && (r+2 == n || p[r+2] == '/'):
// .. element: remove to last /
r += 2
r += 3
if w > 1 {
// can backtrack
+1
View File
@@ -22,6 +22,7 @@ var cleanTests = []struct {
// missing root
{"", "/"},
{"a/", "/a/"},
{"abc", "/abc"},
{"abc/def", "/abc/def"},
{"a/b/c", "/a/b/c"},
+66 -30
View File
@@ -55,7 +55,7 @@
//
// Catch-all parameters match anything until the path end, including the
// directory index (the '/' before the catch-all). Since they match anything
// until the end, catch-all paramerters must always be the final path element.
// until the end, catch-all parameters must always be the final path element.
// Path: /files/*filepath
//
// Requests:
@@ -138,14 +138,20 @@ type Router struct {
// handler.
HandleMethodNotAllowed bool
// Configurable http.HandlerFunc which is called when no matching route is
// found. If it is not set, http.NotFound is used.
NotFound http.HandlerFunc
// If enabled, the router automatically replies to OPTIONS requests.
// Custom OPTIONS handlers take priority over automatic replies.
HandleOPTIONS bool
// Configurable http.HandlerFunc which is called when a request
// Configurable http.Handler which is called when no matching route is
// found. If it is not set, http.NotFound is used.
NotFound http.Handler
// Configurable http.Handler which is called when a request
// cannot be routed and HandleMethodNotAllowed is true.
// If it is not set, http.Error with http.StatusMethodNotAllowed is used.
MethodNotAllowed http.HandlerFunc
// The "Allow" header with allowed request methods is set before the handler
// is called.
MethodNotAllowed http.Handler
// Function to handle panics recovered from http handlers.
// It should be used to generate a error page and return the http error code
@@ -165,6 +171,7 @@ func New() *Router {
RedirectTrailingSlash: true,
RedirectFixedPath: true,
HandleMethodNotAllowed: true,
HandleOPTIONS: true,
}
}
@@ -229,16 +236,6 @@ func (r *Router) Handle(method, path string, handle Handle) {
root.addRoute(path, handle)
}
// Handler is an adapter which allows the usage of an http.Handler as a
// request handle.
func (r *Router) Handler(method, path string, handler http.Handler) {
r.Handle(method, path,
func(w http.ResponseWriter, req *http.Request, _ Params) {
handler.ServeHTTP(w, req)
},
)
}
// HandlerFunc is an adapter which allows the usage of an http.HandlerFunc as a
// request handle.
func (r *Router) HandlerFunc(method, path string, handler http.HandlerFunc) {
@@ -286,15 +283,53 @@ func (r *Router) Lookup(method, path string) (Handle, Params, bool) {
return nil, nil, false
}
func (r *Router) allowed(path, reqMethod string) (allow string) {
if path == "*" { // server-wide
for method := range r.trees {
if method == "OPTIONS" {
continue
}
// add request method to list of allowed methods
if len(allow) == 0 {
allow = method
} else {
allow += ", " + method
}
}
} else { // specific path
for method := range r.trees {
// Skip the requested method - we already tried this one
if method == reqMethod || method == "OPTIONS" {
continue
}
handle, _, _ := r.trees[method].getValue(path)
if handle != nil {
// add request method to list of allowed methods
if len(allow) == 0 {
allow = method
} else {
allow += ", " + method
}
}
}
}
if len(allow) > 0 {
allow += ", OPTIONS"
}
return
}
// ServeHTTP makes the router implement the http.Handler interface.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
if r.PanicHandler != nil {
defer r.recv(w, req)
}
if root := r.trees[req.Method]; root != nil {
path := req.URL.Path
path := req.URL.Path
if root := r.trees[req.Method]; root != nil {
if handle, ps, tsr := root.getValue(path); handle != nil {
handle(w, req, ps)
return
@@ -331,18 +366,19 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
}
// Handle 405
if r.HandleMethodNotAllowed {
for method := range r.trees {
// Skip the requested method - we already tried this one
if method == req.Method {
continue
}
handle, _, _ := r.trees[method].getValue(req.URL.Path)
if handle != nil {
if req.Method == "OPTIONS" && r.HandleOPTIONS {
// Handle OPTIONS requests
if allow := r.allowed(path, req.Method); len(allow) > 0 {
w.Header().Set("Allow", allow)
return
}
} else {
// Handle 405
if r.HandleMethodNotAllowed {
if allow := r.allowed(path, req.Method); len(allow) > 0 {
w.Header().Set("Allow", allow)
if r.MethodNotAllowed != nil {
r.MethodNotAllowed(w, req)
r.MethodNotAllowed.ServeHTTP(w, req)
} else {
http.Error(w,
http.StatusText(http.StatusMethodNotAllowed),
@@ -356,7 +392,7 @@ func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
// Handle 404
if r.NotFound != nil {
r.NotFound(w, req)
r.NotFound.ServeHTTP(w, req)
} else {
http.NotFound(w, req)
}
+173 -21
View File
@@ -68,11 +68,11 @@ func TestRouter(t *testing.T) {
}
type handlerStruct struct {
handeled *bool
handled *bool
}
func (h handlerStruct) ServeHTTP(w http.ResponseWriter, r *http.Request) {
*h.handeled = true
*h.handled = true
}
func TestRouterAPI(t *testing.T) {
@@ -174,26 +174,175 @@ func TestRouterRoot(t *testing.T) {
}
}
func TestRouterChaining(t *testing.T) {
router1 := New()
router2 := New()
router1.NotFound = router2
fooHit := false
router1.POST("/foo", func(w http.ResponseWriter, req *http.Request, _ Params) {
fooHit = true
w.WriteHeader(http.StatusOK)
})
barHit := false
router2.POST("/bar", func(w http.ResponseWriter, req *http.Request, _ Params) {
barHit = true
w.WriteHeader(http.StatusOK)
})
r, _ := http.NewRequest("POST", "/foo", nil)
w := httptest.NewRecorder()
router1.ServeHTTP(w, r)
if !(w.Code == http.StatusOK && fooHit) {
t.Errorf("Regular routing failed with router chaining.")
t.FailNow()
}
r, _ = http.NewRequest("POST", "/bar", nil)
w = httptest.NewRecorder()
router1.ServeHTTP(w, r)
if !(w.Code == http.StatusOK && barHit) {
t.Errorf("Chained routing failed with router chaining.")
t.FailNow()
}
r, _ = http.NewRequest("POST", "/qax", nil)
w = httptest.NewRecorder()
router1.ServeHTTP(w, r)
if !(w.Code == http.StatusNotFound) {
t.Errorf("NotFound behavior failed with router chaining.")
t.FailNow()
}
}
func TestRouterOPTIONS(t *testing.T) {
handlerFunc := func(_ http.ResponseWriter, _ *http.Request, _ Params) {}
router := New()
router.POST("/path", handlerFunc)
// test not allowed
// * (server)
r, _ := http.NewRequest("OPTIONS", "*", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusOK) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
// path
r, _ = http.NewRequest("OPTIONS", "/path", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusOK) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
r, _ = http.NewRequest("OPTIONS", "/doesnotexist", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusNotFound) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
}
// add another method
router.GET("/path", handlerFunc)
// test again
// * (server)
r, _ = http.NewRequest("OPTIONS", "*", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusOK) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, GET, OPTIONS" && allow != "GET, POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
// path
r, _ = http.NewRequest("OPTIONS", "/path", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusOK) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, GET, OPTIONS" && allow != "GET, POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
// custom handler
var custom bool
router.OPTIONS("/path", func(w http.ResponseWriter, r *http.Request, _ Params) {
custom = true
})
// test again
// * (server)
r, _ = http.NewRequest("OPTIONS", "*", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusOK) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, GET, OPTIONS" && allow != "GET, POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
if custom {
t.Error("custom handler called on *")
}
// path
r, _ = http.NewRequest("OPTIONS", "/path", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusOK) {
t.Errorf("OPTIONS handling failed: Code=%d, Header=%v", w.Code, w.Header())
}
if !custom {
t.Error("custom handler not called")
}
}
func TestRouterNotAllowed(t *testing.T) {
handlerFunc := func(_ http.ResponseWriter, _ *http.Request, _ Params) {}
router := New()
router.POST("/path", handlerFunc)
// Test not allowed
// test not allowed
r, _ := http.NewRequest("GET", "/path", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusMethodNotAllowed) {
t.Errorf("NotAllowed handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
// add another method
router.DELETE("/path", handlerFunc)
router.OPTIONS("/path", handlerFunc) // must be ignored
// test again
r, _ = http.NewRequest("GET", "/path", nil)
w = httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == http.StatusMethodNotAllowed) {
t.Errorf("NotAllowed handling failed: Code=%d, Header=%v", w.Code, w.Header())
} else if allow := w.Header().Get("Allow"); allow != "POST, DELETE, OPTIONS" && allow != "DELETE, POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
// test custom handler
w = httptest.NewRecorder()
responseText := "custom method"
router.MethodNotAllowed = func(w http.ResponseWriter, req *http.Request) {
router.MethodNotAllowed = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(http.StatusTeapot)
w.Write([]byte(responseText))
}
})
router.ServeHTTP(w, r)
if got := w.Body.String(); !(got == responseText) {
t.Errorf("unexpected response got %q want %q", got, responseText)
@@ -201,6 +350,9 @@ func TestRouterNotAllowed(t *testing.T) {
if w.Code != http.StatusTeapot {
t.Errorf("unexpected response code %d want %d", w.Code, http.StatusTeapot)
}
if allow := w.Header().Get("Allow"); allow != "POST, DELETE, OPTIONS" && allow != "DELETE, POST, OPTIONS" {
t.Error("unexpected Allow header value: " + allow)
}
}
func TestRouterNotFound(t *testing.T) {
@@ -212,35 +364,35 @@ func TestRouterNotFound(t *testing.T) {
router.GET("/", handlerFunc)
testRoutes := []struct {
route string
code int
header string
route string
code int
location string
}{
{"/path/", 301, "map[Location:[/path]]"}, // TSR -/
{"/dir", 301, "map[Location:[/dir/]]"}, // TSR +/
{"", 301, "map[Location:[/]]"}, // TSR +/
{"/PATH", 301, "map[Location:[/path]]"}, // Fixed Case
{"/DIR/", 301, "map[Location:[/dir/]]"}, // Fixed Case
{"/PATH/", 301, "map[Location:[/path]]"}, // Fixed Case -/
{"/DIR", 301, "map[Location:[/dir/]]"}, // Fixed Case +/
{"/../path", 301, "map[Location:[/path]]"}, // CleanPath
{"/nope", 404, ""}, // NotFound
{"/path/", 301, "/path"}, // TSR -/
{"/dir", 301, "/dir/"}, // TSR +/
{"", 301, "/"}, // TSR +/
{"/PATH", 301, "/path"}, // Fixed Case
{"/DIR/", 301, "/dir/"}, // Fixed Case
{"/PATH/", 301, "/path"}, // Fixed Case -/
{"/DIR", 301, "/dir/"}, // Fixed Case +/
{"/../path", 301, "/path"}, // CleanPath
{"/nope", 404, ""}, // NotFound
}
for _, tr := range testRoutes {
r, _ := http.NewRequest("GET", tr.route, nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, r)
if !(w.Code == tr.code && (w.Code == 404 || fmt.Sprint(w.Header()) == tr.header)) {
t.Errorf("NotFound handling route %s failed: Code=%d, Header=%v", tr.route, w.Code, w.Header())
if !(w.Code == tr.code && (w.Code == 404 || fmt.Sprint(w.Header().Get("Location")) == tr.location)) {
t.Errorf("NotFound handling route %s failed: Code=%d, Header=%v", tr.route, w.Code, w.Header().Get("Location"))
}
}
// Test custom not found handler
var notFound bool
router.NotFound = func(rw http.ResponseWriter, r *http.Request) {
router.NotFound = http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
rw.WriteHeader(404)
notFound = true
}
})
r, _ := http.NewRequest("GET", "/nope", nil)
w := httptest.NewRecorder()
router.ServeHTTP(w, r)
+141 -40
View File
@@ -7,6 +7,7 @@ package httprouter
import (
"strings"
"unicode"
"unicode/utf8"
)
func min(a, b int) int {
@@ -33,9 +34,10 @@ func countParams(path string) uint8 {
type nodeType uint8
const (
static nodeType = 0
param nodeType = 1
catchAll nodeType = 2
static nodeType = iota // default
root
param
catchAll
)
type node struct {
@@ -58,9 +60,7 @@ func (n *node) incrementChildPrio(pos int) int {
newPos := pos
for newPos > 0 && n.children[newPos-1].priority < prio {
// swap node positions
tmpN := n.children[newPos-1]
n.children[newPos-1] = n.children[newPos]
n.children[newPos] = tmpN
n.children[newPos-1], n.children[newPos] = n.children[newPos], n.children[newPos-1]
newPos--
}
@@ -105,6 +105,7 @@ func (n *node) addRoute(path string, handle Handle) {
child := node{
path: n.path[i:],
wildChild: n.wildChild,
nType: static,
indices: n.indices,
children: n.children,
handle: n.handle,
@@ -141,16 +142,25 @@ func (n *node) addRoute(path string, handle Handle) {
numParams--
// Check if the wildcard matches
if len(path) >= len(n.path) && n.path == path[:len(n.path)] {
// check for longer wildcard, e.g. :name and :names
if len(n.path) >= len(path) || path[len(n.path)] == '/' {
continue walk
if len(path) >= len(n.path) && n.path == path[:len(n.path)] &&
// Check for longer wildcard, e.g. :name and :names
(len(n.path) >= len(path) || path[len(n.path)] == '/') {
continue walk
} else {
// Wildcard conflict
var pathSeg string
if n.nType == catchAll {
pathSeg = path
} else {
pathSeg = strings.SplitN(path, "/", 2)[0]
}
prefix := fullPath[:strings.Index(fullPath, pathSeg)] + n.path
panic("'" + pathSeg +
"' in new path '" + fullPath +
"' conflicts with existing wildcard '" + n.path +
"' in existing prefix '" + prefix +
"'")
}
panic("path segment '" + path +
"' conflicts with existing wildcard '" + n.path +
"' in path '" + fullPath + "'")
}
c := path[0]
@@ -187,7 +197,7 @@ func (n *node) addRoute(path string, handle Handle) {
} else if i == len(path) { // Make node a (in-path) leaf
if n.handle != nil {
panic("a handle is already registered for path ''" + fullPath + "'")
panic("a handle is already registered for path '" + fullPath + "'")
}
n.handle = handle
}
@@ -195,6 +205,7 @@ func (n *node) addRoute(path string, handle Handle) {
}
} else { // Empty tree
n.insertChild(numParams, path, fullPath, handle)
n.nType = root
}
}
@@ -317,7 +328,7 @@ func (n *node) insertChild(numParams uint8, path, fullPath string, handle Handle
// made if a handle exists with an extra (without the) trailing slash for the
// given path.
func (n *node) getValue(path string) (handle Handle, p Params, tsr bool) {
walk: // Outer loop for walking the tree
walk: // outer loop for walking the tree
for {
if len(path) > len(n.path) {
if path[:len(n.path)] == n.path {
@@ -411,6 +422,11 @@ walk: // Outer loop for walking the tree
return
}
if path == "/" && n.wildChild && n.nType != root {
tsr = true
return
}
// No handle found. Check if a handle for this path + a
// trailing slash exists for trailing slash recommendation
for i := 0; i < len(n.indices); i++ {
@@ -439,34 +455,117 @@ walk: // Outer loop for walking the tree
// It returns the case-corrected path and a bool indicating whether the lookup
// was successful.
func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPath []byte, found bool) {
ciPath = make([]byte, 0, len(path)+1) // preallocate enough memory
return n.findCaseInsensitivePathRec(
path,
strings.ToLower(path),
make([]byte, 0, len(path)+1), // preallocate enough memory for new path
[4]byte{}, // empty rune buffer
fixTrailingSlash,
)
}
// Outer loop for walking the tree
for len(path) >= len(n.path) && strings.ToLower(path[:len(n.path)]) == strings.ToLower(n.path) {
path = path[len(n.path):]
// shift bytes in array by n bytes left
func shiftNRuneBytes(rb [4]byte, n int) [4]byte {
switch n {
case 0:
return rb
case 1:
return [4]byte{rb[1], rb[2], rb[3], 0}
case 2:
return [4]byte{rb[2], rb[3]}
case 3:
return [4]byte{rb[3]}
default:
return [4]byte{}
}
}
// recursive case-insensitive lookup function used by n.findCaseInsensitivePath
func (n *node) findCaseInsensitivePathRec(path, loPath string, ciPath []byte, rb [4]byte, fixTrailingSlash bool) ([]byte, bool) {
loNPath := strings.ToLower(n.path)
walk: // outer loop for walking the tree
for len(loPath) >= len(loNPath) && (len(loNPath) == 0 || loPath[1:len(loNPath)] == loNPath[1:]) {
// add common path to result
ciPath = append(ciPath, n.path...)
if len(path) > 0 {
if path = path[len(n.path):]; len(path) > 0 {
loOld := loPath
loPath = loPath[len(loNPath):]
// If this node does not have a wildcard (param or catchAll) child,
// we can just look up the next child node and continue to walk down
// the tree
if !n.wildChild {
r := unicode.ToLower(rune(path[0]))
for i, index := range n.indices {
// must use recursive approach since both index and
// ToLower(index) could exist. We must check both.
if r == unicode.ToLower(index) {
out, found := n.children[i].findCaseInsensitivePath(path, fixTrailingSlash)
if found {
return append(ciPath, out...), true
// skip rune bytes already processed
rb = shiftNRuneBytes(rb, len(loNPath))
if rb[0] != 0 {
// old rune not finished
for i := 0; i < len(n.indices); i++ {
if n.indices[i] == rb[0] {
// continue with child node
n = n.children[i]
loNPath = strings.ToLower(n.path)
continue walk
}
}
} else {
// process a new rune
var rv rune
// find rune start
// runes are up to 4 byte long,
// -4 would definitely be another rune
var off int
for max := min(len(loNPath), 3); off < max; off++ {
if i := len(loNPath) - off; utf8.RuneStart(loOld[i]) {
// read rune from cached lowercase path
rv, _ = utf8.DecodeRuneInString(loOld[i:])
break
}
}
// calculate lowercase bytes of current rune
utf8.EncodeRune(rb[:], rv)
// skipp already processed bytes
rb = shiftNRuneBytes(rb, off)
for i := 0; i < len(n.indices); i++ {
// lowercase matches
if n.indices[i] == rb[0] {
// must use a recursive approach since both the
// uppercase byte and the lowercase byte might exist
// as an index
if out, found := n.children[i].findCaseInsensitivePathRec(
path, loPath, ciPath, rb, fixTrailingSlash,
); found {
return out, true
}
break
}
}
// same for uppercase rune, if it differs
if up := unicode.ToUpper(rv); up != rv {
utf8.EncodeRune(rb[:], up)
rb = shiftNRuneBytes(rb, off)
for i := 0; i < len(n.indices); i++ {
// uppercase matches
if n.indices[i] == rb[0] {
// continue with child node
n = n.children[i]
loNPath = strings.ToLower(n.path)
continue walk
}
}
}
}
// Nothing found. We can recommend to redirect to the same URL
// without a trailing slash if a leaf exists for that path
found = (fixTrailingSlash && path == "/" && n.handle != nil)
return
return ciPath, (fixTrailingSlash && path == "/" && n.handle != nil)
}
n = n.children[0]
@@ -484,8 +583,11 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa
// we need to go deeper!
if k < len(path) {
if len(n.children) > 0 {
path = path[k:]
// continue with child node
n = n.children[0]
loNPath = strings.ToLower(n.path)
loPath = loPath[k:]
path = path[k:]
continue
}
@@ -493,7 +595,7 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa
if fixTrailingSlash && len(path) == k+1 {
return ciPath, true
}
return
return ciPath, false
}
if n.handle != nil {
@@ -506,7 +608,7 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa
return append(ciPath, '/'), true
}
}
return
return ciPath, false
case catchAll:
return append(ciPath, path...), true
@@ -531,11 +633,11 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa
(n.nType == catchAll && n.children[0].handle != nil) {
return append(ciPath, '/'), true
}
return
return ciPath, false
}
}
}
return
return ciPath, false
}
}
@@ -545,11 +647,10 @@ func (n *node) findCaseInsensitivePath(path string, fixTrailingSlash bool) (ciPa
if path == "/" {
return ciPath, true
}
if len(path)+1 == len(n.path) && n.path[len(path)] == '/' &&
strings.ToLower(path) == strings.ToLower(n.path[:len(path)]) &&
n.handle != nil {
if len(loPath)+1 == len(loNPath) && loNPath[len(loPath)] == '/' &&
loPath[1:] == loNPath[1:len(loPath)] && n.handle != nil {
return append(ciPath, n.path...), true
}
}
return
return ciPath, false
}
+90 -2
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"net/http"
"reflect"
"regexp"
"strings"
"testing"
)
@@ -22,7 +23,7 @@ func printChildren(n *node, prefix string) {
}
}
// Used as a workaround since we can't compare functions or their adresses
// Used as a workaround since we can't compare functions or their addresses
var fakeHandlerValue string
func fakeHandler(val string) Handle {
@@ -89,7 +90,7 @@ func checkMaxParams(t *testing.T, n *node) uint8 {
maxParams = params
}
}
if n.nType != static && !n.wildChild {
if n.nType > root && !n.wildChild {
maxParams++
}
@@ -394,6 +395,9 @@ func TestTreeTrailingSlashRedirect(t *testing.T) {
"/1/:id/2",
"/aa",
"/a/",
"/admin",
"/admin/:category",
"/admin/:category/:page",
"/doc",
"/doc/go_faq.html",
"/doc/go1.html",
@@ -423,6 +427,9 @@ func TestTreeTrailingSlashRedirect(t *testing.T) {
"/0/go/",
"/1/go",
"/a",
"/admin/",
"/admin/config/",
"/admin/config/permissions/",
"/doc/",
}
for _, route := range tsrRoutes {
@@ -452,6 +459,24 @@ func TestTreeTrailingSlashRedirect(t *testing.T) {
}
}
func TestTreeRootTrailingSlashRedirect(t *testing.T) {
tree := &node{}
recv := catchPanic(func() {
tree.addRoute("/:test", fakeHandler("/:test"))
})
if recv != nil {
t.Fatalf("panic inserting test route: %v", recv)
}
handler, _, tsr := tree.getValue("/")
if handler != nil {
t.Fatalf("non-nil handler")
} else if tsr {
t.Errorf("expected no TSR recommendation")
}
}
func TestTreeFindCaseInsensitivePath(t *testing.T) {
tree := &node{}
@@ -478,6 +503,16 @@ func TestTreeFindCaseInsensitivePath(t *testing.T) {
"/doc/go/away",
"/no/a",
"/no/b",
"/Π",
"/u/apfêl/",
"/u/äpfêl/",
"/u/öpfêl",
"/v/Äpfêl/",
"/v/Öpfêl",
"/w/♬", // 3 byte
"/w/♭/", // 3 byte, last byte differs
"/w/𠜎", // 4 byte
"/w/𠜏/", // 4 byte
}
for _, route := range routes {
@@ -556,6 +591,20 @@ func TestTreeFindCaseInsensitivePath(t *testing.T) {
{"/DOC/", "/doc", true, true},
{"/NO", "", false, true},
{"/DOC/GO", "", false, true},
{"/π", "/Π", true, false},
{"/π/", "/Π", true, true},
{"/u/ÄPFÊL/", "/u/äpfêl/", true, false},
{"/u/ÄPFÊL", "/u/äpfêl/", true, true},
{"/u/ÖPFÊL/", "/u/öpfêl", true, true},
{"/u/ÖPFÊL", "/u/öpfêl", true, false},
{"/v/äpfêL/", "/v/Äpfêl/", true, false},
{"/v/äpfêL", "/v/Äpfêl/", true, true},
{"/v/öpfêL/", "/v/Öpfêl", true, true},
{"/v/öpfêL", "/v/Öpfêl", true, false},
{"/w/♬/", "/w/♬", true, true},
{"/w/♭", "/w/♭/", true, true},
{"/w/𠜎/", "/w/𠜎", true, true},
{"/w/𠜏", "/w/𠜏/", true, true},
}
// With fixTrailingSlash = true
for _, test := range tests {
@@ -609,3 +658,42 @@ func TestTreeInvalidNodeType(t *testing.T) {
t.Fatalf("Expected panic '"+panicMsg+"', got '%v'", recv)
}
}
func TestTreeWildcardConflictEx(t *testing.T) {
conflicts := [...]struct {
route string
segPath string
existPath string
existSegPath string
}{
{"/who/are/foo", "/foo", `/who/are/\*you`, `/\*you`},
{"/who/are/foo/", "/foo/", `/who/are/\*you`, `/\*you`},
{"/who/are/foo/bar", "/foo/bar", `/who/are/\*you`, `/\*you`},
{"/conxxx", "xxx", `/con:tact`, `:tact`},
{"/conooo/xxx", "ooo", `/con:tact`, `:tact`},
}
for _, conflict := range conflicts {
// I have to re-create a 'tree', because the 'tree' will be
// in an inconsistent state when the loop recovers from the
// panic which threw by 'addRoute' function.
tree := &node{}
routes := [...]string{
"/con:tact",
"/who/are/*you",
"/who/foo/hello",
}
for _, route := range routes {
tree.addRoute(route, fakeHandler(route))
}
recv := catchPanic(func() {
tree.addRoute(conflict.route, fakeHandler(conflict.route))
})
if !regexp.MustCompile(fmt.Sprintf("'%s' in new path .* conflicts with existing wildcard '%s' in existing prefix '%s'", conflict.segPath, conflict.existSegPath, conflict.existPath)).MatchString(fmt.Sprint(recv)) {
t.Fatalf("invalid wildcard conflict error (%v)", recv)
}
}
}
+1
View File
@@ -0,0 +1 @@
*.bin -text -diff
+8 -5
View File
@@ -7,11 +7,9 @@ os:
- osx
go:
- 1.4
- 1.5
- 1.6
- 1.7
- tip
- 1.9.x
- 1.10.x
- master
install:
- go get -t ./...
@@ -22,3 +20,8 @@ script:
- go test -cpu=2 -tags=noasm ./...
- go test -cpu=1,2,4 -short -race ./...
- go test -cpu=2,4 -short -race -tags=noasm ./...
matrix:
allow_failures:
- go: 'master'
fast_finish: true
+57 -186
View File
@@ -10,9 +10,19 @@ It offers slightly better compression at lower compression settings, and up to 3
* [Re-balancing Deflate Compression Levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/)
[![Build Status](https://travis-ci.org/klauspost/compress.svg?branch=master)](https://travis-ci.org/klauspost/compress)
[![Sourcegraph Badge](https://sourcegraph.com/github.com/klauspost/compress/-/badge.svg)](https://sourcegraph.com/github.com/klauspost/compress?badge)
# changelog
* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625(https://github.com/golang/go/issues/18625).
* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression).
* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below.
* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0).
* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change.
* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change.
* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function.
* May 28, 2017: Reduce allocations when resetting decoder.
* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7.
* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625).
* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before.
* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update.
* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level.
@@ -50,7 +60,7 @@ The packages are drop-in replacements for standard libraries. Simply replace the
| `compress/gzip` | `github.com/klauspost/compress/gzip` |
| `compress/zlib` | `github.com/klauspost/compress/zlib` |
| `archive/zip` | `github.com/klauspost/compress/zip` |
| `compress/deflate` | `github.com/klauspost/compress/deflate` |
| `compress/flate` | `github.com/klauspost/compress/flate` |
You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
@@ -58,151 +68,72 @@ The packages contains the same as the standard library, so you can use the godoc
Currently there is only minor speedup on decompression (mostly CRC32 calculation).
# deflate optimizations
# Performance Update 2018
* Minimum matches are 4 bytes, this leads to fewer searches and better compression. (In Go 1.7)
* Stronger hash (iSCSI CRC32) for matches on x64 with SSE 4.2 support. This leads to fewer hash collisions. (Go 1.7 also has improved hashes)
* Literal byte matching using SSE 4.2 for faster match comparisons. (not in Go)
* Bulk hashing on matches. (In Go 1.7)
* Much faster dictionary indexing with `NewWriterDict()`/`Reset()`. (In Go 1.7)
* Make Bit Coder faster by assuming we are on a 64 bit CPU. (In Go 1.7)
* Level 1 compression replaced by converted "Snappy" algorithm. (In Go 1.7)
* Uncompressible content is detected and skipped faster. (Only in BestSpeed in Go)
* A lot of branching eliminated by having two encoders for levels 4-6 and 7-9. (not in Go)
* All heap memory allocations eliminated. (In Go 1.7)
It has been a while since we have been looking at the speed of this package compared to the standard library, so I thought I would re-do my tests and give some overall recommendations based on the current state. All benchmarks have been performed with Go 1.10 on my Desktop Intel(R) Core(TM) i7-2600 CPU @3.40GHz. Since I last ran the tests, I have gotten more RAM, which means tests with big files are no longer limited by my SSD.
```
benchmark old ns/op new ns/op delta
BenchmarkEncodeDigitsSpeed1e4-4 554029 265175 -52.14%
BenchmarkEncodeDigitsSpeed1e5-4 3908558 2416595 -38.17%
BenchmarkEncodeDigitsSpeed1e6-4 37546692 24875330 -33.75%
BenchmarkEncodeDigitsDefault1e4-4 781510 486322 -37.77%
BenchmarkEncodeDigitsDefault1e5-4 15530248 6740175 -56.60%
BenchmarkEncodeDigitsDefault1e6-4 174915710 76498625 -56.27%
BenchmarkEncodeDigitsCompress1e4-4 769995 485652 -36.93%
BenchmarkEncodeDigitsCompress1e5-4 15450113 6929589 -55.15%
BenchmarkEncodeDigitsCompress1e6-4 175114660 73348495 -58.11%
BenchmarkEncodeTwainSpeed1e4-4 560122 275977 -50.73%
BenchmarkEncodeTwainSpeed1e5-4 3740978 2506095 -33.01%
BenchmarkEncodeTwainSpeed1e6-4 35542802 21904440 -38.37%
BenchmarkEncodeTwainDefault1e4-4 828534 549026 -33.74%
BenchmarkEncodeTwainDefault1e5-4 13667153 7528455 -44.92%
BenchmarkEncodeTwainDefault1e6-4 141191770 79952170 -43.37%
BenchmarkEncodeTwainCompress1e4-4 830050 545694 -34.26%
BenchmarkEncodeTwainCompress1e5-4 16620852 8460600 -49.10%
BenchmarkEncodeTwainCompress1e6-4 193326820 90808750 -53.03%
The raw results are in my [updated spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). Due to cgo changes and upstream updates i could not get the cgo version of gzip to compile. Instead I included the [zstd](https://github.com/datadog/zstd) cgo implementation. If I get cgo gzip to work again, I might replace the results in the sheet.
benchmark old MB/s new MB/s speedup
BenchmarkEncodeDigitsSpeed1e4-4 18.05 37.71 2.09x
BenchmarkEncodeDigitsSpeed1e5-4 25.58 41.38 1.62x
BenchmarkEncodeDigitsSpeed1e6-4 26.63 40.20 1.51x
BenchmarkEncodeDigitsDefault1e4-4 12.80 20.56 1.61x
BenchmarkEncodeDigitsDefault1e5-4 6.44 14.84 2.30x
BenchmarkEncodeDigitsDefault1e6-4 5.72 13.07 2.28x
BenchmarkEncodeDigitsCompress1e4-4 12.99 20.59 1.59x
BenchmarkEncodeDigitsCompress1e5-4 6.47 14.43 2.23x
BenchmarkEncodeDigitsCompress1e6-4 5.71 13.63 2.39x
BenchmarkEncodeTwainSpeed1e4-4 17.85 36.23 2.03x
BenchmarkEncodeTwainSpeed1e5-4 26.73 39.90 1.49x
BenchmarkEncodeTwainSpeed1e6-4 28.14 45.65 1.62x
BenchmarkEncodeTwainDefault1e4-4 12.07 18.21 1.51x
BenchmarkEncodeTwainDefault1e5-4 7.32 13.28 1.81x
BenchmarkEncodeTwainDefault1e6-4 7.08 12.51 1.77x
BenchmarkEncodeTwainCompress1e4-4 12.05 18.33 1.52x
BenchmarkEncodeTwainCompress1e5-4 6.02 11.82 1.96x
BenchmarkEncodeTwainCompress1e6-4 5.17 11.01 2.13x
```
* "Speed" is compression level 1
* "Default" is compression level 6
* "Compress" is compression level 9
* Test files are [Digits](https://github.com/klauspost/compress/blob/master/testdata/e.txt) (no matches) and [Twain](https://github.com/klauspost/compress/blob/master/testdata/Mark.Twain-Tom.Sawyer.txt) (plain text) .
The columns to take note of are: *MB/s* - the throughput. *Reduction* - the data size reduction in percent of the original. *Rel Speed* relative speed compared to the standard libary at the same level. *Smaller* - how many percent smaller is the compressed output compared to stdlib. Negative means the output was bigger. *Loss* means the loss (or gain) in compression as a percentage difference of the input.
As can be seen it shows a very good speedup all across the line.
The `gzstd` (standard library gzip) and `gzkp` (this package gzip) only uses one CPU core. [`pgzip`](https://github.com/klauspost/pgzip), [`bgzf`](https://github.com/biogo/hts/bgzf) uses all 4 cores. [`zstd`](https://github.com/DataDog/zstd) uses one core, and is a beast (but not Go, yet).
`Twain` is a much more realistic benchmark, and will be closer to JSON/HTML performance. Here speed is equivalent or faster, up to 2 times.
**Without assembly**. This is what you can expect on systems that does not have amd64 and SSE 4:
```
benchmark old ns/op new ns/op delta
BenchmarkEncodeDigitsSpeed1e4-4 554029 249558 -54.96%
BenchmarkEncodeDigitsSpeed1e5-4 3908558 2295216 -41.28%
BenchmarkEncodeDigitsSpeed1e6-4 37546692 22594905 -39.82%
BenchmarkEncodeDigitsDefault1e4-4 781510 579850 -25.80%
BenchmarkEncodeDigitsDefault1e5-4 15530248 10096561 -34.99%
BenchmarkEncodeDigitsDefault1e6-4 174915710 111470780 -36.27%
BenchmarkEncodeDigitsCompress1e4-4 769995 579708 -24.71%
BenchmarkEncodeDigitsCompress1e5-4 15450113 10266373 -33.55%
BenchmarkEncodeDigitsCompress1e6-4 175114660 110170120 -37.09%
BenchmarkEncodeTwainSpeed1e4-4 560122 260679 -53.46%
BenchmarkEncodeTwainSpeed1e5-4 3740978 2097372 -43.94%
BenchmarkEncodeTwainSpeed1e6-4 35542802 20353449 -42.74%
BenchmarkEncodeTwainDefault1e4-4 828534 646016 -22.03%
BenchmarkEncodeTwainDefault1e5-4 13667153 10056369 -26.42%
BenchmarkEncodeTwainDefault1e6-4 141191770 105268770 -25.44%
BenchmarkEncodeTwainCompress1e4-4 830050 642401 -22.61%
BenchmarkEncodeTwainCompress1e5-4 16620852 11157081 -32.87%
BenchmarkEncodeTwainCompress1e6-4 193326820 121780770 -37.01%
## Overall differences.
benchmark old MB/s new MB/s speedup
BenchmarkEncodeDigitsSpeed1e4-4 18.05 40.07 2.22x
BenchmarkEncodeDigitsSpeed1e5-4 25.58 43.57 1.70x
BenchmarkEncodeDigitsSpeed1e6-4 26.63 44.26 1.66x
BenchmarkEncodeDigitsDefault1e4-4 12.80 17.25 1.35x
BenchmarkEncodeDigitsDefault1e5-4 6.44 9.90 1.54x
BenchmarkEncodeDigitsDefault1e6-4 5.72 8.97 1.57x
BenchmarkEncodeDigitsCompress1e4-4 12.99 17.25 1.33x
BenchmarkEncodeDigitsCompress1e5-4 6.47 9.74 1.51x
BenchmarkEncodeDigitsCompress1e6-4 5.71 9.08 1.59x
BenchmarkEncodeTwainSpeed1e4-4 17.85 38.36 2.15x
BenchmarkEncodeTwainSpeed1e5-4 26.73 47.68 1.78x
BenchmarkEncodeTwainSpeed1e6-4 28.14 49.13 1.75x
BenchmarkEncodeTwainDefault1e4-4 12.07 15.48 1.28x
BenchmarkEncodeTwainDefault1e5-4 7.32 9.94 1.36x
BenchmarkEncodeTwainDefault1e6-4 7.08 9.50 1.34x
BenchmarkEncodeTwainCompress1e4-4 12.05 15.57 1.29x
BenchmarkEncodeTwainCompress1e5-4 6.02 8.96 1.49x
BenchmarkEncodeTwainCompress1e6-4 5.17 8.21 1.59x
```
So even without the assembly optimizations there is a general speedup across the board.
There appears to be a roughly 5-10% speed advantage over the standard library when comparing at similar compression levels.
## level 1-3 "snappy" compression
The biggest difference you will see is the result of [re-balancing](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) the compression levels. I wanted by library to give a smoother transition between the compression levels than the standard library.
Levels 1 "Best Speed", 2 and 3 are completely replaced by a converted version of the algorithm found in Snappy, modified to be fully
compatible with the deflate bitstream (and thus still compatible with all existing zlib/gzip libraries and tools).
This version is considerably faster than the "old" deflate at level 1. It does however come at a compression loss, usually in the order of 3-4% compared to the old level 1. However, the speed is usually 1.75 times that of the fastest deflate mode.
This package attempts to provide a more smooth transition, where "1" is taking a lot of shortcuts, "5" is the reasonable trade-off and "9" is the "give me the best compression", and the values in between gives something reasonable in between. The standard library has big differences in levels 1-4, but levels 5-9 having no significant gains - often spending a lot more time than can be justified by the achieved compression.
In my previous experiments the most common case for "level 1" was that it provided no significant speedup, only lower compression compared to level 2 and sometimes even 3. However, the modified Snappy algorithm provides a very good sweet spot. Usually about 75% faster and with only little compression loss. Therefore I decided to *replace* level 1 with this mode entirely.
There are links to all the test data in the [spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) in the top left field on each tab.
Input is split into blocks of 64kb of, and they are encoded independently (no backreferences across blocks) for the best speed. Contrary to Snappy the output is entropy-encoded, so you will almost always see better compression than Snappy. But Snappy is still about twice as fast as Snappy in deflate mode.
## Web Content
Level 2 and 3 have also been replaced. Level 2 is capable is matching between blocks and level 3 checks up to two hashes for matches it will try.
This test set aims to emulate typical use in a web server. The test-set is 4GB data in 53k files, and is a mixture of (mostly) HTML, JS, CSS.
## compression levels
Since level 1 and 9 are close to being the same code, they are quite close. But looking at the levels in-between the differences are quite big.
This table shows the compression at each level, and the percentage of the output size compared to output
at the similar level with the standard library. Compression data is `Twain`, see above.
Looking at level 6, this package is 88% faster, but will output about 6% more data. For a web server, this means you can serve 88% more data, but have to pay for 6% more bandwidth. You can draw your own conclusions on what would be the most expensive for your case.
(Not up-to-date after rebalancing)
## Object files
| Level | Bytes | % size |
|-------|--------|--------|
| 1 | 194622 | 103.7% |
| 2 | 174684 | 96.85% |
| 3 | 170301 | 98.45% |
| 4 | 165253 | 97.69% |
| 5 | 161274 | 98.65% |
| 6 | 160464 | 99.71% |
| 7 | 160304 | 99.87% |
| 8 | 160279 | 99.99% |
| 9 | 160279 | 99.99% |
This test is for typical data files stored on a server. In this case it is a collection of Go precompiled objects. They are very compressible.
To interpret and example, this version of deflate compresses input of 407287 bytes to 161274 bytes at level 5, which is 98.6% of the size of what the standard library produces; 161274 bytes.
The picture is similar to the web content, but with small differences since this is very compressible. Levels 2-3 offer good speed, but is sacrificing quite a bit of compression.
The standard library seems suboptimal on level 3 and 4 - offering both worse compression and speed than level 6 & 7 of this package respectively.
## Highly Compressible File
This is a JSON file with very high redundancy. The reduction starts at 95% on level 1, so in real life terms we are dealing with something like a highly redundant stream of data, etc.
It is definitely visible that we are dealing with specialized content here, so the results are very scattered. This package does not do very well at levels 1-4, but picks up significantly at level 5 and levels 7 and 8 offering great speed for the achieved compression.
So if you know you content is extremely compressible you might want to go slightly higher than the defaults. The standard library has a huge gap between levels 3 and 4 in terms of speed (2.75x slowdown), so it offers little "middle ground".
## Medium-High Compressible
This is a pretty common test corpus: [enwik9](http://mattmahoney.net/dc/textdata.html). It contains the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. This is a very good test of typical text based compression and more data heavy streams.
We see a similar picture here as in "Web Content". On equal levels some compression is sacrificed for more speed. Level 5 seems to be the best trade-off between speed and size, beating stdlib level 3 in both.
## Medium Compressible
I will combine two test sets, one [10GB file set](http://mattmahoney.net/dc/10gb.html) and a VM disk image (~8GB). Both contain different data types and represent a typical backup scenario.
The most notable thing is how quickly the standard libary drops to very low compression speeds around level 5-6 without any big gains in compression. Since this type of data is fairly common, this does not seem like good behavior.
## Un-compressible Content
This is mainly a test of how good the algorithms are at detecting un-compressible input. The standard library only offers this feature with very conservative settings at level 1. Obviously there is no reason for the algorithms to try to compress input that cannot be compressed. The only downside is that it might skip some compressible data on false detections.
This means that from level 4 you can expect a compression level increase of a few percent. Level 1 is about 3% worse, as descibed above.
# linear time compression (huffman only)
This compression library adds a special compression level, named `ConstantCompression`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character.
This compression library adds a special compression level, named `HuffmanOnly`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character.
This means that often used characters, like 'e' and ' ' (space) in text use the fewest bits to represent, and rare characters like '¤' takes more bits to represent. For more information see [wikipedia](https://en.wikipedia.org/wiki/Huffman_coding) or this nice [video](https://youtu.be/ZdooBTdW5bM).
@@ -217,66 +148,6 @@ For more information see my blog post on [Fast Linear Time Compression](http://b
This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
# gzip/zip optimizations
* Uses the faster deflate
* Uses SSE 4.2 CRC32 calculations.
Speed increase is up to 3x of the standard library, but usually around 2x.
This is close to a real world benchmark as you will get. A 2.3MB JSON file. (NOTE: not up-to-date)
```
benchmark old ns/op new ns/op delta
BenchmarkGzipL1-4 95212470 59938275 -37.05%
BenchmarkGzipL2-4 102069730 76349195 -25.20%
BenchmarkGzipL3-4 115472770 82492215 -28.56%
BenchmarkGzipL4-4 153197780 107570890 -29.78%
BenchmarkGzipL5-4 203930260 134387930 -34.10%
BenchmarkGzipL6-4 233172100 145495400 -37.60%
BenchmarkGzipL7-4 297190260 197926950 -33.40%
BenchmarkGzipL8-4 512819750 376244733 -26.63%
BenchmarkGzipL9-4 563366800 403266833 -28.42%
benchmark old MB/s new MB/s speedup
BenchmarkGzipL1-4 52.11 82.78 1.59x
BenchmarkGzipL2-4 48.61 64.99 1.34x
BenchmarkGzipL3-4 42.97 60.15 1.40x
BenchmarkGzipL4-4 32.39 46.13 1.42x
BenchmarkGzipL5-4 24.33 36.92 1.52x
BenchmarkGzipL6-4 21.28 34.10 1.60x
BenchmarkGzipL7-4 16.70 25.07 1.50x
BenchmarkGzipL8-4 9.68 13.19 1.36x
BenchmarkGzipL9-4 8.81 12.30 1.40x
```
Multithreaded compression using [pgzip](https://github.com/klauspost/pgzip) comparison, Quadcore, CPU = 8:
(Not updated, old numbers)
```
benchmark old ns/op new ns/op delta
BenchmarkGzipL1 96155500 25981486 -72.98%
BenchmarkGzipL2 101905830 24601408 -75.86%
BenchmarkGzipL3 113506490 26321506 -76.81%
BenchmarkGzipL4 143708220 31761818 -77.90%
BenchmarkGzipL5 188210770 39602266 -78.96%
BenchmarkGzipL6 209812000 40402313 -80.74%
BenchmarkGzipL7 270015440 56103210 -79.22%
BenchmarkGzipL8 461359700 91255220 -80.22%
BenchmarkGzipL9 498361833 88755075 -82.19%
benchmark old MB/s new MB/s speedup
BenchmarkGzipL1 51.60 190.97 3.70x
BenchmarkGzipL2 48.69 201.69 4.14x
BenchmarkGzipL3 43.71 188.51 4.31x
BenchmarkGzipL4 34.53 156.22 4.52x
BenchmarkGzipL5 26.36 125.29 4.75x
BenchmarkGzipL6 23.65 122.81 5.19x
BenchmarkGzipL7 18.38 88.44 4.81x
BenchmarkGzipL8 10.75 54.37 5.06x
BenchmarkGzipL9 9.96 55.90 5.61x
```
# snappy package
The standard snappy package has now been improved. This repo contains a copy of the snappy repo.
+63
View File
@@ -0,0 +1,63 @@
package compress
import "math"
// Estimate returns a normalized compressibility estimate of block b.
// Values close to zero are likely uncompressible.
// Values above 0.1 are likely to be compressible.
// Values above 0.5 are very compressible.
// Very small lengths will return 0.
func Estimate(b []byte) float64 {
if len(b) < 16 {
return 0
}
// Correctly predicted order 1
hits := 0
lastMatch := false
var o1 [256]byte
var hist [256]int
c1 := byte(0)
for _, c := range b {
if c == o1[c1] {
// We only count a hit if there was two correct predictions in a row.
if lastMatch {
hits++
}
lastMatch = true
} else {
lastMatch = false
}
o1[c1] = c
c1 = c
hist[c]++
}
// Use x^0.6 to give better spread
prediction := math.Pow(float64(hits)/float64(len(b)), 0.6)
// Calculate histogram distribution
variance := float64(0)
avg := float64(len(b)) / 256
for _, v := range hist {
Δ := float64(v) - avg
variance += Δ * Δ
}
stddev := math.Sqrt(float64(variance)) / float64(len(b))
exp := math.Sqrt(1 / float64(len(b)))
// Subtract expected stddev
stddev -= exp
if stddev < 0 {
stddev = 0
}
stddev *= 1 + exp
// Use x^0.4 to give better spread
entropy := math.Pow(stddev, 0.4)
// 50/50 weight between prediction and histogram distribution
return math.Pow((prediction+entropy)/2, 0.9)
}
+121
View File
@@ -0,0 +1,121 @@
package compress
import (
"crypto/rand"
"encoding/base32"
"testing"
)
func BenchmarkEstimate(b *testing.B) {
b.ReportAllocs()
// (predictable, low entropy distibution)
b.Run("zeroes-5k", func(b *testing.B) {
var testData = make([]byte, 5000)
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (predictable, high entropy distibution)
b.Run("predictable-5k", func(b *testing.B) {
var testData = make([]byte, 5000)
for i := range testData {
testData[i] = byte(float64(i) / float64(len(testData)) * 256)
}
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (not predictable, high entropy distibution)
b.Run("random-500b", func(b *testing.B) {
var testData = make([]byte, 500)
rand.Read(testData)
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (not predictable, high entropy distibution)
b.Run("random-5k", func(b *testing.B) {
var testData = make([]byte, 5000)
rand.Read(testData)
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (not predictable, high entropy distibution)
b.Run("random-50k", func(b *testing.B) {
var testData = make([]byte, 50000)
rand.Read(testData)
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (not predictable, high entropy distibution)
b.Run("random-500k", func(b *testing.B) {
var testData = make([]byte, 500000)
rand.Read(testData)
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (not predictable, medium entropy distibution)
b.Run("base-32-5k", func(b *testing.B) {
var testData = make([]byte, 5000)
rand.Read(testData)
s := base32.StdEncoding.EncodeToString(testData)
testData = []byte(s)
testData = testData[:5000]
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
// (medium predictable, medium entropy distibution)
b.Run("text", func(b *testing.B) {
var testData = []byte(`If compression is done per-chunk, care should be taken that it doesn't leave restic backups open to watermarking/fingerprinting attacks.
This is essentially the same problem we discussed related to fingerprinting the CDC deduplication process:
With "naive" CDC, a "known plaintext" file can be verified to exist within the backup if the size of individual blocks can be observed by an attacker, by using CDC on the file in parallel and comparing the resulting amount of chunks and individual chunk lengths.
As discussed earlier, this can be somewhat mitigated by salting the CDC algorithm with a secret value, as done in attic.
With salted CDC, I assume compression would happen on each individual chunk, after splitting the problematic file into chunks. Restic chunks are in the range of 512 KB to 8 MB (but not evenly distributed - right?).
Attacker knows that the CDC algorithm uses a secret salt, so the attacker generates a range of chunks consisting of the first 512 KB to 8 MB of the file, one for each valid chunk length. The attacker is also able to determine the lengths of compressed chunks.
The attacker then compresses that chunk using the compression algorithm.
The attacker compares the lengths of the resulting chunks to the first chunk in the restic backup sets.
IF a matching block length is found, the attacker repeats the exercise with the next chunk, and the next chunk, and the next chunk, ... and the next chunk.
It is my belief that with sufficiently large files, and considering the fact that the CDC algorithm is "biased" (in lack of better of words) towards generating blocks of about 1 MB, this would be sufficient to ascertain whether or not a certain large file exists in the backup.
AS always, a paranoid and highly unscientific stream of consciousness.
Thoughts?`)
testData = append(testData, testData...)
testData = append(testData, testData...)
b.SetBytes(int64(len(testData)))
b.ResetTimer()
for i := 0; i < b.N; i++ {
Estimate(testData)
}
b.Log(Estimate(testData))
})
}
+1
View File
@@ -1,5 +1,6 @@
//+build !noasm
//+build !appengine
//+build !gccgo
// Copyright 2015, Klaus Post, see LICENSE for details.
+1
View File
@@ -1,5 +1,6 @@
//+build !noasm
//+build !appengine
//+build !gccgo
// Copyright 2015, Klaus Post, see LICENSE for details.
+1 -1
View File
@@ -1,4 +1,4 @@
//+build !amd64 noasm appengine
//+build !amd64 noasm appengine gccgo
// Copyright 2015, Klaus Post, see LICENSE for details.
+71 -37
View File
@@ -10,12 +10,14 @@ package flate
import (
"bufio"
"io"
"math/bits"
"strconv"
"sync"
)
const (
maxCodeLen = 16 // max length of Huffman code
maxCodeLen = 16 // max length of Huffman code
maxCodeLenMask = 15 // mask for max length of Huffman code
// The next three numbers come from the RFC section 3.2.7, with the
// additional proviso in section 3.2.5 which implies that distance codes
// 30 and 31 should never occur in compressed data.
@@ -101,10 +103,10 @@ const (
)
type huffmanDecoder struct {
min int // the minimum code length
chunks [huffmanNumChunks]uint32 // chunks as described above
links [][]uint32 // overflow links
linkMask uint32 // mask the width of the link table
min int // the minimum code length
chunks *[huffmanNumChunks]uint32 // chunks as described above
links [][]uint32 // overflow links
linkMask uint32 // mask the width of the link table
}
// Initialize Huffman decoding tables from array of code lengths.
@@ -112,21 +114,24 @@ type huffmanDecoder struct {
// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
// degenerate case where the tree has only a single symbol with length 1. Empty
// trees are permitted.
func (h *huffmanDecoder) init(bits []int) bool {
func (h *huffmanDecoder) init(lengths []int) bool {
// Sanity enables additional runtime tests during Huffman
// table construction. It's intended to be used during
// development to supplement the currently ad-hoc unit tests.
const sanity = false
if h.chunks == nil {
h.chunks = &[huffmanNumChunks]uint32{}
}
if h.min != 0 {
*h = huffmanDecoder{}
*h = huffmanDecoder{chunks: h.chunks, links: h.links}
}
// Count number of codes of each length,
// compute min and max length.
var count [maxCodeLen]int
var min, max int
for _, n := range bits {
for _, n := range lengths {
if n == 0 {
continue
}
@@ -136,7 +141,7 @@ func (h *huffmanDecoder) init(bits []int) bool {
if n > max {
max = n
}
count[n]++
count[n&maxCodeLenMask]++
}
// Empty tree. The decompressor.huffSym function will fail later if the tree
@@ -154,8 +159,8 @@ func (h *huffmanDecoder) init(bits []int) bool {
var nextcode [maxCodeLen]int
for i := min; i <= max; i++ {
code <<= 1
nextcode[i] = code
code += count[i]
nextcode[i&maxCodeLenMask] = code
code += count[i&maxCodeLenMask]
}
// Check that the coding is complete (i.e., that we've
@@ -168,33 +173,49 @@ func (h *huffmanDecoder) init(bits []int) bool {
}
h.min = min
chunks := h.chunks[:]
for i := range chunks {
chunks[i] = 0
}
if max > huffmanChunkBits {
numLinks := 1 << (uint(max) - huffmanChunkBits)
h.linkMask = uint32(numLinks - 1)
// create link tables
link := nextcode[huffmanChunkBits+1] >> 1
h.links = make([][]uint32, huffmanNumChunks-link)
if cap(h.links) < huffmanNumChunks-link {
h.links = make([][]uint32, huffmanNumChunks-link)
} else {
h.links = h.links[:huffmanNumChunks-link]
}
for j := uint(link); j < huffmanNumChunks; j++ {
reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8
reverse := int(bits.Reverse16(uint16(j)))
reverse >>= uint(16 - huffmanChunkBits)
off := j - uint(link)
if sanity && h.chunks[reverse] != 0 {
panic("impossible: overwriting existing chunk")
}
h.chunks[reverse] = uint32(off<<huffmanValueShift | (huffmanChunkBits + 1))
h.links[off] = make([]uint32, numLinks)
if cap(h.links[off]) < numLinks {
h.links[off] = make([]uint32, numLinks)
} else {
links := h.links[off][:0]
h.links[off] = links[:numLinks]
}
}
} else {
h.links = h.links[:0]
}
for i, n := range bits {
for i, n := range lengths {
if n == 0 {
continue
}
code := nextcode[n]
nextcode[n]++
chunk := uint32(i<<huffmanValueShift | n)
reverse := int(reverseByte[code>>8]) | int(reverseByte[code&0xff])<<8
reverse := int(bits.Reverse16(uint16(code)))
reverse >>= uint(16 - n)
if n <= huffmanChunkBits {
for off := reverse; off < len(h.chunks); off += 1 << uint(n) {
@@ -589,7 +610,7 @@ readLiteral:
return
}
}
dist = int(reverseByte[(f.b&0x1F)<<3])
dist = int(bits.Reverse8(uint8(f.b & 0x1F << 3)))
f.b >>= 5
f.nb -= 5
} else {
@@ -661,10 +682,7 @@ func (f *decompressor) dataBlock() {
nr, err := io.ReadFull(f.r, f.buf[0:4])
f.roffset += int64(nr)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
f.err = err
f.err = noEOF(err)
return
}
n := int(f.buf[0]) | int(f.buf[1])<<8
@@ -697,10 +715,7 @@ func (f *decompressor) copyData() {
f.copyLen -= cnt
f.dict.writeMark(cnt)
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
f.err = err
f.err = noEOF(err)
return
}
@@ -722,13 +737,18 @@ func (f *decompressor) finishBlock() {
f.step = (*decompressor).nextBlock
}
// noEOF returns err, unless err == io.EOF, in which case it returns io.ErrUnexpectedEOF.
func noEOF(e error) error {
if e == io.EOF {
return io.ErrUnexpectedEOF
}
return e
}
func (f *decompressor) moreBits() error {
c, err := f.r.ReadByte()
if err != nil {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
return noEOF(err)
}
f.roffset++
f.b |= uint32(c) << f.nb
@@ -743,25 +763,37 @@ func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) {
// cases, the chunks slice will be 0 for the invalid sequence, leading it
// satisfy the n == 0 check below.
n := uint(h.min)
// Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers,
// but is smart enough to keep local variables in registers, so use nb and b,
// inline call to moreBits and reassign b,nb back to f on return.
nb, b := f.nb, f.b
for {
for f.nb < n {
if err := f.moreBits(); err != nil {
return 0, err
for nb < n {
c, err := f.r.ReadByte()
if err != nil {
f.b = b
f.nb = nb
return 0, noEOF(err)
}
f.roffset++
b |= uint32(c) << (nb & 31)
nb += 8
}
chunk := h.chunks[f.b&(huffmanNumChunks-1)]
chunk := h.chunks[b&(huffmanNumChunks-1)]
n = uint(chunk & huffmanCountMask)
if n > huffmanChunkBits {
chunk = h.links[chunk>>huffmanValueShift][(f.b>>huffmanChunkBits)&h.linkMask]
chunk = h.links[chunk>>huffmanValueShift][(b>>huffmanChunkBits)&h.linkMask]
n = uint(chunk & huffmanCountMask)
}
if n <= f.nb {
if n <= nb {
if n == 0 {
f.b = b
f.nb = nb
f.err = CorruptInputError(f.roffset)
return 0, f.err
}
f.b >>= n
f.nb -= n
f.b = b >> (n & 31)
f.nb = nb - n
return int(chunk >> huffmanValueShift), nil
}
}
@@ -799,6 +831,8 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error {
r: makeReader(r),
bits: f.bits,
codebits: f.codebits,
h1: f.h1,
h2: f.h2,
dict: f.dict,
step: (*decompressor).nextBlock,
}
+10 -1
View File
@@ -25,6 +25,7 @@ func TestNlitOutOfRange(t *testing.T) {
const (
digits = iota
twain
random
)
var testfiles = []string{
@@ -34,6 +35,8 @@ var testfiles = []string{
digits: "../testdata/e.txt",
// Twain is Project Gutenberg's edition of Mark Twain's classic English novel.
twain: "../testdata/Mark.Twain-Tom.Sawyer.txt",
// Random bytes
random: "../testdata/sharnd.out",
}
func benchmarkDecode(b *testing.B, testfile, level, n int) {
@@ -63,8 +66,11 @@ func benchmarkDecode(b *testing.B, testfile, level, n int) {
buf0, compressed, w = nil, nil, nil
runtime.GC()
b.StartTimer()
r := NewReader(bytes.NewReader(buf1))
res := r.(Resetter)
for i := 0; i < b.N; i++ {
io.Copy(ioutil.Discard, NewReader(bytes.NewReader(buf1)))
res.Reset(bytes.NewReader(buf1), nil)
io.Copy(ioutil.Discard, r)
}
}
@@ -95,3 +101,6 @@ func BenchmarkDecodeTwainDefault1e6(b *testing.B) { benchmarkDecode(b, twain,
func BenchmarkDecodeTwainCompress1e4(b *testing.B) { benchmarkDecode(b, twain, compress, 1e4) }
func BenchmarkDecodeTwainCompress1e5(b *testing.B) { benchmarkDecode(b, twain, compress, 1e5) }
func BenchmarkDecodeTwainCompress1e6(b *testing.B) { benchmarkDecode(b, twain, compress, 1e6) }
func BenchmarkDecodeRandomSpeed1e4(b *testing.B) { benchmarkDecode(b, random, speed, 1e4) }
func BenchmarkDecodeRandomSpeed1e5(b *testing.B) { benchmarkDecode(b, random, speed, 1e5) }
func BenchmarkDecodeRandomSpeed1e6(b *testing.B) { benchmarkDecode(b, random, speed, 1e6) }

Some files were not shown because too many files have changed in this diff Show More