From 8f998801761db12abe06a10d4e7de70bb977e86d Mon Sep 17 00:00:00 2001 From: Derek Gonyeo Date: Mon, 13 Jun 2016 17:57:05 -0700 Subject: [PATCH] godeps: added docker2aci and dependencies --- Godeps/Godeps.json | 85 ++ .../src/github.com/appc/docker2aci/LICENSE | 202 +++ .../appc/docker2aci/lib/common/common.go | 62 + .../appc/docker2aci/lib/conversion_store.go | 126 ++ .../appc/docker2aci/lib/docker2aci.go | 417 +++++ .../lib/internal/backend/file/file.go | 332 ++++ .../internal/backend/repository/repository.go | 164 ++ .../backend/repository/repository1.go | 391 +++++ .../backend/repository/repository2.go | 466 ++++++ .../lib/internal/docker/docker_functions.go | 161 ++ .../appc/docker2aci/lib/internal/internal.go | 591 +++++++ .../lib/internal/tarball/tarfile.go | 42 + .../docker2aci/lib/internal/tarball/walk.go | 43 + .../lib/internal/types/docker_types.go | 92 ++ .../docker2aci/lib/internal/types/types.go | 27 + .../appc/docker2aci/lib/internal/util/util.go | 83 + .../github.com/appc/docker2aci/lib/version.go | 20 + .../github.com/appc/docker2aci/pkg/log/log.go | 46 + .../src/github.com/coreos/pkg/LICENSE | 202 +++ .../src/github.com/coreos/pkg/NOTICE | 5 + .../coreos/pkg/progressutil/iocopy.go | 189 +++ .../coreos/pkg/progressutil/progressbar.go | 263 ++++ .../github.com/docker/distribution/LICENSE | 202 +++ .../docker/distribution/digest/digest.go | 139 ++ .../docker/distribution/digest/digester.go | 155 ++ .../docker/distribution/digest/doc.go | 42 + .../docker/distribution/digest/set.go | 245 +++ .../docker/distribution/digest/verifiers.go | 44 + .../distribution/reference/reference.go | 334 ++++ .../docker/distribution/reference/regexp.go | 124 ++ .../src/github.com/klauspost/compress/LICENSE | 27 + .../klauspost/compress/flate/copy.go | 32 + .../klauspost/compress/flate/crc32_amd64.go | 39 + .../klauspost/compress/flate/crc32_amd64.s | 219 +++ .../klauspost/compress/flate/crc32_noasm.go | 34 + .../klauspost/compress/flate/deflate.go | 1357 +++++++++++++++++ .../klauspost/compress/flate/fixedhuff.go | 78 + .../klauspost/compress/flate/gen.go | 265 ++++ .../compress/flate/huffman_bit_writer.go | 717 +++++++++ .../klauspost/compress/flate/huffman_code.go | 363 +++++ .../klauspost/compress/flate/inflate.go | 846 ++++++++++ .../klauspost/compress/flate/reverse_bits.go | 48 + .../klauspost/compress/flate/snappy.go | 558 +++++++ .../klauspost/compress/flate/token.go | 105 ++ .../src/github.com/klauspost/cpuid/.gitignore | 24 + .../github.com/klauspost/cpuid/.travis.yml | 8 + .../src/github.com/klauspost/cpuid/LICENSE | 22 + .../src/github.com/klauspost/cpuid/README.md | 145 ++ .../src/github.com/klauspost/cpuid/cpuid.go | 1022 +++++++++++++ .../github.com/klauspost/cpuid/cpuid_386.s | 42 + .../github.com/klauspost/cpuid/cpuid_amd64.s | 42 + .../klauspost/cpuid/detect_intel.go | 17 + .../github.com/klauspost/cpuid/detect_ref.go | 23 + .../github.com/klauspost/cpuid/generate.go | 3 + .../github.com/klauspost/cpuid/private-gen.go | 476 ++++++ .../src/github.com/klauspost/crc32/.gitignore | 24 + .../github.com/klauspost/crc32/.travis.yml | 12 + .../src/github.com/klauspost/crc32/LICENSE | 28 + .../src/github.com/klauspost/crc32/README.md | 84 + .../src/github.com/klauspost/crc32/crc32.go | 186 +++ .../github.com/klauspost/crc32/crc32_amd64.go | 62 + .../github.com/klauspost/crc32/crc32_amd64.s | 237 +++ .../klauspost/crc32/crc32_amd64p32.go | 40 + .../klauspost/crc32/crc32_amd64p32.s | 67 + .../klauspost/crc32/crc32_generic.go | 29 + .../src/github.com/klauspost/pgzip/.gitignore | 24 + .../github.com/klauspost/pgzip/.travis.yml | 18 + .../src/github.com/klauspost/pgzip/GO_LICENSE | 27 + .../src/github.com/klauspost/pgzip/LICENSE | 22 + .../src/github.com/klauspost/pgzip/README.md | 115 ++ .../src/github.com/klauspost/pgzip/circle.yml | 7 + .../src/github.com/klauspost/pgzip/gunzip.go | 564 +++++++ .../src/github.com/klauspost/pgzip/gzip.go | 485 ++++++ 73 files changed, 13837 insertions(+) create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/LICENSE create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/file/file.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository1.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository2.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/docker/docker_functions.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/internal.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/tarfile.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/walk.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/docker_types.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/types.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/util/util.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/lib/version.go create mode 100644 Godeps/_workspace/src/github.com/appc/docker2aci/pkg/log/log.go create mode 100644 Godeps/_workspace/src/github.com/coreos/pkg/LICENSE create mode 100644 Godeps/_workspace/src/github.com/coreos/pkg/NOTICE create mode 100644 Godeps/_workspace/src/github.com/coreos/pkg/progressutil/iocopy.go create mode 100644 Godeps/_workspace/src/github.com/coreos/pkg/progressutil/progressbar.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/LICENSE create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/digest/digest.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/digest/digester.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/digest/doc.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/digest/set.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/digest/verifiers.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/reference/reference.go create mode 100644 Godeps/_workspace/src/github.com/docker/distribution/reference/regexp.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/LICENSE create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/copy.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.s create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_noasm.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/deflate.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/fixedhuff.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/gen.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_bit_writer.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_code.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/inflate.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/reverse_bits.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/snappy.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/compress/flate/token.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/.gitignore create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/LICENSE create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/README.md create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_386.s create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_amd64.s create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/detect_intel.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/detect_ref.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/generate.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/cpuid/private-gen.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/.gitignore create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/LICENSE create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/README.md create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/crc32.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.s create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.s create mode 100644 Godeps/_workspace/src/github.com/klauspost/crc32/crc32_generic.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/.gitignore create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/.travis.yml create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/GO_LICENSE create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/LICENSE create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/README.md create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/circle.yml create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/gunzip.go create mode 100644 Godeps/_workspace/src/github.com/klauspost/pgzip/gzip.go diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 82a6ccc1..0d343411 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -6,6 +6,56 @@ "./..." ], "Deps": [ + { + "ImportPath": "github.com/appc/docker2aci/lib", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/common", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal/backend/file", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal/backend/repository", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal/docker", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal/tarball", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal/types", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/lib/internal/util", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, + { + "ImportPath": "github.com/appc/docker2aci/pkg/log", + "Comment": "v0.11.1", + "Rev": "355af275fa8d48b4ecdd361a5feaf173abff81e7" + }, { "ImportPath": "github.com/appc/spec/aci", "Comment": "v0.8.4", @@ -54,6 +104,11 @@ "ImportPath": "github.com/coreos/ioprogress", "Rev": "e7fc03058804de5488baed8df5b89f3924b9ec9a" }, + { + "ImportPath": "github.com/coreos/pkg/progressutil", + "Comment": "v2-4-g938dc0a", + "Rev": "938dc0af5bf1b9907f3be6fefe4d2592a5658030" + }, { "ImportPath": "github.com/coreos/rkt/pkg/fileutil", "Comment": "v1.7.0", @@ -94,6 +149,16 @@ "Comment": "v1.0.4", "Rev": "71acacd42f85e5e82f70a55327789582a5200a90" }, + { + "ImportPath": "github.com/docker/distribution/digest", + "Comment": "v2.4.0-rc.1-139-g8b7c698", + "Rev": "8b7c698036e65e6be11b8d11f01ea8e9ef6abfb6" + }, + { + "ImportPath": "github.com/docker/distribution/reference", + "Comment": "v2.4.0-rc.1-139-g8b7c698", + "Rev": "8b7c698036e65e6be11b8d11f01ea8e9ef6abfb6" + }, { "ImportPath": "github.com/hashicorp/errwrap", "Rev": "7554cd9344cec97297fa6649b055a8c98c2a1e55" @@ -102,6 +167,26 @@ "ImportPath": "github.com/inconshreveable/mousetrap", "Rev": "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" }, + { + "ImportPath": "github.com/klauspost/compress/flate", + "Comment": "v1.0-12-g14eb9c4", + "Rev": "14eb9c4951195779ecfbec34431a976de7335b0a" + }, + { + "ImportPath": "github.com/klauspost/cpuid", + "Comment": "v1.0", + "Rev": "09cded8978dc9e80714c4d85b0322337b0a1e5e0" + }, + { + "ImportPath": "github.com/klauspost/crc32", + "Comment": "v1.0", + "Rev": "19b0b332c9e4516a6370a0456e6182c3b5036720" + }, + { + "ImportPath": "github.com/klauspost/pgzip", + "Comment": "v1.0", + "Rev": "95e8170c5d4da28db9c64dfc9ec3138ea4466fd4" + }, { "ImportPath": "github.com/kylelemons/godebug/diff", "Rev": "21cb3784d9bda523911b96719efba02b7e983256" diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/LICENSE b/Godeps/_workspace/src/github.com/appc/docker2aci/LICENSE new file mode 100644 index 00000000..e06d2081 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go new file mode 100644 index 00000000..5fd1d460 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/common/common.go @@ -0,0 +1,62 @@ +// Copyright 2016 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package common provides misc types and variables. +package common + +import ( + "github.com/appc/docker2aci/lib/internal/docker" + "github.com/appc/docker2aci/lib/internal/types" +) + +type Compression int + +const ( + NoCompression = iota + GzipCompression +) + +type ParsedDockerURL types.ParsedDockerURL + +const ( + AppcDockerRegistryURL = "appc.io/docker/registryurl" + AppcDockerRepository = "appc.io/docker/repository" + AppcDockerTag = "appc.io/docker/tag" + AppcDockerImageID = "appc.io/docker/imageid" + AppcDockerParentImageID = "appc.io/docker/parentimageid" + AppcDockerEntrypoint = "appc.io/docker/entrypoint" + AppcDockerCmd = "appc.io/docker/cmd" +) + +type ErrSeveralImages struct { + Msg string + Images []string +} + +// InsecureConfig represents the different insecure options available +type InsecureConfig struct { + SkipVerify bool + AllowHTTP bool +} + +func (e *ErrSeveralImages) Error() string { + return e.Msg +} + +// ParseDockerURL takes a Docker URL and returns a ParsedDockerURL with its +// index URL, image name, and tag. +func ParseDockerURL(arg string) (*ParsedDockerURL, error) { + p, err := docker.ParseDockerURL(arg) + return (*ParsedDockerURL)(p), err +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go new file mode 100644 index 00000000..4944f138 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/conversion_store.go @@ -0,0 +1,126 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker2aci + +import ( + "crypto/sha512" + "fmt" + "hash" + "io" + "io/ioutil" + "os" + + "github.com/appc/spec/aci" + "github.com/appc/spec/schema" + "github.com/appc/spec/schema/types" +) + +const ( + hashPrefix = "sha512-" +) + +type aciInfo struct { + path string + key string + ImageManifest *schema.ImageManifest +} + +// conversionStore is an simple implementation of the acirenderer.ACIRegistry +// interface. It stores the Docker layers converted to ACI so we can take +// advantage of acirenderer to generate a squashed ACI Image. +type conversionStore struct { + acis map[string]*aciInfo +} + +func newConversionStore() *conversionStore { + return &conversionStore{acis: make(map[string]*aciInfo)} +} + +func (ms *conversionStore) WriteACI(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + cr, err := aci.NewCompressedReader(f) + if err != nil { + return "", err + } + defer cr.Close() + + h := sha512.New() + r := io.TeeReader(cr, h) + + // read the file so we can get the hash + if _, err := io.Copy(ioutil.Discard, r); err != nil { + return "", fmt.Errorf("error reading ACI: %v", err) + } + + im, err := aci.ManifestFromImage(f) + if err != nil { + return "", err + } + + key := ms.HashToKey(h) + ms.acis[key] = &aciInfo{path: path, key: key, ImageManifest: im} + return key, nil +} + +func (ms *conversionStore) GetImageManifest(key string) (*schema.ImageManifest, error) { + aci, ok := ms.acis[key] + if !ok { + return nil, fmt.Errorf("aci with key: %s not found", key) + } + return aci.ImageManifest, nil +} + +func (ms *conversionStore) GetACI(name types.ACIdentifier, labels types.Labels) (string, error) { + for _, aci := range ms.acis { + // we implement this function to comply with the interface so don't + // bother implementing a proper label check + if aci.ImageManifest.Name.String() == name.String() { + return aci.key, nil + } + } + return "", fmt.Errorf("aci not found") +} + +func (ms *conversionStore) ReadStream(key string) (io.ReadCloser, error) { + img, ok := ms.acis[key] + if !ok { + return nil, fmt.Errorf("stream for key: %s not found", key) + } + f, err := os.Open(img.path) + if err != nil { + return nil, fmt.Errorf("error opening aci: %s", img.path) + } + + tr, err := aci.NewCompressedReader(f) + if err != nil { + return nil, err + } + + return tr, nil +} + +func (ms *conversionStore) ResolveKey(key string) (string, error) { + return key, nil +} + +func (ms *conversionStore) HashToKey(h hash.Hash) string { + s := h.Sum(nil) + return fmt.Sprintf("%s%x", hashPrefix, s) +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go new file mode 100644 index 00000000..4cb0ecc9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/docker2aci.go @@ -0,0 +1,417 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package docker2aci implements a simple library for converting docker images to +// App Container Images (ACIs). +package docker2aci + +import ( + "archive/tar" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/appc/docker2aci/lib/common" + "github.com/appc/docker2aci/lib/internal" + "github.com/appc/docker2aci/lib/internal/backend/file" + "github.com/appc/docker2aci/lib/internal/backend/repository" + "github.com/appc/docker2aci/lib/internal/docker" + "github.com/appc/docker2aci/lib/internal/tarball" + "github.com/appc/docker2aci/lib/internal/types" + "github.com/appc/docker2aci/lib/internal/util" + "github.com/appc/docker2aci/pkg/log" + "github.com/appc/spec/pkg/acirenderer" + "github.com/appc/spec/schema" + appctypes "github.com/appc/spec/schema/types" + gzip "github.com/klauspost/pgzip" +) + +// CommonConfig represents the shared configuration options for converting +// Docker images. +type CommonConfig struct { + Squash bool // squash the layers in one file + OutputDir string // where to put the resulting ACI + TmpDir string // directory to use for temporary files + Compression common.Compression // which compression to use for the resulting file(s) +} + +// RemoteConfig represents the remote repository specific configuration for +// converting Docker images. +type RemoteConfig struct { + CommonConfig + Username string // username to use if the image to convert needs authentication + Password string // password to use if the image to convert needs authentication + Insecure common.InsecureConfig // Insecure options +} + +// FileConfig represents the saved file specific configuration for converting +// Docker images. +type FileConfig struct { + CommonConfig + DockerURL string // select an image if there are several images/tags in the file, Syntax: "{docker registry URL}/{image name}:{tag}" +} + +// ConvertRemoteRepo generates ACI images from docker registry URLs. It takes +// as input a dockerURL of the form: +// +// {registry URL}/{repository}:{reference[tag|digest]} +// +// It then gets all the layers of the requested image and converts each of +// them to ACI. +// It returns the list of generated ACI paths. +func ConvertRemoteRepo(dockerURL string, config RemoteConfig) ([]string, error) { + repositoryBackend := repository.NewRepositoryBackend(config.Username, config.Password, config.Insecure) + return convertReal(repositoryBackend, dockerURL, config.Squash, config.OutputDir, config.TmpDir, config.Compression) +} + +// ConvertSavedFile generates ACI images from a file generated with "docker +// save". If there are several images/tags in the file, a particular image can +// be chosen via FileConfig.DockerURL. +// +// It returns the list of generated ACI paths. +func ConvertSavedFile(dockerSavedFile string, config FileConfig) ([]string, error) { + f, err := os.Open(dockerSavedFile) + if err != nil { + return nil, fmt.Errorf("error opening file: %v", err) + } + defer f.Close() + + fileBackend := file.NewFileBackend(f) + return convertReal(fileBackend, config.DockerURL, config.Squash, config.OutputDir, config.TmpDir, config.Compression) +} + +// GetIndexName returns the docker index server from a docker URL. +func GetIndexName(dockerURL string) string { + index, _ := docker.SplitReposName(dockerURL) + return index +} + +// GetDockercfgAuth reads a ~/.dockercfg file and returns the username and password +// of the given docker index server. +func GetDockercfgAuth(indexServer string) (string, string, error) { + return docker.GetAuthInfo(indexServer) +} + +func convertReal(backend internal.Docker2ACIBackend, dockerURL string, squash bool, outputDir string, tmpDir string, compression common.Compression) ([]string, error) { + log.Debug("Getting image info...") + ancestry, parsedDockerURL, err := backend.GetImageInfo(dockerURL) + if err != nil { + return nil, err + } + + layersOutputDir := outputDir + if squash { + layersOutputDir, err = ioutil.TempDir(tmpDir, "docker2aci-") + if err != nil { + return nil, fmt.Errorf("error creating dir: %v", err) + } + defer os.RemoveAll(layersOutputDir) + } + + conversionStore := newConversionStore() + + // only compress individual layers if we're not squashing + layerCompression := compression + if squash { + layerCompression = common.NoCompression + } + + aciLayerPaths, aciManifests, err := backend.BuildACI(ancestry, parsedDockerURL, layersOutputDir, tmpDir, layerCompression) + if err != nil { + return nil, err + } + + var images acirenderer.Images + for i, aciLayerPath := range aciLayerPaths { + key, err := conversionStore.WriteACI(aciLayerPath) + if err != nil { + return nil, fmt.Errorf("error inserting in the conversion store: %v", err) + } + + images = append(images, acirenderer.Image{Im: aciManifests[i], Key: key, Level: uint16(len(aciLayerPaths) - 1 - i)}) + } + + // acirenderer expects images in order from upper to base layer + images = util.ReverseImages(images) + if squash { + squashedImagePath, err := squashLayers(images, conversionStore, *parsedDockerURL, outputDir, compression) + if err != nil { + return nil, fmt.Errorf("error squashing image: %v", err) + } + aciLayerPaths = []string{squashedImagePath} + } + + return aciLayerPaths, nil +} + +// squashLayers receives a list of ACI layer file names ordered from base image +// to application image and squashes them into one ACI +func squashLayers(images []acirenderer.Image, aciRegistry acirenderer.ACIRegistry, parsedDockerURL types.ParsedDockerURL, outputDir string, compression common.Compression) (path string, err error) { + log.Debug("Squashing layers...") + log.Debug("Rendering ACI...") + renderedACI, err := acirenderer.GetRenderedACIFromList(images, aciRegistry) + if err != nil { + return "", fmt.Errorf("error rendering squashed image: %v", err) + } + manifests, err := getManifests(renderedACI, aciRegistry) + if err != nil { + return "", fmt.Errorf("error getting manifests: %v", err) + } + + squashedFilename := getSquashedFilename(parsedDockerURL) + squashedImagePath := filepath.Join(outputDir, squashedFilename) + + squashedTempFile, err := ioutil.TempFile(outputDir, "docker2aci-squashedFile-") + if err != nil { + return "", err + } + defer func() { + if err == nil { + err = squashedTempFile.Close() + } else { + // remove temp file on error + // we ignore its error to not mask the real error + os.Remove(squashedTempFile.Name()) + } + }() + + log.Debug("Writing squashed ACI...") + if err := writeSquashedImage(squashedTempFile, renderedACI, aciRegistry, manifests, compression); err != nil { + return "", fmt.Errorf("error writing squashed image: %v", err) + } + + log.Debug("Validating squashed ACI...") + if err := internal.ValidateACI(squashedTempFile.Name()); err != nil { + return "", fmt.Errorf("error validating image: %v", err) + } + + if err := os.Rename(squashedTempFile.Name(), squashedImagePath); err != nil { + return "", err + } + + log.Debug("ACI squashed!") + return squashedImagePath, nil +} + +func getSquashedFilename(parsedDockerURL types.ParsedDockerURL) string { + squashedFilename := strings.Replace(parsedDockerURL.ImageName, "/", "-", -1) + if parsedDockerURL.Tag != "" { + squashedFilename += "-" + parsedDockerURL.Tag + } + squashedFilename += ".aci" + + return squashedFilename +} + +func getManifests(renderedACI acirenderer.RenderedACI, aciRegistry acirenderer.ACIRegistry) ([]schema.ImageManifest, error) { + var manifests []schema.ImageManifest + + for _, aci := range renderedACI { + im, err := aciRegistry.GetImageManifest(aci.Key) + if err != nil { + return nil, err + } + manifests = append(manifests, *im) + } + + return manifests, nil +} + +func writeSquashedImage(outputFile *os.File, renderedACI acirenderer.RenderedACI, aciProvider acirenderer.ACIProvider, manifests []schema.ImageManifest, compression common.Compression) error { + var tarWriterTarget io.WriteCloser = outputFile + + switch compression { + case common.NoCompression: + case common.GzipCompression: + tarWriterTarget = gzip.NewWriter(outputFile) + defer tarWriterTarget.Close() + default: + return fmt.Errorf("unexpected compression enum value: %d", compression) + } + + outputWriter := tar.NewWriter(tarWriterTarget) + defer outputWriter.Close() + + finalManifest := mergeManifests(manifests) + + if err := internal.WriteManifest(outputWriter, finalManifest); err != nil { + return err + } + + if err := internal.WriteRootfsDir(outputWriter); err != nil { + return err + } + + type hardLinkEntry struct { + firstLinkCleanName string + firstLinkHeader tar.Header + keepOriginal bool + walked bool + } + // map aciFileKey -> cleanTarget -> hardLinkEntry + hardLinks := make(map[string]map[string]hardLinkEntry) + + // first pass: read all the entries and build the hardLinks map in memory + // but don't write on disk + for _, aciFile := range renderedACI { + rs, err := aciProvider.ReadStream(aciFile.Key) + if err != nil { + return err + } + defer rs.Close() + + hardLinks[aciFile.Key] = map[string]hardLinkEntry{} + + squashWalker := func(t *tarball.TarFile) error { + cleanName := filepath.Clean(t.Name()) + // the rootfs and the squashed manifest are added separately + if cleanName == "manifest" || cleanName == "rootfs" { + return nil + } + _, keep := aciFile.FileMap[cleanName] + if keep && t.Header.Typeflag == tar.TypeLink { + cleanTarget := filepath.Clean(t.Linkname()) + if _, ok := hardLinks[aciFile.Key][cleanTarget]; !ok { + _, keepOriginal := aciFile.FileMap[cleanTarget] + hardLinks[aciFile.Key][cleanTarget] = hardLinkEntry{cleanName, *t.Header, keepOriginal, false} + } + } + return nil + } + + tr := tar.NewReader(rs) + if err := tarball.Walk(*tr, squashWalker); err != nil { + return err + } + } + + // second pass: write on disk + for _, aciFile := range renderedACI { + rs, err := aciProvider.ReadStream(aciFile.Key) + if err != nil { + return err + } + defer rs.Close() + + squashWalker := func(t *tarball.TarFile) error { + cleanName := filepath.Clean(t.Name()) + // the rootfs and the squashed manifest are added separately + if cleanName == "manifest" || cleanName == "rootfs" { + return nil + } + _, keep := aciFile.FileMap[cleanName] + + if link, ok := hardLinks[aciFile.Key][cleanName]; ok { + if keep != link.keepOriginal { + return fmt.Errorf("logic error: should we keep file %q?", cleanName) + } + if keep { + if err := outputWriter.WriteHeader(t.Header); err != nil { + return fmt.Errorf("error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("error copying file into the tar out: %v", err) + } + } else { + // The current file does not remain but there is a hard link pointing to + // it. Write the current file but with the filename of the first hard link + // pointing to it. That first hard link will not be written later, see + // variable "alreadyWritten". + link.firstLinkHeader.Size = t.Header.Size + link.firstLinkHeader.Typeflag = t.Header.Typeflag + link.firstLinkHeader.Linkname = "" + + if err := outputWriter.WriteHeader(&link.firstLinkHeader); err != nil { + return fmt.Errorf("error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("error copying file into the tar out: %v", err) + } + } + } else if keep { + alreadyWritten := false + if t.Header.Typeflag == tar.TypeLink { + cleanTarget := filepath.Clean(t.Linkname()) + if link, ok := hardLinks[aciFile.Key][cleanTarget]; ok { + if !link.keepOriginal { + if link.walked { + t.Header.Linkname = link.firstLinkCleanName + } else { + alreadyWritten = true + } + } + link.walked = true + hardLinks[aciFile.Key][cleanTarget] = link + } + } + + if !alreadyWritten { + if err := outputWriter.WriteHeader(t.Header); err != nil { + return fmt.Errorf("error writing header: %v", err) + } + if _, err := io.Copy(outputWriter, t.TarStream); err != nil { + return fmt.Errorf("error copying file into the tar out: %v", err) + } + } + } + return nil + } + + tr := tar.NewReader(rs) + if err := tarball.Walk(*tr, squashWalker); err != nil { + return err + } + } + + return nil +} + +func mergeManifests(manifests []schema.ImageManifest) schema.ImageManifest { + // FIXME(iaguis) we take app layer's manifest as the final manifest for now + manifest := manifests[0] + + manifest.Dependencies = nil + + layerIndex := -1 + for i, l := range manifest.Labels { + if l.Name.String() == "layer" { + layerIndex = i + } + } + + if layerIndex != -1 { + manifest.Labels = append(manifest.Labels[:layerIndex], manifest.Labels[layerIndex+1:]...) + } + + nameWithoutLayerID := appctypes.MustACIdentifier(stripLayerID(manifest.Name.String())) + + manifest.Name = *nameWithoutLayerID + + // once the image is squashed, we don't need a pathWhitelist + manifest.PathWhitelist = nil + + return manifest +} + +// striplayerID strips the layer ID from an app name: +// +// myregistry.com/organization/app-name-85738f8f9a7f1b04b5329c590ebcb9e425925c6d0984089c43a022de4f19c281 +// myregistry.com/organization/app-name +func stripLayerID(layerName string) string { + n := strings.LastIndex(layerName, "-") + return layerName[:n] +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/file/file.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/file/file.go new file mode 100644 index 00000000..a83402df --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/file/file.go @@ -0,0 +1,332 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package file is an implementation of Docker2ACIBackend for files saved via +// "docker save". +// +// Note: this package is an implementation detail and shouldn't be used outside +// of docker2aci. +package file + +import ( + "archive/tar" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + + "github.com/appc/docker2aci/lib/common" + "github.com/appc/docker2aci/lib/internal" + "github.com/appc/docker2aci/lib/internal/docker" + "github.com/appc/docker2aci/lib/internal/tarball" + "github.com/appc/docker2aci/lib/internal/types" + "github.com/appc/docker2aci/pkg/log" + "github.com/appc/spec/schema" +) + +type FileBackend struct { + file *os.File +} + +func NewFileBackend(file *os.File) *FileBackend { + return &FileBackend{ + file: file, + } +} + +func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, *types.ParsedDockerURL, error) { + parsedDockerURL, err := docker.ParseDockerURL(dockerURL) + if err != nil { + // a missing Docker URL could mean that the file only contains one + // image, so we ignore the error here, we'll handle it in getImageID + } + + appImageID, parsedDockerURL, err := getImageID(lb.file, parsedDockerURL) + if err != nil { + return nil, nil, err + } + + ancestry, err := getAncestry(lb.file, appImageID) + if err != nil { + return nil, nil, fmt.Errorf("error getting ancestry: %v", err) + } + + return ancestry, parsedDockerURL, nil +} + +func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { + var aciLayerPaths []string + var aciManifests []*schema.ImageManifest + var curPwl []string + for i := len(layerIDs) - 1; i >= 0; i-- { + tmpDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") + if err != nil { + return nil, nil, fmt.Errorf("error creating dir: %v", err) + } + defer os.RemoveAll(tmpDir) + + j, err := getJson(lb.file, layerIDs[i]) + if err != nil { + return nil, nil, fmt.Errorf("error getting image json: %v", err) + } + + layerData := types.DockerImageData{} + if err := json.Unmarshal(j, &layerData); err != nil { + return nil, nil, fmt.Errorf("error unmarshaling layer data: %v", err) + } + + tmpLayerPath := path.Join(tmpDir, layerIDs[i]) + tmpLayerPath += ".tar" + + layerFile, err := extractEmbeddedLayer(lb.file, layerIDs[i], tmpLayerPath) + if err != nil { + return nil, nil, fmt.Errorf("error getting layer from file: %v", err) + } + defer layerFile.Close() + + log.Debug("Generating layer ACI...") + aciPath, manifest, err := internal.GenerateACI(i, layerData, dockerURL, outputDir, layerFile, curPwl, compression) + if err != nil { + return nil, nil, fmt.Errorf("error generating ACI: %v", err) + } + + aciLayerPaths = append(aciLayerPaths, aciPath) + aciManifests = append(aciManifests, manifest) + curPwl = manifest.PathWhitelist + } + + return aciLayerPaths, aciManifests, nil +} + +func getImageID(file *os.File, dockerURL *types.ParsedDockerURL) (string, *types.ParsedDockerURL, error) { + type tags map[string]string + type apps map[string]tags + + _, err := file.Seek(0, 0) + if err != nil { + return "", nil, fmt.Errorf("error seeking file: %v", err) + } + + var imageID string + var appName string + reposWalker := func(t *tarball.TarFile) error { + if filepath.Clean(t.Name()) == "repositories" { + repob, err := ioutil.ReadAll(t.TarStream) + if err != nil { + return fmt.Errorf("error reading repositories file: %v", err) + } + + var repositories apps + if err := json.Unmarshal(repob, &repositories); err != nil { + return fmt.Errorf("error unmarshaling repositories file") + } + + if dockerURL == nil { + n := len(repositories) + switch { + case n == 1: + for key, _ := range repositories { + appName = key + } + case n > 1: + var appNames []string + for key, _ := range repositories { + appNames = append(appNames, key) + } + return &common.ErrSeveralImages{ + Msg: "several images found", + Images: appNames, + } + default: + return fmt.Errorf("no images found") + } + } else { + appName = dockerURL.ImageName + } + + tag := "latest" + if dockerURL != nil { + tag = dockerURL.Tag + } + + app, ok := repositories[appName] + if !ok { + return fmt.Errorf("app %q not found", appName) + } + + _, ok = app[tag] + if !ok { + if len(app) == 1 { + for key, _ := range app { + tag = key + } + } else { + return fmt.Errorf("tag %q not found", tag) + } + } + + if dockerURL == nil { + dockerURL = &types.ParsedDockerURL{ + IndexURL: "", + Tag: tag, + ImageName: appName, + } + } + + imageID = string(app[tag]) + } + + return nil + } + + tr := tar.NewReader(file) + if err := tarball.Walk(*tr, reposWalker); err != nil { + return "", nil, err + } + + if imageID == "" { + return "", nil, fmt.Errorf("repositories file not found") + } + + return imageID, dockerURL, nil +} + +func getJson(file *os.File, layerID string) ([]byte, error) { + jsonPath := path.Join(layerID, "json") + return getTarFileBytes(file, jsonPath) +} + +func getTarFileBytes(file *os.File, path string) ([]byte, error) { + _, err := file.Seek(0, 0) + if err != nil { + fmt.Errorf("error seeking file: %v", err) + } + + var fileBytes []byte + fileWalker := func(t *tarball.TarFile) error { + if filepath.Clean(t.Name()) == path { + fileBytes, err = ioutil.ReadAll(t.TarStream) + if err != nil { + return err + } + } + + return nil + } + + tr := tar.NewReader(file) + if err := tarball.Walk(*tr, fileWalker); err != nil { + return nil, err + } + + if fileBytes == nil { + return nil, fmt.Errorf("file %q not found", path) + } + + return fileBytes, nil +} + +func extractEmbeddedLayer(file *os.File, layerID string, outputPath string) (*os.File, error) { + log.Info("Extracting ", layerID[:12], "\n") + + _, err := file.Seek(0, 0) + if err != nil { + fmt.Errorf("error seeking file: %v", err) + } + + layerTarPath := path.Join(layerID, "layer.tar") + + var layerFile *os.File + fileWalker := func(t *tarball.TarFile) error { + if filepath.Clean(t.Name()) == layerTarPath { + layerFile, err = os.Create(outputPath) + if err != nil { + return fmt.Errorf("error creating layer: %v", err) + } + + _, err = io.Copy(layerFile, t.TarStream) + if err != nil { + return fmt.Errorf("error getting layer: %v", err) + } + } + + return nil + } + + tr := tar.NewReader(file) + if err := tarball.Walk(*tr, fileWalker); err != nil { + return nil, err + } + + if layerFile == nil { + return nil, fmt.Errorf("file %q not found", layerTarPath) + } + + return layerFile, nil +} + +func getAncestry(file *os.File, imgID string) ([]string, error) { + var ancestry []string + + curImgID := imgID + + var err error + for curImgID != "" { + ancestry = append(ancestry, curImgID) + curImgID, err = getParent(file, curImgID) + if err != nil { + return nil, err + } + } + + return ancestry, nil +} + +func getParent(file *os.File, imgID string) (string, error) { + var parent string + + _, err := file.Seek(0, 0) + if err != nil { + return "", fmt.Errorf("error seeking file: %v", err) + } + + jsonPath := filepath.Join(imgID, "json") + parentWalker := func(t *tarball.TarFile) error { + if filepath.Clean(t.Name()) == jsonPath { + jsonb, err := ioutil.ReadAll(t.TarStream) + if err != nil { + return fmt.Errorf("error reading layer json: %v", err) + } + + var dockerData types.DockerImageData + if err := json.Unmarshal(jsonb, &dockerData); err != nil { + return fmt.Errorf("error unmarshaling layer data: %v", err) + } + + parent = dockerData.Parent + } + + return nil + } + + tr := tar.NewReader(file) + if err := tarball.Walk(*tr, parentWalker); err != nil { + return "", err + } + + return parent, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository.go new file mode 100644 index 00000000..57533857 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository.go @@ -0,0 +1,164 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package repository is an implementation of Docker2ACIBackend for Docker +// remote registries. +// +// Note: this package is an implementation detail and shouldn't be used outside +// of docker2aci. +package repository + +import ( + "fmt" + "net/http" + "path" + + "github.com/appc/docker2aci/lib/common" + "github.com/appc/docker2aci/lib/internal/docker" + "github.com/appc/docker2aci/lib/internal/types" + "github.com/appc/docker2aci/lib/internal/util" + "github.com/appc/spec/schema" +) + +type registryVersion int + +const ( + registryV1 registryVersion = iota + registryV2 +) + +type RepositoryBackend struct { + repoData *RepoData + username string + password string + insecure common.InsecureConfig + hostsV2Support map[string]bool + hostsV2AuthTokens map[string]map[string]string + schema string + imageManifests map[types.ParsedDockerURL]v2Manifest +} + +func NewRepositoryBackend(username string, password string, insecure common.InsecureConfig) *RepositoryBackend { + return &RepositoryBackend{ + username: username, + password: password, + insecure: insecure, + hostsV2Support: make(map[string]bool), + hostsV2AuthTokens: make(map[string]map[string]string), + imageManifests: make(map[types.ParsedDockerURL]v2Manifest), + } +} + +func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, *types.ParsedDockerURL, error) { + dockerURL, err := docker.ParseDockerURL(url) + if err != nil { + return nil, nil, err + } + + var supportsV2, ok bool + var URLSchema string + if supportsV2, ok = rb.hostsV2Support[dockerURL.IndexURL]; !ok { + var err error + URLSchema, supportsV2, err = rb.supportsRegistry(dockerURL.IndexURL, registryV2) + if err != nil { + return nil, nil, err + } + rb.schema = URLSchema + "://" + rb.hostsV2Support[dockerURL.IndexURL] = supportsV2 + } + + if supportsV2 { + return rb.getImageInfoV2(dockerURL) + } else { + URLSchema, supportsV1, err := rb.supportsRegistry(dockerURL.IndexURL, registryV1) + if err != nil { + return nil, nil, err + } + if !supportsV1 { + return nil, nil, fmt.Errorf("registry doesn't support API v2 nor v1") + } + rb.schema = URLSchema + "://" + return rb.getImageInfoV1(dockerURL) + } +} + +func (rb *RepositoryBackend) BuildACI(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { + if rb.hostsV2Support[dockerURL.IndexURL] { + return rb.buildACIV2(layerIDs, dockerURL, outputDir, tmpBaseDir, compression) + } else { + return rb.buildACIV1(layerIDs, dockerURL, outputDir, tmpBaseDir, compression) + } +} + +func checkRegistryStatus(statusCode int, hdr http.Header, version registryVersion) (bool, error) { + switch statusCode { + case http.StatusOK, http.StatusUnauthorized: + ok := true + if version == registryV2 { + // v2 API requires this check + ok = hdr.Get("Docker-Distribution-API-Version") == "registry/2.0" + } + return ok, nil + case http.StatusNotFound: + return false, nil + } + + return false, fmt.Errorf("unexpected http code: %d", statusCode) +} + +func (rb *RepositoryBackend) supportsRegistry(indexURL string, version registryVersion) (schema string, ok bool, err error) { + var URLPath string + switch version { + case registryV1: + // the v1 API defines this URL to check if the registry's status + URLPath = "v1/_ping" + case registryV2: + URLPath = "v2" + } + URLStr := path.Join(indexURL, URLPath) + + fetch := func(schema string) (res *http.Response, err error) { + url := schema + "://" + URLStr + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + rb.setBasicAuth(req) + + client := util.GetTLSClient(rb.insecure.SkipVerify) + res, err = client.Do(req) + return + } + + schema = "https" + res, err := fetch(schema) + if err == nil { + ok, err = checkRegistryStatus(res.StatusCode, res.Header, version) + defer res.Body.Close() + } + if err != nil || !ok { + if rb.insecure.AllowHTTP { + schema = "http" + res, err = fetch(schema) + if err == nil { + ok, err = checkRegistryStatus(res.StatusCode, res.Header, version) + defer res.Body.Close() + } + } + return schema, ok, err + } + + return schema, ok, err +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository1.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository1.go new file mode 100644 index 00000000..bc54e47c --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository1.go @@ -0,0 +1,391 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "strconv" + "strings" + "time" + + "github.com/appc/docker2aci/lib/common" + "github.com/appc/docker2aci/lib/internal" + "github.com/appc/docker2aci/lib/internal/types" + "github.com/appc/docker2aci/lib/internal/util" + "github.com/appc/docker2aci/pkg/log" + "github.com/appc/spec/schema" + "github.com/coreos/ioprogress" +) + +type RepoData struct { + Tokens []string + Endpoints []string + Cookie []string +} + +func (rb *RepositoryBackend) getImageInfoV1(dockerURL *types.ParsedDockerURL) ([]string, *types.ParsedDockerURL, error) { + repoData, err := rb.getRepoDataV1(dockerURL.IndexURL, dockerURL.ImageName) + if err != nil { + return nil, nil, fmt.Errorf("error getting repository data: %v", err) + } + + // TODO(iaguis) check more endpoints + appImageID, err := rb.getImageIDFromTagV1(repoData.Endpoints[0], dockerURL.ImageName, dockerURL.Tag, repoData) + if err != nil { + return nil, nil, fmt.Errorf("error getting ImageID from tag %s: %v", dockerURL.Tag, err) + } + + ancestry, err := rb.getAncestryV1(appImageID, repoData.Endpoints[0], repoData) + if err != nil { + return nil, nil, err + } + + rb.repoData = repoData + + return ancestry, dockerURL, nil +} + +func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { + layerFiles := make([]*os.File, len(layerIDs)) + layerDatas := make([]types.DockerImageData, len(layerIDs)) + + tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") + if err != nil { + return nil, nil, err + } + defer os.RemoveAll(tmpParentDir) + + var doneChannels []chan error + for i, layerID := range layerIDs { + doneChan := make(chan error) + doneChannels = append(doneChannels, doneChan) + // https://github.com/golang/go/wiki/CommonMistakes + i := i // golang-- + layerID := layerID + go func() { + tmpDir, err := ioutil.TempDir(tmpParentDir, "") + if err != nil { + doneChan <- fmt.Errorf("error creating dir: %v", err) + return + } + + j, size, err := rb.getJsonV1(layerID, rb.repoData.Endpoints[0], rb.repoData) + if err != nil { + doneChan <- fmt.Errorf("error getting image json: %v", err) + return + } + + layerDatas[i] = types.DockerImageData{} + if err := json.Unmarshal(j, &layerDatas[i]); err != nil { + doneChan <- fmt.Errorf("error unmarshaling layer data: %v", err) + return + } + + layerFiles[i], err = rb.getLayerV1(layerID, rb.repoData.Endpoints[0], rb.repoData, size, tmpDir) + if err != nil { + doneChan <- fmt.Errorf("error getting the remote layer: %v", err) + return + } + doneChan <- nil + }() + } + for _, doneChan := range doneChannels { + err := <-doneChan + if err != nil { + return nil, nil, err + } + } + var aciLayerPaths []string + var aciManifests []*schema.ImageManifest + var curPwl []string + + for i := len(layerIDs) - 1; i >= 0; i-- { + log.Debug("Generating layer ACI...") + aciPath, manifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression) + if err != nil { + return nil, nil, fmt.Errorf("error generating ACI: %v", err) + } + aciLayerPaths = append(aciLayerPaths, aciPath) + aciManifests = append(aciManifests, manifest) + curPwl = manifest.PathWhitelist + + layerFiles[i].Close() + } + + return aciLayerPaths, aciManifests, nil +} + +func (rb *RepositoryBackend) getRepoDataV1(indexURL string, remote string) (*RepoData, error) { + client := util.GetTLSClient(rb.insecure.SkipVerify) + repositoryURL := rb.schema + path.Join(indexURL, "v1", "repositories", remote, "images") + + req, err := http.NewRequest("GET", repositoryURL, nil) + if err != nil { + return nil, err + } + + if rb.username != "" && rb.password != "" { + req.SetBasicAuth(rb.username, rb.password) + } + + req.Header.Set("X-Docker-Token", "true") + + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code: %d, URL: %s", res.StatusCode, req.URL) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + } + + var cookies []string + if res.Header.Get("Set-Cookie") != "" { + cookies = res.Header["Set-Cookie"] + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = makeEndpointsListV1(res.Header["X-Docker-Endpoints"]) + } else { + // Assume same endpoint + endpoints = append(endpoints, indexURL) + } + + return &RepoData{ + Endpoints: endpoints, + Tokens: tokens, + Cookie: cookies, + }, nil +} + +func (rb *RepositoryBackend) getImageIDFromTagV1(registry string, appName string, tag string, repoData *RepoData) (string, error) { + client := util.GetTLSClient(rb.insecure.SkipVerify) + // we get all the tags instead of directly getting the imageID of the + // requested one (.../tags/TAG) because even though it's specified in the + // Docker API, some registries (e.g. Google Container Registry) don't + // implement it. + req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "repositories", appName, "tags"), nil) + if err != nil { + return "", fmt.Errorf("failed to get Image ID: %s, URL: %s", err, req.URL) + } + + setAuthTokenV1(req, repoData.Tokens) + setCookieV1(req, repoData.Cookie) + res, err := client.Do(req) + if err != nil { + return "", fmt.Errorf("failed to get Image ID: %s, URL: %s", err, req.URL) + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return "", fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + j, err := ioutil.ReadAll(res.Body) + + if err != nil { + return "", err + } + + var tags map[string]string + + if err := json.Unmarshal(j, &tags); err != nil { + return "", fmt.Errorf("error unmarshaling: %v", err) + } + + imageID, ok := tags[tag] + if !ok { + return "", fmt.Errorf("tag %s not found", tag) + } + + return imageID, nil +} + +func (rb *RepositoryBackend) getAncestryV1(imgID, registry string, repoData *RepoData) ([]string, error) { + client := util.GetTLSClient(rb.insecure.SkipVerify) + req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "images", imgID, "ancestry"), nil) + if err != nil { + return nil, err + } + + setAuthTokenV1(req, repoData.Tokens) + setCookieV1(req, repoData.Cookie) + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + var ancestry []string + + j, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Failed to read downloaded json: %s (%s)", err, j) + } + + if err := json.Unmarshal(j, &ancestry); err != nil { + return nil, fmt.Errorf("error unmarshaling: %v", err) + } + + return ancestry, nil +} + +func (rb *RepositoryBackend) getJsonV1(imgID, registry string, repoData *RepoData) ([]byte, int64, error) { + client := util.GetTLSClient(rb.insecure.SkipVerify) + req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "images", imgID, "json"), nil) + if err != nil { + return nil, -1, err + } + setAuthTokenV1(req, repoData.Tokens) + setCookieV1(req, repoData.Cookie) + res, err := client.Do(req) + if err != nil { + return nil, -1, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + return nil, -1, fmt.Errorf("HTTP code: %d, URL: %s", res.StatusCode, req.URL) + } + + imageSize := int64(-1) + + if hdr := res.Header.Get("X-Docker-Size"); hdr != "" { + imageSize, err = strconv.ParseInt(hdr, 10, 64) + if err != nil { + return nil, -1, err + } + } + + b, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, -1, fmt.Errorf("failed to read downloaded json: %v (%s)", err, b) + } + + return b, imageSize, nil +} + +func (rb *RepositoryBackend) getLayerV1(imgID, registry string, repoData *RepoData, imgSize int64, tmpDir string) (*os.File, error) { + client := util.GetTLSClient(rb.insecure.SkipVerify) + req, err := http.NewRequest("GET", rb.schema+path.Join(registry, "images", imgID, "layer"), nil) + if err != nil { + return nil, err + } + + setAuthTokenV1(req, repoData.Tokens) + setCookieV1(req, repoData.Cookie) + + res, err := client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode != 200 { + res.Body.Close() + return nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + // if we didn't receive the size via X-Docker-Size when we retrieved the + // layer's json, try Content-Length + if imgSize == -1 { + if hdr := res.Header.Get("Content-Length"); hdr != "" { + imgSize, err = strconv.ParseInt(hdr, 10, 64) + if err != nil { + return nil, err + } + } + } + + prefix := "Downloading " + imgID[:12] + fmtBytesSize := 18 + barSize := int64(80 - len(prefix) - fmtBytesSize) + bar := ioprogress.DrawTextFormatBarForW(barSize, os.Stderr) + fmtfunc := func(progress, total int64) string { + return fmt.Sprintf( + "%s: %s %s", + prefix, + bar(progress, total), + ioprogress.DrawTextFormatBytes(progress, total), + ) + } + + progressReader := &ioprogress.Reader{ + Reader: res.Body, + Size: imgSize, + DrawFunc: ioprogress.DrawTerminalf(os.Stderr, fmtfunc), + DrawInterval: 500 * time.Millisecond, + } + + layerFile, err := ioutil.TempFile(tmpDir, "dockerlayer-") + if err != nil { + return nil, err + } + + _, err = io.Copy(layerFile, progressReader) + if err != nil { + return nil, err + } + + if err := layerFile.Sync(); err != nil { + return nil, err + } + + return layerFile, nil +} + +func setAuthTokenV1(req *http.Request, token []string) { + if req.Header.Get("Authorization") == "" { + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + } +} + +func setCookieV1(req *http.Request, cookie []string) { + if req.Header.Get("Cookie") == "" { + req.Header.Set("Cookie", strings.Join(cookie, "")) + } +} + +func makeEndpointsListV1(headers []string) []string { + var endpoints []string + + for _, ep := range headers { + endpointsList := strings.Split(ep, ",") + for _, endpointEl := range endpointsList { + endpoints = append( + endpoints, + path.Join(strings.TrimSpace(endpointEl), "v1")) + } + } + + return endpoints +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository2.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository2.go new file mode 100644 index 00000000..356d6f9b --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/backend/repository/repository2.go @@ -0,0 +1,466 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/appc/docker2aci/lib/common" + "github.com/appc/docker2aci/lib/internal" + "github.com/appc/docker2aci/lib/internal/types" + "github.com/appc/docker2aci/lib/internal/util" + "github.com/appc/docker2aci/pkg/log" + "github.com/appc/spec/schema" + "github.com/coreos/pkg/progressutil" +) + +const ( + defaultIndexURL = "registry-1.docker.io" +) + +var validHex = regexp.MustCompile(`^([a-f0-9]{64})$`) + +type v2Manifest struct { + Name string `json:"name"` + Tag string `json:"tag"` + FSLayers []struct { + BlobSum string `json:"blobSum"` + } `json:"fsLayers"` + History []struct { + V1Compatibility string `json:"v1Compatibility"` + } `json:"history"` + Signature []byte `json:"signature"` +} + +func (rb *RepositoryBackend) getImageInfoV2(dockerURL *types.ParsedDockerURL) ([]string, *types.ParsedDockerURL, error) { + manifest, layers, err := rb.getManifestV2(dockerURL) + if err != nil { + return nil, nil, err + } + + rb.imageManifests[*dockerURL] = *manifest + + return layers, dockerURL, nil +} + +func (rb *RepositoryBackend) buildACIV2(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) { + layerFiles := make([]*os.File, len(layerIDs)) + layerDatas := make([]types.DockerImageData, len(layerIDs)) + + tmpParentDir, err := ioutil.TempDir(tmpBaseDir, "docker2aci-") + if err != nil { + return nil, nil, err + } + defer os.RemoveAll(tmpParentDir) + + copier := progressutil.NewCopyProgressPrinter() + + var errChannels []chan error + closers := make([]io.ReadCloser, len(layerIDs)) + var wg sync.WaitGroup + for i, layerID := range layerIDs { + wg.Add(1) + errChan := make(chan error, 1) + errChannels = append(errChannels, errChan) + // https://github.com/golang/go/wiki/CommonMistakes + i := i // golang-- + layerID := layerID + go func() { + defer wg.Done() + + manifest := rb.imageManifests[*dockerURL] + + layerIndex, err := getLayerIndex(layerID, manifest) + if err != nil { + errChan <- err + return + } + + if len(manifest.History) <= layerIndex { + errChan <- fmt.Errorf("history not found for layer %s", layerID) + return + } + + layerDatas[i] = types.DockerImageData{} + if err := json.Unmarshal([]byte(manifest.History[layerIndex].V1Compatibility), &layerDatas[i]); err != nil { + errChan <- fmt.Errorf("error unmarshaling layer data: %v", err) + return + } + + tmpDir, err := ioutil.TempDir(tmpParentDir, "") + if err != nil { + errChan <- fmt.Errorf("error creating dir: %v", err) + return + } + + layerFiles[i], closers[i], err = rb.getLayerV2(layerID, dockerURL, tmpDir, copier) + if err != nil { + errChan <- fmt.Errorf("error getting the remote layer: %v", err) + return + } + errChan <- nil + }() + } + // Need to wait for all of the readers to be added to the copier (which happens during rb.getLayerV2) + wg.Wait() + err = copier.PrintAndWait(os.Stderr, 500*time.Millisecond, nil) + if err != nil { + return nil, nil, err + } + for _, closer := range closers { + if closer != nil { + closer.Close() + } + } + for _, errChan := range errChannels { + err := <-errChan + if err != nil { + return nil, nil, err + } + } + for _, layerFile := range layerFiles { + err := layerFile.Sync() + if err != nil { + return nil, nil, err + } + } + var aciLayerPaths []string + var aciManifests []*schema.ImageManifest + var curPwl []string + for i := len(layerIDs) - 1; i >= 0; i-- { + log.Debug("Generating layer ACI...") + aciPath, aciManifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression) + if err != nil { + return nil, nil, fmt.Errorf("error generating ACI: %v", err) + } + aciLayerPaths = append(aciLayerPaths, aciPath) + aciManifests = append(aciManifests, aciManifest) + curPwl = aciManifest.PathWhitelist + + layerFiles[i].Close() + } + + return aciLayerPaths, aciManifests, nil +} + +func (rb *RepositoryBackend) getManifestV2(dockerURL *types.ParsedDockerURL) (*v2Manifest, []string, error) { + url := rb.schema + path.Join(dockerURL.IndexURL, "v2", dockerURL.ImageName, "manifests", dockerURL.Tag) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, nil, err + } + + rb.setBasicAuth(req) + + res, err := rb.makeRequest(req, dockerURL.ImageName) + if err != nil { + return nil, nil, err + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, req.URL) + } + + manblob, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, nil, err + } + + manifest := &v2Manifest{} + + err = json.Unmarshal(manblob, manifest) + if err != nil { + return nil, nil, err + } + + if manifest.Name != dockerURL.ImageName { + return nil, nil, fmt.Errorf("name doesn't match what was requested, expected: %s, downloaded: %s", dockerURL.ImageName, manifest.Name) + } + + if manifest.Tag != dockerURL.Tag { + return nil, nil, fmt.Errorf("tag doesn't match what was requested, expected: %s, downloaded: %s", dockerURL.Tag, manifest.Tag) + } + + if err := fixManifestLayers(manifest); err != nil { + return nil, nil, err + } + + //TODO: verify signature here + + layers := make([]string, len(manifest.FSLayers)) + + for i, layer := range manifest.FSLayers { + layers[i] = layer.BlobSum + } + + return manifest, layers, nil +} + +func fixManifestLayers(manifest *v2Manifest) error { + type imageV1 struct { + ID string + Parent string + } + imgs := make([]*imageV1, len(manifest.FSLayers)) + for i := range manifest.FSLayers { + img := &imageV1{} + + if err := json.Unmarshal([]byte(manifest.History[i].V1Compatibility), img); err != nil { + return err + } + + imgs[i] = img + if err := validateV1ID(img.ID); err != nil { + return err + } + } + + if imgs[len(imgs)-1].Parent != "" { + return errors.New("Invalid parent ID in the base layer of the image.") + } + + // check general duplicates to error instead of a deadlock + idmap := make(map[string]struct{}) + + var lastID string + for _, img := range imgs { + // skip IDs that appear after each other, we handle those later + if _, exists := idmap[img.ID]; img.ID != lastID && exists { + return fmt.Errorf("ID %+v appears multiple times in manifest", img.ID) + } + lastID = img.ID + idmap[lastID] = struct{}{} + } + + // backwards loop so that we keep the remaining indexes after removing items + for i := len(imgs) - 2; i >= 0; i-- { + if imgs[i].ID == imgs[i+1].ID { // repeated ID. remove and continue + manifest.FSLayers = append(manifest.FSLayers[:i], manifest.FSLayers[i+1:]...) + manifest.History = append(manifest.History[:i], manifest.History[i+1:]...) + } else if imgs[i].Parent != imgs[i+1].ID { + return fmt.Errorf("Invalid parent ID. Expected %v, got %v.", imgs[i+1].ID, imgs[i].Parent) + } + } + + return nil +} + +func validateV1ID(id string) error { + if ok := validHex.MatchString(id); !ok { + return fmt.Errorf("image ID %q is invalid", id) + } + return nil +} + +func getLayerIndex(layerID string, manifest v2Manifest) (int, error) { + for i, layer := range manifest.FSLayers { + if layer.BlobSum == layerID { + return i, nil + } + } + return -1, fmt.Errorf("layer not found in manifest: %s", layerID) +} + +func (rb *RepositoryBackend) getLayerV2(layerID string, dockerURL *types.ParsedDockerURL, tmpDir string, copier *progressutil.CopyProgressPrinter) (*os.File, io.ReadCloser, error) { + url := rb.schema + path.Join(dockerURL.IndexURL, "v2", dockerURL.ImageName, "blobs", layerID) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, nil, err + } + + rb.setBasicAuth(req) + + res, err := rb.makeRequest(req, dockerURL.ImageName) + if err != nil { + return nil, nil, err + } + + if res.StatusCode == http.StatusTemporaryRedirect || res.StatusCode == http.StatusFound { + location := res.Header.Get("Location") + if location != "" { + req, err = http.NewRequest("GET", location, nil) + if err != nil { + return nil, nil, err + } + res, err = rb.makeRequest(req, dockerURL.ImageName) + if err != nil { + return nil, nil, err + } + defer res.Body.Close() + } + } + + if res.StatusCode != http.StatusOK { + return nil, nil, fmt.Errorf("HTTP code: %d. URL: %s", res.StatusCode, req.URL) + } + + var in io.Reader + in = res.Body + + var size int64 + + if hdr := res.Header.Get("Content-Length"); hdr != "" { + size, err = strconv.ParseInt(hdr, 10, 64) + if err != nil { + return nil, nil, err + } + } + + name := "Downloading " + layerID[:18] + + layerFile, err := ioutil.TempFile(tmpDir, "dockerlayer-") + if err != nil { + return nil, nil, err + } + + err = copier.AddCopy(in, name, size, layerFile) + if err != nil { + return nil, nil, err + } + + return layerFile, res.Body, nil +} + +func (rb *RepositoryBackend) makeRequest(req *http.Request, repo string) (*http.Response, error) { + setBearerHeader := false + hostAuthTokens, ok := rb.hostsV2AuthTokens[req.URL.Host] + if ok { + authToken, ok := hostAuthTokens[repo] + if ok { + req.Header.Set("Authorization", "Bearer "+authToken) + setBearerHeader = true + } + } + + client := util.GetTLSClient(rb.insecure.SkipVerify) + res, err := client.Do(req) + if err != nil { + return nil, err + } + + if res.StatusCode == http.StatusUnauthorized && setBearerHeader { + return res, err + } + + hdr := res.Header.Get("www-authenticate") + if res.StatusCode != http.StatusUnauthorized || hdr == "" { + return res, err + } + + tokens := strings.Split(hdr, ",") + if len(tokens) != 3 || + !strings.HasPrefix(strings.ToLower(tokens[0]), "bearer realm") { + return res, err + } + res.Body.Close() + + var realm, service, scope string + for _, token := range tokens { + if strings.HasPrefix(strings.ToLower(token), "bearer realm") { + realm = strings.Trim(token[len("bearer realm="):], "\"") + } + if strings.HasPrefix(token, "service") { + service = strings.Trim(token[len("service="):], "\"") + } + if strings.HasPrefix(token, "scope") { + scope = strings.Trim(token[len("scope="):], "\"") + } + } + + if realm == "" { + return nil, fmt.Errorf("missing realm in bearer auth challenge") + } + if service == "" { + return nil, fmt.Errorf("missing service in bearer auth challenge") + } + // The scope can be empty if we're not getting a token for a specific repo + if scope == "" && repo != "" { + // If the scope is empty and it shouldn't be, we can infer it based on the repo + scope = fmt.Sprintf("repository:%s:pull", repo) + } + + authReq, err := http.NewRequest("GET", realm, nil) + if err != nil { + return nil, err + } + + getParams := authReq.URL.Query() + getParams.Add("service", service) + if scope != "" { + getParams.Add("scope", scope) + } + authReq.URL.RawQuery = getParams.Encode() + + rb.setBasicAuth(authReq) + + res, err = client.Do(authReq) + if err != nil { + return nil, err + } + defer res.Body.Close() + + switch res.StatusCode { + case http.StatusUnauthorized: + return nil, fmt.Errorf("unable to retrieve auth token: 401 unauthorized") + case http.StatusOK: + break + default: + return nil, fmt.Errorf("unexpected http code: %d, URL: %s", res.StatusCode, authReq.URL) + } + + tokenBlob, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + tokenStruct := struct { + Token string `json:"token"` + }{} + + err = json.Unmarshal(tokenBlob, &tokenStruct) + if err != nil { + return nil, err + } + + hostAuthTokens, ok = rb.hostsV2AuthTokens[req.URL.Host] + if !ok { + hostAuthTokens = make(map[string]string) + rb.hostsV2AuthTokens[req.URL.Host] = hostAuthTokens + } + + hostAuthTokens[repo] = tokenStruct.Token + + return rb.makeRequest(req, repo) +} + +func (rb *RepositoryBackend) setBasicAuth(req *http.Request) { + if rb.username != "" && rb.password != "" { + req.SetBasicAuth(rb.username, rb.password) + } +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/docker/docker_functions.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/docker/docker_functions.go new file mode 100644 index 00000000..70bca8e4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/docker/docker_functions.go @@ -0,0 +1,161 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path" + "runtime" + "strings" + + "github.com/appc/docker2aci/lib/internal/types" + + "github.com/docker/distribution/reference" +) + +const ( + dockercfgFileName = "config.json" + dockercfgFileNameOld = ".dockercfg" + defaultIndexURL = "registry-1.docker.io" + defaultIndexURLAuth = "https://index.docker.io/v1/" + defaultTag = "latest" + defaultRepoPrefix = "library/" +) + +// SplitReposName breaks a repo name into an index name and remote name. +func SplitReposName(name string) (indexName, remoteName string) { + i := strings.IndexRune(name, '/') + if i == -1 || (!strings.ContainsAny(name[:i], ".:") && name[:i] != "localhost") { + indexName, remoteName = defaultIndexURL, name + } else { + indexName, remoteName = name[:i], name[i+1:] + } + if indexName == defaultIndexURL && !strings.ContainsRune(remoteName, '/') { + remoteName = defaultRepoPrefix + remoteName + } + return +} + +// Get a repos name and returns the right reposName + tag +// The tag can be confusing because of a port in a repository name. +// Ex: localhost.localdomain:5000/samalba/hipache:latest +func parseRepositoryTag(repos string) (string, string) { + n := strings.LastIndex(repos, ":") + if n < 0 { + return repos, "" + } + if tag := repos[n+1:]; !strings.Contains(tag, "/") { + return repos[:n], tag + } + return repos, "" +} + +func decodeDockerAuth(s string) (string, string, error) { + decoded, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return "", "", err + } + parts := strings.SplitN(string(decoded), ":", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf("invalid auth configuration file") + } + user := parts[0] + password := strings.Trim(parts[1], "\x00") + return user, password, nil +} + +func getHomeDir() string { + if runtime.GOOS == "windows" { + return os.Getenv("USERPROFILE") + } + return os.Getenv("HOME") +} + +// GetDockercfgAuth reads a ~/.dockercfg file and returns the username and password +// of the given docker index server. +func GetAuthInfo(indexServer string) (string, string, error) { + // official docker registry + if indexServer == defaultIndexURL { + indexServer = defaultIndexURLAuth + } + dockerCfgPath := path.Join(getHomeDir(), ".docker", dockercfgFileName) + if _, err := os.Stat(dockerCfgPath); err == nil { + j, err := ioutil.ReadFile(dockerCfgPath) + if err != nil { + return "", "", err + } + var dockerAuth types.DockerConfigFile + if err := json.Unmarshal(j, &dockerAuth); err != nil { + return "", "", err + } + // try the normal case + if c, ok := dockerAuth.AuthConfigs[indexServer]; ok { + return decodeDockerAuth(c.Auth) + } + } else if os.IsNotExist(err) { + oldDockerCfgPath := path.Join(getHomeDir(), dockercfgFileNameOld) + if _, err := os.Stat(oldDockerCfgPath); err != nil { + return "", "", nil //missing file is not an error + } + j, err := ioutil.ReadFile(oldDockerCfgPath) + if err != nil { + return "", "", err + } + var dockerAuthOld map[string]types.DockerAuthConfigOld + if err := json.Unmarshal(j, &dockerAuthOld); err != nil { + return "", "", err + } + if c, ok := dockerAuthOld[indexServer]; ok { + return decodeDockerAuth(c.Auth) + } + } else { + // if file is there but we can't stat it for any reason other + // than it doesn't exist then stop + return "", "", fmt.Errorf("%s - %v", dockerCfgPath, err) + } + return "", "", nil +} + +// ParseDockerURL takes a Docker URL and returns a ParsedDockerURL with its +// index URL, image name, and tag. +func ParseDockerURL(arg string) (*types.ParsedDockerURL, error) { + r, err := reference.ParseNamed(arg) + if err != nil { + return nil, err + } + + tag := defaultTag + var digest string + switch x := r.(type) { + case reference.Canonical: + digest = x.Digest().String() + case reference.NamedTagged: + tag = x.Tag() + } + + indexURL, remoteName := SplitReposName(r.Name()) + + p := &types.ParsedDockerURL{ + IndexURL: indexURL, + ImageName: remoteName, + Tag: tag, + Digest: digest, + } + return p, nil +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/internal.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/internal.go new file mode 100644 index 00000000..29cd8c87 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/internal.go @@ -0,0 +1,591 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package internal provides functions shared by different parts of docker2aci. +// +// Note: this package is an implementation detail and shouldn't be used outside +// of docker2aci. +package internal + +import ( + "archive/tar" + "encoding/json" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/appc/docker2aci/lib/common" + "github.com/appc/docker2aci/lib/internal/tarball" + "github.com/appc/docker2aci/lib/internal/types" + "github.com/appc/docker2aci/lib/internal/util" + "github.com/appc/docker2aci/pkg/log" + "github.com/appc/spec/aci" + "github.com/appc/spec/schema" + appctypes "github.com/appc/spec/schema/types" + gzip "github.com/klauspost/pgzip" +) + +// Docker2ACIBackend is the interface that abstracts converting Docker layers +// to ACI from where they're stored (remote or file). +// +// GetImageInfo takes a Docker URL and returns a list of layers and the parsed +// Docker URL. +// +// BuildACI takes a Docker layer, converts it to ACI and returns its output +// path and its converted ImageManifest. +type Docker2ACIBackend interface { + GetImageInfo(dockerUrl string) ([]string, *types.ParsedDockerURL, error) + BuildACI(layerIDs []string, dockerURL *types.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) +} + +// GenerateACI takes a Docker layer and generates an ACI from it. +func GenerateACI(layerNumber int, layerData types.DockerImageData, dockerURL *types.ParsedDockerURL, outputDir string, layerFile *os.File, curPwl []string, compression common.Compression) (string, *schema.ImageManifest, error) { + manifest, err := GenerateManifest(layerData, dockerURL) + if err != nil { + return "", nil, fmt.Errorf("error generating the manifest: %v", err) + } + + imageName := strings.Replace(dockerURL.ImageName, "/", "-", -1) + aciPath := imageName + "-" + layerData.ID + if dockerURL.Tag != "" { + aciPath += "-" + dockerURL.Tag + } + if layerData.OS != "" { + aciPath += "-" + layerData.OS + if layerData.Architecture != "" { + aciPath += "-" + layerData.Architecture + } + } + aciPath += "-" + strconv.Itoa(layerNumber) + aciPath += ".aci" + + aciPath = path.Join(outputDir, aciPath) + manifest, err = writeACI(layerFile, *manifest, curPwl, aciPath, compression) + if err != nil { + return "", nil, fmt.Errorf("error writing ACI: %v", err) + } + + if err := ValidateACI(aciPath); err != nil { + return "", nil, fmt.Errorf("invalid ACI generated: %v", err) + } + + return aciPath, manifest, nil +} + +// ValidateACI checks whether the ACI in aciPath is valid. +func ValidateACI(aciPath string) error { + aciFile, err := os.Open(aciPath) + if err != nil { + return err + } + defer aciFile.Close() + + tr, err := aci.NewCompressedTarReader(aciFile) + if err != nil { + return err + } + defer tr.Close() + + if err := aci.ValidateArchive(tr.Reader); err != nil { + return err + } + + return nil +} + +// GenerateManifest converts the docker manifest format to an appc +// ImageManifest. +func GenerateManifest(layerData types.DockerImageData, dockerURL *types.ParsedDockerURL) (*schema.ImageManifest, error) { + dockerConfig := layerData.Config + genManifest := &schema.ImageManifest{} + + appURL := "" + appURL = dockerURL.IndexURL + "/" + appURL += dockerURL.ImageName + "-" + layerData.ID + appURL, err := appctypes.SanitizeACIdentifier(appURL) + if err != nil { + return nil, err + } + name := appctypes.MustACIdentifier(appURL) + genManifest.Name = *name + + acVersion, err := appctypes.NewSemVer(schema.AppContainerVersion.String()) + if err != nil { + panic("invalid appc spec version") + } + genManifest.ACVersion = *acVersion + + genManifest.ACKind = appctypes.ACKind(schema.ImageManifestKind) + + var ( + labels appctypes.Labels + parentLabels appctypes.Labels + annotations appctypes.Annotations + ) + + layer := appctypes.MustACIdentifier("layer") + labels = append(labels, appctypes.Label{Name: *layer, Value: layerData.ID}) + + tag := dockerURL.Tag + version := appctypes.MustACIdentifier("version") + labels = append(labels, appctypes.Label{Name: *version, Value: tag}) + + if layerData.OS != "" { + os := appctypes.MustACIdentifier("os") + labels = append(labels, appctypes.Label{Name: *os, Value: layerData.OS}) + parentLabels = append(parentLabels, appctypes.Label{Name: *os, Value: layerData.OS}) + + if layerData.Architecture != "" { + arch := appctypes.MustACIdentifier("arch") + labels = append(labels, appctypes.Label{Name: *arch, Value: layerData.Architecture}) + parentLabels = append(parentLabels, appctypes.Label{Name: *arch, Value: layerData.Architecture}) + } + } + + if layerData.Author != "" { + authorsKey := appctypes.MustACIdentifier("authors") + annotations = append(annotations, appctypes.Annotation{Name: *authorsKey, Value: layerData.Author}) + } + epoch := time.Unix(0, 0) + if !layerData.Created.Equal(epoch) { + createdKey := appctypes.MustACIdentifier("created") + annotations = append(annotations, appctypes.Annotation{Name: *createdKey, Value: layerData.Created.Format(time.RFC3339)}) + } + if layerData.Comment != "" { + commentKey := appctypes.MustACIdentifier("docker-comment") + annotations = append(annotations, appctypes.Annotation{Name: *commentKey, Value: layerData.Comment}) + } + + if dockerURL.IndexURL != "" { + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerRegistryURL), Value: dockerURL.IndexURL}) + } + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerRepository), Value: dockerURL.ImageName}) + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerImageID), Value: layerData.ID}) + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerParentImageID), Value: layerData.Parent}) + + if dockerConfig != nil { + if len(dockerConfig.Entrypoint) > 0 { + entry, err := json.Marshal(dockerConfig.Entrypoint) + if err != nil { + return nil, err + } + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerEntrypoint), Value: string(entry)}) + } + if len(dockerConfig.Cmd) > 0 { + cmd, err := json.Marshal(dockerConfig.Cmd) + if err != nil { + return nil, err + } + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerCmd), Value: string(cmd)}) + } + + exec := getExecCommand(dockerConfig.Entrypoint, dockerConfig.Cmd) + if exec != nil { + user, group := parseDockerUser(dockerConfig.User) + var env appctypes.Environment + for _, v := range dockerConfig.Env { + parts := strings.SplitN(v, "=", 2) + env.Set(parts[0], parts[1]) + } + app := &appctypes.App{ + Exec: exec, + User: user, + Group: group, + Environment: env, + WorkingDirectory: dockerConfig.WorkingDir, + } + + app.MountPoints, err = convertVolumesToMPs(dockerConfig.Volumes) + if err != nil { + return nil, err + } + + app.Ports, err = convertPorts(dockerConfig.ExposedPorts, dockerConfig.PortSpecs) + if err != nil { + return nil, err + } + + genManifest.App = app + } + } + + if layerData.Parent != "" { + indexPrefix := "" + // omit docker hub index URL in app name + indexPrefix = dockerURL.IndexURL + "/" + parentImageNameString := indexPrefix + dockerURL.ImageName + "-" + layerData.Parent + parentImageNameString, err := appctypes.SanitizeACIdentifier(parentImageNameString) + if err != nil { + return nil, err + } + parentImageName := appctypes.MustACIdentifier(parentImageNameString) + + genManifest.Dependencies = append(genManifest.Dependencies, appctypes.Dependency{ImageName: *parentImageName, Labels: parentLabels}) + + annotations = append(annotations, appctypes.Annotation{Name: *appctypes.MustACIdentifier(common.AppcDockerTag), Value: dockerURL.Tag}) + } + + genManifest.Labels = labels + genManifest.Annotations = annotations + + return genManifest, nil +} + +type appcPortSorter []appctypes.Port + +func (s appcPortSorter) Len() int { + return len(s) +} + +func (s appcPortSorter) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s appcPortSorter) Less(i, j int) bool { + return s[i].Name.String() < s[j].Name.String() +} + +func convertPorts(dockerExposedPorts map[string]struct{}, dockerPortSpecs []string) ([]appctypes.Port, error) { + ports := []appctypes.Port{} + + for ep := range dockerExposedPorts { + appcPort, err := parseDockerPort(ep) + if err != nil { + return nil, err + } + ports = append(ports, *appcPort) + } + + if dockerExposedPorts == nil && dockerPortSpecs != nil { + log.Debug("warning: docker image uses deprecated PortSpecs field") + for _, ep := range dockerPortSpecs { + appcPort, err := parseDockerPort(ep) + if err != nil { + return nil, err + } + ports = append(ports, *appcPort) + } + } + + sort.Sort(appcPortSorter(ports)) + + return ports, nil +} + +func parseDockerPort(dockerPort string) (*appctypes.Port, error) { + var portString string + proto := "tcp" + sp := strings.Split(dockerPort, "/") + if len(sp) < 2 { + portString = dockerPort + } else { + proto = sp[1] + portString = sp[0] + } + + port, err := strconv.ParseUint(portString, 10, 0) + if err != nil { + return nil, fmt.Errorf("error parsing port %q: %v", portString, err) + } + + sn, err := appctypes.SanitizeACName(dockerPort) + if err != nil { + return nil, err + } + + appcPort := &appctypes.Port{ + Name: *appctypes.MustACName(sn), + Protocol: proto, + Port: uint(port), + } + + return appcPort, nil +} + +type appcVolSorter []appctypes.MountPoint + +func (s appcVolSorter) Len() int { + return len(s) +} + +func (s appcVolSorter) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s appcVolSorter) Less(i, j int) bool { + return s[i].Name.String() < s[j].Name.String() +} + +func convertVolumesToMPs(dockerVolumes map[string]struct{}) ([]appctypes.MountPoint, error) { + mps := []appctypes.MountPoint{} + dup := make(map[string]int) + + for p := range dockerVolumes { + n := filepath.Join("volume", p) + sn, err := appctypes.SanitizeACName(n) + if err != nil { + return nil, err + } + + // check for duplicate names + if i, ok := dup[sn]; ok { + dup[sn] = i + 1 + sn = fmt.Sprintf("%s-%d", sn, i) + } else { + dup[sn] = 1 + } + + mp := appctypes.MountPoint{ + Name: *appctypes.MustACName(sn), + Path: p, + } + + mps = append(mps, mp) + } + + sort.Sort(appcVolSorter(mps)) + + return mps, nil +} + +func writeACI(layer io.ReadSeeker, manifest schema.ImageManifest, curPwl []string, output string, compression common.Compression) (*schema.ImageManifest, error) { + aciFile, err := os.Create(output) + if err != nil { + return nil, fmt.Errorf("error creating ACI file: %v", err) + } + defer aciFile.Close() + + var w io.WriteCloser = aciFile + if compression == common.GzipCompression { + w = gzip.NewWriter(aciFile) + defer w.Close() + } + trw := tar.NewWriter(w) + defer trw.Close() + + if err := WriteRootfsDir(trw); err != nil { + return nil, fmt.Errorf("error writing rootfs entry: %v", err) + } + + fileMap := make(map[string]struct{}) + var whiteouts []string + convWalker := func(t *tarball.TarFile) error { + name := t.Name() + if name == "./" { + return nil + } + t.Header.Name = path.Join("rootfs", name) + absolutePath := strings.TrimPrefix(t.Header.Name, "rootfs") + + if filepath.Clean(absolutePath) == "/dev" && t.Header.Typeflag != tar.TypeDir { + return fmt.Errorf(`invalid layer: "/dev" is not a directory`) + } + + fileMap[absolutePath] = struct{}{} + if strings.Contains(t.Header.Name, "/.wh.") { + whiteouts = append(whiteouts, strings.Replace(absolutePath, ".wh.", "", 1)) + return nil + } + if t.Header.Typeflag == tar.TypeLink { + t.Header.Linkname = path.Join("rootfs", t.Linkname()) + } + + if err := trw.WriteHeader(t.Header); err != nil { + return err + } + if _, err := io.Copy(trw, t.TarStream); err != nil { + return err + } + + if !util.In(curPwl, absolutePath) { + curPwl = append(curPwl, absolutePath) + } + + return nil + } + tr, err := aci.NewCompressedTarReader(layer) + if err == nil { + defer tr.Close() + // write files in rootfs/ + if err := tarball.Walk(*tr.Reader, convWalker); err != nil { + return nil, err + } + } else { + // ignore errors: empty layers in tars generated by docker save are not + // valid tar files so we ignore errors trying to open them. Converted + // ACIs will have the manifest and an empty rootfs directory in any + // case. + } + newPwl := subtractWhiteouts(curPwl, whiteouts) + + newPwl, err = writeStdioSymlinks(trw, fileMap, newPwl) + if err != nil { + return nil, err + } + // Let's copy the newly generated PathWhitelist to avoid unintended + // side-effects + manifest.PathWhitelist = make([]string, len(newPwl)) + copy(manifest.PathWhitelist, newPwl) + + if err := WriteManifest(trw, manifest); err != nil { + return nil, fmt.Errorf("error writing manifest: %v", err) + } + + return &manifest, nil +} + +func getExecCommand(entrypoint []string, cmd []string) appctypes.Exec { + return append(entrypoint, cmd...) +} + +func parseDockerUser(dockerUser string) (string, string) { + // if the docker user is empty assume root user and group + if dockerUser == "" { + return "0", "0" + } + + dockerUserParts := strings.Split(dockerUser, ":") + + // when only the user is given, the docker spec says that the default and + // supplementary groups of the user in /etc/passwd should be applied. + // Assume root group for now in this case. + if len(dockerUserParts) < 2 { + return dockerUserParts[0], "0" + } + + return dockerUserParts[0], dockerUserParts[1] +} + +func subtractWhiteouts(pathWhitelist []string, whiteouts []string) []string { + matchPaths := []string{} + for _, path := range pathWhitelist { + // If one of the parent dirs of the current path matches the + // whiteout then also this path should be removed + curPath := path + for curPath != "/" { + for _, whiteout := range whiteouts { + if curPath == whiteout { + matchPaths = append(matchPaths, path) + } + } + curPath = filepath.Dir(curPath) + } + } + for _, matchPath := range matchPaths { + idx := util.IndexOf(pathWhitelist, matchPath) + if idx != -1 { + pathWhitelist = append(pathWhitelist[:idx], pathWhitelist[idx+1:]...) + } + } + + sort.Sort(sort.StringSlice(pathWhitelist)) + + return pathWhitelist +} + +// WriteManifest writes a schema.ImageManifest entry on a tar.Writer. +func WriteManifest(outputWriter *tar.Writer, manifest schema.ImageManifest) error { + b, err := json.Marshal(manifest) + if err != nil { + return err + } + + hdr := getGenericTarHeader() + hdr.Name = "manifest" + hdr.Mode = 0644 + hdr.Size = int64(len(b)) + hdr.Typeflag = tar.TypeReg + + if err := outputWriter.WriteHeader(hdr); err != nil { + return err + } + if _, err := outputWriter.Write(b); err != nil { + return err + } + + return nil +} + +// WriteRootfsDir writes a "rootfs" dir entry on a tar.Writer. +func WriteRootfsDir(tarWriter *tar.Writer) error { + hdr := getGenericTarHeader() + hdr.Name = "rootfs" + hdr.Mode = 0755 + hdr.Size = int64(0) + hdr.Typeflag = tar.TypeDir + + return tarWriter.WriteHeader(hdr) +} + +type symlink struct { + linkname string + target string +} + +// writeStdioSymlinks adds the /dev/stdin, /dev/stdout, /dev/stderr, and +// /dev/fd symlinks expected by Docker to the converted ACIs so apps can find +// them as expected +func writeStdioSymlinks(tarWriter *tar.Writer, fileMap map[string]struct{}, pwl []string) ([]string, error) { + stdioSymlinks := []symlink{ + {"/dev/stdin", "/proc/self/fd/0"}, + // Docker makes /dev/{stdout,stderr} point to /proc/self/fd/{1,2} but + // we point to /dev/console instead in order to support the case when + // stdout/stderr is a Unix socket (e.g. for the journal). + {"/dev/stdout", "/dev/console"}, + {"/dev/stderr", "/dev/console"}, + {"/dev/fd", "/proc/self/fd"}, + } + + for _, s := range stdioSymlinks { + name := s.linkname + target := s.target + if _, exists := fileMap[name]; exists { + continue + } + hdr := &tar.Header{ + Name: filepath.Join("rootfs", name), + Mode: 0777, + Typeflag: tar.TypeSymlink, + Linkname: target, + } + if err := tarWriter.WriteHeader(hdr); err != nil { + return nil, err + } + if !util.In(pwl, name) { + pwl = append(pwl, name) + } + } + + return pwl, nil +} + +func getGenericTarHeader() *tar.Header { + // FIXME(iaguis) Use docker image time instead of the Unix Epoch? + hdr := &tar.Header{ + Uid: 0, + Gid: 0, + ModTime: time.Unix(0, 0), + Uname: "0", + Gname: "0", + ChangeTime: time.Unix(0, 0), + } + + return hdr +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/tarfile.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/tarfile.go new file mode 100644 index 00000000..c72d18d9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/tarfile.go @@ -0,0 +1,42 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package tarball provides functions to manipulate tar files. +// +// Note: this package is an implementation detail and shouldn't be used outside +// of docker2aci. +package tarball + +import ( + "archive/tar" + "io" +) + +// TarFile is a representation of a file in a tarball. It consists of two parts, +// the Header and the Stream. The Header is a regular tar header, the Stream +// is a byte stream that can be used to read the file's contents. +type TarFile struct { + Header *tar.Header + TarStream io.Reader +} + +// Name returns the name of the file as reported by the header. +func (t *TarFile) Name() string { + return t.Header.Name +} + +// Linkname returns the Linkname of the file as reported by the header. +func (t *TarFile) Linkname() string { + return t.Header.Linkname +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/walk.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/walk.go new file mode 100644 index 00000000..c12d2e34 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/tarball/walk.go @@ -0,0 +1,43 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tarball + +import ( + "archive/tar" + "fmt" + "io" +) + +// WalkFunc is a func for handling each file (header and byte stream) in a tarball +type WalkFunc func(t *TarFile) error + +// Walk walks through the files in the tarball represented by tarstream and +// passes each of them to the WalkFunc provided as an argument +func Walk(tarReader tar.Reader, walkFunc func(t *TarFile) error) error { + for { + hdr, err := tarReader.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return fmt.Errorf("Error reading tar entry: %v", err) + } + if err := walkFunc(&TarFile{Header: hdr, TarStream: &tarReader}); err != nil { + return err + } + } + return nil +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/docker_types.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/docker_types.go new file mode 100644 index 00000000..7af1e17e --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/docker_types.go @@ -0,0 +1,92 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package types + +import "time" + +// DockerImageData stores the JSON structure of a Docker image. +// Taken and adapted from upstream Docker. +type DockerImageData struct { + ID string `json:"id"` + Parent string `json:"parent,omitempty"` + Comment string `json:"comment,omitempty"` + Created time.Time `json:"created"` + Container string `json:"container,omitempty"` + ContainerConfig DockerImageConfig `json:"container_config,omitempty"` + DockerVersion string `json:"docker_version,omitempty"` + Author string `json:"author,omitempty"` + Config *DockerImageConfig `json:"config,omitempty"` + Architecture string `json:"architecture,omitempty"` + OS string `json:"os,omitempty"` + Checksum string `json:"checksum"` +} + +// Note: the Config structure should hold only portable information about the container. +// Here, "portable" means "independent from the host we are running on". +// Non-portable information *should* appear in HostConfig. +// Taken and adapted from upstream Docker. +type DockerImageConfig struct { + Hostname string + Domainname string + User string + Memory int64 // Memory limit (in bytes) + MemorySwap int64 // Total memory usage (memory + swap); set `-1' to disable swap + CpuShares int64 // CPU shares (relative weight vs. other containers) + Cpuset string // Cpuset 0-2, 0,1 + AttachStdin bool + AttachStdout bool + AttachStderr bool + PortSpecs []string // Deprecated - Can be in the format of 8080/tcp + ExposedPorts map[string]struct{} + Tty bool // Attach standard streams to a tty, including stdin if it is not closed. + OpenStdin bool // Open stdin + StdinOnce bool // If true, close stdin after the 1 attached client disconnects. + Env []string + Cmd []string + Image string // Name of the image as it was passed by the operator (eg. could be symbolic) + Volumes map[string]struct{} + WorkingDir string + Entrypoint []string + NetworkDisabled bool + MacAddress string + OnBuild []string +} + +// DockerAuthConfigOld represents the deprecated ~/.dockercfg auth +// configuration. +// Taken from upstream Docker. +type DockerAuthConfigOld struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Auth string `json:"auth"` + Email string `json:"email"` + ServerAddress string `json:"serveraddress,omitempty"` +} + +// DockerAuthConfig represents a config.json auth entry. +// Taken from upstream Docker. +type DockerAuthConfig struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Auth string `json:"auth,omitempty"` + ServerAddress string `json:"serveraddress,omitempty"` + RegistryToken string `json:"registrytoken,omitempty"` +} + +// DockerConfigFile represents a config.json auth file. +// Taken from upstream docker. +type DockerConfigFile struct { + AuthConfigs map[string]DockerAuthConfig `json:"auths"` +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/types.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/types.go new file mode 100644 index 00000000..cc4fc12a --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/types/types.go @@ -0,0 +1,27 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package types defines Docker image, URL and configuration types. +// +// Note: this package is an implementation detail and shouldn't be used outside +// of docker2aci. +package types + +// ParsedDockerURL represents a parsed Docker URL. +type ParsedDockerURL struct { + IndexURL string + ImageName string + Tag string + Digest string +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/util/util.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/util/util.go new file mode 100644 index 00000000..ebaafca6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/internal/util/util.go @@ -0,0 +1,83 @@ +// Copyright 2015 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package util defines convenience functions for handling slices and debugging. +// +// Note: this package is an implementation detail and shouldn't be used outside +// of docker2aci. +package util + +import ( + "crypto/tls" + "fmt" + "net/http" + + "github.com/appc/spec/pkg/acirenderer" +) + +// Quote takes a slice of strings and returns another slice with them quoted. +func Quote(l []string) []string { + var quoted []string + + for _, s := range l { + quoted = append(quoted, fmt.Sprintf("%q", s)) + } + + return quoted +} + +// ReverseImages takes an acirenderer.Images and reverses it. +func ReverseImages(s acirenderer.Images) acirenderer.Images { + var o acirenderer.Images + for i := len(s) - 1; i >= 0; i-- { + o = append(o, s[i]) + } + + return o +} + +// In checks whether el is in list. +func In(list []string, el string) bool { + return IndexOf(list, el) != -1 +} + +// IndexOf returns the index of el in list, or -1 if it's not found. +func IndexOf(list []string, el string) int { + for i, x := range list { + if el == x { + return i + } + } + return -1 +} + +// GetTLSClient gets an HTTP client that behaves like the default HTTP +// client, but optionally skips the TLS certificate verification. +func GetTLSClient(skipTLSCheck bool) *http.Client { + if !skipTLSCheck { + return http.DefaultClient + } + client := *http.DefaultClient + // Default transport is hidden behind the RoundTripper + // interface, so we can't easily make a copy of it. If this + // ever panics, we will have to adapt. + realTransport := http.DefaultTransport.(*http.Transport) + tr := *realTransport + if tr.TLSClientConfig == nil { + tr.TLSClientConfig = &tls.Config{} + } + tr.TLSClientConfig.InsecureSkipVerify = true + client.Transport = &tr + return &client +} diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/lib/version.go b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/version.go new file mode 100644 index 00000000..487c4bc5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/lib/version.go @@ -0,0 +1,20 @@ +// Copyright 2016 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package docker2aci + +import "github.com/appc/spec/schema" + +var Version = "0.11.1" +var AppcVersion = schema.AppContainerVersion diff --git a/Godeps/_workspace/src/github.com/appc/docker2aci/pkg/log/log.go b/Godeps/_workspace/src/github.com/appc/docker2aci/pkg/log/log.go new file mode 100644 index 00000000..2a02c474 --- /dev/null +++ b/Godeps/_workspace/src/github.com/appc/docker2aci/pkg/log/log.go @@ -0,0 +1,46 @@ +// Copyright 2016 The appc Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package log + +import ( + "fmt" + "io" + "os" + "strings" +) + +var debugEnabled bool + +func printTo(w io.Writer, i ...interface{}) { + s := fmt.Sprint(i...) + fmt.Fprintln(w, strings.TrimSuffix(s, "\n")) +} + +// Info prints a message to stderr. +func Info(i ...interface{}) { + printTo(os.Stderr, i...) +} + +// Debug prints a message to stderr if debug is enabled. +func Debug(i ...interface{}) { + if debugEnabled { + printTo(os.Stderr, i...) + } +} + +// InitDebug enables debug output. +func InitDebug() { + debugEnabled = true +} diff --git a/Godeps/_workspace/src/github.com/coreos/pkg/LICENSE b/Godeps/_workspace/src/github.com/coreos/pkg/LICENSE new file mode 100644 index 00000000..e06d2081 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/pkg/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Godeps/_workspace/src/github.com/coreos/pkg/NOTICE b/Godeps/_workspace/src/github.com/coreos/pkg/NOTICE new file mode 100644 index 00000000..b39ddfa5 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/pkg/NOTICE @@ -0,0 +1,5 @@ +CoreOS Project +Copyright 2014 CoreOS, Inc + +This product includes software developed at CoreOS, Inc. +(http://www.coreos.com/). diff --git a/Godeps/_workspace/src/github.com/coreos/pkg/progressutil/iocopy.go b/Godeps/_workspace/src/github.com/coreos/pkg/progressutil/iocopy.go new file mode 100644 index 00000000..04cd0dfa --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/pkg/progressutil/iocopy.go @@ -0,0 +1,189 @@ +// Copyright 2016 CoreOS Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progressutil + +import ( + "errors" + "fmt" + "io" + "sync" + "time" +) + +var ( + ErrAlreadyStarted = errors.New("cannot add copies after PrintAndWait has been called") +) + +type copyReader struct { + reader io.Reader + current int64 + total int64 + pb *ProgressBar +} + +func (cr *copyReader) Read(p []byte) (int, error) { + n, err := cr.reader.Read(p) + cr.current += int64(n) + err1 := cr.updateProgressBar() + if err == nil { + err = err1 + } + return n, err +} + +func (cr *copyReader) updateProgressBar() error { + cr.pb.SetPrintAfter(cr.formattedProgress()) + + progress := float64(cr.current) / float64(cr.total) + if progress > 1 { + progress = 1 + } + return cr.pb.SetCurrentProgress(progress) +} + +// NewCopyProgressPrinter returns a new CopyProgressPrinter +func NewCopyProgressPrinter() *CopyProgressPrinter { + return &CopyProgressPrinter{results: make(chan error), cancel: make(chan struct{})} +} + +// CopyProgressPrinter will perform an arbitrary number of io.Copy calls, while +// continually printing the progress of each copy. +type CopyProgressPrinter struct { + results chan error + cancel chan struct{} + + // `lock` mutex protects all fields below it in CopyProgressPrinter struct + lock sync.Mutex + readers []*copyReader + started bool + pbp *ProgressBarPrinter +} + +// AddCopy adds a copy for this CopyProgressPrinter to perform. An io.Copy call +// will be made to copy bytes from reader to dest, and name and size will be +// used to label the progress bar and display how much progress has been made. +// If size is 0, the total size of the reader is assumed to be unknown. +// AddCopy can only be called before PrintAndWait; otherwise, ErrAlreadyStarted +// will be returned. +func (cpp *CopyProgressPrinter) AddCopy(reader io.Reader, name string, size int64, dest io.Writer) error { + cpp.lock.Lock() + defer cpp.lock.Unlock() + + if cpp.started { + return ErrAlreadyStarted + } + if cpp.pbp == nil { + cpp.pbp = &ProgressBarPrinter{} + cpp.pbp.PadToBeEven = true + } + + cr := ©Reader{ + reader: reader, + current: 0, + total: size, + pb: cpp.pbp.AddProgressBar(), + } + cr.pb.SetPrintBefore(name) + cr.pb.SetPrintAfter(cr.formattedProgress()) + + cpp.readers = append(cpp.readers, cr) + + go func() { + _, err := io.Copy(dest, cr) + select { + case <-cpp.cancel: + return + case cpp.results <- err: + return + } + }() + return nil +} + +// PrintAndWait will print the progress for each copy operation added with +// AddCopy to printTo every printInterval. This will continue until every added +// copy is finished, or until cancel is written to. +// PrintAndWait may only be called once; any subsequent calls will immediately +// return ErrAlreadyStarted. After PrintAndWait has been called, no more +// copies may be added to the CopyProgressPrinter. +func (cpp *CopyProgressPrinter) PrintAndWait(printTo io.Writer, printInterval time.Duration, cancel chan struct{}) error { + cpp.lock.Lock() + if cpp.started { + cpp.lock.Unlock() + return ErrAlreadyStarted + } + cpp.started = true + cpp.lock.Unlock() + + n := len(cpp.readers) + if n == 0 { + // Nothing to do. + return nil + } + + defer close(cpp.cancel) + t := time.NewTicker(printInterval) + allDone := false + for i := 0; i < n; { + select { + case <-cancel: + return nil + case <-t.C: + _, err := cpp.pbp.Print(printTo) + if err != nil { + return err + } + case err := <-cpp.results: + i++ + // Once completion is signaled, further on this just drains + // (unlikely) errors from the channel. + if err == nil && !allDone { + allDone, err = cpp.pbp.Print(printTo) + } + if err != nil { + return err + } + } + } + return nil +} + +func (cr *copyReader) formattedProgress() string { + var totalStr string + if cr.total == 0 { + totalStr = "?" + } else { + totalStr = ByteUnitStr(cr.total) + } + return fmt.Sprintf("%s / %s", ByteUnitStr(cr.current), totalStr) +} + +var byteUnits = []string{"B", "KB", "MB", "GB", "TB", "PB"} + +// ByteUnitStr pretty prints a number of bytes. +func ByteUnitStr(n int64) string { + var unit string + size := float64(n) + for i := 1; i < len(byteUnits); i++ { + if size < 1000 { + unit = byteUnits[i-1] + break + } + + size = size / 1000 + } + + return fmt.Sprintf("%.3g %s", size, unit) +} diff --git a/Godeps/_workspace/src/github.com/coreos/pkg/progressutil/progressbar.go b/Godeps/_workspace/src/github.com/coreos/pkg/progressutil/progressbar.go new file mode 100644 index 00000000..d15fc5b6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/coreos/pkg/progressutil/progressbar.go @@ -0,0 +1,263 @@ +// Copyright 2016 CoreOS Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package progressutil + +import ( + "fmt" + "io" + "os" + "strings" + "sync" + + "golang.org/x/crypto/ssh/terminal" +) + +var ( + // ErrorProgressOutOfBounds is returned if the progress is set to a value + // not between 0 and 1. + ErrorProgressOutOfBounds = fmt.Errorf("progress is out of bounds (0 to 1)") + + // ErrorNoBarsAdded is returned when no progress bars have been added to a + // ProgressBarPrinter before PrintAndWait is called. + ErrorNoBarsAdded = fmt.Errorf("AddProgressBar hasn't been called yet") +) + +// ProgressBar represents one progress bar in a ProgressBarPrinter. Should not +// be created directly, use the AddProgressBar on a ProgressBarPrinter to +// create these. +type ProgressBar struct { + lock sync.Mutex + + currentProgress float64 + printBefore string + printAfter string + done bool +} + +func (pb *ProgressBar) clone() *ProgressBar { + pb.lock.Lock() + pbClone := &ProgressBar{ + currentProgress: pb.currentProgress, + printBefore: pb.printBefore, + printAfter: pb.printAfter, + done: pb.done, + } + pb.lock.Unlock() + return pbClone +} + +func (pb *ProgressBar) GetCurrentProgress() float64 { + pb.lock.Lock() + val := pb.currentProgress + pb.lock.Unlock() + return val +} + +// SetCurrentProgress sets the progress of this ProgressBar. The progress must +// be between 0 and 1 inclusive. +func (pb *ProgressBar) SetCurrentProgress(progress float64) error { + if progress < 0 || progress > 1 { + return ErrorProgressOutOfBounds + } + pb.lock.Lock() + pb.currentProgress = progress + pb.lock.Unlock() + return nil +} + +// GetDone returns whether or not this progress bar is done +func (pb *ProgressBar) GetDone() bool { + pb.lock.Lock() + val := pb.done + pb.lock.Unlock() + return val +} + +// SetDone sets whether or not this progress bar is done +func (pb *ProgressBar) SetDone(val bool) { + pb.lock.Lock() + pb.done = val + pb.lock.Unlock() +} + +// GetPrintBefore gets the text printed on the line before the progress bar. +func (pb *ProgressBar) GetPrintBefore() string { + pb.lock.Lock() + val := pb.printBefore + pb.lock.Unlock() + return val +} + +// SetPrintBefore sets the text printed on the line before the progress bar. +func (pb *ProgressBar) SetPrintBefore(before string) { + pb.lock.Lock() + pb.printBefore = before + pb.lock.Unlock() +} + +// GetPrintAfter gets the text printed on the line after the progress bar. +func (pb *ProgressBar) GetPrintAfter() string { + pb.lock.Lock() + val := pb.printAfter + pb.lock.Unlock() + return val +} + +// SetPrintAfter sets the text printed on the line after the progress bar. +func (pb *ProgressBar) SetPrintAfter(after string) { + pb.lock.Lock() + pb.printAfter = after + pb.lock.Unlock() +} + +// ProgressBarPrinter will print out the progress of some number of +// ProgressBars. +type ProgressBarPrinter struct { + lock sync.Mutex + + // DisplayWidth can be set to influence how large the progress bars are. + // The bars will be scaled to attempt to produce lines of this number of + // characters, but lines of different lengths may still be printed. When + // this value is 0 (aka unset), 80 character columns are assumed. + DisplayWidth int + // PadToBeEven, when set to true, will make Print pad the printBefore text + // with trailing spaces and the printAfter text with leading spaces to make + // the progress bars the same length. + PadToBeEven bool + numLinesInLastPrint int + progressBars []*ProgressBar + maxBefore int + maxAfter int + + // printToTTYAlways forces this ProgressBarPrinter to always behave as if + // in a tty. Used for tests. + printToTTYAlways bool +} + +// AddProgressBar will create a new ProgressBar, register it with this +// ProgressBarPrinter, and return it. This must be called at least once before +// PrintAndWait is called. +func (pbp *ProgressBarPrinter) AddProgressBar() *ProgressBar { + pb := &ProgressBar{} + pbp.lock.Lock() + pbp.progressBars = append(pbp.progressBars, pb) + pbp.lock.Unlock() + return pb +} + +// Print will print out progress information for each ProgressBar that has been +// added to this ProgressBarPrinter. The progress will be written to printTo, +// and if printTo is a terminal it will draw progress bars. AddProgressBar +// must be called at least once before Print is called. If printing to a +// terminal, all draws after the first one will move the cursor up to draw over +// the previously printed bars. +func (pbp *ProgressBarPrinter) Print(printTo io.Writer) (bool, error) { + pbp.lock.Lock() + var bars []*ProgressBar + for _, bar := range pbp.progressBars { + bars = append(bars, bar.clone()) + } + numColumns := pbp.DisplayWidth + pbp.lock.Unlock() + + if len(bars) == 0 { + return false, ErrorNoBarsAdded + } + + if numColumns == 0 { + numColumns = 80 + } + + if pbp.isTerminal(printTo) { + moveCursorUp(printTo, pbp.numLinesInLastPrint) + } + + for _, bar := range bars { + beforeSize := len(bar.GetPrintBefore()) + afterSize := len(bar.GetPrintAfter()) + if beforeSize > pbp.maxBefore { + pbp.maxBefore = beforeSize + } + if afterSize > pbp.maxAfter { + pbp.maxAfter = afterSize + } + } + + allDone := true + for _, bar := range bars { + if pbp.isTerminal(printTo) { + bar.printToTerminal(printTo, numColumns, pbp.PadToBeEven, pbp.maxBefore, pbp.maxAfter) + } else { + bar.printToNonTerminal(printTo) + } + allDone = allDone && bar.GetCurrentProgress() == 1 + } + + pbp.numLinesInLastPrint = len(bars) + + return allDone, nil +} + +// moveCursorUp moves the cursor up numLines in the terminal +func moveCursorUp(printTo io.Writer, numLines int) { + if numLines > 0 { + fmt.Fprintf(printTo, "\033[%dA", numLines) + } +} + +func (pb *ProgressBar) printToTerminal(printTo io.Writer, numColumns int, padding bool, maxBefore, maxAfter int) { + before := pb.GetPrintBefore() + after := pb.GetPrintAfter() + + if padding { + before = before + strings.Repeat(" ", maxBefore-len(before)) + after = strings.Repeat(" ", maxAfter-len(after)) + after + } + + progressBarSize := numColumns - (len(fmt.Sprintf("%s [] %s", before, after))) + progressBar := "" + if progressBarSize > 0 { + currentProgress := int(pb.GetCurrentProgress() * float64(progressBarSize)) + progressBar = fmt.Sprintf("[%s%s] ", + strings.Repeat("=", currentProgress), + strings.Repeat(" ", progressBarSize-currentProgress)) + } else { + // If we can't fit the progress bar, better to not pad the before/after. + before = pb.GetPrintBefore() + after = pb.GetPrintAfter() + } + + fmt.Fprintf(printTo, "%s %s%s\n", before, progressBar, after) +} + +func (pb *ProgressBar) printToNonTerminal(printTo io.Writer) { + if !pb.GetDone() { + fmt.Fprintf(printTo, "%s %s\n", pb.printBefore, pb.printAfter) + if pb.GetCurrentProgress() == 1 { + pb.SetDone(true) + } + } +} + +// isTerminal returns True when w is going to a tty, and false otherwise. +func (pbp *ProgressBarPrinter) isTerminal(w io.Writer) bool { + if pbp.printToTTYAlways { + return true + } + if f, ok := w.(*os.File); ok { + return terminal.IsTerminal(int(f.Fd())) + } + return false +} diff --git a/Godeps/_workspace/src/github.com/docker/distribution/LICENSE b/Godeps/_workspace/src/github.com/docker/distribution/LICENSE new file mode 100644 index 00000000..e06d2081 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/Godeps/_workspace/src/github.com/docker/distribution/digest/digest.go b/Godeps/_workspace/src/github.com/docker/distribution/digest/digest.go new file mode 100644 index 00000000..31d821bb --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/digest/digest.go @@ -0,0 +1,139 @@ +package digest + +import ( + "fmt" + "hash" + "io" + "regexp" + "strings" +) + +const ( + // DigestSha256EmptyTar is the canonical sha256 digest of empty data + DigestSha256EmptyTar = "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +) + +// Digest allows simple protection of hex formatted digest strings, prefixed +// by their algorithm. Strings of type Digest have some guarantee of being in +// the correct format and it provides quick access to the components of a +// digest string. +// +// The following is an example of the contents of Digest types: +// +// sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc +// +// This allows to abstract the digest behind this type and work only in those +// terms. +type Digest string + +// NewDigest returns a Digest from alg and a hash.Hash object. +func NewDigest(alg Algorithm, h hash.Hash) Digest { + return NewDigestFromBytes(alg, h.Sum(nil)) +} + +// NewDigestFromBytes returns a new digest from the byte contents of p. +// Typically, this can come from hash.Hash.Sum(...) or xxx.SumXXX(...) +// functions. This is also useful for rebuilding digests from binary +// serializations. +func NewDigestFromBytes(alg Algorithm, p []byte) Digest { + return Digest(fmt.Sprintf("%s:%x", alg, p)) +} + +// NewDigestFromHex returns a Digest from alg and a the hex encoded digest. +func NewDigestFromHex(alg, hex string) Digest { + return Digest(fmt.Sprintf("%s:%s", alg, hex)) +} + +// DigestRegexp matches valid digest types. +var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-fA-F0-9]+`) + +// DigestRegexpAnchored matches valid digest types, anchored to the start and end of the match. +var DigestRegexpAnchored = regexp.MustCompile(`^` + DigestRegexp.String() + `$`) + +var ( + // ErrDigestInvalidFormat returned when digest format invalid. + ErrDigestInvalidFormat = fmt.Errorf("invalid checksum digest format") + + // ErrDigestInvalidLength returned when digest has invalid length. + ErrDigestInvalidLength = fmt.Errorf("invalid checksum digest length") + + // ErrDigestUnsupported returned when the digest algorithm is unsupported. + ErrDigestUnsupported = fmt.Errorf("unsupported digest algorithm") +) + +// ParseDigest parses s and returns the validated digest object. An error will +// be returned if the format is invalid. +func ParseDigest(s string) (Digest, error) { + d := Digest(s) + + return d, d.Validate() +} + +// FromReader returns the most valid digest for the underlying content using +// the canonical digest algorithm. +func FromReader(rd io.Reader) (Digest, error) { + return Canonical.FromReader(rd) +} + +// FromBytes digests the input and returns a Digest. +func FromBytes(p []byte) Digest { + return Canonical.FromBytes(p) +} + +// Validate checks that the contents of d is a valid digest, returning an +// error if not. +func (d Digest) Validate() error { + s := string(d) + + if !DigestRegexpAnchored.MatchString(s) { + return ErrDigestInvalidFormat + } + + i := strings.Index(s, ":") + if i < 0 { + return ErrDigestInvalidFormat + } + + // case: "sha256:" with no hex. + if i+1 == len(s) { + return ErrDigestInvalidFormat + } + + switch algorithm := Algorithm(s[:i]); algorithm { + case SHA256, SHA384, SHA512: + if algorithm.Size()*2 != len(s[i+1:]) { + return ErrDigestInvalidLength + } + break + default: + return ErrDigestUnsupported + } + + return nil +} + +// Algorithm returns the algorithm portion of the digest. This will panic if +// the underlying digest is not in a valid format. +func (d Digest) Algorithm() Algorithm { + return Algorithm(d[:d.sepIndex()]) +} + +// Hex returns the hex digest portion of the digest. This will panic if the +// underlying digest is not in a valid format. +func (d Digest) Hex() string { + return string(d[d.sepIndex()+1:]) +} + +func (d Digest) String() string { + return string(d) +} + +func (d Digest) sepIndex() int { + i := strings.Index(string(d), ":") + + if i < 0 { + panic("could not find ':' in digest: " + d) + } + + return i +} diff --git a/Godeps/_workspace/src/github.com/docker/distribution/digest/digester.go b/Godeps/_workspace/src/github.com/docker/distribution/digest/digester.go new file mode 100644 index 00000000..f3105a45 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/digest/digester.go @@ -0,0 +1,155 @@ +package digest + +import ( + "crypto" + "fmt" + "hash" + "io" +) + +// Algorithm identifies and implementation of a digester by an identifier. +// Note the that this defines both the hash algorithm used and the string +// encoding. +type Algorithm string + +// supported digest types +const ( + SHA256 Algorithm = "sha256" // sha256 with hex encoding + SHA384 Algorithm = "sha384" // sha384 with hex encoding + SHA512 Algorithm = "sha512" // sha512 with hex encoding + + // Canonical is the primary digest algorithm used with the distribution + // project. Other digests may be used but this one is the primary storage + // digest. + Canonical = SHA256 +) + +var ( + // TODO(stevvooe): Follow the pattern of the standard crypto package for + // registration of digests. Effectively, we are a registerable set and + // common symbol access. + + // algorithms maps values to hash.Hash implementations. Other algorithms + // may be available but they cannot be calculated by the digest package. + algorithms = map[Algorithm]crypto.Hash{ + SHA256: crypto.SHA256, + SHA384: crypto.SHA384, + SHA512: crypto.SHA512, + } +) + +// Available returns true if the digest type is available for use. If this +// returns false, New and Hash will return nil. +func (a Algorithm) Available() bool { + h, ok := algorithms[a] + if !ok { + return false + } + + // check availability of the hash, as well + return h.Available() +} + +func (a Algorithm) String() string { + return string(a) +} + +// Size returns number of bytes returned by the hash. +func (a Algorithm) Size() int { + h, ok := algorithms[a] + if !ok { + return 0 + } + return h.Size() +} + +// Set implemented to allow use of Algorithm as a command line flag. +func (a *Algorithm) Set(value string) error { + if value == "" { + *a = Canonical + } else { + // just do a type conversion, support is queried with Available. + *a = Algorithm(value) + } + + return nil +} + +// New returns a new digester for the specified algorithm. If the algorithm +// does not have a digester implementation, nil will be returned. This can be +// checked by calling Available before calling New. +func (a Algorithm) New() Digester { + return &digester{ + alg: a, + hash: a.Hash(), + } +} + +// Hash returns a new hash as used by the algorithm. If not available, the +// method will panic. Check Algorithm.Available() before calling. +func (a Algorithm) Hash() hash.Hash { + if !a.Available() { + // NOTE(stevvooe): A missing hash is usually a programming error that + // must be resolved at compile time. We don't import in the digest + // package to allow users to choose their hash implementation (such as + // when using stevvooe/resumable or a hardware accelerated package). + // + // Applications that may want to resolve the hash at runtime should + // call Algorithm.Available before call Algorithm.Hash(). + panic(fmt.Sprintf("%v not available (make sure it is imported)", a)) + } + + return algorithms[a].New() +} + +// FromReader returns the digest of the reader using the algorithm. +func (a Algorithm) FromReader(rd io.Reader) (Digest, error) { + digester := a.New() + + if _, err := io.Copy(digester.Hash(), rd); err != nil { + return "", err + } + + return digester.Digest(), nil +} + +// FromBytes digests the input and returns a Digest. +func (a Algorithm) FromBytes(p []byte) Digest { + digester := a.New() + + if _, err := digester.Hash().Write(p); err != nil { + // Writes to a Hash should never fail. None of the existing + // hash implementations in the stdlib or hashes vendored + // here can return errors from Write. Having a panic in this + // condition instead of having FromBytes return an error value + // avoids unnecessary error handling paths in all callers. + panic("write to hash function returned error: " + err.Error()) + } + + return digester.Digest() +} + +// TODO(stevvooe): Allow resolution of verifiers using the digest type and +// this registration system. + +// Digester calculates the digest of written data. Writes should go directly +// to the return value of Hash, while calling Digest will return the current +// value of the digest. +type Digester interface { + Hash() hash.Hash // provides direct access to underlying hash instance. + Digest() Digest +} + +// digester provides a simple digester definition that embeds a hasher. +type digester struct { + alg Algorithm + hash hash.Hash +} + +func (d *digester) Hash() hash.Hash { + return d.hash +} + +func (d *digester) Digest() Digest { + return NewDigest(d.alg, d.hash) +} diff --git a/Godeps/_workspace/src/github.com/docker/distribution/digest/doc.go b/Godeps/_workspace/src/github.com/docker/distribution/digest/doc.go new file mode 100644 index 00000000..f64b0db3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/digest/doc.go @@ -0,0 +1,42 @@ +// Package digest provides a generalized type to opaquely represent message +// digests and their operations within the registry. The Digest type is +// designed to serve as a flexible identifier in a content-addressable system. +// More importantly, it provides tools and wrappers to work with +// hash.Hash-based digests with little effort. +// +// Basics +// +// The format of a digest is simply a string with two parts, dubbed the +// "algorithm" and the "digest", separated by a colon: +// +// : +// +// An example of a sha256 digest representation follows: +// +// sha256:7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc +// +// In this case, the string "sha256" is the algorithm and the hex bytes are +// the "digest". +// +// Because the Digest type is simply a string, once a valid Digest is +// obtained, comparisons are cheap, quick and simple to express with the +// standard equality operator. +// +// Verification +// +// The main benefit of using the Digest type is simple verification against a +// given digest. The Verifier interface, modeled after the stdlib hash.Hash +// interface, provides a common write sink for digest verification. After +// writing is complete, calling the Verifier.Verified method will indicate +// whether or not the stream of bytes matches the target digest. +// +// Missing Features +// +// In addition to the above, we intend to add the following features to this +// package: +// +// 1. A Digester type that supports write sink digest calculation. +// +// 2. Suspend and resume of ongoing digest calculations to support efficient digest verification in the registry. +// +package digest diff --git a/Godeps/_workspace/src/github.com/docker/distribution/digest/set.go b/Godeps/_workspace/src/github.com/docker/distribution/digest/set.go new file mode 100644 index 00000000..4b9313c1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/digest/set.go @@ -0,0 +1,245 @@ +package digest + +import ( + "errors" + "sort" + "strings" + "sync" +) + +var ( + // ErrDigestNotFound is used when a matching digest + // could not be found in a set. + ErrDigestNotFound = errors.New("digest not found") + + // ErrDigestAmbiguous is used when multiple digests + // are found in a set. None of the matching digests + // should be considered valid matches. + ErrDigestAmbiguous = errors.New("ambiguous digest string") +) + +// Set is used to hold a unique set of digests which +// may be easily referenced by easily referenced by a string +// representation of the digest as well as short representation. +// The uniqueness of the short representation is based on other +// digests in the set. If digests are omitted from this set, +// collisions in a larger set may not be detected, therefore it +// is important to always do short representation lookups on +// the complete set of digests. To mitigate collisions, an +// appropriately long short code should be used. +type Set struct { + mutex sync.RWMutex + entries digestEntries +} + +// NewSet creates an empty set of digests +// which may have digests added. +func NewSet() *Set { + return &Set{ + entries: digestEntries{}, + } +} + +// checkShortMatch checks whether two digests match as either whole +// values or short values. This function does not test equality, +// rather whether the second value could match against the first +// value. +func checkShortMatch(alg Algorithm, hex, shortAlg, shortHex string) bool { + if len(hex) == len(shortHex) { + if hex != shortHex { + return false + } + if len(shortAlg) > 0 && string(alg) != shortAlg { + return false + } + } else if !strings.HasPrefix(hex, shortHex) { + return false + } else if len(shortAlg) > 0 && string(alg) != shortAlg { + return false + } + return true +} + +// Lookup looks for a digest matching the given string representation. +// If no digests could be found ErrDigestNotFound will be returned +// with an empty digest value. If multiple matches are found +// ErrDigestAmbiguous will be returned with an empty digest value. +func (dst *Set) Lookup(d string) (Digest, error) { + dst.mutex.RLock() + defer dst.mutex.RUnlock() + if len(dst.entries) == 0 { + return "", ErrDigestNotFound + } + var ( + searchFunc func(int) bool + alg Algorithm + hex string + ) + dgst, err := ParseDigest(d) + if err == ErrDigestInvalidFormat { + hex = d + searchFunc = func(i int) bool { + return dst.entries[i].val >= d + } + } else { + hex = dgst.Hex() + alg = dgst.Algorithm() + searchFunc = func(i int) bool { + if dst.entries[i].val == hex { + return dst.entries[i].alg >= alg + } + return dst.entries[i].val >= hex + } + } + idx := sort.Search(len(dst.entries), searchFunc) + if idx == len(dst.entries) || !checkShortMatch(dst.entries[idx].alg, dst.entries[idx].val, string(alg), hex) { + return "", ErrDigestNotFound + } + if dst.entries[idx].alg == alg && dst.entries[idx].val == hex { + return dst.entries[idx].digest, nil + } + if idx+1 < len(dst.entries) && checkShortMatch(dst.entries[idx+1].alg, dst.entries[idx+1].val, string(alg), hex) { + return "", ErrDigestAmbiguous + } + + return dst.entries[idx].digest, nil +} + +// Add adds the given digest to the set. An error will be returned +// if the given digest is invalid. If the digest already exists in the +// set, this operation will be a no-op. +func (dst *Set) Add(d Digest) error { + if err := d.Validate(); err != nil { + return err + } + dst.mutex.Lock() + defer dst.mutex.Unlock() + entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d} + searchFunc := func(i int) bool { + if dst.entries[i].val == entry.val { + return dst.entries[i].alg >= entry.alg + } + return dst.entries[i].val >= entry.val + } + idx := sort.Search(len(dst.entries), searchFunc) + if idx == len(dst.entries) { + dst.entries = append(dst.entries, entry) + return nil + } else if dst.entries[idx].digest == d { + return nil + } + + entries := append(dst.entries, nil) + copy(entries[idx+1:], entries[idx:len(entries)-1]) + entries[idx] = entry + dst.entries = entries + return nil +} + +// Remove removes the given digest from the set. An err will be +// returned if the given digest is invalid. If the digest does +// not exist in the set, this operation will be a no-op. +func (dst *Set) Remove(d Digest) error { + if err := d.Validate(); err != nil { + return err + } + dst.mutex.Lock() + defer dst.mutex.Unlock() + entry := &digestEntry{alg: d.Algorithm(), val: d.Hex(), digest: d} + searchFunc := func(i int) bool { + if dst.entries[i].val == entry.val { + return dst.entries[i].alg >= entry.alg + } + return dst.entries[i].val >= entry.val + } + idx := sort.Search(len(dst.entries), searchFunc) + // Not found if idx is after or value at idx is not digest + if idx == len(dst.entries) || dst.entries[idx].digest != d { + return nil + } + + entries := dst.entries + copy(entries[idx:], entries[idx+1:]) + entries = entries[:len(entries)-1] + dst.entries = entries + + return nil +} + +// All returns all the digests in the set +func (dst *Set) All() []Digest { + dst.mutex.RLock() + defer dst.mutex.RUnlock() + retValues := make([]Digest, len(dst.entries)) + for i := range dst.entries { + retValues[i] = dst.entries[i].digest + } + + return retValues +} + +// ShortCodeTable returns a map of Digest to unique short codes. The +// length represents the minimum value, the maximum length may be the +// entire value of digest if uniqueness cannot be achieved without the +// full value. This function will attempt to make short codes as short +// as possible to be unique. +func ShortCodeTable(dst *Set, length int) map[Digest]string { + dst.mutex.RLock() + defer dst.mutex.RUnlock() + m := make(map[Digest]string, len(dst.entries)) + l := length + resetIdx := 0 + for i := 0; i < len(dst.entries); i++ { + var short string + extended := true + for extended { + extended = false + if len(dst.entries[i].val) <= l { + short = dst.entries[i].digest.String() + } else { + short = dst.entries[i].val[:l] + for j := i + 1; j < len(dst.entries); j++ { + if checkShortMatch(dst.entries[j].alg, dst.entries[j].val, "", short) { + if j > resetIdx { + resetIdx = j + } + extended = true + } else { + break + } + } + if extended { + l++ + } + } + } + m[dst.entries[i].digest] = short + if i >= resetIdx { + l = length + } + } + return m +} + +type digestEntry struct { + alg Algorithm + val string + digest Digest +} + +type digestEntries []*digestEntry + +func (d digestEntries) Len() int { + return len(d) +} + +func (d digestEntries) Less(i, j int) bool { + if d[i].val != d[j].val { + return d[i].val < d[j].val + } + return d[i].alg < d[j].alg +} + +func (d digestEntries) Swap(i, j int) { + d[i], d[j] = d[j], d[i] +} diff --git a/Godeps/_workspace/src/github.com/docker/distribution/digest/verifiers.go b/Godeps/_workspace/src/github.com/docker/distribution/digest/verifiers.go new file mode 100644 index 00000000..9af3be13 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/digest/verifiers.go @@ -0,0 +1,44 @@ +package digest + +import ( + "hash" + "io" +) + +// Verifier presents a general verification interface to be used with message +// digests and other byte stream verifications. Users instantiate a Verifier +// from one of the various methods, write the data under test to it then check +// the result with the Verified method. +type Verifier interface { + io.Writer + + // Verified will return true if the content written to Verifier matches + // the digest. + Verified() bool +} + +// NewDigestVerifier returns a verifier that compares the written bytes +// against a passed in digest. +func NewDigestVerifier(d Digest) (Verifier, error) { + if err := d.Validate(); err != nil { + return nil, err + } + + return hashVerifier{ + hash: d.Algorithm().Hash(), + digest: d, + }, nil +} + +type hashVerifier struct { + digest Digest + hash hash.Hash +} + +func (hv hashVerifier) Write(p []byte) (n int, err error) { + return hv.hash.Write(p) +} + +func (hv hashVerifier) Verified() bool { + return hv.digest == NewDigest(hv.digest.Algorithm(), hv.hash) +} diff --git a/Godeps/_workspace/src/github.com/docker/distribution/reference/reference.go b/Godeps/_workspace/src/github.com/docker/distribution/reference/reference.go new file mode 100644 index 00000000..bb09fa25 --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/reference/reference.go @@ -0,0 +1,334 @@ +// Package reference provides a general type to represent any way of referencing images within the registry. +// Its main purpose is to abstract tags and digests (content-addressable hash). +// +// Grammar +// +// reference := name [ ":" tag ] [ "@" digest ] +// name := [hostname '/'] component ['/' component]* +// hostname := hostcomponent ['.' hostcomponent]* [':' port-number] +// hostcomponent := /([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])/ +// port-number := /[0-9]+/ +// component := alpha-numeric [separator alpha-numeric]* +// alpha-numeric := /[a-z0-9]+/ +// separator := /[_.]|__|[-]*/ +// +// tag := /[\w][\w.-]{0,127}/ +// +// digest := digest-algorithm ":" digest-hex +// digest-algorithm := digest-algorithm-component [ digest-algorithm-separator digest-algorithm-component ] +// digest-algorithm-separator := /[+.-_]/ +// digest-algorithm-component := /[A-Za-z][A-Za-z0-9]*/ +// digest-hex := /[0-9a-fA-F]{32,}/ ; At least 128 bit digest value +package reference + +import ( + "errors" + "fmt" + + "github.com/docker/distribution/digest" +) + +const ( + // NameTotalLengthMax is the maximum total number of characters in a repository name. + NameTotalLengthMax = 255 +) + +var ( + // ErrReferenceInvalidFormat represents an error while trying to parse a string as a reference. + ErrReferenceInvalidFormat = errors.New("invalid reference format") + + // ErrTagInvalidFormat represents an error while trying to parse a string as a tag. + ErrTagInvalidFormat = errors.New("invalid tag format") + + // ErrDigestInvalidFormat represents an error while trying to parse a string as a tag. + ErrDigestInvalidFormat = errors.New("invalid digest format") + + // ErrNameEmpty is returned for empty, invalid repository names. + ErrNameEmpty = errors.New("repository name must have at least one component") + + // ErrNameTooLong is returned when a repository name is longer than NameTotalLengthMax. + ErrNameTooLong = fmt.Errorf("repository name must not be more than %v characters", NameTotalLengthMax) +) + +// Reference is an opaque object reference identifier that may include +// modifiers such as a hostname, name, tag, and digest. +type Reference interface { + // String returns the full reference + String() string +} + +// Field provides a wrapper type for resolving correct reference types when +// working with encoding. +type Field struct { + reference Reference +} + +// AsField wraps a reference in a Field for encoding. +func AsField(reference Reference) Field { + return Field{reference} +} + +// Reference unwraps the reference type from the field to +// return the Reference object. This object should be +// of the appropriate type to further check for different +// reference types. +func (f Field) Reference() Reference { + return f.reference +} + +// MarshalText serializes the field to byte text which +// is the string of the reference. +func (f Field) MarshalText() (p []byte, err error) { + return []byte(f.reference.String()), nil +} + +// UnmarshalText parses text bytes by invoking the +// reference parser to ensure the appropriately +// typed reference object is wrapped by field. +func (f *Field) UnmarshalText(p []byte) error { + r, err := Parse(string(p)) + if err != nil { + return err + } + + f.reference = r + return nil +} + +// Named is an object with a full name +type Named interface { + Reference + Name() string +} + +// Tagged is an object which has a tag +type Tagged interface { + Reference + Tag() string +} + +// NamedTagged is an object including a name and tag. +type NamedTagged interface { + Named + Tag() string +} + +// Digested is an object which has a digest +// in which it can be referenced by +type Digested interface { + Reference + Digest() digest.Digest +} + +// Canonical reference is an object with a fully unique +// name including a name with hostname and digest +type Canonical interface { + Named + Digest() digest.Digest +} + +// SplitHostname splits a named reference into a +// hostname and name string. If no valid hostname is +// found, the hostname is empty and the full value +// is returned as name +func SplitHostname(named Named) (string, string) { + name := named.Name() + match := anchoredNameRegexp.FindStringSubmatch(name) + if match == nil || len(match) != 3 { + return "", name + } + return match[1], match[2] +} + +// Parse parses s and returns a syntactically valid Reference. +// If an error was encountered it is returned, along with a nil Reference. +// NOTE: Parse will not handle short digests. +func Parse(s string) (Reference, error) { + matches := ReferenceRegexp.FindStringSubmatch(s) + if matches == nil { + if s == "" { + return nil, ErrNameEmpty + } + // TODO(dmcgowan): Provide more specific and helpful error + return nil, ErrReferenceInvalidFormat + } + + if len(matches[1]) > NameTotalLengthMax { + return nil, ErrNameTooLong + } + + ref := reference{ + name: matches[1], + tag: matches[2], + } + if matches[3] != "" { + var err error + ref.digest, err = digest.ParseDigest(matches[3]) + if err != nil { + return nil, err + } + } + + r := getBestReferenceType(ref) + if r == nil { + return nil, ErrNameEmpty + } + + return r, nil +} + +// ParseNamed parses s and returns a syntactically valid reference implementing +// the Named interface. The reference must have a name, otherwise an error is +// returned. +// If an error was encountered it is returned, along with a nil Reference. +// NOTE: ParseNamed will not handle short digests. +func ParseNamed(s string) (Named, error) { + ref, err := Parse(s) + if err != nil { + return nil, err + } + named, isNamed := ref.(Named) + if !isNamed { + return nil, fmt.Errorf("reference %s has no name", ref.String()) + } + return named, nil +} + +// WithName returns a named object representing the given string. If the input +// is invalid ErrReferenceInvalidFormat will be returned. +func WithName(name string) (Named, error) { + if len(name) > NameTotalLengthMax { + return nil, ErrNameTooLong + } + if !anchoredNameRegexp.MatchString(name) { + return nil, ErrReferenceInvalidFormat + } + return repository(name), nil +} + +// WithTag combines the name from "name" and the tag from "tag" to form a +// reference incorporating both the name and the tag. +func WithTag(name Named, tag string) (NamedTagged, error) { + if !anchoredTagRegexp.MatchString(tag) { + return nil, ErrTagInvalidFormat + } + return taggedReference{ + name: name.Name(), + tag: tag, + }, nil +} + +// WithDigest combines the name from "name" and the digest from "digest" to form +// a reference incorporating both the name and the digest. +func WithDigest(name Named, digest digest.Digest) (Canonical, error) { + if !anchoredDigestRegexp.MatchString(digest.String()) { + return nil, ErrDigestInvalidFormat + } + return canonicalReference{ + name: name.Name(), + digest: digest, + }, nil +} + +func getBestReferenceType(ref reference) Reference { + if ref.name == "" { + // Allow digest only references + if ref.digest != "" { + return digestReference(ref.digest) + } + return nil + } + if ref.tag == "" { + if ref.digest != "" { + return canonicalReference{ + name: ref.name, + digest: ref.digest, + } + } + return repository(ref.name) + } + if ref.digest == "" { + return taggedReference{ + name: ref.name, + tag: ref.tag, + } + } + + return ref +} + +type reference struct { + name string + tag string + digest digest.Digest +} + +func (r reference) String() string { + return r.name + ":" + r.tag + "@" + r.digest.String() +} + +func (r reference) Name() string { + return r.name +} + +func (r reference) Tag() string { + return r.tag +} + +func (r reference) Digest() digest.Digest { + return r.digest +} + +type repository string + +func (r repository) String() string { + return string(r) +} + +func (r repository) Name() string { + return string(r) +} + +type digestReference digest.Digest + +func (d digestReference) String() string { + return d.String() +} + +func (d digestReference) Digest() digest.Digest { + return digest.Digest(d) +} + +type taggedReference struct { + name string + tag string +} + +func (t taggedReference) String() string { + return t.name + ":" + t.tag +} + +func (t taggedReference) Name() string { + return t.name +} + +func (t taggedReference) Tag() string { + return t.tag +} + +type canonicalReference struct { + name string + digest digest.Digest +} + +func (c canonicalReference) String() string { + return c.name + "@" + c.digest.String() +} + +func (c canonicalReference) Name() string { + return c.name +} + +func (c canonicalReference) Digest() digest.Digest { + return c.digest +} diff --git a/Godeps/_workspace/src/github.com/docker/distribution/reference/regexp.go b/Godeps/_workspace/src/github.com/docker/distribution/reference/regexp.go new file mode 100644 index 00000000..9a7d366b --- /dev/null +++ b/Godeps/_workspace/src/github.com/docker/distribution/reference/regexp.go @@ -0,0 +1,124 @@ +package reference + +import "regexp" + +var ( + // alphaNumericRegexp defines the alpha numeric atom, typically a + // component of names. This only allows lower case characters and digits. + alphaNumericRegexp = match(`[a-z0-9]+`) + + // separatorRegexp defines the separators allowed to be embedded in name + // components. This allow one period, one or two underscore and multiple + // dashes. + separatorRegexp = match(`(?:[._]|__|[-]*)`) + + // nameComponentRegexp restricts registry path component names to start + // with at least one letter or number, with following parts able to be + // separated by one period, one or two underscore and multiple dashes. + nameComponentRegexp = expression( + alphaNumericRegexp, + optional(repeated(separatorRegexp, alphaNumericRegexp))) + + // hostnameComponentRegexp restricts the registry hostname component of a + // repository name to start with a component as defined by hostnameRegexp + // and followed by an optional port. + hostnameComponentRegexp = match(`(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])`) + + // hostnameRegexp defines the structure of potential hostname components + // that may be part of image names. This is purposely a subset of what is + // allowed by DNS to ensure backwards compatibility with Docker image + // names. + hostnameRegexp = expression( + hostnameComponentRegexp, + optional(repeated(literal(`.`), hostnameComponentRegexp)), + optional(literal(`:`), match(`[0-9]+`))) + + // TagRegexp matches valid tag names. From docker/docker:graph/tags.go. + TagRegexp = match(`[\w][\w.-]{0,127}`) + + // anchoredTagRegexp matches valid tag names, anchored at the start and + // end of the matched string. + anchoredTagRegexp = anchored(TagRegexp) + + // DigestRegexp matches valid digests. + DigestRegexp = match(`[A-Za-z][A-Za-z0-9]*(?:[-_+.][A-Za-z][A-Za-z0-9]*)*[:][[:xdigit:]]{32,}`) + + // anchoredDigestRegexp matches valid digests, anchored at the start and + // end of the matched string. + anchoredDigestRegexp = anchored(DigestRegexp) + + // NameRegexp is the format for the name component of references. The + // regexp has capturing groups for the hostname and name part omitting + // the separating forward slash from either. + NameRegexp = expression( + optional(hostnameRegexp, literal(`/`)), + nameComponentRegexp, + optional(repeated(literal(`/`), nameComponentRegexp))) + + // anchoredNameRegexp is used to parse a name value, capturing the + // hostname and trailing components. + anchoredNameRegexp = anchored( + optional(capture(hostnameRegexp), literal(`/`)), + capture(nameComponentRegexp, + optional(repeated(literal(`/`), nameComponentRegexp)))) + + // ReferenceRegexp is the full supported format of a reference. The regexp + // is anchored and has capturing groups for name, tag, and digest + // components. + ReferenceRegexp = anchored(capture(NameRegexp), + optional(literal(":"), capture(TagRegexp)), + optional(literal("@"), capture(DigestRegexp))) +) + +// match compiles the string to a regular expression. +var match = regexp.MustCompile + +// literal compiles s into a literal regular expression, escaping any regexp +// reserved characters. +func literal(s string) *regexp.Regexp { + re := match(regexp.QuoteMeta(s)) + + if _, complete := re.LiteralPrefix(); !complete { + panic("must be a literal") + } + + return re +} + +// expression defines a full expression, where each regular expression must +// follow the previous. +func expression(res ...*regexp.Regexp) *regexp.Regexp { + var s string + for _, re := range res { + s += re.String() + } + + return match(s) +} + +// optional wraps the expression in a non-capturing group and makes the +// production optional. +func optional(res ...*regexp.Regexp) *regexp.Regexp { + return match(group(expression(res...)).String() + `?`) +} + +// repeated wraps the regexp in a non-capturing group to get one or more +// matches. +func repeated(res ...*regexp.Regexp) *regexp.Regexp { + return match(group(expression(res...)).String() + `+`) +} + +// group wraps the regexp in a non-capturing group. +func group(res ...*regexp.Regexp) *regexp.Regexp { + return match(`(?:` + expression(res...).String() + `)`) +} + +// capture wraps the expression in a capturing group. +func capture(res ...*regexp.Regexp) *regexp.Regexp { + return match(`(` + expression(res...).String() + `)`) +} + +// anchored anchors the regular expression by adding start and end delimiters. +func anchored(res ...*regexp.Regexp) *regexp.Regexp { + return match(`^` + expression(res...).String() + `$`) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/LICENSE b/Godeps/_workspace/src/github.com/klauspost/compress/LICENSE new file mode 100644 index 00000000..74487567 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/copy.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/copy.go new file mode 100644 index 00000000..a3200a8f --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/copy.go @@ -0,0 +1,32 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +// forwardCopy is like the built-in copy function except that it always goes +// forward from the start, even if the dst and src overlap. +// It is equivalent to: +// for i := 0; i < n; i++ { +// mem[dst+i] = mem[src+i] +// } +func forwardCopy(mem []byte, dst, src, n int) { + if dst <= src { + copy(mem[dst:dst+n], mem[src:src+n]) + return + } + for { + if dst >= src+n { + copy(mem[dst:dst+n], mem[src:src+n]) + return + } + // There is some forward overlap. The destination + // will be filled with a repeated pattern of mem[src:src+k]. + // We copy one instance of the pattern here, then repeat. + // Each time around this loop k will double. + k := dst - src + copy(mem[dst:dst+k], mem[src:src+k]) + n -= k + dst += k + } +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.go new file mode 100644 index 00000000..45d52f62 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.go @@ -0,0 +1,39 @@ +//+build !noasm +//+build !appengine + +// Copyright 2015, Klaus Post, see LICENSE for details. + +package flate + +import ( + "github.com/klauspost/cpuid" +) + +// crc32sse returns a hash for the first 4 bytes of the slice +// len(a) must be >= 4. +//go:noescape +func crc32sse(a []byte) hash + +// crc32sseAll calculates hashes for each 4-byte set in a. +// dst must be east len(a) - 4 in size. +// The size is not checked by the assembly. +//go:noescape +func crc32sseAll(a []byte, dst []hash) + +// matchLenSSE4 returns the number of matching bytes in a and b +// up to length 'max'. Both slices must be at least 'max' +// bytes in size. +// It uses the PCMPESTRI SSE 4.2 instruction. +//go:noescape +func matchLenSSE4(a, b []byte, max int) int + +// histogram accumulates a histogram of b in h. +// h must be at least 256 entries in length, +// and must be cleared before calling this function. +//go:noescape +func histogram(b []byte, h []int32) + +// Detect SSE 4.2 feature. +func init() { + useSSE42 = cpuid.CPU.SSE42() +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.s b/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.s new file mode 100644 index 00000000..bfa0fdac --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_amd64.s @@ -0,0 +1,219 @@ +//+build !noasm +//+build !appengine + +// Copyright 2015, Klaus Post, see LICENSE for details. + +// func crc32sse(a []byte) hash +TEXT ·crc32sse(SB), 4, $0 + MOVQ a+0(FP), R10 + XORQ BX, BX + + // CRC32 dword (R10), EBX + BYTE $0xF2; BYTE $0x41; BYTE $0x0f + BYTE $0x38; BYTE $0xf1; BYTE $0x1a + + MOVL BX, ret+24(FP) + RET + +// func crc32sseAll(a []byte, dst []hash) +TEXT ·crc32sseAll(SB), 4, $0 + MOVQ a+0(FP), R8 // R8: src + MOVQ a_len+8(FP), R10 // input length + MOVQ dst+24(FP), R9 // R9: dst + SUBQ $4, R10 + JS end + JZ one_crc + MOVQ R10, R13 + SHRQ $2, R10 // len/4 + ANDQ $3, R13 // len&3 + XORQ BX, BX + ADDQ $1, R13 + TESTQ R10, R10 + JZ rem_loop + +crc_loop: + MOVQ (R8), R11 + XORQ BX, BX + XORQ DX, DX + XORQ DI, DI + MOVQ R11, R12 + SHRQ $8, R11 + MOVQ R12, AX + MOVQ R11, CX + SHRQ $16, R12 + SHRQ $16, R11 + MOVQ R12, SI + + // CRC32 EAX, EBX + BYTE $0xF2; BYTE $0x0f + BYTE $0x38; BYTE $0xf1; BYTE $0xd8 + + // CRC32 ECX, EDX + BYTE $0xF2; BYTE $0x0f + BYTE $0x38; BYTE $0xf1; BYTE $0xd1 + + // CRC32 ESI, EDI + BYTE $0xF2; BYTE $0x0f + BYTE $0x38; BYTE $0xf1; BYTE $0xfe + MOVL BX, (R9) + MOVL DX, 4(R9) + MOVL DI, 8(R9) + + XORQ BX, BX + MOVL R11, AX + + // CRC32 EAX, EBX + BYTE $0xF2; BYTE $0x0f + BYTE $0x38; BYTE $0xf1; BYTE $0xd8 + MOVL BX, 12(R9) + + ADDQ $16, R9 + ADDQ $4, R8 + XORQ BX, BX + SUBQ $1, R10 + JNZ crc_loop + +rem_loop: + MOVL (R8), AX + + // CRC32 EAX, EBX + BYTE $0xF2; BYTE $0x0f + BYTE $0x38; BYTE $0xf1; BYTE $0xd8 + + MOVL BX, (R9) + ADDQ $4, R9 + ADDQ $1, R8 + XORQ BX, BX + SUBQ $1, R13 + JNZ rem_loop + +end: + RET + +one_crc: + MOVQ $1, R13 + XORQ BX, BX + JMP rem_loop + +// func matchLenSSE4(a, b []byte, max int) int +TEXT ·matchLenSSE4(SB), 4, $0 + MOVQ a+0(FP), SI // RSI: &a + MOVQ b+24(FP), DI // RDI: &b + MOVQ max+48(FP), R10 // R10: max + XORQ R11, R11 // R11: match length + MOVQ R10, R12 // R12: Remainder + SHRQ $4, R10 // max / 16 + MOVQ $16, AX // Set length for PCMPESTRI + MOVQ $16, DX // Set length for PCMPESTRI + ANDQ $15, R12 // max & 15 + TESTQ R10, R10 + JZ matchlen_verysmall + +loopback_matchlen: + MOVOU (SI), X0 // a[x] + MOVOU (DI), X1 // b[x] + + // PCMPESTRI $0x18, X1, X0 + // 0x18 = _SIDD_UBYTE_OPS (0x0) | _SIDD_CMP_EQUAL_EACH (0x8) | _SIDD_NEGATIVE_POLARITY (0x10) + BYTE $0x66; BYTE $0x0f; BYTE $0x3a + BYTE $0x61; BYTE $0xc1; BYTE $0x18 + + JC match_ended + + ADDQ $16, SI + ADDQ $16, DI + ADDQ $16, R11 + + SUBQ $1, R10 + JNZ loopback_matchlen + + // Check the remainder using REP CMPSB +matchlen_verysmall: + TESTQ R12, R12 + JZ done_matchlen + MOVQ R12, CX + ADDQ R12, R11 + + // Compare CX bytes at [SI] [DI] + // Subtract one from CX for every match. + // Terminates when CX is zero (checked pre-compare) + CLD + REP; CMPSB + + // Check if last was a match. + JZ done_matchlen + + // Subtract remanding bytes. + SUBQ CX, R11 + SUBQ $1, R11 + MOVQ R11, ret+56(FP) + RET + +match_ended: + ADDQ CX, R11 + +done_matchlen: + MOVQ R11, ret+56(FP) + RET + +// func histogram(b []byte, h []int32) +TEXT ·histogram(SB), 4, $0 + MOVQ b+0(FP), SI // SI: &b + MOVQ b_len+8(FP), R9 // R9: len(b) + MOVQ h+24(FP), DI // DI: Histogram + MOVQ R9, R8 + SHRQ $3, R8 + JZ hist1 + XORQ R11, R11 + +loop_hist8: + MOVQ (SI), R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + MOVB R10, R11 + INCL (DI)(R11*4) + SHRQ $8, R10 + + INCL (DI)(R10*4) + + ADDQ $8, SI + DECQ R8 + JNZ loop_hist8 + +hist1: + ANDQ $7, R9 + JZ end_hist + XORQ R10, R10 + +loop_hist1: + MOVB (SI), R10 + INCL (DI)(R10*4) + INCQ SI + DECQ R9 + JNZ loop_hist1 + +end_hist: + RET diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_noasm.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_noasm.go new file mode 100644 index 00000000..1c6d23ee --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/crc32_noasm.go @@ -0,0 +1,34 @@ +//+build !amd64 noasm appengine + +// Copyright 2015, Klaus Post, see LICENSE for details. + +package flate + +func init() { + useSSE42 = false +} + +// crc32sse should never be called. +func crc32sse(a []byte) hash { + panic("no assembler") +} + +// crc32sseAll should never be called. +func crc32sseAll(a []byte, dst []hash) { + panic("no assembler") +} + +// matchLenSSE4 should never be called. +func matchLenSSE4(a, b []byte, max int) int { + panic("no assembler") + return 0 +} + +// histogram accumulates a histogram of b in h. +// h must be at least 256 entries in length, +// and must be cleared before calling this function. +func histogram(b []byte, h []int32) { + for _, t := range b { + h[t]++ + } +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/deflate.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/deflate.go new file mode 100644 index 00000000..5b84d805 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/deflate.go @@ -0,0 +1,1357 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Copyright (c) 2015 Klaus Post +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "fmt" + "io" + "math" +) + +const ( + NoCompression = 0 + BestSpeed = 1 + BestCompression = 9 + DefaultCompression = -1 + ConstantCompression = -2 // Does only Huffman encoding + logWindowSize = 15 + windowSize = 1 << logWindowSize + windowMask = windowSize - 1 + logMaxOffsetSize = 15 // Standard DEFLATE + minMatchLength = 4 // The smallest match that the compressor looks for + maxMatchLength = 258 // The longest match for the compressor + minOffsetSize = 1 // The shortest offset that makes any sense + + // The maximum number of tokens we put into a single flat block, just too + // stop things from getting too large. + maxFlateBlockTokens = 1 << 14 + maxStoreBlockSize = 65535 + hashBits = 17 // After 17 performance degrades + hashSize = 1 << hashBits + hashMask = (1 << hashBits) - 1 + hashShift = (hashBits + minMatchLength - 1) / minMatchLength + maxHashOffset = 1 << 24 + + skipNever = math.MaxInt32 +) + +var useSSE42 bool + +type compressionLevel struct { + good, lazy, nice, chain, fastSkipHashing, level int +} + +// Compression levels have been rebalanced from zlib deflate defaults +// to give a bigger spread in speed and compression. +// See https://blog.klauspost.com/rebalancing-deflate-compression-levels/ +var levels = []compressionLevel{ + {}, // 0 + // Level 1+2 uses snappy algorithm - values not used + {0, 0, 0, 0, 0, 1}, + {0, 0, 0, 0, 0, 2}, + // For levels 3-6 we don't bother trying with lazy matches. + // Lazy matching is at least 30% slower, with 1.5% increase. + {4, 0, 8, 4, 4, 3}, + {4, 0, 12, 6, 5, 4}, + {6, 0, 24, 16, 6, 5}, + {8, 0, 32, 32, 7, 6}, + // Levels 7-9 use increasingly more lazy matching + // and increasingly stringent conditions for "good enough". + {4, 8, 16, 16, skipNever, 7}, + {6, 16, 32, 64, skipNever, 8}, + {32, 258, 258, 4096, skipNever, 9}, +} + +type hashid uint32 + +type compressor struct { + compressionLevel + + w *huffmanBitWriter + bulkHasher func([]byte, []hash) + + // compression algorithm + fill func(*compressor, []byte) int // copy data to window + step func(*compressor) // process window + sync bool // requesting flush + + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + chainHead int + hashHead []hashid + hashPrev []hashid + hashOffset int + + // input window: unprocessed data is window[index:windowEnd] + index int + window []byte + windowEnd int + blockStart int // window index where current tokens start + byteAvailable bool // if true, still need to process window[index-1]. + + // queued output tokens + tokens tokens + + // deflate state + length int + offset int + hash hash + maxInsertIndex int + err error + ii uint16 // position of last match, intended to overflow to reset. + + snap snappyEnc + hashMatch [maxMatchLength + minMatchLength]hash +} + +type hash int32 + +func (d *compressor) fillDeflate(b []byte) int { + if d.index >= 2*windowSize-(minMatchLength+maxMatchLength) { + // shift the window by windowSize + copy(d.window, d.window[windowSize:2*windowSize]) + d.index -= windowSize + d.windowEnd -= windowSize + if d.blockStart >= windowSize { + d.blockStart -= windowSize + } else { + d.blockStart = math.MaxInt32 + } + d.hashOffset += windowSize + if d.hashOffset > maxHashOffset { + delta := d.hashOffset - 1 + d.hashOffset -= delta + d.chainHead -= delta + for i, v := range d.hashPrev { + if int(v) > delta { + d.hashPrev[i] = hashid(int(v) - delta) + } else { + d.hashPrev[i] = 0 + } + } + for i, v := range d.hashHead { + if int(v) > delta { + d.hashHead[i] = hashid(int(v) - delta) + } else { + d.hashHead[i] = 0 + } + } + } + } + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) writeBlock(tok tokens, index int, eof bool) error { + if index > 0 || eof { + var window []byte + if d.blockStart <= index { + window = d.window[d.blockStart:index] + } + d.blockStart = index + d.w.writeBlock(tok, eof, window) + return d.w.err + } + return nil +} + +// writeBlockSkip writes the current block and uses the number of tokens +// to determine if the block should be stored on no matches, or +// only huffman encoded. +func (d *compressor) writeBlockSkip(tok tokens, index int, eof bool) error { + if index > 0 || eof { + if d.blockStart <= index { + window := d.window[d.blockStart:index] + // If we removed less than a 64th of all literals + // we huffman compress the block. + if tok.n > len(window)-(tok.n>>6) { + d.w.writeBlockHuff(eof, window) + } else { + // Write a dynamic huffman block. + d.w.writeBlockDynamic(tok, eof, window) + } + } else { + d.w.writeBlock(tok, eof, nil) + } + d.blockStart = index + return d.w.err + } + return nil +} + +// fillWindow will fill the current window with the supplied +// dictionary and calculate all hashes. +// This is much faster than doing a full encode. +// Should only be used after a start/reset. +func (d *compressor) fillWindow(b []byte) { + // Do not fill window if we are in store-only mode, + // use constant or Snappy compression. + switch d.compressionLevel.level { + case 0, 1, 2: + return + } + // If we are given too much, cut it. + if len(b) > windowSize { + b = b[len(b)-windowSize:] + } + // Add all to window. + n := copy(d.window[d.windowEnd:], b) + + // Calculate 256 hashes at the time (more L1 cache hits) + loops := (n + 256 - minMatchLength) / 256 + for j := 0; j < loops; j++ { + startindex := j * 256 + end := startindex + 256 + minMatchLength - 1 + if end > n { + end = n + } + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + + if dstSize <= 0 { + continue + } + + dst := d.hashMatch[:dstSize] + d.bulkHasher(tocheck, dst) + var newH hash + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = d.hashHead[newH] + // Set the head of the hash chain to us. + d.hashHead[newH] = hashid(di + d.hashOffset) + } + d.hash = newH + } + // Update window information. + d.windowEnd += n + d.index = n +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead +func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + minMatchLook := maxMatchLength + if lookahead < minMatchLook { + minMatchLook = lookahead + } + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := len(win) - pos + if d.nice < nice { + nice = d.nice + } + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = prevLength + if length >= d.good { + tries >>= 2 + } + + wEnd := win[pos+length] + wPos := win[pos:] + minIndex := pos - windowSize + + for i := prevHead; tries > 0; tries-- { + if wEnd == win[i+length] { + n := matchLen(win[i:], wPos, minMatchLook) + + if n > length && (n > minMatchLength || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + i = int(d.hashPrev[i&windowMask]) - d.hashOffset + if i < minIndex || i < 0 { + break + } + } + return +} + +// Try to find a match starting at index whose length is greater than prevSize. +// We only look at chainCount possibilities before giving up. +// pos = d.index, prevHead = d.chainHead-d.hashOffset, prevLength=minMatchLength-1, lookahead +func (d *compressor) findMatchSSE(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) { + minMatchLook := maxMatchLength + if lookahead < minMatchLook { + minMatchLook = lookahead + } + + win := d.window[0 : pos+minMatchLook] + + // We quit when we get a match that's at least nice long + nice := len(win) - pos + if d.nice < nice { + nice = d.nice + } + + // If we've got a match that's good enough, only look in 1/4 the chain. + tries := d.chain + length = prevLength + if length >= d.good { + tries >>= 2 + } + + wEnd := win[pos+length] + wPos := win[pos:] + minIndex := pos - windowSize + + for i := prevHead; tries > 0; tries-- { + if wEnd == win[i+length] { + n := matchLenSSE4(win[i:], wPos, minMatchLook) + + if n > length && (n > minMatchLength || pos-i <= 4096) { + length = n + offset = pos - i + ok = true + if n >= nice { + // The match is good enough that we don't try to find a better one. + break + } + wEnd = win[pos+n] + } + } + if i == minIndex { + // hashPrev[i & windowMask] has already been overwritten, so stop now. + break + } + i = int(d.hashPrev[i&windowMask]) - d.hashOffset + if i < minIndex || i < 0 { + break + } + } + return +} + +func (d *compressor) writeStoredBlock(buf []byte) error { + if d.w.writeStoredHeader(len(buf), false); d.w.err != nil { + return d.w.err + } + d.w.writeBytes(buf) + return d.w.err +} + +// oldHash is the hash function used when no native crc32 calculation +// or similar is present. +func oldHash(b []byte) hash { + return hash(b[0])<<(hashShift*3) + hash(b[1])<<(hashShift*2) + hash(b[2])< d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + return + } + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + if d.tokens.n > 0 { + if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + return + } + } + if d.index < d.maxInsertIndex { + // Update the hash + d.hash = oldHash(d.window[d.index:d.index+minMatchLength]) & hashMask + ch := d.hashHead[d.hash] + d.chainHead = int(ch) + d.hashPrev[d.index&windowMask] = ch + d.hashHead[d.hash] = hashid(d.index + d.hashOffset) + } + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize + if minIndex < 0 { + minIndex = 0 + } + + if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if d.length >= minMatchLength { + d.ii = 0 + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3 + d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize)) + d.tokens.n++ + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if d.length <= d.fastSkipHashing { + var newIndex int + newIndex = d.index + d.length + // Calculate missing hashes + end := newIndex + if end > d.maxInsertIndex { + end = d.maxInsertIndex + } + end += minMatchLength - 1 + startindex := d.index + 1 + if startindex > d.maxInsertIndex { + startindex = d.maxInsertIndex + } + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + if dstSize > 0 { + dst := d.hashMatch[:dstSize] + oldBulkHash(tocheck, dst) + var newH hash + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = d.hashHead[newH] + // Set the head of the hash chain to us. + d.hashHead[newH] = hashid(di + d.hashOffset) + } + d.hash = newH + } + d.index = newIndex + } else { + // For matches this long, we don't bother inserting each individual + // item into the table. + d.index += d.length + if d.index < d.maxInsertIndex { + d.hash = oldHash(d.window[d.index:d.index+minMatchLength]) & hashMask + } + } + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } else { + d.ii++ + end := d.index + int(d.ii>>uint(d.fastSkipHashing)) + 1 + if end > d.windowEnd { + end = d.windowEnd + } + for i := d.index; i < end; i++ { + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i])) + d.tokens.n++ + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } + d.index = end + } + } +} + +// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, +// meaning it always has lazy matching on. +func (d *compressor) deflateLazy() { + // Sanity enables additional runtime tests. + // It's intended to be used during development + // to supplement the currently ad-hoc unit tests. + const sanity = false + + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + if d.index < d.maxInsertIndex { + d.hash = oldHash(d.window[d.index:d.index+minMatchLength]) & hashMask + } + + for { + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + return + } + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + d.byteAvailable = false + } + if d.tokens.n > 0 { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + return + } + } + if d.index < d.maxInsertIndex { + // Update the hash + d.hash = oldHash(d.window[d.index:d.index+minMatchLength]) & hashMask + ch := d.hashHead[d.hash] + d.chainHead = int(ch) + d.hashPrev[d.index&windowMask] = ch + d.hashHead[d.hash] = hashid(d.index + d.hashOffset) + } + prevLength := d.length + prevOffset := d.offset + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize + if minIndex < 0 { + minIndex = 0 + } + + if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { + if newLength, newOffset, ok := d.findMatch(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if prevLength >= minMatchLength && d.length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) + d.tokens.n++ + + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + var newIndex int + newIndex = d.index + prevLength - 1 + // Calculate missing hashes + end := newIndex + if end > d.maxInsertIndex { + end = d.maxInsertIndex + } + end += minMatchLength - 1 + startindex := d.index + 1 + if startindex > d.maxInsertIndex { + startindex = d.maxInsertIndex + } + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + if dstSize > 0 { + dst := d.hashMatch[:dstSize] + oldBulkHash(tocheck, dst) + var newH hash + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = d.hashHead[newH] + // Set the head of the hash chain to us. + d.hashHead[newH] = hashid(di + d.hashOffset) + } + d.hash = newH + } + + d.index = newIndex + d.byteAvailable = false + d.length = minMatchLength - 1 + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } else { + // Reset, if we got a match this run. + if d.length >= minMatchLength { + d.ii = 0 + } + // We have a byte waiting. Emit it. + if d.byteAvailable { + d.ii++ + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + d.index++ + + // If we have a long run of no matches, skip additional bytes + // Resets when d.ii overflows after 64KB. + if d.ii > 31 { + n := int(d.ii >> 6) + for j := 0; j < n; j++ { + if d.index >= d.windowEnd-1 { + break + } + + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + d.index++ + } + // Flush last byte + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + d.byteAvailable = false + // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } + } else { + d.index++ + d.byteAvailable = true + } + } + } +} + +// Assumes that d.fastSkipHashing != skipNever, +// otherwise use deflateLazySSE +func (d *compressor) deflateSSE() { + + // Sanity enables additional runtime tests. + // It's intended to be used during development + // to supplement the currently ad-hoc unit tests. + const sanity = false + + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + if d.index < d.maxInsertIndex { + d.hash = oldHash(d.window[d.index:d.index+minMatchLength]) & hashMask + } + + for { + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + return + } + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + if d.tokens.n > 0 { + if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + return + } + } + if d.index < d.maxInsertIndex { + // Update the hash + d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask + ch := d.hashHead[d.hash] + d.chainHead = int(ch) + d.hashPrev[d.index&windowMask] = ch + d.hashHead[d.hash] = hashid(d.index + d.hashOffset) + } + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize + if minIndex < 0 { + minIndex = 0 + } + + if d.chainHead-d.hashOffset >= minIndex && lookahead > minMatchLength-1 { + if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if d.length >= minMatchLength { + d.ii = 0 + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + // "d.length-3" should NOT be "d.length-minMatchLength", since the format always assume 3 + d.tokens.tokens[d.tokens.n] = matchToken(uint32(d.length-3), uint32(d.offset-minOffsetSize)) + d.tokens.n++ + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + if d.length <= d.fastSkipHashing { + var newIndex int + newIndex = d.index + d.length + // Calculate missing hashes + end := newIndex + if end > d.maxInsertIndex { + end = d.maxInsertIndex + } + end += minMatchLength - 1 + startindex := d.index + 1 + if startindex > d.maxInsertIndex { + startindex = d.maxInsertIndex + } + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + if dstSize > 0 { + dst := d.hashMatch[:dstSize] + + crc32sseAll(tocheck, dst) + var newH hash + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = d.hashHead[newH] + // Set the head of the hash chain to us. + d.hashHead[newH] = hashid(di + d.hashOffset) + } + d.hash = newH + } + d.index = newIndex + } else { + // For matches this long, we don't bother inserting each individual + // item into the table. + d.index += d.length + if d.index < d.maxInsertIndex { + d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask + } + } + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlockSkip(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } else { + d.ii++ + end := d.index + int(d.ii>>uint(d.fastSkipHashing)) + 1 + if end > d.windowEnd { + end = d.windowEnd + } + for i := d.index; i < end; i++ { + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[i])) + d.tokens.n++ + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlockSkip(d.tokens, i+1, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } + d.index = end + } + } +} + +// deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever, +// meaning it always has lazy matching on. +func (d *compressor) deflateLazySSE() { + // Sanity enables additional runtime tests. + // It's intended to be used during development + // to supplement the currently ad-hoc unit tests. + const sanity = false + + if d.windowEnd-d.index < minMatchLength+maxMatchLength && !d.sync { + return + } + + d.maxInsertIndex = d.windowEnd - (minMatchLength - 1) + if d.index < d.maxInsertIndex { + d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask + } + + for { + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + lookahead := d.windowEnd - d.index + if lookahead < minMatchLength+maxMatchLength { + if !d.sync { + return + } + if sanity && d.index > d.windowEnd { + panic("index > windowEnd") + } + if lookahead == 0 { + // Flush current output block if any. + if d.byteAvailable { + // There is still one pending token that needs to be flushed + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + d.byteAvailable = false + } + if d.tokens.n > 0 { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + return + } + } + if d.index < d.maxInsertIndex { + // Update the hash + d.hash = crc32sse(d.window[d.index:d.index+minMatchLength]) & hashMask + ch := d.hashHead[d.hash] + d.chainHead = int(ch) + d.hashPrev[d.index&windowMask] = ch + d.hashHead[d.hash] = hashid(d.index + d.hashOffset) + } + prevLength := d.length + prevOffset := d.offset + d.length = minMatchLength - 1 + d.offset = 0 + minIndex := d.index - windowSize + if minIndex < 0 { + minIndex = 0 + } + + if d.chainHead-d.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy { + if newLength, newOffset, ok := d.findMatchSSE(d.index, d.chainHead-d.hashOffset, minMatchLength-1, lookahead); ok { + d.length = newLength + d.offset = newOffset + } + } + if prevLength >= minMatchLength && d.length <= prevLength { + // There was a match at the previous step, and the current match is + // not better. Output the previous match. + d.tokens.tokens[d.tokens.n] = matchToken(uint32(prevLength-3), uint32(prevOffset-minOffsetSize)) + d.tokens.n++ + + // Insert in the hash table all strings up to the end of the match. + // index and index-1 are already inserted. If there is not enough + // lookahead, the last two strings are not inserted into the hash + // table. + var newIndex int + newIndex = d.index + prevLength - 1 + // Calculate missing hashes + end := newIndex + if end > d.maxInsertIndex { + end = d.maxInsertIndex + } + end += minMatchLength - 1 + startindex := d.index + 1 + if startindex > d.maxInsertIndex { + startindex = d.maxInsertIndex + } + tocheck := d.window[startindex:end] + dstSize := len(tocheck) - minMatchLength + 1 + if dstSize > 0 { + dst := d.hashMatch[:dstSize] + crc32sseAll(tocheck, dst) + var newH hash + for i, val := range dst { + di := i + startindex + newH = val & hashMask + // Get previous value with the same hash. + // Our chain should point to the previous value. + d.hashPrev[di&windowMask] = d.hashHead[newH] + // Set the head of the hash chain to us. + d.hashHead[newH] = hashid(di + d.hashOffset) + } + d.hash = newH + } + + d.index = newIndex + d.byteAvailable = false + d.length = minMatchLength - 1 + if d.tokens.n == maxFlateBlockTokens { + // The block includes the current character + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } else { + // Reset, if we got a match this run. + if d.length >= minMatchLength { + d.ii = 0 + } + // We have a byte waiting. Emit it. + if d.byteAvailable { + d.ii++ + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + d.index++ + + // If we have a long run of no matches, skip additional bytes + // Resets when d.ii overflows after 64KB. + if d.ii > 31 { + n := int(d.ii >> 6) + for j := 0; j < n; j++ { + if d.index >= d.windowEnd-1 { + break + } + + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + d.index++ + } + // Flush last byte + d.tokens.tokens[d.tokens.n] = literalToken(uint32(d.window[d.index-1])) + d.tokens.n++ + d.byteAvailable = false + // d.length = minMatchLength - 1 // not needed, since d.ii is reset above, so it should never be > minMatchLength + if d.tokens.n == maxFlateBlockTokens { + if d.err = d.writeBlock(d.tokens, d.index, false); d.err != nil { + return + } + d.tokens.n = 0 + } + } + } else { + d.index++ + d.byteAvailable = true + } + } + } +} + +func (d *compressor) fillStore(b []byte) int { + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +func (d *compressor) store() { + if d.windowEnd > 0 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + } + d.windowEnd = 0 +} + +// fillHuff will fill the buffer with data for huffman-only compression. +// The number of bytes copied is returned. +func (d *compressor) fillHuff(b []byte) int { + n := copy(d.window[d.windowEnd:], b) + d.windowEnd += n + return n +} + +// storeHuff will compress and store the currently added data, +// if enough has been accumulated or we at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) storeHuff() { + // We only compress if we have maxStoreBlockSize or we are at end-of-stream + if d.windowEnd < maxStoreBlockSize && !d.sync { + return + } + if d.windowEnd == 0 { + return + } + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + d.windowEnd = 0 +} + +// storeHuff will compress and store the currently added data, +// if enough has been accumulated or we at the end of the stream. +// Any error that occurred will be in d.err +func (d *compressor) storeSnappy() { + // We only compress if we have maxStoreBlockSize. + if d.windowEnd < maxStoreBlockSize { + if !d.sync { + return + } + // Handle extremely small sizes. + if d.windowEnd < 128 { + if d.windowEnd == 0 { + return + } + if d.windowEnd <= 32 { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + d.tokens.n = 0 + d.windowEnd = 0 + } else { + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + } + d.tokens.n = 0 + d.windowEnd = 0 + return + } + } + + d.snap.Encode(&d.tokens, d.window[:d.windowEnd]) + // If we made zero matches, store the block as is. + if d.tokens.n == d.windowEnd { + d.err = d.writeStoredBlock(d.window[:d.windowEnd]) + // If we removed less than 1/16th, huffman compress the block. + } else if d.tokens.n > d.windowEnd-(d.windowEnd>>4) { + d.w.writeBlockHuff(false, d.window[:d.windowEnd]) + d.err = d.w.err + } else { + d.w.writeBlockDynamic(d.tokens, false, d.window[:d.windowEnd]) + d.err = d.w.err + } + d.tokens.n = 0 + d.windowEnd = 0 +} + +// write will add input byte to the stream. +// Unless an error occurs all bytes will be consumed. +func (d *compressor) write(b []byte) (n int, err error) { + if d.err != nil { + return 0, d.err + } + n = len(b) + for len(b) > 0 { + d.step(d) + b = b[d.fill(d, b):] + if d.err != nil { + return 0, d.err + } + } + return n, d.err +} + +func (d *compressor) syncFlush() error { + d.sync = true + if d.err != nil { + return d.err + } + d.step(d) + if d.err == nil { + d.w.writeStoredHeader(0, false) + d.w.flush() + d.err = d.w.err + } + d.sync = false + return d.err +} + +func (d *compressor) init(w io.Writer, level int) (err error) { + d.w = newHuffmanBitWriter(w) + + switch { + case level == NoCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillStore + d.step = (*compressor).store + case level == ConstantCompression: + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillHuff + d.step = (*compressor).storeHuff + case level >= 1 && level <= 3: + d.snap = newSnappy(level) + d.window = make([]byte, maxStoreBlockSize) + d.fill = (*compressor).fillHuff + d.step = (*compressor).storeSnappy + d.tokens.tokens = make([]token, maxStoreBlockSize+1) + case level == DefaultCompression: + level = 5 + fallthrough + case 4 <= level && level <= 9: + d.compressionLevel = levels[level] + d.initDeflate() + d.fill = (*compressor).fillDeflate + if d.fastSkipHashing == skipNever { + if useSSE42 { + d.step = (*compressor).deflateLazySSE + } else { + d.step = (*compressor).deflateLazy + } + } else { + if useSSE42 { + d.step = (*compressor).deflateSSE + } else { + d.step = (*compressor).deflate + + } + } + default: + return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level) + } + return nil +} + +// Used for zeroing the hash slice +var hzeroes [256]hashid + +// reset the state of the compressor. +func (d *compressor) reset(w io.Writer) { + d.w.reset(w) + d.sync = false + d.err = nil + // We only need to reset a few things for Snappy. + if d.snap != nil { + d.snap.Reset() + d.windowEnd = 0 + d.tokens.n = 0 + return + } + switch d.compressionLevel.chain { + case 0: + // level was NoCompression or ConstantCompresssion. + d.windowEnd = 0 + default: + d.chainHead = -1 + for s := d.hashHead; len(s) > 0; { + n := copy(s, hzeroes[:]) + s = s[n:] + } + for s := d.hashPrev; len(s) > 0; s = s[len(hzeroes):] { + copy(s, hzeroes[:]) + } + d.hashOffset = 1 + + d.index, d.windowEnd = 0, 0 + d.blockStart, d.byteAvailable = 0, false + + d.tokens.n = 0 + d.length = minMatchLength - 1 + d.offset = 0 + d.hash = 0 + d.ii = 0 + d.maxInsertIndex = 0 + } +} + +func (d *compressor) close() error { + if d.err != nil { + return d.err + } + d.sync = true + d.step(d) + if d.err != nil { + return d.err + } + if d.w.writeStoredHeader(0, true); d.w.err != nil { + return d.w.err + } + d.w.flush() + return d.w.err +} + +// NewWriter returns a new Writer compressing data at the given level. +// Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression); +// higher levels typically run slower but compress more. +// Level 0 (NoCompression) does not attempt any compression; it only adds the +// necessary DEFLATE framing. +// Level -1 (DefaultCompression) uses the default compression level. +// Level -2 (ConstantCompression) will use Huffman compression only, giving +// a very fast compression for all types of input, but sacrificing considerable +// compression efficiency. +// +// If level is in the range [-2, 9] then the error returned will be nil. +// Otherwise the error returned will be non-nil. +func NewWriter(w io.Writer, level int) (*Writer, error) { + var dw Writer + if err := dw.d.init(w, level); err != nil { + return nil, err + } + return &dw, nil +} + +// NewWriterDict is like NewWriter but initializes the new +// Writer with a preset dictionary. The returned Writer behaves +// as if the dictionary had been written to it without producing +// any compressed output. The compressed data written to w +// can only be decompressed by a Reader initialized with the +// same dictionary. +func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) { + dw := &dictWriter{w} + zw, err := NewWriter(dw, level) + if err != nil { + return nil, err + } + zw.d.fillWindow(dict) + zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method. + return zw, err +} + +type dictWriter struct { + w io.Writer +} + +func (w *dictWriter) Write(b []byte) (n int, err error) { + return w.w.Write(b) +} + +// A Writer takes data written to it and writes the compressed +// form of that data to an underlying writer (see NewWriter). +type Writer struct { + d compressor + dict []byte +} + +// Write writes data to w, which will eventually write the +// compressed form of data to its underlying writer. +func (w *Writer) Write(data []byte) (n int, err error) { + return w.d.write(data) +} + +// Flush flushes any pending compressed data to the underlying writer. +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. +// Flush does not return until the data has been written. +// If the underlying writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (w *Writer) Flush() error { + // For more about flushing: + // http://www.bolet.org/~pornin/deflate-flush.html + return w.d.syncFlush() +} + +// Close flushes and closes the writer. +func (w *Writer) Close() error { + return w.d.close() +} + +// Reset discards the writer's state and makes it equivalent to +// the result of NewWriter or NewWriterDict called with dst +// and w's level and dictionary. +func (w *Writer) Reset(dst io.Writer) { + if dw, ok := w.d.w.w.(*dictWriter); ok { + // w was created with NewWriterDict + dw.w = dst + w.d.reset(dw) + w.d.fillWindow(w.dict) + } else { + // w was created with NewWriter + w.d.reset(dst) + } +} + +// ResetDict discards the writer's state and makes it equivalent to +// the result of NewWriter or NewWriterDict called with dst +// and w's level, but sets a specific dictionary. +func (w *Writer) ResetDict(dst io.Writer, dict []byte) { + w.dict = dict + w.d.reset(dst) + w.d.fillWindow(w.dict) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/fixedhuff.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/fixedhuff.go new file mode 100644 index 00000000..7df8b9a2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/fixedhuff.go @@ -0,0 +1,78 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT + +var fixedHuffmanDecoder = huffmanDecoder{ + 7, + [huffmanNumChunks]uint32{ + 0x1007, 0x0508, 0x0108, 0x1188, 0x1107, 0x0708, 0x0308, 0x0c09, + 0x1087, 0x0608, 0x0208, 0x0a09, 0x0008, 0x0808, 0x0408, 0x0e09, + 0x1047, 0x0588, 0x0188, 0x0909, 0x1147, 0x0788, 0x0388, 0x0d09, + 0x10c7, 0x0688, 0x0288, 0x0b09, 0x0088, 0x0888, 0x0488, 0x0f09, + 0x1027, 0x0548, 0x0148, 0x11c8, 0x1127, 0x0748, 0x0348, 0x0c89, + 0x10a7, 0x0648, 0x0248, 0x0a89, 0x0048, 0x0848, 0x0448, 0x0e89, + 0x1067, 0x05c8, 0x01c8, 0x0989, 0x1167, 0x07c8, 0x03c8, 0x0d89, + 0x10e7, 0x06c8, 0x02c8, 0x0b89, 0x00c8, 0x08c8, 0x04c8, 0x0f89, + 0x1017, 0x0528, 0x0128, 0x11a8, 0x1117, 0x0728, 0x0328, 0x0c49, + 0x1097, 0x0628, 0x0228, 0x0a49, 0x0028, 0x0828, 0x0428, 0x0e49, + 0x1057, 0x05a8, 0x01a8, 0x0949, 0x1157, 0x07a8, 0x03a8, 0x0d49, + 0x10d7, 0x06a8, 0x02a8, 0x0b49, 0x00a8, 0x08a8, 0x04a8, 0x0f49, + 0x1037, 0x0568, 0x0168, 0x11e8, 0x1137, 0x0768, 0x0368, 0x0cc9, + 0x10b7, 0x0668, 0x0268, 0x0ac9, 0x0068, 0x0868, 0x0468, 0x0ec9, + 0x1077, 0x05e8, 0x01e8, 0x09c9, 0x1177, 0x07e8, 0x03e8, 0x0dc9, + 0x10f7, 0x06e8, 0x02e8, 0x0bc9, 0x00e8, 0x08e8, 0x04e8, 0x0fc9, + 0x1007, 0x0518, 0x0118, 0x1198, 0x1107, 0x0718, 0x0318, 0x0c29, + 0x1087, 0x0618, 0x0218, 0x0a29, 0x0018, 0x0818, 0x0418, 0x0e29, + 0x1047, 0x0598, 0x0198, 0x0929, 0x1147, 0x0798, 0x0398, 0x0d29, + 0x10c7, 0x0698, 0x0298, 0x0b29, 0x0098, 0x0898, 0x0498, 0x0f29, + 0x1027, 0x0558, 0x0158, 0x11d8, 0x1127, 0x0758, 0x0358, 0x0ca9, + 0x10a7, 0x0658, 0x0258, 0x0aa9, 0x0058, 0x0858, 0x0458, 0x0ea9, + 0x1067, 0x05d8, 0x01d8, 0x09a9, 0x1167, 0x07d8, 0x03d8, 0x0da9, + 0x10e7, 0x06d8, 0x02d8, 0x0ba9, 0x00d8, 0x08d8, 0x04d8, 0x0fa9, + 0x1017, 0x0538, 0x0138, 0x11b8, 0x1117, 0x0738, 0x0338, 0x0c69, + 0x1097, 0x0638, 0x0238, 0x0a69, 0x0038, 0x0838, 0x0438, 0x0e69, + 0x1057, 0x05b8, 0x01b8, 0x0969, 0x1157, 0x07b8, 0x03b8, 0x0d69, + 0x10d7, 0x06b8, 0x02b8, 0x0b69, 0x00b8, 0x08b8, 0x04b8, 0x0f69, + 0x1037, 0x0578, 0x0178, 0x11f8, 0x1137, 0x0778, 0x0378, 0x0ce9, + 0x10b7, 0x0678, 0x0278, 0x0ae9, 0x0078, 0x0878, 0x0478, 0x0ee9, + 0x1077, 0x05f8, 0x01f8, 0x09e9, 0x1177, 0x07f8, 0x03f8, 0x0de9, + 0x10f7, 0x06f8, 0x02f8, 0x0be9, 0x00f8, 0x08f8, 0x04f8, 0x0fe9, + 0x1007, 0x0508, 0x0108, 0x1188, 0x1107, 0x0708, 0x0308, 0x0c19, + 0x1087, 0x0608, 0x0208, 0x0a19, 0x0008, 0x0808, 0x0408, 0x0e19, + 0x1047, 0x0588, 0x0188, 0x0919, 0x1147, 0x0788, 0x0388, 0x0d19, + 0x10c7, 0x0688, 0x0288, 0x0b19, 0x0088, 0x0888, 0x0488, 0x0f19, + 0x1027, 0x0548, 0x0148, 0x11c8, 0x1127, 0x0748, 0x0348, 0x0c99, + 0x10a7, 0x0648, 0x0248, 0x0a99, 0x0048, 0x0848, 0x0448, 0x0e99, + 0x1067, 0x05c8, 0x01c8, 0x0999, 0x1167, 0x07c8, 0x03c8, 0x0d99, + 0x10e7, 0x06c8, 0x02c8, 0x0b99, 0x00c8, 0x08c8, 0x04c8, 0x0f99, + 0x1017, 0x0528, 0x0128, 0x11a8, 0x1117, 0x0728, 0x0328, 0x0c59, + 0x1097, 0x0628, 0x0228, 0x0a59, 0x0028, 0x0828, 0x0428, 0x0e59, + 0x1057, 0x05a8, 0x01a8, 0x0959, 0x1157, 0x07a8, 0x03a8, 0x0d59, + 0x10d7, 0x06a8, 0x02a8, 0x0b59, 0x00a8, 0x08a8, 0x04a8, 0x0f59, + 0x1037, 0x0568, 0x0168, 0x11e8, 0x1137, 0x0768, 0x0368, 0x0cd9, + 0x10b7, 0x0668, 0x0268, 0x0ad9, 0x0068, 0x0868, 0x0468, 0x0ed9, + 0x1077, 0x05e8, 0x01e8, 0x09d9, 0x1177, 0x07e8, 0x03e8, 0x0dd9, + 0x10f7, 0x06e8, 0x02e8, 0x0bd9, 0x00e8, 0x08e8, 0x04e8, 0x0fd9, + 0x1007, 0x0518, 0x0118, 0x1198, 0x1107, 0x0718, 0x0318, 0x0c39, + 0x1087, 0x0618, 0x0218, 0x0a39, 0x0018, 0x0818, 0x0418, 0x0e39, + 0x1047, 0x0598, 0x0198, 0x0939, 0x1147, 0x0798, 0x0398, 0x0d39, + 0x10c7, 0x0698, 0x0298, 0x0b39, 0x0098, 0x0898, 0x0498, 0x0f39, + 0x1027, 0x0558, 0x0158, 0x11d8, 0x1127, 0x0758, 0x0358, 0x0cb9, + 0x10a7, 0x0658, 0x0258, 0x0ab9, 0x0058, 0x0858, 0x0458, 0x0eb9, + 0x1067, 0x05d8, 0x01d8, 0x09b9, 0x1167, 0x07d8, 0x03d8, 0x0db9, + 0x10e7, 0x06d8, 0x02d8, 0x0bb9, 0x00d8, 0x08d8, 0x04d8, 0x0fb9, + 0x1017, 0x0538, 0x0138, 0x11b8, 0x1117, 0x0738, 0x0338, 0x0c79, + 0x1097, 0x0638, 0x0238, 0x0a79, 0x0038, 0x0838, 0x0438, 0x0e79, + 0x1057, 0x05b8, 0x01b8, 0x0979, 0x1157, 0x07b8, 0x03b8, 0x0d79, + 0x10d7, 0x06b8, 0x02b8, 0x0b79, 0x00b8, 0x08b8, 0x04b8, 0x0f79, + 0x1037, 0x0578, 0x0178, 0x11f8, 0x1137, 0x0778, 0x0378, 0x0cf9, + 0x10b7, 0x0678, 0x0278, 0x0af9, 0x0078, 0x0878, 0x0478, 0x0ef9, + 0x1077, 0x05f8, 0x01f8, 0x09f9, 0x1177, 0x07f8, 0x03f8, 0x0df9, + 0x10f7, 0x06f8, 0x02f8, 0x0bf9, 0x00f8, 0x08f8, 0x04f8, 0x0ff9, + }, + nil, 0, +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/gen.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/gen.go new file mode 100644 index 00000000..154c89a4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/gen.go @@ -0,0 +1,265 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// This program generates fixedhuff.go +// Invoke as +// +// go run gen.go -output fixedhuff.go + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "io/ioutil" + "log" +) + +var filename = flag.String("output", "fixedhuff.go", "output file name") + +const maxCodeLen = 16 + +// Note: the definition of the huffmanDecoder struct is copied from +// inflate.go, as it is private to the implementation. + +// chunk & 15 is number of bits +// chunk >> 4 is value, including table link + +const ( + huffmanChunkBits = 9 + huffmanNumChunks = 1 << huffmanChunkBits + huffmanCountMask = 15 + huffmanValueShift = 4 +) + +type huffmanDecoder struct { + min int // the minimum code length + chunks [huffmanNumChunks]uint32 // chunks as described above + links [][]uint32 // overflow links + linkMask uint32 // mask the width of the link table +} + +// Initialize Huffman decoding tables from array of code lengths. +// Following this function, h is guaranteed to be initialized into a complete +// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a +// degenerate case where the tree has only a single symbol with length 1. Empty +// trees are permitted. +func (h *huffmanDecoder) init(bits []int) bool { + // Sanity enables additional runtime tests during Huffman + // table construction. It's intended to be used during + // development to supplement the currently ad-hoc unit tests. + const sanity = false + + if h.min != 0 { + *h = huffmanDecoder{} + } + + // Count number of codes of each length, + // compute min and max length. + var count [maxCodeLen]int + var min, max int + for _, n := range bits { + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if n > max { + max = n + } + count[n]++ + } + + // Empty tree. The decompressor.huffSym function will fail later if the tree + // is used. Technically, an empty tree is only valid for the HDIST tree and + // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree + // is guaranteed to fail since it will attempt to use the tree to decode the + // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is + // guaranteed to fail later since the compressed data section must be + // composed of at least one symbol (the end-of-block marker). + if max == 0 { + return true + } + + code := 0 + var nextcode [maxCodeLen]int + for i := min; i <= max; i++ { + code <<= 1 + nextcode[i] = code + code += count[i] + } + + // Check that the coding is complete (i.e., that we've + // assigned all 2-to-the-max possible bit sequences). + // Exception: To be compatible with zlib, we also need to + // accept degenerate single-code codings. See also + // TestDegenerateHuffmanCoding. + if code != 1< huffmanChunkBits { + numLinks := 1 << (uint(max) - huffmanChunkBits) + h.linkMask = uint32(numLinks - 1) + + // create link tables + link := nextcode[huffmanChunkBits+1] >> 1 + h.links = make([][]uint32, huffmanNumChunks-link) + for j := uint(link); j < huffmanNumChunks; j++ { + reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8 + reverse >>= uint(16 - huffmanChunkBits) + off := j - uint(link) + if sanity && h.chunks[reverse] != 0 { + panic("impossible: overwriting existing chunk") + } + h.chunks[reverse] = uint32(off<>8]) | int(reverseByte[code&0xff])<<8 + reverse >>= uint(16 - n) + if n <= huffmanChunkBits { + for off := reverse; off < len(h.chunks); off += 1 << uint(n) { + // We should never need to overwrite + // an existing chunk. Also, 0 is + // never a valid chunk, because the + // lower 4 "count" bits should be + // between 1 and 15. + if sanity && h.chunks[off] != 0 { + panic("impossible: overwriting existing chunk") + } + h.chunks[off] = chunk + } + } else { + j := reverse & (huffmanNumChunks - 1) + if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { + // Longer codes should have been + // associated with a link table above. + panic("impossible: not an indirect chunk") + } + value := h.chunks[j] >> huffmanValueShift + linktab := h.links[value] + reverse >>= huffmanChunkBits + for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { + if sanity && linktab[off] != 0 { + panic("impossible: overwriting existing chunk") + } + linktab[off] = chunk + } + } + } + + if sanity { + // Above we've sanity checked that we never overwrote + // an existing entry. Here we additionally check that + // we filled the tables completely. + for i, chunk := range h.chunks { + if chunk == 0 { + // As an exception, in the degenerate + // single-code case, we allow odd + // chunks to be missing. + if code == 1 && i%2 == 1 { + continue + } + panic("impossible: missing chunk") + } + } + for _, linktab := range h.links { + for _, chunk := range linktab { + if chunk == 0 { + panic("impossible: missing chunk") + } + } + } + } + + return true +} + +func main() { + flag.Parse() + + var h huffmanDecoder + var bits [288]int + initReverseByte() + for i := 0; i < 144; i++ { + bits[i] = 8 + } + for i := 144; i < 256; i++ { + bits[i] = 9 + } + for i := 256; i < 280; i++ { + bits[i] = 7 + } + for i := 280; i < 288; i++ { + bits[i] = 8 + } + h.init(bits[:]) + if h.links != nil { + log.Fatal("Unexpected links table in fixed Huffman decoder") + } + + var buf bytes.Buffer + + fmt.Fprintf(&buf, `// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file.`+"\n\n") + + fmt.Fprintln(&buf, "package flate") + fmt.Fprintln(&buf) + fmt.Fprintln(&buf, "// autogenerated by go run gen.go -output fixedhuff.go, DO NOT EDIT") + fmt.Fprintln(&buf) + fmt.Fprintln(&buf, "var fixedHuffmanDecoder = huffmanDecoder{") + fmt.Fprintf(&buf, "\t%d,\n", h.min) + fmt.Fprintln(&buf, "\t[huffmanNumChunks]uint32{") + for i := 0; i < huffmanNumChunks; i++ { + if i&7 == 0 { + fmt.Fprintf(&buf, "\t\t") + } else { + fmt.Fprintf(&buf, " ") + } + fmt.Fprintf(&buf, "0x%04x,", h.chunks[i]) + if i&7 == 7 { + fmt.Fprintln(&buf) + } + } + fmt.Fprintln(&buf, "\t},") + fmt.Fprintln(&buf, "\tnil, 0,") + fmt.Fprintln(&buf, "}") + + data, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatal(err) + } + err = ioutil.WriteFile(*filename, data, 0644) + if err != nil { + log.Fatal(err) + } +} + +var reverseByte [256]byte + +func initReverseByte() { + for x := 0; x < 256; x++ { + var result byte + for i := uint(0); i < 8; i++ { + result |= byte(((x >> i) & 1) << (7 - i)) + } + reverseByte[x] = result + } +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_bit_writer.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_bit_writer.go new file mode 100644 index 00000000..5f7ffdd2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -0,0 +1,717 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "io" + "math" +) + +const ( + // The largest offset code. + offsetCodeCount = 30 + + // The special code used to mark the end of a block. + endBlockMarker = 256 + + // The first length code. + lengthCodesStart = 257 + + // The number of codegen codes. + codegenCodeCount = 19 + badCode = 255 + + // Output byte buffer size + // Must be multiple of 6 (48 bits) + 8 + bufferSize = 240 + 8 +) + +// The number of extra bits needed by length code X - LENGTH_CODES_START. +var lengthExtraBits = []int8{ + /* 257 */ 0, 0, 0, + /* 260 */ 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, + /* 270 */ 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, + /* 280 */ 4, 5, 5, 5, 5, 0, +} + +// The length indicated by length code X - LENGTH_CODES_START. +var lengthBase = []uint32{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 10, + 12, 14, 16, 20, 24, 28, 32, 40, 48, 56, + 64, 80, 96, 112, 128, 160, 192, 224, 255, +} + +// offset code word extra bits. +var offsetExtraBits = []int8{ + 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, + 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, + 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, + /* extended window */ + 14, 14, 15, 15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, +} + +var offsetBase = []uint32{ + /* normal deflate */ + 0x000000, 0x000001, 0x000002, 0x000003, 0x000004, + 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018, + 0x000020, 0x000030, 0x000040, 0x000060, 0x000080, + 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300, + 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000, + 0x001800, 0x002000, 0x003000, 0x004000, 0x006000, + + /* extended window */ + 0x008000, 0x00c000, 0x010000, 0x018000, 0x020000, + 0x030000, 0x040000, 0x060000, 0x080000, 0x0c0000, + 0x100000, 0x180000, 0x200000, 0x300000, +} + +// The odd order in which the codegen code sizes are written. +var codegenOrder = []uint32{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +type huffmanBitWriter struct { + w io.Writer + // Data waiting to be written is bytes[0:nbytes] + // and then the low nbits of bits. + bits uint64 + nbits uint + bytes [bufferSize]byte + nbytes int + literalFreq []int32 + offsetFreq []int32 + codegen []uint8 + codegenFreq []int32 + literalEncoding *huffmanEncoder + offsetEncoding *huffmanEncoder + codegenEncoding *huffmanEncoder + err error +} + +func newHuffmanBitWriter(w io.Writer) *huffmanBitWriter { + return &huffmanBitWriter{ + w: w, + literalFreq: make([]int32, maxNumLit), + offsetFreq: make([]int32, offsetCodeCount), + codegen: make([]uint8, maxNumLit+offsetCodeCount+1), + codegenFreq: make([]int32, codegenCodeCount), + literalEncoding: newHuffmanEncoder(maxNumLit), + codegenEncoding: newHuffmanEncoder(codegenCodeCount), + offsetEncoding: newHuffmanEncoder(offsetCodeCount), + } +} + +func (w *huffmanBitWriter) reset(writer io.Writer) { + w.w = writer + w.bits, w.nbits, w.nbytes, w.err = 0, 0, 0, nil + w.bytes = [bufferSize]byte{} +} + +func (w *huffmanBitWriter) flush() { + if w.err != nil { + w.nbits = 0 + return + } + n := w.nbytes + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + if w.nbits > 8 { // Avoid underflow + w.nbits -= 8 + } else { + w.nbits = 0 + } + n++ + } + w.bits = 0 + _, w.err = w.w.Write(w.bytes[0:n]) + w.nbytes = 0 +} + +func (w *huffmanBitWriter) writeBits(b int32, nb uint) { + w.bits |= uint64(b) << w.nbits + w.nbits += nb + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + w.bytes[n] = byte(bits) + w.bytes[n+1] = byte(bits >> 8) + w.bytes[n+2] = byte(bits >> 16) + w.bytes[n+3] = byte(bits >> 24) + w.bytes[n+4] = byte(bits >> 32) + w.bytes[n+5] = byte(bits >> 40) + n += 6 + if n >= bufferSize-8 { + _, w.err = w.w.Write(w.bytes[:bufferSize-8]) + n = 0 + } + w.nbytes = n + } +} + +func (w *huffmanBitWriter) writeBytes(bytes []byte) { + if w.err != nil { + return + } + n := w.nbytes + for w.nbits != 0 { + w.bytes[n] = byte(w.bits) + w.bits >>= 8 + w.nbits -= 8 + n++ + } + if w.nbits != 0 { + w.err = InternalError("writeBytes with unfinished bits") + return + } + if n != 0 { + _, w.err = w.w.Write(w.bytes[0:n]) + if w.err != nil { + return + } + } + w.nbytes = 0 + _, w.err = w.w.Write(bytes) +} + +// RFC 1951 3.2.7 specifies a special run-length encoding for specifying +// the literal and offset lengths arrays (which are concatenated into a single +// array). This method generates that run-length encoding. +// +// The result is written into the codegen array, and the frequencies +// of each code is written into the codegenFreq array. +// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional +// information. Code badCode is an end marker +// +// numLiterals The number of literals in literalEncoding +// numOffsets The number of offsets in offsetEncoding +func (w *huffmanBitWriter) generateCodegen(numLiterals int, numOffsets int, offenc *huffmanEncoder) { + for i := range w.codegenFreq { + w.codegenFreq[i] = 0 + } + // Note that we are using codegen both as a temporary variable for holding + // a copy of the frequencies, and as the place where we put the result. + // This is fine because the output is always shorter than the input used + // so far. + codegen := w.codegen // cache + // Copy the concatenated code sizes to codegen. Put a marker at the end. + cgnl := codegen[0:numLiterals] + for i := range cgnl { + cgnl[i] = uint8(w.literalEncoding.codes[i].bits()) + } + + cgnl = codegen[numLiterals : numLiterals+numOffsets] + for i := range cgnl { + cgnl[i] = uint8(offenc.codes[i].bits()) + } + codegen[numLiterals+numOffsets] = badCode + + size := codegen[0] + count := 1 + outIndex := 0 + for inIndex := 1; size != badCode; inIndex++ { + // INVARIANT: We have seen "count" copies of size that have not yet + // had output generated for them. + nextSize := codegen[inIndex] + if nextSize == size { + count++ + continue + } + // We need to generate codegen indicating "count" of size. + if size != 0 { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + count-- + for count >= 3 { + n := 6 + if n > count { + n = count + } + codegen[outIndex] = 16 + outIndex++ + codegen[outIndex] = uint8(n - 3) + outIndex++ + w.codegenFreq[16]++ + count -= n + } + } else { + for count >= 11 { + n := 138 + if n > count { + n = count + } + codegen[outIndex] = 18 + outIndex++ + codegen[outIndex] = uint8(n - 11) + outIndex++ + w.codegenFreq[18]++ + count -= n + } + if count >= 3 { + // count >= 3 && count <= 10 + codegen[outIndex] = 17 + outIndex++ + codegen[outIndex] = uint8(count - 3) + outIndex++ + w.codegenFreq[17]++ + count = 0 + } + } + count-- + for ; count >= 0; count-- { + codegen[outIndex] = size + outIndex++ + w.codegenFreq[size]++ + } + // Set up invariant for next time through the loop. + size = nextSize + count = 1 + } + // Marker indicating the end of the codegen. + codegen[outIndex] = badCode +} + +func (w *huffmanBitWriter) writeCode(c hcode) { + if w.err != nil { + return + } + w.bits |= uint64(c.code()) << w.nbits + w.nbits += c.bits() + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + w.bytes[n] = byte(bits) + w.bytes[n+1] = byte(bits >> 8) + w.bytes[n+2] = byte(bits >> 16) + w.bytes[n+3] = byte(bits >> 24) + w.bytes[n+4] = byte(bits >> 32) + w.bytes[n+5] = byte(bits >> 40) + n += 6 + if n >= bufferSize-8 { + _, w.err = w.w.Write(w.bytes[:bufferSize-8]) + n = 0 + } + w.nbytes = n + } + +} + +// Write the header of a dynamic Huffman block to the output stream. +// +// numLiterals The number of literals specified in codegen +// numOffsets The number of offsets specified in codegen +// numCodegens The number of codegens used in codegen +func (w *huffmanBitWriter) writeDynamicHeader(numLiterals int, numOffsets int, numCodegens int, isEof bool) { + if w.err != nil { + return + } + var firstBits int32 = 4 + if isEof { + firstBits = 5 + } + w.writeBits(firstBits, 3) + w.writeBits(int32(numLiterals-257), 5) + w.writeBits(int32(numOffsets-1), 5) + w.writeBits(int32(numCodegens-4), 4) + + for i := 0; i < numCodegens; i++ { + value := w.codegenEncoding.codes[codegenOrder[i]].bits() + w.writeBits(int32(value), 3) + } + + i := 0 + for { + var codeWord int = int(w.codegen[i]) + i++ + if codeWord == badCode { + break + } + // The low byte contains the actual code to generate. + w.writeCode(w.codegenEncoding.codes[uint32(codeWord)]) + + switch codeWord { + case 16: + w.writeBits(int32(w.codegen[i]), 2) + i++ + break + case 17: + w.writeBits(int32(w.codegen[i]), 3) + i++ + break + case 18: + w.writeBits(int32(w.codegen[i]), 7) + i++ + break + } + } +} + +func (w *huffmanBitWriter) writeStoredHeader(length int, isEof bool) { + if w.err != nil { + return + } + var flag int32 + if isEof { + flag = 1 + } + w.writeBits(flag, 3) + w.flush() + w.writeBits(int32(length), 16) + w.writeBits(int32(^uint16(length)), 16) +} + +func (w *huffmanBitWriter) writeFixedHeader(isEof bool) { + if w.err != nil { + return + } + // Indicate that we are a fixed Huffman block + var value int32 = 2 + if isEof { + value = 3 + } + w.writeBits(value, 3) +} + +func (w *huffmanBitWriter) writeBlock(tok tokens, eof bool, input []byte) { + if w.err != nil { + return + } + for i := range w.literalFreq { + w.literalFreq[i] = 0 + } + for i := range w.offsetFreq { + w.offsetFreq[i] = 0 + } + + tok.tokens[tok.n] = endBlockMarker + tokens := tok.tokens[0 : tok.n+1] + + for _, t := range tokens { + if t < matchType { + w.literalFreq[t.literal()]++ + } else { + length := t.length() + offset := t.offset() + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + } + } + + // get the number of literals + numLiterals := len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets := len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + + storedBytes := 0 + if input != nil { + storedBytes = len(input) + } + var extraBits int64 + var storedSize int64 = math.MaxInt64 + if storedBytes <= maxStoreBlockSize && input != nil { + storedSize = int64((storedBytes + 5) * 8) + // We only bother calculating the costs of the extra bits required by + // the length of offset fields (which will be the same for both fixed + // and dynamic encoding), if we need to compare those two encodings + // against stored encoding. + for lengthCode := lengthCodesStart + 8; lengthCode < numLiterals; lengthCode++ { + // First eight length codes have extra size = 0. + extraBits += int64(w.literalFreq[lengthCode]) * int64(lengthExtraBits[lengthCode-lengthCodesStart]) + } + for offsetCode := 4; offsetCode < numOffsets; offsetCode++ { + // First four offset codes have extra size = 0. + extraBits += int64(w.offsetFreq[offsetCode]) * int64(offsetExtraBits[offsetCode]) + } + } + + // Figure out smallest code. + // Fixed Huffman baseline. + var size = int64(3) + + fixedLiteralEncoding.bitLength(w.literalFreq) + + fixedOffsetEncoding.bitLength(w.offsetFreq) + + extraBits + var literalEncoding = fixedLiteralEncoding + var offsetEncoding = fixedOffsetEncoding + + // Dynamic Huffman? + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq, 7) + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + dynamicHeader := int64(3+5+5+4+(3*numCodegens)) + + w.codegenEncoding.bitLength(w.codegenFreq) + + int64(extraBits) + + int64(w.codegenFreq[16]*2) + + int64(w.codegenFreq[17]*3) + + int64(w.codegenFreq[18]*7) + dynamicSize := dynamicHeader + + w.literalEncoding.bitLength(w.literalFreq) + + w.offsetEncoding.bitLength(w.offsetFreq) + + if dynamicSize < size { + size = dynamicSize + literalEncoding = w.literalEncoding + offsetEncoding = w.offsetEncoding + } + + // Stored bytes? + if storedSize < size { + w.writeStoredHeader(storedBytes, eof) + w.writeBytes(input[0:storedBytes]) + return + } + + // Huffman. + if literalEncoding == fixedLiteralEncoding { + w.writeFixedHeader(eof) + } else { + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + } + + leCodes := literalEncoding.codes + oeCodes := offsetEncoding.codes + for _, t := range tokens { + if t < matchType { + w.writeCode(leCodes[t.literal()]) + } else { + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(leCodes[lengthCode+lengthCodesStart]) + extraLengthBits := uint(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(oeCodes[offsetCode]) + extraOffsetBits := uint(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + } + } +} + +// writeBlockDynamic will write a block as dynamic Huffman table +// compressed. This should be used, if the caller has a reasonable expectation +// that this block contains compressible data. +func (w *huffmanBitWriter) writeBlockDynamic(tok tokens, eof bool, input []byte) { + if w.err != nil { + return + } + for i := range w.literalFreq { + w.literalFreq[i] = 0 + } + for i := range w.offsetFreq { + w.offsetFreq[i] = 0 + } + + tok.tokens[tok.n] = endBlockMarker + tokens := tok.tokens[0 : tok.n+1] + + for _, t := range tokens { + if t < matchType { + w.literalFreq[t.literal()]++ + } else { + length := t.length() + offset := t.offset() + w.literalFreq[lengthCodesStart+lengthCode(length)]++ + w.offsetFreq[offsetCode(offset)]++ + } + } + + // get the number of literals + numLiterals := len(w.literalFreq) + for w.literalFreq[numLiterals-1] == 0 { + numLiterals-- + } + // get the number of offsets + numOffsets := len(w.offsetFreq) + for numOffsets > 0 && w.offsetFreq[numOffsets-1] == 0 { + numOffsets-- + } + if numOffsets == 0 { + // We haven't found a single match. If we want to go with the dynamic encoding, + // we should count at least one offset to be sure that the offset huffman tree could be encoded. + w.offsetFreq[0] = 1 + numOffsets = 1 + } + + w.literalEncoding.generate(w.literalFreq, 15) + w.offsetEncoding.generate(w.offsetFreq, 15) + + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, w.offsetEncoding) + w.codegenEncoding.generate(w.codegenFreq, 7) + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + var literalEncoding = w.literalEncoding + var offsetEncoding = w.offsetEncoding + + // Write Huffman table. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + leCodes := literalEncoding.codes + oeCodes := offsetEncoding.codes + + for _, t := range tokens { + if t < matchType { + w.writeCode(leCodes[t.literal()]) + } else { + // Write the length + length := t.length() + lengthCode := lengthCode(length) + w.writeCode(leCodes[lengthCode+lengthCodesStart]) + extraLengthBits := uint(lengthExtraBits[lengthCode]) + if extraLengthBits > 0 { + extraLength := int32(length - lengthBase[lengthCode]) + w.writeBits(extraLength, extraLengthBits) + } + // Write the offset + offset := t.offset() + offsetCode := offsetCode(offset) + w.writeCode(oeCodes[offsetCode]) + extraOffsetBits := uint(offsetExtraBits[offsetCode]) + if extraOffsetBits > 0 { + extraOffset := int32(offset - offsetBase[offsetCode]) + w.writeBits(extraOffset, extraOffsetBits) + } + } + } +} + +// static offset encoder used for huffman only encoding. +var huffOffset *huffmanEncoder + +func init() { + var w = newHuffmanBitWriter(nil) + w.offsetFreq[0] = 1 + huffOffset = newHuffmanEncoder(offsetCodeCount) + huffOffset.generate(w.offsetFreq, 15) +} + +// writeBlockHuff will write a block of bytes as either +// Huffman encoded literals or uncompressed bytes if the +// results only gains very little from compression. +func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte) { + if w.err != nil { + return + } + + // Clear histogram + for i := range w.literalFreq { + w.literalFreq[i] = 0 + } + + // Add everything as literals + histogram(input, w.literalFreq) + + w.literalFreq[endBlockMarker] = 1 + + const numLiterals = endBlockMarker + 1 + const numOffsets = 1 + + w.literalEncoding.generate(w.literalFreq, 15) + + // Figure out smallest code. + // Always use dynamic Huffman or Store + var numCodegens int + + // Generate codegen and codegenFrequencies, which indicates how to encode + // the literalEncoding and the offsetEncoding. + w.generateCodegen(numLiterals, numOffsets, huffOffset) + w.codegenEncoding.generate(w.codegenFreq, 7) + numCodegens = len(w.codegenFreq) + for numCodegens > 4 && w.codegenFreq[codegenOrder[numCodegens-1]] == 0 { + numCodegens-- + } + headerSize := int64(3+5+5+4+(3*numCodegens)) + + w.codegenEncoding.bitLength(w.codegenFreq) + + int64(w.codegenFreq[16]*2) + + int64(w.codegenFreq[17]*3) + + int64(w.codegenFreq[18]*7) + + // Includes EOB marker + size := headerSize + w.literalEncoding.bitLength(w.literalFreq) + + // Calculate stored size + var storedSize int64 = math.MaxInt64 + var storedBytes = len(input) + if storedBytes <= maxStoreBlockSize { + storedSize = int64(storedBytes+5) * 8 + } + + // Store bytes, if we don't get a reasonable improvement. + if storedSize < (size + size>>4) { + w.writeStoredHeader(storedBytes, eof) + w.writeBytes(input) + return + } + + // Huffman. + w.writeDynamicHeader(numLiterals, numOffsets, numCodegens, eof) + encoding := w.literalEncoding.codes + for _, t := range input { + // Bitwriting inlined, ~30% speedup + c := encoding[t] + w.bits |= uint64(c.code()) << w.nbits + w.nbits += c.bits() + if w.nbits >= 48 { + bits := w.bits + w.bits >>= 48 + w.nbits -= 48 + n := w.nbytes + w.bytes[n] = byte(bits) + w.bytes[n+1] = byte(bits >> 8) + w.bytes[n+2] = byte(bits >> 16) + w.bytes[n+3] = byte(bits >> 24) + w.bytes[n+4] = byte(bits >> 32) + w.bytes[n+5] = byte(bits >> 40) + n += 6 + if n >= bufferSize-8 { + _, w.err = w.w.Write(w.bytes[:bufferSize-8]) + if w.err != nil { + return + } + w.nbytes = 0 + } else { + w.nbytes = n + } + } + } + w.writeCode(encoding[endBlockMarker]) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_code.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_code.go new file mode 100644 index 00000000..9dba0faf --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/huffman_code.go @@ -0,0 +1,363 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +import ( + "math" + "sort" +) + +type hcode uint32 + +type huffmanEncoder struct { + codes []hcode + freqcache []literalNode + bitCount [17]int32 + lns literalNodeSorter + lfs literalFreqSorter +} + +type literalNode struct { + literal uint16 + freq int32 +} + +// A levelInfo describes the state of the constructed tree for a given depth. +type levelInfo struct { + // Our level. for better printing + level int32 + + // The frequency of the last node at this level + lastFreq int32 + + // The frequency of the next character to add to this level + nextCharFreq int32 + + // The frequency of the next pair (from level below) to add to this level. + // Only valid if the "needed" value of the next lower level is 0. + nextPairFreq int32 + + // The number of chains remaining to generate for this level before moving + // up to the next level + needed int32 +} + +func (h hcode) codeBits() (code uint16, bits uint8) { + return uint16(h), uint8(h >> 16) +} + +func (h *hcode) set(code uint16, bits uint8) { + *h = hcode(code) | hcode(uint32(bits)<<16) +} + +func (h *hcode) setBits(bits uint8) { + *h = hcode(*h&0xffff) | hcode(uint32(bits)<<16) +} + +func toCode(code uint16, bits uint8) hcode { + return hcode(code) | hcode(uint32(bits)<<16) +} + +func (h hcode) code() (code uint16) { + return uint16(h) +} + +func (h hcode) bits() (bits uint) { + return uint(h >> 16) +} + +func maxNode() literalNode { return literalNode{math.MaxUint16, math.MaxInt32} } + +func newHuffmanEncoder(size int) *huffmanEncoder { + return &huffmanEncoder{codes: make([]hcode, size), freqcache: nil} +} + +// Generates a HuffmanCode corresponding to the fixed literal table +func generateFixedLiteralEncoding() *huffmanEncoder { + h := newHuffmanEncoder(maxNumLit) + codes := h.codes + var ch uint16 + for ch = 0; ch < maxNumLit; ch++ { + var bits uint16 + var size uint8 + switch { + case ch < 144: + // size 8, 000110000 .. 10111111 + bits = ch + 48 + size = 8 + break + case ch < 256: + // size 9, 110010000 .. 111111111 + bits = ch + 400 - 144 + size = 9 + break + case ch < 280: + // size 7, 0000000 .. 0010111 + bits = ch - 256 + size = 7 + break + default: + // size 8, 11000000 .. 11000111 + bits = ch + 192 - 280 + size = 8 + } + codes[ch] = toCode(reverseBits(bits, size), size) + } + return h +} + +func generateFixedOffsetEncoding() *huffmanEncoder { + h := newHuffmanEncoder(30) + codes := h.codes + for ch := uint16(0); ch < 30; ch++ { + codes[ch] = toCode(reverseBits(ch, 5), 5) + } + return h +} + +var fixedLiteralEncoding *huffmanEncoder = generateFixedLiteralEncoding() +var fixedOffsetEncoding *huffmanEncoder = generateFixedOffsetEncoding() + +func (h *huffmanEncoder) bitLength(freq []int32) int64 { + var total int64 + for i, f := range freq { + if f != 0 { + total += int64(f) * int64(h.codes[i].bits()) + } + } + return total +} + +const maxBitsLimit = 16 + +// Return the number of literals assigned to each bit size in the Huffman encoding +// +// This method is only called when list.length >= 3 +// The cases of 0, 1, and 2 literals are handled by special case code. +// +// list An array of the literals with non-zero frequencies +// and their associated frequencies. The array is in order of increasing +// frequency, and has as its last element a special element with frequency +// MaxInt32 +// maxBits The maximum number of bits that should be used to encode any literal. +// Must be less than 16. +// return An integer array in which array[i] indicates the number of literals +// that should be encoded in i bits. +func (h *huffmanEncoder) bitCounts(list []literalNode, maxBits int32) []int32 { + if maxBits >= maxBitsLimit { + panic("flate: maxBits too large") + } + n := int32(len(list)) + list = list[0 : n+1] + list[n] = maxNode() + + // The tree can't have greater depth than n - 1, no matter what. This + // saves a little bit of work in some small cases + if maxBits > n-1 { + maxBits = n - 1 + } + + // Create information about each of the levels. + // A bogus "Level 0" whose sole purpose is so that + // level1.prev.needed==0. This makes level1.nextPairFreq + // be a legitimate value that never gets chosen. + var levels [maxBitsLimit]levelInfo + // leafCounts[i] counts the number of literals at the left + // of ancestors of the rightmost node at level i. + // leafCounts[i][j] is the number of literals at the left + // of the level j ancestor. + var leafCounts [maxBitsLimit][maxBitsLimit]int32 + + for level := int32(1); level <= maxBits; level++ { + // For every level, the first two items are the first two characters. + // We initialize the levels as if we had already figured this out. + levels[level] = levelInfo{ + level: level, + lastFreq: list[1].freq, + nextCharFreq: list[2].freq, + nextPairFreq: list[0].freq + list[1].freq, + } + leafCounts[level][level] = 2 + if level == 1 { + levels[level].nextPairFreq = math.MaxInt32 + } + } + + // We need a total of 2*n - 2 items at top level and have already generated 2. + levels[maxBits].needed = 2*n - 4 + + level := maxBits + for { + l := &levels[level] + if l.nextPairFreq == math.MaxInt32 && l.nextCharFreq == math.MaxInt32 { + // We've run out of both leafs and pairs. + // End all calculations for this level. + // To make sure we never come back to this level or any lower level, + // set nextPairFreq impossibly large. + l.needed = 0 + levels[level+1].nextPairFreq = math.MaxInt32 + level++ + continue + } + + prevFreq := l.lastFreq + if l.nextCharFreq < l.nextPairFreq { + // The next item on this row is a leaf node. + n := leafCounts[level][level] + 1 + l.lastFreq = l.nextCharFreq + // Lower leafCounts are the same of the previous node. + leafCounts[level][level] = n + l.nextCharFreq = list[n].freq + } else { + // The next item on this row is a pair from the previous row. + // nextPairFreq isn't valid until we generate two + // more values in the level below + l.lastFreq = l.nextPairFreq + // Take leaf counts from the lower level, except counts[level] remains the same. + copy(leafCounts[level][:level], leafCounts[level-1][:level]) + levels[l.level-1].needed = 2 + } + + if l.needed--; l.needed == 0 { + // We've done everything we need to do for this level. + // Continue calculating one level up. Fill in nextPairFreq + // of that level with the sum of the two nodes we've just calculated on + // this level. + if l.level == maxBits { + // All done! + break + } + levels[l.level+1].nextPairFreq = prevFreq + l.lastFreq + level++ + } else { + // If we stole from below, move down temporarily to replenish it. + for levels[level-1].needed > 0 { + level-- + } + } + } + + // Somethings is wrong if at the end, the top level is null or hasn't used + // all of the leaves. + if leafCounts[maxBits][maxBits] != n { + panic("leafCounts[maxBits][maxBits] != n") + } + + bitCount := h.bitCount[:maxBits+1] + //make([]int32, maxBits+1) + bits := 1 + counts := &leafCounts[maxBits] + for level := maxBits; level > 0; level-- { + // chain.leafCount gives the number of literals requiring at least "bits" + // bits to encode. + bitCount[bits] = counts[level] - counts[level-1] + bits++ + } + return bitCount +} + +// Look at the leaves and assign them a bit count and an encoding as specified +// in RFC 1951 3.2.2 +func (h *huffmanEncoder) assignEncodingAndSize(bitCount []int32, list []literalNode) { + code := uint16(0) + for n, bits := range bitCount { + code <<= 1 + if n == 0 || bits == 0 { + continue + } + // The literals list[len(list)-bits] .. list[len(list)-bits] + // are encoded using "bits" bits, and get the values + // code, code + 1, .... The code values are + // assigned in literal order (not frequency order). + chunk := list[len(list)-int(bits):] + + h.lns.Sort(chunk) + for _, node := range chunk { + h.codes[node.literal] = toCode(reverseBits(code, uint8(n)), uint8(n)) + code++ + } + list = list[0 : len(list)-int(bits)] + } +} + +// Update this Huffman Code object to be the minimum code for the specified frequency count. +// +// freq An array of frequencies, in which frequency[i] gives the frequency of literal i. +// maxBits The maximum number of bits to use for any literal. +func (h *huffmanEncoder) generate(freq []int32, maxBits int32) { + if h.freqcache == nil { + h.freqcache = make([]literalNode, 300) + } + list := h.freqcache[:len(freq)+1] + // Number of non-zero literals + count := 0 + // Set list to be the set of all non-zero literals and their frequencies + for i, f := range freq { + if f != 0 { + list[count] = literalNode{uint16(i), f} + count++ + } else { + list[count] = literalNode{} + //h.codeBits[i] = 0 + h.codes[i].setBits(0) + } + } + list[len(freq)] = literalNode{} + // If freq[] is shorter than codeBits[], fill rest of codeBits[] with zeros + // FIXME: Doesn't do what it says on the tin (klauspost) + //h.codeBits = h.codeBits[0:len(freq)] + + list = list[0:count] + if count <= 2 { + // Handle the small cases here, because they are awkward for the general case code. With + // two or fewer literals, everything has bit length 1. + for i, node := range list { + // "list" is in order of increasing literal value. + h.codes[node.literal].set(uint16(i), 1) + //h.codeBits[node.literal] = 1 + //h.code[node.literal] = uint16(i) + } + return + } + h.lfs.Sort(list) + + // Get the number of literals for each bit count + bitCount := h.bitCounts(list, maxBits) + // And do the assignment + h.assignEncodingAndSize(bitCount, list) +} + +type literalNodeSorter []literalNode + +func (s *literalNodeSorter) Sort(a []literalNode) { + *s = literalNodeSorter(a) + sort.Sort(s) +} + +func (s literalNodeSorter) Len() int { return len(s) } + +func (s literalNodeSorter) Less(i, j int) bool { + return s[i].literal < s[j].literal +} + +func (s literalNodeSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type literalFreqSorter []literalNode + +func (s *literalFreqSorter) Sort(a []literalNode) { + *s = literalFreqSorter(a) + sort.Sort(s) +} + +func (s literalFreqSorter) Len() int { return len(s) } + +func (s literalFreqSorter) Less(i, j int) bool { + if s[i].freq == s[j].freq { + return s[i].literal < s[j].literal + } + return s[i].freq < s[j].freq +} + +func (s literalFreqSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/inflate.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/inflate.go new file mode 100644 index 00000000..91e27e7e --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/inflate.go @@ -0,0 +1,846 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:generate go run gen.go -output fixedhuff.go + +// Package flate implements the DEFLATE compressed data format, described in +// RFC 1951. The gzip and zlib packages implement access to DEFLATE-based file +// formats. +package flate + +import ( + "bufio" + "io" + "strconv" +) + +const ( + maxCodeLen = 16 // max length of Huffman code + maxHist = 32768 // max history required + // The next three numbers come from the RFC section 3.2.7, with the + // additional proviso in section 3.2.5 which implies that distance codes + // 30 and 31 should never occur in compressed data. + maxNumLit = 286 + maxNumDist = 30 + numCodes = 19 // number of codes in Huffman meta-code +) + +// A CorruptInputError reports the presence of corrupt input at a given offset. +type CorruptInputError int64 + +func (e CorruptInputError) Error() string { + return "flate: corrupt input before offset " + strconv.FormatInt(int64(e), 10) +} + +// An InternalError reports an error in the flate code itself. +type InternalError string + +func (e InternalError) Error() string { return "flate: internal error: " + string(e) } + +// A ReadError reports an error encountered while reading input. +type ReadError struct { + Offset int64 // byte offset where error occurred + Err error // error returned by underlying Read +} + +func (e *ReadError) Error() string { + return "flate: read error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() +} + +// A WriteError reports an error encountered while writing output. +type WriteError struct { + Offset int64 // byte offset where error occurred + Err error // error returned by underlying Write +} + +func (e *WriteError) Error() string { + return "flate: write error at offset " + strconv.FormatInt(e.Offset, 10) + ": " + e.Err.Error() +} + +// Resetter resets a ReadCloser returned by NewReader or NewReaderDict to +// to switch to a new underlying Reader. This permits reusing a ReadCloser +// instead of allocating a new one. +type Resetter interface { + // Reset discards any buffered data and resets the Resetter as if it was + // newly initialized with the given reader. + Reset(r io.Reader, dict []byte) error +} + +// Note that much of the implementation of huffmanDecoder is also copied +// into gen.go (in package main) for the purpose of precomputing the +// fixed huffman tables so they can be included statically. + +// The data structure for decoding Huffman tables is based on that of +// zlib. There is a lookup table of a fixed bit width (huffmanChunkBits), +// For codes smaller than the table width, there are multiple entries +// (each combination of trailing bits has the same value). For codes +// larger than the table width, the table contains a link to an overflow +// table. The width of each entry in the link table is the maximum code +// size minus the chunk width. + +// Note that you can do a lookup in the table even without all bits +// filled. Since the extra bits are zero, and the DEFLATE Huffman codes +// have the property that shorter codes come before longer ones, the +// bit length estimate in the result is a lower bound on the actual +// number of bits. + +// chunk & 15 is number of bits +// chunk >> 4 is value, including table link + +const ( + huffmanChunkBits = 9 + huffmanNumChunks = 1 << huffmanChunkBits + huffmanCountMask = 15 + huffmanValueShift = 4 +) + +type huffmanDecoder struct { + min int // the minimum code length + chunks [huffmanNumChunks]uint32 // chunks as described above + links [][]uint32 // overflow links + linkMask uint32 // mask the width of the link table +} + +// Initialize Huffman decoding tables from array of code lengths. +// Following this function, h is guaranteed to be initialized into a complete +// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a +// degenerate case where the tree has only a single symbol with length 1. Empty +// trees are permitted. +func (h *huffmanDecoder) init(bits []int) bool { + // Sanity enables additional runtime tests during Huffman + // table construction. It's intended to be used during + // development to supplement the currently ad-hoc unit tests. + const sanity = false + + if h.min != 0 { + *h = huffmanDecoder{} + } + + // Count number of codes of each length, + // compute min and max length. + var count [maxCodeLen]int + var min, max int + for _, n := range bits { + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if n > max { + max = n + } + count[n]++ + } + + // Empty tree. The decompressor.huffSym function will fail later if the tree + // is used. Technically, an empty tree is only valid for the HDIST tree and + // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree + // is guaranteed to fail since it will attempt to use the tree to decode the + // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is + // guaranteed to fail later since the compressed data section must be + // composed of at least one symbol (the end-of-block marker). + if max == 0 { + return true + } + + code := 0 + var nextcode [maxCodeLen]int + for i := min; i <= max; i++ { + code <<= 1 + nextcode[i] = code + code += count[i] + } + + // Check that the coding is complete (i.e., that we've + // assigned all 2-to-the-max possible bit sequences). + // Exception: To be compatible with zlib, we also need to + // accept degenerate single-code codings. See also + // TestDegenerateHuffmanCoding. + if code != 1< huffmanChunkBits { + numLinks := 1 << (uint(max) - huffmanChunkBits) + h.linkMask = uint32(numLinks - 1) + + // create link tables + link := nextcode[huffmanChunkBits+1] >> 1 + h.links = make([][]uint32, huffmanNumChunks-link) + for j := uint(link); j < huffmanNumChunks; j++ { + reverse := int(reverseByte[j>>8]) | int(reverseByte[j&0xff])<<8 + reverse >>= uint(16 - huffmanChunkBits) + off := j - uint(link) + if sanity && h.chunks[reverse] != 0 { + panic("impossible: overwriting existing chunk") + } + h.chunks[reverse] = uint32(off<>8]) | int(reverseByte[code&0xff])<<8 + reverse >>= uint(16 - n) + if n <= huffmanChunkBits { + for off := reverse; off < len(h.chunks); off += 1 << uint(n) { + // We should never need to overwrite + // an existing chunk. Also, 0 is + // never a valid chunk, because the + // lower 4 "count" bits should be + // between 1 and 15. + if sanity && h.chunks[off] != 0 { + panic("impossible: overwriting existing chunk") + } + h.chunks[off] = chunk + } + } else { + j := reverse & (huffmanNumChunks - 1) + if sanity && h.chunks[j]&huffmanCountMask != huffmanChunkBits+1 { + // Longer codes should have been + // associated with a link table above. + panic("impossible: not an indirect chunk") + } + value := h.chunks[j] >> huffmanValueShift + linktab := h.links[value] + reverse >>= huffmanChunkBits + for off := reverse; off < len(linktab); off += 1 << uint(n-huffmanChunkBits) { + if sanity && linktab[off] != 0 { + panic("impossible: overwriting existing chunk") + } + linktab[off] = chunk + } + } + } + + if sanity { + // Above we've sanity checked that we never overwrote + // an existing entry. Here we additionally check that + // we filled the tables completely. + for i, chunk := range h.chunks { + if chunk == 0 { + // As an exception, in the degenerate + // single-code case, we allow odd + // chunks to be missing. + if code == 1 && i%2 == 1 { + continue + } + panic("impossible: missing chunk") + } + } + for _, linktab := range h.links { + for _, chunk := range linktab { + if chunk == 0 { + panic("impossible: missing chunk") + } + } + } + } + + return true +} + +// The actual read interface needed by NewReader. +// If the passed in io.Reader does not also have ReadByte, +// the NewReader will introduce its own buffering. +type Reader interface { + io.Reader + io.ByteReader +} + +// Decompress state. +type decompressor struct { + // Input source. + r Reader + roffset int64 + woffset int64 + + // Input bits, in top of b. + b uint32 + nb uint + + // Huffman decoders for literal/length, distance. + h1, h2 huffmanDecoder + + // Length arrays used to define Huffman codes. + bits *[maxNumLit + maxNumDist]int + codebits *[numCodes]int + + // Output history, buffer. + hist *[maxHist]byte + hp int // current output position in buffer + hw int // have written hist[0:hw] already + hfull bool // buffer has filled at least once + + // Temporary buffer (avoids repeated allocation). + buf [4]byte + + // Next step in the decompression, + // and decompression state. + step func(*decompressor) + final bool + err error + toRead []byte + hl, hd *huffmanDecoder + copyLen int + copyDist int +} + +func (f *decompressor) nextBlock() { + if f.final { + if f.hw != f.hp { + f.flush((*decompressor).nextBlock) + return + } + f.err = io.EOF + return + } + for f.nb < 1+2 { + if f.err = f.moreBits(); f.err != nil { + return + } + } + f.final = f.b&1 == 1 + f.b >>= 1 + typ := f.b & 3 + f.b >>= 2 + f.nb -= 1 + 2 + switch typ { + case 0: + f.dataBlock() + case 1: + // compressed, fixed Huffman tables + f.hl = &fixedHuffmanDecoder + f.hd = nil + f.huffmanBlock() + case 2: + // compressed, dynamic Huffman tables + if f.err = f.readHuffman(); f.err != nil { + break + } + f.hl = &f.h1 + f.hd = &f.h2 + f.huffmanBlock() + default: + // 3 is reserved. + f.err = CorruptInputError(f.roffset) + } +} + +func (f *decompressor) Read(b []byte) (int, error) { + for { + if len(f.toRead) > 0 { + n := copy(b, f.toRead) + f.toRead = f.toRead[n:] + return n, nil + } + if f.err != nil { + return 0, f.err + } + f.step(f) + } +} + +// Support the io.WriteTo interface for io.Copy and friends. +func (f *decompressor) WriteTo(w io.Writer) (int64, error) { + total := int64(0) + for { + if f.err != nil { + if f.err == io.EOF { + return total, nil + } + return total, f.err + } + if len(f.toRead) > 0 { + var n int + n, f.err = w.Write(f.toRead) + if f.err != nil { + return total, f.err + } + if n != len(f.toRead) { + return total, io.ErrShortWrite + } + f.toRead = f.toRead[:0] + total += int64(n) + } + f.step(f) + } +} + +func (f *decompressor) Close() error { + if f.err == io.EOF { + return nil + } + return f.err +} + +// RFC 1951 section 3.2.7. +// Compression with dynamic Huffman codes + +var codeOrder = [...]int{16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15} + +func (f *decompressor) readHuffman() error { + // HLIT[5], HDIST[5], HCLEN[4]. + for f.nb < 5+5+4 { + if err := f.moreBits(); err != nil { + return err + } + } + nlit := int(f.b&0x1F) + 257 + if nlit > maxNumLit { + return CorruptInputError(f.roffset) + } + f.b >>= 5 + ndist := int(f.b&0x1F) + 1 + if ndist > maxNumDist { + return CorruptInputError(f.roffset) + } + f.b >>= 5 + nclen := int(f.b&0xF) + 4 + // numCodes is 19, so nclen is always valid. + f.b >>= 4 + f.nb -= 5 + 5 + 4 + + // (HCLEN+4)*3 bits: code lengths in the magic codeOrder order. + for i := 0; i < nclen; i++ { + for f.nb < 3 { + if err := f.moreBits(); err != nil { + return err + } + } + f.codebits[codeOrder[i]] = int(f.b & 0x7) + f.b >>= 3 + f.nb -= 3 + } + for i := nclen; i < len(codeOrder); i++ { + f.codebits[codeOrder[i]] = 0 + } + if !f.h1.init(f.codebits[0:]) { + return CorruptInputError(f.roffset) + } + + // HLIT + 257 code lengths, HDIST + 1 code lengths, + // using the code length Huffman code. + for i, n := 0, nlit+ndist; i < n; { + x, err := f.huffSym(&f.h1) + if err != nil { + return err + } + if x < 16 { + // Actual length. + f.bits[i] = x + i++ + continue + } + // Repeat previous length or zero. + var rep int + var nb uint + var b int + switch x { + default: + return InternalError("unexpected length code") + case 16: + rep = 3 + nb = 2 + if i == 0 { + return CorruptInputError(f.roffset) + } + b = f.bits[i-1] + case 17: + rep = 3 + nb = 3 + b = 0 + case 18: + rep = 11 + nb = 7 + b = 0 + } + for f.nb < nb { + if err := f.moreBits(); err != nil { + return err + } + } + rep += int(f.b & uint32(1<>= nb + f.nb -= nb + if i+rep > n { + return CorruptInputError(f.roffset) + } + for j := 0; j < rep; j++ { + f.bits[i] = b + i++ + } + } + + if !f.h1.init(f.bits[0:nlit]) || !f.h2.init(f.bits[nlit:nlit+ndist]) { + return CorruptInputError(f.roffset) + } + + // In order to preserve the property that we never read any extra bytes + // after the end of the DEFLATE stream, huffSym conservatively reads min + // bits at a time until it decodes the symbol. However, since every block + // must end with an EOB marker, we can use that as the minimum number of + // bits to read and guarantee we never read past the end of the stream. + if f.bits[endBlockMarker] > 0 { + f.h1.min = f.bits[endBlockMarker] // Length of EOB marker + } + + return nil +} + +// Decode a single Huffman block from f. +// hl and hd are the Huffman states for the lit/length values +// and the distance values, respectively. If hd == nil, using the +// fixed distance encoding associated with fixed Huffman blocks. +func (f *decompressor) huffmanBlock() { + for { + v, err := f.huffSym(f.hl) + if err != nil { + f.err = err + return + } + var n uint // number of bits extra + var length int + switch { + case v < 256: + f.hist[f.hp] = byte(v) + f.hp++ + if f.hp == len(f.hist) { + // After the flush, continue this loop. + f.flush((*decompressor).huffmanBlock) + return + } + continue + case v == 256: + // Done with huffman block; read next block. + f.step = (*decompressor).nextBlock + return + // otherwise, reference to older data + case v < 265: + length = v - (257 - 3) + n = 0 + case v < 269: + length = v*2 - (265*2 - 11) + n = 1 + case v < 273: + length = v*4 - (269*4 - 19) + n = 2 + case v < 277: + length = v*8 - (273*8 - 35) + n = 3 + case v < 281: + length = v*16 - (277*16 - 67) + n = 4 + case v < 285: + length = v*32 - (281*32 - 131) + n = 5 + case v < maxNumLit: + length = 258 + n = 0 + default: + f.err = CorruptInputError(f.roffset) + return + } + if n > 0 { + for f.nb < n { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + length += int(f.b & uint32(1<>= n + f.nb -= n + } + + var dist int + if f.hd == nil { + for f.nb < 5 { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + dist = int(reverseByte[(f.b&0x1F)<<3]) + f.b >>= 5 + f.nb -= 5 + } else { + if dist, err = f.huffSym(f.hd); err != nil { + f.err = err + return + } + } + + switch { + case dist < 4: + dist++ + case dist < maxNumDist: + nb := uint(dist-2) >> 1 + // have 1 bit in bottom of dist, need nb more. + extra := (dist & 1) << nb + for f.nb < nb { + if err = f.moreBits(); err != nil { + f.err = err + return + } + } + extra |= int(f.b & uint32(1<>= nb + f.nb -= nb + dist = 1<<(nb+1) + 1 + extra + default: + f.err = CorruptInputError(f.roffset) + return + } + + // Copy history[-dist:-dist+length] into output. + if dist > len(f.hist) { + f.err = InternalError("bad history distance") + return + } + + // No check on length; encoding can be prescient. + if !f.hfull && dist > f.hp { + f.err = CorruptInputError(f.roffset) + return + } + + f.copyLen, f.copyDist = length, dist + if f.copyHist() { + return + } + } +} + +// copyHist copies f.copyLen bytes from f.hist (f.copyDist bytes ago) to itself. +// It reports whether the f.hist buffer is full. +func (f *decompressor) copyHist() bool { + p := f.hp - f.copyDist + if p < 0 { + p += len(f.hist) + } + for f.copyLen > 0 { + n := f.copyLen + if x := len(f.hist) - f.hp; n > x { + n = x + } + if x := len(f.hist) - p; n > x { + n = x + } + forwardCopy(f.hist[:], f.hp, p, n) + p += n + f.hp += n + f.copyLen -= n + if f.hp == len(f.hist) { + // After flush continue copying out of history. + f.flush((*decompressor).copyHuff) + return true + } + if p == len(f.hist) { + p = 0 + } + } + return false +} + +func (f *decompressor) copyHuff() { + if f.copyHist() { + return + } + f.huffmanBlock() +} + +// Copy a single uncompressed data block from input to output. +func (f *decompressor) dataBlock() { + // Uncompressed. + // Discard current half-byte. + f.nb = 0 + f.b = 0 + + // Length then ones-complement of length. + nr, err := io.ReadFull(f.r, f.buf[0:4]) + f.roffset += int64(nr) + if err != nil { + f.err = &ReadError{f.roffset, err} + return + } + n := int(f.buf[0]) | int(f.buf[1])<<8 + nn := int(f.buf[2]) | int(f.buf[3])<<8 + if uint16(nn) != uint16(^n) { + f.err = CorruptInputError(f.roffset) + return + } + + if n == 0 { + // 0-length block means sync + f.flush((*decompressor).nextBlock) + return + } + + f.copyLen = n + f.copyData() +} + +// copyData copies f.copyLen bytes from the underlying reader into f.hist. +// It pauses for reads when f.hist is full. +func (f *decompressor) copyData() { + n := f.copyLen + for n > 0 { + m := len(f.hist) - f.hp + if m > n { + m = n + } + m, err := io.ReadFull(f.r, f.hist[f.hp:f.hp+m]) + f.roffset += int64(m) + if err != nil { + f.err = &ReadError{f.roffset, err} + return + } + n -= m + f.hp += m + if f.hp == len(f.hist) { + f.copyLen = n + f.flush((*decompressor).copyData) + return + } + } + f.step = (*decompressor).nextBlock +} + +func (f *decompressor) setDict(dict []byte) { + if len(dict) > len(f.hist) { + // Will only remember the tail. + dict = dict[len(dict)-len(f.hist):] + } + + f.hp = copy(f.hist[:], dict) + if f.hp == len(f.hist) { + f.hp = 0 + f.hfull = true + } + f.hw = f.hp +} + +func (f *decompressor) moreBits() error { + c, err := f.r.ReadByte() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err + } + f.roffset++ + f.b |= uint32(c) << f.nb + f.nb += 8 + return nil +} + +// Read the next Huffman-encoded symbol from f according to h. +func (f *decompressor) huffSym(h *huffmanDecoder) (int, error) { + // Since a huffmanDecoder can be empty or be composed of a degenerate tree + // with single element, huffSym must error on these two edge cases. In both + // cases, the chunks slice will be 0 for the invalid sequence, leading it + // satisfy the n == 0 check below. + n := uint(h.min) + for { + for f.nb < n { + if err := f.moreBits(); err != nil { + return 0, err + } + } + chunk := h.chunks[f.b&(huffmanNumChunks-1)] + n = uint(chunk & huffmanCountMask) + if n > huffmanChunkBits { + chunk = h.links[chunk>>huffmanValueShift][(f.b>>huffmanChunkBits)&h.linkMask] + n = uint(chunk & huffmanCountMask) + } + if n <= f.nb { + if n == 0 { + f.err = CorruptInputError(f.roffset) + return 0, f.err + } + f.b >>= n + f.nb -= n + return int(chunk >> huffmanValueShift), nil + } + } +} + +// Flush any buffered output to the underlying writer. +func (f *decompressor) flush(step func(*decompressor)) { + f.toRead = f.hist[f.hw:f.hp] + f.woffset += int64(f.hp - f.hw) + f.hw = f.hp + if f.hp == len(f.hist) { + f.hp = 0 + f.hw = 0 + f.hfull = true + } + f.step = step +} + +func makeReader(r io.Reader) Reader { + if rr, ok := r.(Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +func (f *decompressor) Reset(r io.Reader, dict []byte) error { + *f = decompressor{ + r: makeReader(r), + bits: f.bits, + codebits: f.codebits, + hist: f.hist, + step: (*decompressor).nextBlock, + } + if dict != nil { + f.setDict(dict) + } + return nil +} + +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. +// If r does not also implement io.ByteReader, +// the decompressor may read more data than necessary from r. +// It is the caller's responsibility to call Close on the ReadCloser +// when finished reading. +// +// The ReadCloser returned by NewReader also implements Resetter. +func NewReader(r io.Reader) io.ReadCloser { + var f decompressor + f.bits = new([maxNumLit + maxNumDist]int) + f.codebits = new([numCodes]int) + f.r = makeReader(r) + f.hist = new([maxHist]byte) + f.step = (*decompressor).nextBlock + return &f +} + +// NewReaderDict is like NewReader but initializes the reader +// with a preset dictionary. The returned Reader behaves as if +// the uncompressed data stream started with the given dictionary, +// which has already been read. NewReaderDict is typically used +// to read data compressed by NewWriterDict. +// +// The ReadCloser returned by NewReader also implements Resetter. +func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { + var f decompressor + f.r = makeReader(r) + f.hist = new([maxHist]byte) + f.bits = new([maxNumLit + maxNumDist]int) + f.codebits = new([numCodes]int) + f.step = (*decompressor).nextBlock + f.setDict(dict) + return &f +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/reverse_bits.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/reverse_bits.go new file mode 100644 index 00000000..c1a02720 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/reverse_bits.go @@ -0,0 +1,48 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +var reverseByte = [256]byte{ + 0x00, 0x80, 0x40, 0xc0, 0x20, 0xa0, 0x60, 0xe0, + 0x10, 0x90, 0x50, 0xd0, 0x30, 0xb0, 0x70, 0xf0, + 0x08, 0x88, 0x48, 0xc8, 0x28, 0xa8, 0x68, 0xe8, + 0x18, 0x98, 0x58, 0xd8, 0x38, 0xb8, 0x78, 0xf8, + 0x04, 0x84, 0x44, 0xc4, 0x24, 0xa4, 0x64, 0xe4, + 0x14, 0x94, 0x54, 0xd4, 0x34, 0xb4, 0x74, 0xf4, + 0x0c, 0x8c, 0x4c, 0xcc, 0x2c, 0xac, 0x6c, 0xec, + 0x1c, 0x9c, 0x5c, 0xdc, 0x3c, 0xbc, 0x7c, 0xfc, + 0x02, 0x82, 0x42, 0xc2, 0x22, 0xa2, 0x62, 0xe2, + 0x12, 0x92, 0x52, 0xd2, 0x32, 0xb2, 0x72, 0xf2, + 0x0a, 0x8a, 0x4a, 0xca, 0x2a, 0xaa, 0x6a, 0xea, + 0x1a, 0x9a, 0x5a, 0xda, 0x3a, 0xba, 0x7a, 0xfa, + 0x06, 0x86, 0x46, 0xc6, 0x26, 0xa6, 0x66, 0xe6, + 0x16, 0x96, 0x56, 0xd6, 0x36, 0xb6, 0x76, 0xf6, + 0x0e, 0x8e, 0x4e, 0xce, 0x2e, 0xae, 0x6e, 0xee, + 0x1e, 0x9e, 0x5e, 0xde, 0x3e, 0xbe, 0x7e, 0xfe, + 0x01, 0x81, 0x41, 0xc1, 0x21, 0xa1, 0x61, 0xe1, + 0x11, 0x91, 0x51, 0xd1, 0x31, 0xb1, 0x71, 0xf1, + 0x09, 0x89, 0x49, 0xc9, 0x29, 0xa9, 0x69, 0xe9, + 0x19, 0x99, 0x59, 0xd9, 0x39, 0xb9, 0x79, 0xf9, + 0x05, 0x85, 0x45, 0xc5, 0x25, 0xa5, 0x65, 0xe5, + 0x15, 0x95, 0x55, 0xd5, 0x35, 0xb5, 0x75, 0xf5, + 0x0d, 0x8d, 0x4d, 0xcd, 0x2d, 0xad, 0x6d, 0xed, + 0x1d, 0x9d, 0x5d, 0xdd, 0x3d, 0xbd, 0x7d, 0xfd, + 0x03, 0x83, 0x43, 0xc3, 0x23, 0xa3, 0x63, 0xe3, + 0x13, 0x93, 0x53, 0xd3, 0x33, 0xb3, 0x73, 0xf3, + 0x0b, 0x8b, 0x4b, 0xcb, 0x2b, 0xab, 0x6b, 0xeb, + 0x1b, 0x9b, 0x5b, 0xdb, 0x3b, 0xbb, 0x7b, 0xfb, + 0x07, 0x87, 0x47, 0xc7, 0x27, 0xa7, 0x67, 0xe7, + 0x17, 0x97, 0x57, 0xd7, 0x37, 0xb7, 0x77, 0xf7, + 0x0f, 0x8f, 0x4f, 0xcf, 0x2f, 0xaf, 0x6f, 0xef, + 0x1f, 0x9f, 0x5f, 0xdf, 0x3f, 0xbf, 0x7f, 0xff, +} + +func reverseUint16(v uint16) uint16 { + return uint16(reverseByte[v>>8]) | uint16(reverseByte[v&0xFF])<<8 +} + +func reverseBits(number uint16, bitLength byte) uint16 { + return reverseUint16(number << uint8(16-bitLength)) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/snappy.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/snappy.go new file mode 100644 index 00000000..a80ebe9e --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/snappy.go @@ -0,0 +1,558 @@ +// Copyright 2011 The Snappy-Go Authors. All rights reserved. +// Modified for deflate by Klaus Post (c) 2015. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +// We limit how far copy back-references can go, the same as the C++ code. +const maxOffset = 1 << 15 + +// emitLiteral writes a literal chunk and returns the number of bytes written. +func emitLiteral(dst *tokens, lit []byte) { + ol := dst.n + for i, v := range lit { + dst.tokens[i+ol] = token(v) + } + dst.n += len(lit) +} + +// emitCopy writes a copy chunk and returns the number of bytes written. +func emitCopy(dst *tokens, offset, length int) { + dst.tokens[dst.n] = matchToken(uint32(length-3), uint32(offset-minOffsetSize)) + dst.n++ +} + +type snappyEnc interface { + Encode(dst *tokens, src []byte) + Reset() +} + +func newSnappy(level int) snappyEnc { + if useSSE42 { + e := &snappySSE4{snappyGen: snappyGen{cur: 1}} + switch level { + case 3: + e.enc = e.encodeL3 + return e + } + } + e := &snappyGen{cur: 1} + switch level { + case 1: + e.enc = e.encodeL1 + case 2: + e.enc = e.encodeL2 + case 3: + e.enc = e.encodeL3 + default: + panic("invalid level specified") + } + return e +} + +const tableBits = 14 // Bits used in the table +const tableSize = 1 << tableBits // Size of the table + +// snappyGen maintains the table for matches, +// and the previous byte block for level 2. +// This is the generic implementation. +type snappyGen struct { + table [tableSize]int64 + block [maxStoreBlockSize]byte + prev []byte + cur int + enc func(dst *tokens, src []byte) +} + +func (e *snappyGen) Encode(dst *tokens, src []byte) { + e.enc(dst, src) +} + +// EncodeL1 uses Snappy-like compression, but stores as Huffman +// blocks. +func (e *snappyGen) encodeL1(dst *tokens, src []byte) { + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + emitLiteral(dst, src) + } + e.cur += 4 + return + } + + // Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. + if e.cur > 1<<30 { + e.cur = 1 + } + + // Iterate over the source bytes. + var ( + s int // The iterator position. + t int // The last position with the same hash as s. + lit int // The start position of any pending literal bytes. + ) + + for s+3 < len(src) { + // Update the hash table. + b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3] + h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24 + p := &e.table[(h*0x1e35a7bd)>>(32-tableBits)] + // We need to to store values in [-1, inf) in table. + // To save some initialization time, we make sure that + // e.cur is never zero. + t, *p = int(*p)-e.cur, int64(s+e.cur) + + offset := uint(s - t - 1) + + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + if t < 0 || offset >= (maxOffset-1) || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] { + // Skip 1 byte for 16 consecutive missed. + s += 1 + ((s - lit) >> 4) + continue + } + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + emitLiteral(dst, src[lit:s]) + } + // Extend the match to be as long as possible. + s0 := s + s1 := s + maxMatchLength + if s1 > len(src) { + s1 = len(src) + } + s, t = s+4, t+4 + for s < s1 && src[s] == src[t] { + s++ + t++ + } + // Emit the copied bytes. + // inlined: emitCopy(dst, s-t, s-s0) + + dst.tokens[dst.n] = matchToken(uint32(s-s0-3), uint32(s-t-minOffsetSize)) + dst.n++ + lit = s + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + emitLiteral(dst, src[lit:]) + } + e.cur += len(src) +} + +// EncodeL2 uses a similar algorithm to level 1, but is capable +// of matching across blocks giving better compression at a small slowdown. +func (e *snappyGen) encodeL2(dst *tokens, src []byte) { + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + emitLiteral(dst, src) + } + e.prev = nil + e.cur += len(src) + return + } + + // Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. + if e.cur > 1<<30 { + e.cur = 1 + } + + // Iterate over the source bytes. + var ( + s int // The iterator position. + t int // The last position with the same hash as s. + lit int // The start position of any pending literal bytes. + ) + + for s+3 < len(src) { + // Update the hash table. + b0, b1, b2, b3 := src[s], src[s+1], src[s+2], src[s+3] + h := uint32(b0) | uint32(b1)<<8 | uint32(b2)<<16 | uint32(b3)<<24 + p := &e.table[(h*0x1e35a7bd)>>(32-tableBits)] + // We need to to store values in [-1, inf) in table. + // To save some initialization time, we make sure that + // e.cur is never zero. + t, *p = int(*p)-e.cur, int64(s+e.cur) + + // If t is positive, the match starts in the current block + if t >= 0 { + + offset := uint(s - t - 1) + // Check that the offset is valid and that we match at least 4 bytes + if offset >= (maxOffset-1) || b0 != src[t] || b1 != src[t+1] || b2 != src[t+2] || b3 != src[t+3] { + // Skip 1 byte for 32 consecutive missed. + s += 1 + ((s - lit) >> 5) + continue + } + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + emitLiteral(dst, src[lit:s]) + } + // Extend the match to be as long as possible. + s0 := s + s1 := s + maxMatchLength + if s1 > len(src) { + s1 = len(src) + } + s, t = s+4, t+4 + for s < s1 && src[s] == src[t] { + s++ + t++ + } + // Emit the copied bytes. + // inlined: emitCopy(dst, s-t, s-s0) + dst.tokens[dst.n] = matchToken(uint32(s-s0-3), uint32(s-t-minOffsetSize)) + dst.n++ + lit = s + continue + } + // We found a match in the previous block. + tp := len(e.prev) + t + if tp < 0 || t > -5 || s-t >= maxOffset || b0 != e.prev[tp] || b1 != e.prev[tp+1] || b2 != e.prev[tp+2] || b3 != e.prev[tp+3] { + // Skip 1 byte for 32 consecutive missed. + s += 1 + ((s - lit) >> 5) + continue + } + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + emitLiteral(dst, src[lit:s]) + } + // Extend the match to be as long as possible. + s0 := s + s1 := s + maxMatchLength + if s1 > len(src) { + s1 = len(src) + } + s, tp = s+4, tp+4 + for s < s1 && src[s] == e.prev[tp] { + s++ + tp++ + if tp == len(e.prev) { + t = 0 + // continue in current buffer + for s < s1 && src[s] == src[t] { + s++ + t++ + } + goto l + } + } + l: + // Emit the copied bytes. + if t < 0 { + t = tp - len(e.prev) + } + dst.tokens[dst.n] = matchToken(uint32(s-s0-3), uint32(s-t-minOffsetSize)) + dst.n++ + lit = s + + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + emitLiteral(dst, src[lit:]) + } + e.cur += len(src) + // Store this block, if it was full length. + if len(src) == maxStoreBlockSize { + copy(e.block[:], src) + e.prev = e.block[:len(src)] + } else { + e.prev = nil + } +} + +// EncodeL3 uses a similar algorithm to level 2, but is capable +// will keep two matches per hash. +// Both hashes are checked if the first isn't ok, and the longest is selected. +func (e *snappyGen) encodeL3(dst *tokens, src []byte) { + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + emitLiteral(dst, src) + } + e.prev = nil + e.cur += len(src) + return + } + + // Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. + if e.cur > 1<<30 { + e.cur = 1 + } + + // Iterate over the source bytes. + var ( + s int // The iterator position. + lit int // The start position of any pending literal bytes. + ) + + for s+3 < len(src) { + // Update the hash table. + h := uint32(src[s]) | uint32(src[s+1])<<8 | uint32(src[s+2])<<16 | uint32(src[s+3])<<24 + p := &e.table[(h*0x1e35a7bd)>>(32-tableBits)] + tmp := *p + p1 := int(tmp & 0xffffffff) // Closest match position + p2 := int(tmp >> 32) // Furthest match position + + // We need to to store values in [-1, inf) in table. + // To save some initialization time, we make sure that + // e.cur is never zero. + t1 := p1 - e.cur + + var l2 int + var t2 int + l1 := e.matchlen(s, t1, src) + // If fist match was ok, don't do the second. + if l1 < 16 { + t2 = p2 - e.cur + l2 = e.matchlen(s, t2, src) + + // If both are short, continue + if l1 < 4 && l2 < 4 { + // Update hash table + *p = int64(s+e.cur) | (int64(p1) << 32) + // Skip 1 byte for 32 consecutive missed. + s += 1 + ((s - lit) >> 5) + continue + } + } + + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + emitLiteral(dst, src[lit:s]) + } + // Update hash table + *p = int64(s+e.cur) | (int64(p1) << 32) + + // Store the longest match l1 will be closest, so we prefer that if equal length + if l1 >= l2 { + dst.tokens[dst.n] = matchToken(uint32(l1-3), uint32(s-t1-minOffsetSize)) + s += l1 + } else { + dst.tokens[dst.n] = matchToken(uint32(l2-3), uint32(s-t2-minOffsetSize)) + s += l2 + } + dst.n++ + lit = s + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + emitLiteral(dst, src[lit:]) + } + e.cur += len(src) + // Store this block, if it was full length. + if len(src) == maxStoreBlockSize { + copy(e.block[:], src) + e.prev = e.block[:len(src)] + } else { + e.prev = nil + } +} + +func (e *snappyGen) matchlen(s, t int, src []byte) int { + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + offset := uint(s - t - 1) + + // If we are inside the current block + if t >= 0 { + if offset >= (maxOffset-1) || + src[s] != src[t] || src[s+1] != src[t+1] || + src[s+2] != src[t+2] || src[s+3] != src[t+3] { + return 0 + } + // Extend the match to be as long as possible. + s0 := s + s1 := s + maxMatchLength + if s1 > len(src) { + s1 = len(src) + } + s, t = s+4, t+4 + for s < s1 && src[s] == src[t] { + s++ + t++ + } + return s - s0 + } + + // We found a match in the previous block. + tp := len(e.prev) + t + if tp < 0 || offset >= (maxOffset-1) || t > -5 || + src[s] != e.prev[tp] || src[s+1] != e.prev[tp+1] || + src[s+2] != e.prev[tp+2] || src[s+3] != e.prev[tp+3] { + return 0 + } + + // Extend the match to be as long as possible. + s0 := s + s1 := s + maxMatchLength + if s1 > len(src) { + s1 = len(src) + } + s, tp = s+4, tp+4 + for s < s1 && src[s] == e.prev[tp] { + s++ + tp++ + if tp == len(e.prev) { + t = 0 + // continue in current buffer + for s < s1 && src[s] == src[t] { + s++ + t++ + } + return s - s0 + } + } + return s - s0 +} + +// Reset the encoding table. +func (e *snappyGen) Reset() { + e.prev = nil +} + +// snappySSE4 extends snappyGen. +// This implementation can use SSE 4.2 for length matching. +type snappySSE4 struct { + snappyGen +} + +// EncodeL3 uses a similar algorithm to level 2, +// but will keep two matches per hash. +// Both hashes are checked if the first isn't ok, and the longest is selected. +func (e *snappySSE4) encodeL3(dst *tokens, src []byte) { + // Return early if src is short. + if len(src) <= 4 { + if len(src) != 0 { + emitLiteral(dst, src) + } + e.prev = nil + e.cur += len(src) + return + } + + // Ensure that e.cur doesn't wrap, mainly an issue on 32 bits. + if e.cur > 1<<30 { + e.cur = 1 + } + + // Iterate over the source bytes. + var ( + s int // The iterator position. + lit int // The start position of any pending literal bytes. + ) + + for s+3 < len(src) { + // Load potential matches from hash table. + h := uint32(src[s]) | uint32(src[s+1])<<8 | uint32(src[s+2])<<16 | uint32(src[s+3])<<24 + p := &e.table[(h*0x1e35a7bd)>>(32-tableBits)] + tmp := *p + p1 := int(tmp & 0xffffffff) // Closest match position + p2 := int(tmp >> 32) // Furthest match position + + // We need to to store values in [-1, inf) in table. + // To save some initialization time, we make sure that + // e.cur is never zero. + t1 := int(p1) - e.cur + + var l2 int + var t2 int + l1 := e.matchlen(s, t1, src) + // If fist match was ok, don't do the second. + if l1 < 16 { + t2 = int(p2) - e.cur + l2 = e.matchlen(s, t2, src) + + // If both are short, continue + if l1 < 4 && l2 < 4 { + // Update hash table + *p = int64(s+e.cur) | (int64(p1) << 32) + // Skip 1 byte for 32 consecutive missed. + s += 1 + ((s - lit) >> 5) + continue + } + } + + // Otherwise, we have a match. First, emit any pending literal bytes. + if lit != s { + emitLiteral(dst, src[lit:s]) + } + // Update hash table + *p = int64(s+e.cur) | (int64(p1) << 32) + + // Store the longest match l1 will be closest, so we prefer that if equal length + if l1 >= l2 { + dst.tokens[dst.n] = matchToken(uint32(l1-3), uint32(s-t1-minOffsetSize)) + s += l1 + } else { + dst.tokens[dst.n] = matchToken(uint32(l2-3), uint32(s-t2-minOffsetSize)) + s += l2 + } + dst.n++ + lit = s + } + + // Emit any final pending literal bytes and return. + if lit != len(src) { + emitLiteral(dst, src[lit:]) + } + e.cur += len(src) + // Store this block, if it was full length. + if len(src) == maxStoreBlockSize { + copy(e.block[:], src) + e.prev = e.block[:len(src)] + } else { + e.prev = nil + } +} + +func (e *snappySSE4) matchlen(s, t int, src []byte) int { + // If t is invalid or src[s:s+4] differs from src[t:t+4], accumulate a literal byte. + offset := uint(s - t - 1) + + // If we are inside the current block + if t >= 0 { + if offset >= (maxOffset - 1) { + return 0 + } + length := len(src) - s + if length > maxMatchLength { + length = maxMatchLength + } + // Extend the match to be as long as possible. + return matchLenSSE4(src[t:], src[s:], length) + } + + // We found a match in the previous block. + tp := len(e.prev) + t + if tp < 0 || offset >= (maxOffset-1) || t > -5 || + src[s] != e.prev[tp] || src[s+1] != e.prev[tp+1] || + src[s+2] != e.prev[tp+2] || src[s+3] != e.prev[tp+3] { + return 0 + } + + // Extend the match to be as long as possible. + s0 := s + s1 := s + maxMatchLength + if s1 > len(src) { + s1 = len(src) + } + s, tp = s+4, tp+4 + for s < s1 && src[s] == e.prev[tp] { + s++ + tp++ + if tp == len(e.prev) { + t = 0 + // continue in current buffer + for s < s1 && src[s] == src[t] { + s++ + t++ + } + return s - s0 + } + } + return s - s0 +} diff --git a/Godeps/_workspace/src/github.com/klauspost/compress/flate/token.go b/Godeps/_workspace/src/github.com/klauspost/compress/flate/token.go new file mode 100644 index 00000000..94fa5eb9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/compress/flate/token.go @@ -0,0 +1,105 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package flate + +const ( + // 2 bits: type 0 = literal 1=EOF 2=Match 3=Unused + // 8 bits: xlength = length - MIN_MATCH_LENGTH + // 22 bits xoffset = offset - MIN_OFFSET_SIZE, or literal + lengthShift = 22 + offsetMask = 1< pair into a match token. +func matchToken(xlength uint32, xoffset uint32) token { + return token(matchType + xlength<> lengthShift) } + +func lengthCode(len uint32) uint32 { return lengthCodes[len] } + +// Returns the offset code corresponding to a specific offset +func offsetCode(off uint32) uint32 { + if off < uint32(len(offsetCodes)) { + return offsetCodes[off] + } else if off>>7 < uint32(len(offsetCodes)) { + return offsetCodes[off>>7] + 14 + } else { + return offsetCodes[off>>14] + 28 + } +} diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/.gitignore b/Godeps/_workspace/src/github.com/klauspost/cpuid/.gitignore new file mode 100644 index 00000000..daf913b1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/.travis.yml b/Godeps/_workspace/src/github.com/klauspost/cpuid/.travis.yml new file mode 100644 index 00000000..bde823d8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/.travis.yml @@ -0,0 +1,8 @@ +language: go + +go: + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - tip diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/LICENSE b/Godeps/_workspace/src/github.com/klauspost/cpuid/LICENSE new file mode 100644 index 00000000..5cec7ee9 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Klaus Post + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/README.md b/Godeps/_workspace/src/github.com/klauspost/cpuid/README.md new file mode 100644 index 00000000..b2b6bee8 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/README.md @@ -0,0 +1,145 @@ +# cpuid +Package cpuid provides information about the CPU running the current program. + +CPU features are detected on startup, and kept for fast access through the life of the application. +Currently x86 / x64 (AMD64) is supported, and no external C (cgo) code is used, which should make the library very easy to use. + +You can access the CPU information by accessing the shared CPU variable of the cpuid library. + +Package home: https://github.com/klauspost/cpuid + +[![GoDoc][1]][2] [![Build Status][3]][4] + +[1]: https://godoc.org/github.com/klauspost/cpuid?status.svg +[2]: https://godoc.org/github.com/klauspost/cpuid +[3]: https://travis-ci.org/klauspost/cpuid.svg +[4]: https://travis-ci.org/klauspost/cpuid + +# features +## CPU Instructions +* **CMOV** (i686 CMOV) +* **NX** (NX (No-Execute) bit) +* **AMD3DNOW** (AMD 3DNOW) +* **AMD3DNOWEXT** (AMD 3DNowExt) +* **MMX** (standard MMX) +* **MMXEXT** (SSE integer functions or AMD MMX ext) +* **SSE** (SSE functions) +* **SSE2** (P4 SSE functions) +* **SSE3** (Prescott SSE3 functions) +* **SSSE3** (Conroe SSSE3 functions) +* **SSE4** (Penryn SSE4.1 functions) +* **SSE4A** (AMD Barcelona microarchitecture SSE4a instructions) +* **SSE42** (Nehalem SSE4.2 functions) +* **AVX** (AVX functions) +* **AVX2** (AVX2 functions) +* **FMA3** (Intel FMA 3) +* **FMA4** (Bulldozer FMA4 functions) +* **XOP** (Bulldozer XOP functions) +* **F16C** (Half-precision floating-point conversion) +* **BMI1** (Bit Manipulation Instruction Set 1) +* **BMI2** (Bit Manipulation Instruction Set 2) +* **TBM** (AMD Trailing Bit Manipulation) +* **LZCNT** (LZCNT instruction) +* **POPCNT** (POPCNT instruction) +* **AESNI** (Advanced Encryption Standard New Instructions) +* **CLMUL** (Carry-less Multiplication) +* **HTT** (Hyperthreading (enabled)) +* **HLE** (Hardware Lock Elision) +* **RTM** (Restricted Transactional Memory) +* **RDRAND** (RDRAND instruction is available) +* **RDSEED** (RDSEED instruction is available) +* **ADX** (Intel ADX (Multi-Precision Add-Carry Instruction Extensions)) +* **SHA** (Intel SHA Extensions) +* **AVX512F** (AVX-512 Foundation) +* **AVX512DQ** (AVX-512 Doubleword and Quadword Instructions) +* **AVX512IFMA** (AVX-512 Integer Fused Multiply-Add Instructions) +* **AVX512PF** (AVX-512 Prefetch Instructions) +* **AVX512ER** (AVX-512 Exponential and Reciprocal Instructions) +* **AVX512CD** (AVX-512 Conflict Detection Instructions) +* **AVX512BW** (AVX-512 Byte and Word Instructions) +* **AVX512VL** (AVX-512 Vector Length Extensions) +* **AVX512VBMI** (AVX-512 Vector Bit Manipulation Instructions) +* **MPX** (Intel MPX (Memory Protection Extensions)) +* **ERMS** (Enhanced REP MOVSB/STOSB) +* **RDTSCP** (RDTSCP Instruction) +* **CX16** (CMPXCHG16B Instruction) +* **SGX** (Software Guard Extensions, with activation details) + +## Performance +* **RDTSCP()** Returns current cycle count. Can be used for benchmarking. +* **SSE2SLOW** (SSE2 is supported, but usually not faster) +* **SSE3SLOW** (SSE3 is supported, but usually not faster) +* **ATOM** (Atom processor, some SSSE3 instructions are slower) +* **Cache line** (Probable size of a cache line). +* **L1, L2, L3 Cache size** on newer Intel/AMD CPUs. + +## Cpu Vendor/VM +* **Intel** +* **AMD** +* **VIA** +* **Transmeta** +* **NSC** +* **KVM** (Kernel-based Virtual Machine) +* **MSVM** (Microsoft Hyper-V or Windows Virtual PC) +* **VMware** +* **XenHVM** + +# installing + +```go get github.com/klauspost/cpuid``` + +# example + +```Go +package main + +import ( + "fmt" + "github.com/klauspost/cpuid" +) + +func main() { + // Print basic CPU information: + fmt.Println("Name:", cpuid.CPU.BrandName) + fmt.Println("PhysicalCores:", cpuid.CPU.PhysicalCores) + fmt.Println("ThreadsPerCore:", cpuid.CPU.ThreadsPerCore) + fmt.Println("LogicalCores:", cpuid.CPU.LogicalCores) + fmt.Println("Family", cpuid.CPU.Family, "Model:", cpuid.CPU.Model) + fmt.Println("Features:", cpuid.CPU.Features) + fmt.Println("Cacheline bytes:", cpuid.CPU.CacheLine) + fmt.Println("L1 Data Cache:", cpuid.CPU.Cache.L1D, "bytes") + fmt.Println("L1 Instruction Cache:", cpuid.CPU.Cache.L1D, "bytes") + fmt.Println("L2 Cache:", cpuid.CPU.Cache.L2, "bytes") + fmt.Println("L3 Cache:", cpuid.CPU.Cache.L3, "bytes") + + // Test if we have a specific feature: + if cpuid.CPU.SSE() { + fmt.Println("We have Streaming SIMD Extensions") + } +} +``` + +Sample output: +``` +>go run main.go +Name: Intel(R) Core(TM) i5-2540M CPU @ 2.60GHz +PhysicalCores: 2 +ThreadsPerCore: 2 +LogicalCores: 4 +Family 6 Model: 42 +Features: CMOV,MMX,MMXEXT,SSE,SSE2,SSE3,SSSE3,SSE4.1,SSE4.2,AVX,AESNI,CLMUL +Cacheline bytes: 64 +We have Streaming SIMD Extensions +``` + +# private package + +In the "private" folder you can find an autogenerated version of the library you can include in your own packages. + +For this purpose all exports are removed, and functions and constants are lowercased. + +This is not a recommended way of using the library, but provided for convenience, if it is difficult for you to use external packages. + +# license + +This code is published under an MIT license. See LICENSE file for more information. diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid.go b/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid.go new file mode 100644 index 00000000..9230ca56 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid.go @@ -0,0 +1,1022 @@ +// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. + +// Package cpuid provides information about the CPU running the current program. +// +// CPU features are detected on startup, and kept for fast access through the life of the application. +// Currently x86 / x64 (AMD64) is supported. +// +// You can access the CPU information by accessing the shared CPU variable of the cpuid library. +// +// Package home: https://github.com/klauspost/cpuid +package cpuid + +import "strings" + +// Vendor is a representation of a CPU vendor. +type Vendor int + +const ( + Other Vendor = iota + Intel + AMD + VIA + Transmeta + NSC + KVM // Kernel-based Virtual Machine + MSVM // Microsoft Hyper-V or Windows Virtual PC + VMware + XenHVM +) + +const ( + CMOV = 1 << iota // i686 CMOV + NX // NX (No-Execute) bit + AMD3DNOW // AMD 3DNOW + AMD3DNOWEXT // AMD 3DNowExt + MMX // standard MMX + MMXEXT // SSE integer functions or AMD MMX ext + SSE // SSE functions + SSE2 // P4 SSE functions + SSE3 // Prescott SSE3 functions + SSSE3 // Conroe SSSE3 functions + SSE4 // Penryn SSE4.1 functions + SSE4A // AMD Barcelona microarchitecture SSE4a instructions + SSE42 // Nehalem SSE4.2 functions + AVX // AVX functions + AVX2 // AVX2 functions + FMA3 // Intel FMA 3 + FMA4 // Bulldozer FMA4 functions + XOP // Bulldozer XOP functions + F16C // Half-precision floating-point conversion + BMI1 // Bit Manipulation Instruction Set 1 + BMI2 // Bit Manipulation Instruction Set 2 + TBM // AMD Trailing Bit Manipulation + LZCNT // LZCNT instruction + POPCNT // POPCNT instruction + AESNI // Advanced Encryption Standard New Instructions + CLMUL // Carry-less Multiplication + HTT // Hyperthreading (enabled) + HLE // Hardware Lock Elision + RTM // Restricted Transactional Memory + RDRAND // RDRAND instruction is available + RDSEED // RDSEED instruction is available + ADX // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) + SHA // Intel SHA Extensions + AVX512F // AVX-512 Foundation + AVX512DQ // AVX-512 Doubleword and Quadword Instructions + AVX512IFMA // AVX-512 Integer Fused Multiply-Add Instructions + AVX512PF // AVX-512 Prefetch Instructions + AVX512ER // AVX-512 Exponential and Reciprocal Instructions + AVX512CD // AVX-512 Conflict Detection Instructions + AVX512BW // AVX-512 Byte and Word Instructions + AVX512VL // AVX-512 Vector Length Extensions + AVX512VBMI // AVX-512 Vector Bit Manipulation Instructions + MPX // Intel MPX (Memory Protection Extensions) + ERMS // Enhanced REP MOVSB/STOSB + RDTSCP // RDTSCP Instruction + CX16 // CMPXCHG16B Instruction + SGX // Software Guard Extensions + + // Performance indicators + SSE2SLOW // SSE2 is supported, but usually not faster + SSE3SLOW // SSE3 is supported, but usually not faster + ATOM // Atom processor, some SSSE3 instructions are slower +) + +var flagNames = map[Flags]string{ + CMOV: "CMOV", // i686 CMOV + NX: "NX", // NX (No-Execute) bit + AMD3DNOW: "AMD3DNOW", // AMD 3DNOW + AMD3DNOWEXT: "AMD3DNOWEXT", // AMD 3DNowExt + MMX: "MMX", // Standard MMX + MMXEXT: "MMXEXT", // SSE integer functions or AMD MMX ext + SSE: "SSE", // SSE functions + SSE2: "SSE2", // P4 SSE2 functions + SSE3: "SSE3", // Prescott SSE3 functions + SSSE3: "SSSE3", // Conroe SSSE3 functions + SSE4: "SSE4.1", // Penryn SSE4.1 functions + SSE4A: "SSE4A", // AMD Barcelona microarchitecture SSE4a instructions + SSE42: "SSE4.2", // Nehalem SSE4.2 functions + AVX: "AVX", // AVX functions + AVX2: "AVX2", // AVX functions + FMA3: "FMA3", // Intel FMA 3 + FMA4: "FMA4", // Bulldozer FMA4 functions + XOP: "XOP", // Bulldozer XOP functions + F16C: "F16C", // Half-precision floating-point conversion + BMI1: "BMI1", // Bit Manipulation Instruction Set 1 + BMI2: "BMI2", // Bit Manipulation Instruction Set 2 + TBM: "TBM", // AMD Trailing Bit Manipulation + LZCNT: "LZCNT", // LZCNT instruction + POPCNT: "POPCNT", // POPCNT instruction + AESNI: "AESNI", // Advanced Encryption Standard New Instructions + CLMUL: "CLMUL", // Carry-less Multiplication + HTT: "HTT", // Hyperthreading (enabled) + HLE: "HLE", // Hardware Lock Elision + RTM: "RTM", // Restricted Transactional Memory + RDRAND: "RDRAND", // RDRAND instruction is available + RDSEED: "RDSEED", // RDSEED instruction is available + ADX: "ADX", // Intel ADX (Multi-Precision Add-Carry Instruction Extensions) + SHA: "SHA", // Intel SHA Extensions + AVX512F: "AVX512F", // AVX-512 Foundation + AVX512DQ: "AVX512DQ", // AVX-512 Doubleword and Quadword Instructions + AVX512IFMA: "AVX512IFMA", // AVX-512 Integer Fused Multiply-Add Instructions + AVX512PF: "AVX512PF", // AVX-512 Prefetch Instructions + AVX512ER: "AVX512ER", // AVX-512 Exponential and Reciprocal Instructions + AVX512CD: "AVX512CD", // AVX-512 Conflict Detection Instructions + AVX512BW: "AVX512BW", // AVX-512 Byte and Word Instructions + AVX512VL: "AVX512VL", // AVX-512 Vector Length Extensions + AVX512VBMI: "AVX512VBMI", // AVX-512 Vector Bit Manipulation Instructions + MPX: "MPX", // Intel MPX (Memory Protection Extensions) + ERMS: "ERMS", // Enhanced REP MOVSB/STOSB + RDTSCP: "RDTSCP", // RDTSCP Instruction + CX16: "CX16", // CMPXCHG16B Instruction + SGX: "SGX", // Software Guard Extensions + + // Performance indicators + SSE2SLOW: "SSE2SLOW", // SSE2 supported, but usually not faster + SSE3SLOW: "SSE3SLOW", // SSE3 supported, but usually not faster + ATOM: "ATOM", // Atom processor, some SSSE3 instructions are slower + +} + +// CPUInfo contains information about the detected system CPU. +type CPUInfo struct { + BrandName string // Brand name reported by the CPU + VendorID Vendor // Comparable CPU vendor ID + Features Flags // Features of the CPU + PhysicalCores int // Number of physical processor cores in your CPU. Will be 0 if undetectable. + ThreadsPerCore int // Number of threads per physical core. Will be 1 if undetectable. + LogicalCores int // Number of physical cores times threads that can run on each core through the use of hyperthreading. Will be 0 if undetectable. + Family int // CPU family number + Model int // CPU model number + CacheLine int // Cache line size in bytes. Will be 0 if undetectable. + Cache struct { + L1I int // L1 Instruction Cache (per core or shared). Will be -1 if undetected + L1D int // L1 Data Cache (per core or shared). Will be -1 if undetected + L2 int // L2 Cache (per core or shared). Will be -1 if undetected + L3 int // L3 Instruction Cache (per core or shared). Will be -1 if undetected + } + SGX SGXSupport + maxFunc uint32 + maxExFunc uint32 +} + +var cpuid func(op uint32) (eax, ebx, ecx, edx uint32) +var cpuidex func(op, op2 uint32) (eax, ebx, ecx, edx uint32) +var xgetbv func(index uint32) (eax, edx uint32) +var rdtscpAsm func() (eax, ebx, ecx, edx uint32) + +// CPU contains information about the CPU as detected on startup, +// or when Detect last was called. +// +// Use this as the primary entry point to you data, +// this way queries are +var CPU CPUInfo + +func init() { + initCPU() + Detect() +} + +// Detect will re-detect current CPU info. +// This will replace the content of the exported CPU variable. +// +// Unless you expect the CPU to change while you are running your program +// you should not need to call this function. +// If you call this, you must ensure that no other goroutine is accessing the +// exported CPU variable. +func Detect() { + CPU.maxFunc = maxFunctionID() + CPU.maxExFunc = maxExtendedFunction() + CPU.BrandName = brandName() + CPU.CacheLine = cacheLine() + CPU.Family, CPU.Model = familyModel() + CPU.Features = support() + CPU.SGX = sgx(CPU.Features&SGX != 0) + CPU.ThreadsPerCore = threadsPerCore() + CPU.LogicalCores = logicalCores() + CPU.PhysicalCores = physicalCores() + CPU.VendorID = vendorID() + CPU.cacheSize() +} + +// Generated here: http://play.golang.org/p/BxFH2Gdc0G + +// Cmov indicates support of CMOV instructions +func (c CPUInfo) Cmov() bool { + return c.Features&CMOV != 0 +} + +// Amd3dnow indicates support of AMD 3DNOW! instructions +func (c CPUInfo) Amd3dnow() bool { + return c.Features&AMD3DNOW != 0 +} + +// Amd3dnowExt indicates support of AMD 3DNOW! Extended instructions +func (c CPUInfo) Amd3dnowExt() bool { + return c.Features&AMD3DNOWEXT != 0 +} + +// MMX indicates support of MMX instructions +func (c CPUInfo) MMX() bool { + return c.Features&MMX != 0 +} + +// MMXExt indicates support of MMXEXT instructions +// (SSE integer functions or AMD MMX ext) +func (c CPUInfo) MMXExt() bool { + return c.Features&MMXEXT != 0 +} + +// SSE indicates support of SSE instructions +func (c CPUInfo) SSE() bool { + return c.Features&SSE != 0 +} + +// SSE2 indicates support of SSE 2 instructions +func (c CPUInfo) SSE2() bool { + return c.Features&SSE2 != 0 +} + +// SSE3 indicates support of SSE 3 instructions +func (c CPUInfo) SSE3() bool { + return c.Features&SSE3 != 0 +} + +// SSSE3 indicates support of SSSE 3 instructions +func (c CPUInfo) SSSE3() bool { + return c.Features&SSSE3 != 0 +} + +// SSE4 indicates support of SSE 4 (also called SSE 4.1) instructions +func (c CPUInfo) SSE4() bool { + return c.Features&SSE4 != 0 +} + +// SSE42 indicates support of SSE4.2 instructions +func (c CPUInfo) SSE42() bool { + return c.Features&SSE42 != 0 +} + +// AVX indicates support of AVX instructions +// and operating system support of AVX instructions +func (c CPUInfo) AVX() bool { + return c.Features&AVX != 0 +} + +// AVX2 indicates support of AVX2 instructions +func (c CPUInfo) AVX2() bool { + return c.Features&AVX2 != 0 +} + +// FMA3 indicates support of FMA3 instructions +func (c CPUInfo) FMA3() bool { + return c.Features&FMA3 != 0 +} + +// FMA4 indicates support of FMA4 instructions +func (c CPUInfo) FMA4() bool { + return c.Features&FMA4 != 0 +} + +// XOP indicates support of XOP instructions +func (c CPUInfo) XOP() bool { + return c.Features&XOP != 0 +} + +// F16C indicates support of F16C instructions +func (c CPUInfo) F16C() bool { + return c.Features&F16C != 0 +} + +// BMI1 indicates support of BMI1 instructions +func (c CPUInfo) BMI1() bool { + return c.Features&BMI1 != 0 +} + +// BMI2 indicates support of BMI2 instructions +func (c CPUInfo) BMI2() bool { + return c.Features&BMI2 != 0 +} + +// TBM indicates support of TBM instructions +// (AMD Trailing Bit Manipulation) +func (c CPUInfo) TBM() bool { + return c.Features&TBM != 0 +} + +// Lzcnt indicates support of LZCNT instruction +func (c CPUInfo) Lzcnt() bool { + return c.Features&LZCNT != 0 +} + +// Popcnt indicates support of POPCNT instruction +func (c CPUInfo) Popcnt() bool { + return c.Features&POPCNT != 0 +} + +// HTT indicates the processor has Hyperthreading enabled +func (c CPUInfo) HTT() bool { + return c.Features&HTT != 0 +} + +// SSE2Slow indicates that SSE2 may be slow on this processor +func (c CPUInfo) SSE2Slow() bool { + return c.Features&SSE2SLOW != 0 +} + +// SSE3Slow indicates that SSE3 may be slow on this processor +func (c CPUInfo) SSE3Slow() bool { + return c.Features&SSE3SLOW != 0 +} + +// AesNi indicates support of AES-NI instructions +// (Advanced Encryption Standard New Instructions) +func (c CPUInfo) AesNi() bool { + return c.Features&AESNI != 0 +} + +// Clmul indicates support of CLMUL instructions +// (Carry-less Multiplication) +func (c CPUInfo) Clmul() bool { + return c.Features&CLMUL != 0 +} + +// NX indicates support of NX (No-Execute) bit +func (c CPUInfo) NX() bool { + return c.Features&NX != 0 +} + +// SSE4A indicates support of AMD Barcelona microarchitecture SSE4a instructions +func (c CPUInfo) SSE4A() bool { + return c.Features&SSE4A != 0 +} + +// HLE indicates support of Hardware Lock Elision +func (c CPUInfo) HLE() bool { + return c.Features&HLE != 0 +} + +// RTM indicates support of Restricted Transactional Memory +func (c CPUInfo) RTM() bool { + return c.Features&RTM != 0 +} + +// Rdrand indicates support of RDRAND instruction is available +func (c CPUInfo) Rdrand() bool { + return c.Features&RDRAND != 0 +} + +// Rdseed indicates support of RDSEED instruction is available +func (c CPUInfo) Rdseed() bool { + return c.Features&RDSEED != 0 +} + +// ADX indicates support of Intel ADX (Multi-Precision Add-Carry Instruction Extensions) +func (c CPUInfo) ADX() bool { + return c.Features&ADX != 0 +} + +// SHA indicates support of Intel SHA Extensions +func (c CPUInfo) SHA() bool { + return c.Features&SHA != 0 +} + +// AVX512F indicates support of AVX-512 Foundation +func (c CPUInfo) AVX512F() bool { + return c.Features&AVX512F != 0 +} + +// AVX512DQ indicates support of AVX-512 Doubleword and Quadword Instructions +func (c CPUInfo) AVX512DQ() bool { + return c.Features&AVX512DQ != 0 +} + +// AVX512IFMA indicates support of AVX-512 Integer Fused Multiply-Add Instructions +func (c CPUInfo) AVX512IFMA() bool { + return c.Features&AVX512IFMA != 0 +} + +// AVX512PF indicates support of AVX-512 Prefetch Instructions +func (c CPUInfo) AVX512PF() bool { + return c.Features&AVX512PF != 0 +} + +// AVX512ER indicates support of AVX-512 Exponential and Reciprocal Instructions +func (c CPUInfo) AVX512ER() bool { + return c.Features&AVX512ER != 0 +} + +// AVX512CD indicates support of AVX-512 Conflict Detection Instructions +func (c CPUInfo) AVX512CD() bool { + return c.Features&AVX512CD != 0 +} + +// AVX512BW indicates support of AVX-512 Byte and Word Instructions +func (c CPUInfo) AVX512BW() bool { + return c.Features&AVX512BW != 0 +} + +// AVX512VL indicates support of AVX-512 Vector Length Extensions +func (c CPUInfo) AVX512VL() bool { + return c.Features&AVX512VL != 0 +} + +// AVX512VBMI indicates support of AVX-512 Vector Bit Manipulation Instructions +func (c CPUInfo) AVX512VBMI() bool { + return c.Features&AVX512VBMI != 0 +} + +// MPX indicates support of Intel MPX (Memory Protection Extensions) +func (c CPUInfo) MPX() bool { + return c.Features&MPX != 0 +} + +// ERMS indicates support of Enhanced REP MOVSB/STOSB +func (c CPUInfo) ERMS() bool { + return c.Features&ERMS != 0 +} + +func (c CPUInfo) RDTSCP() bool { + return c.Features&RDTSCP != 0 +} + +func (c CPUInfo) CX16() bool { + return c.Features&CX16 != 0 +} + +// Atom indicates an Atom processor +func (c CPUInfo) Atom() bool { + return c.Features&ATOM != 0 +} + +// Intel returns true if vendor is recognized as Intel +func (c CPUInfo) Intel() bool { + return c.VendorID == Intel +} + +// AMD returns true if vendor is recognized as AMD +func (c CPUInfo) AMD() bool { + return c.VendorID == AMD +} + +// Transmeta returns true if vendor is recognized as Transmeta +func (c CPUInfo) Transmeta() bool { + return c.VendorID == Transmeta +} + +// NSC returns true if vendor is recognized as National Semiconductor +func (c CPUInfo) NSC() bool { + return c.VendorID == NSC +} + +// VIA returns true if vendor is recognized as VIA +func (c CPUInfo) VIA() bool { + return c.VendorID == VIA +} + +// RTCounter returns the 64-bit time-stamp counter +// Uses the RDTSCP instruction. The value 0 is returned +// if the CPU does not support the instruction. +func (c CPUInfo) RTCounter() uint64 { + if !c.RDTSCP() { + return 0 + } + a, _, _, d := rdtscpAsm() + return uint64(a) | (uint64(d) << 32) +} + +// Ia32TscAux returns the IA32_TSC_AUX part of the RDTSCP. +// This variable is OS dependent, but on Linux contains information +// about the current cpu/core the code is running on. +// If the RDTSCP instruction isn't supported on the CPU, the value 0 is returned. +func (c CPUInfo) Ia32TscAux() uint32 { + if !c.RDTSCP() { + return 0 + } + _, _, ecx, _ := rdtscpAsm() + return ecx +} + +// LogicalCPU will return the Logical CPU the code is currently executing on. +// This is likely to change when the OS re-schedules the running thread +// to another CPU. +// If the current core cannot be detected, -1 will be returned. +func (c CPUInfo) LogicalCPU() int { + if c.maxFunc < 1 { + return -1 + } + _, ebx, _, _ := cpuid(1) + return int(ebx >> 24) +} + +// VM Will return true if the cpu id indicates we are in +// a virtual machine. This is only a hint, and will very likely +// have many false negatives. +func (c CPUInfo) VM() bool { + switch c.VendorID { + case MSVM, KVM, VMware, XenHVM: + return true + } + return false +} + +// Flags contains detected cpu features and caracteristics +type Flags uint64 + +// String returns a string representation of the detected +// CPU features. +func (f Flags) String() string { + return strings.Join(f.Strings(), ",") +} + +// Strings returns and array of the detected features. +func (f Flags) Strings() []string { + s := support() + r := make([]string, 0, 20) + for i := uint(0); i < 64; i++ { + key := Flags(1 << i) + val := flagNames[key] + if s&key != 0 { + r = append(r, val) + } + } + return r +} + +func maxExtendedFunction() uint32 { + eax, _, _, _ := cpuid(0x80000000) + return eax +} + +func maxFunctionID() uint32 { + a, _, _, _ := cpuid(0) + return a +} + +func brandName() string { + if maxExtendedFunction() >= 0x80000004 { + v := make([]uint32, 0, 48) + for i := uint32(0); i < 3; i++ { + a, b, c, d := cpuid(0x80000002 + i) + v = append(v, a, b, c, d) + } + return strings.Trim(string(valAsString(v...)), " ") + } + return "unknown" +} + +func threadsPerCore() int { + mfi := maxFunctionID() + if mfi < 0x4 || vendorID() != Intel { + return 1 + } + + if mfi < 0xb { + _, b, _, d := cpuid(1) + if (d & (1 << 28)) != 0 { + // v will contain logical core count + v := (b >> 16) & 255 + if v > 1 { + a4, _, _, _ := cpuid(4) + // physical cores + v2 := (a4 >> 26) + 1 + if v2 > 0 { + return int(v) / int(v2) + } + } + } + return 1 + } + _, b, _, _ := cpuidex(0xb, 0) + if b&0xffff == 0 { + return 1 + } + return int(b & 0xffff) +} + +func logicalCores() int { + mfi := maxFunctionID() + switch vendorID() { + case Intel: + // Use this on old Intel processors + if mfi < 0xb { + if mfi < 1 { + return 0 + } + // CPUID.1:EBX[23:16] represents the maximum number of addressable IDs (initial APIC ID) + // that can be assigned to logical processors in a physical package. + // The value may not be the same as the number of logical processors that are present in the hardware of a physical package. + _, ebx, _, _ := cpuid(1) + logical := (ebx >> 16) & 0xff + return int(logical) + } + _, b, _, _ := cpuidex(0xb, 1) + return int(b & 0xffff) + case AMD: + _, b, _, _ := cpuid(1) + return int((b >> 16) & 0xff) + default: + return 0 + } +} + +func familyModel() (int, int) { + if maxFunctionID() < 0x1 { + return 0, 0 + } + eax, _, _, _ := cpuid(1) + family := ((eax >> 8) & 0xf) + ((eax >> 20) & 0xff) + model := ((eax >> 4) & 0xf) + ((eax >> 12) & 0xf0) + return int(family), int(model) +} + +func physicalCores() int { + switch vendorID() { + case Intel: + return logicalCores() / threadsPerCore() + case AMD: + if maxExtendedFunction() >= 0x80000008 { + _, _, c, _ := cpuid(0x80000008) + return int(c&0xff) + 1 + } + } + return 0 +} + +// Except from http://en.wikipedia.org/wiki/CPUID#EAX.3D0:_Get_vendor_ID +var vendorMapping = map[string]Vendor{ + "AMDisbetter!": AMD, + "AuthenticAMD": AMD, + "CentaurHauls": VIA, + "GenuineIntel": Intel, + "TransmetaCPU": Transmeta, + "GenuineTMx86": Transmeta, + "Geode by NSC": NSC, + "VIA VIA VIA ": VIA, + "KVMKVMKVMKVM": KVM, + "Microsoft Hv": MSVM, + "VMwareVMware": VMware, + "XenVMMXenVMM": XenHVM, +} + +func vendorID() Vendor { + _, b, c, d := cpuid(0) + v := valAsString(b, d, c) + vend, ok := vendorMapping[string(v)] + if !ok { + return Other + } + return vend +} + +func cacheLine() int { + if maxFunctionID() < 0x1 { + return 0 + } + + _, ebx, _, _ := cpuid(1) + cache := (ebx & 0xff00) >> 5 // cflush size + if cache == 0 && maxExtendedFunction() >= 0x80000006 { + _, _, ecx, _ := cpuid(0x80000006) + cache = ecx & 0xff // cacheline size + } + // TODO: Read from Cache and TLB Information + return int(cache) +} + +func (c *CPUInfo) cacheSize() { + c.Cache.L1D = -1 + c.Cache.L1I = -1 + c.Cache.L2 = -1 + c.Cache.L3 = -1 + vendor := vendorID() + switch vendor { + case Intel: + if maxFunctionID() < 4 { + return + } + for i := uint32(0); ; i++ { + eax, ebx, ecx, _ := cpuidex(4, i) + cacheType := eax & 15 + if cacheType == 0 { + break + } + cacheLevel := (eax >> 5) & 7 + coherency := int(ebx&0xfff) + 1 + partitions := int((ebx>>12)&0x3ff) + 1 + associativity := int((ebx>>22)&0x3ff) + 1 + sets := int(ecx) + 1 + size := associativity * partitions * coherency * sets + switch cacheLevel { + case 1: + if cacheType == 1 { + // 1 = Data Cache + c.Cache.L1D = size + } else if cacheType == 2 { + // 2 = Instruction Cache + c.Cache.L1I = size + } else { + if c.Cache.L1D < 0 { + c.Cache.L1I = size + } + if c.Cache.L1I < 0 { + c.Cache.L1I = size + } + } + case 2: + c.Cache.L2 = size + case 3: + c.Cache.L3 = size + } + } + case AMD: + // Untested. + if maxExtendedFunction() < 0x80000005 { + return + } + _, _, ecx, edx := cpuid(0x80000005) + c.Cache.L1D = int(((ecx >> 24) & 0xFF) * 1024) + c.Cache.L1I = int(((edx >> 24) & 0xFF) * 1024) + + if maxExtendedFunction() < 0x80000006 { + return + } + _, _, ecx, _ = cpuid(0x80000006) + c.Cache.L2 = int(((ecx >> 16) & 0xFFFF) * 1024) + } + + return +} + +type SGXSupport struct { + Available bool + SGX1Supported bool + SGX2Supported bool + MaxEnclaveSizeNot64 int64 + MaxEnclaveSize64 int64 +} + +func sgx(available bool) (rval SGXSupport) { + rval.Available = available + + if !available { + return + } + + a, _, _, d := cpuidex(0x12, 0) + rval.SGX1Supported = a&0x01 != 0 + rval.SGX2Supported = a&0x02 != 0 + rval.MaxEnclaveSizeNot64 = 1 << (d & 0xFF) // pow 2 + rval.MaxEnclaveSize64 = 1 << ((d >> 8) & 0xFF) // pow 2 + + return +} + +func support() Flags { + mfi := maxFunctionID() + vend := vendorID() + if mfi < 0x1 { + return 0 + } + rval := uint64(0) + _, _, c, d := cpuid(1) + if (d & (1 << 15)) != 0 { + rval |= CMOV + } + if (d & (1 << 23)) != 0 { + rval |= MMX + } + if (d & (1 << 25)) != 0 { + rval |= MMXEXT + } + if (d & (1 << 25)) != 0 { + rval |= SSE + } + if (d & (1 << 26)) != 0 { + rval |= SSE2 + } + if (c & 1) != 0 { + rval |= SSE3 + } + if (c & 0x00000200) != 0 { + rval |= SSSE3 + } + if (c & 0x00080000) != 0 { + rval |= SSE4 + } + if (c & 0x00100000) != 0 { + rval |= SSE42 + } + if (c & (1 << 25)) != 0 { + rval |= AESNI + } + if (c & (1 << 1)) != 0 { + rval |= CLMUL + } + if c&(1<<23) != 0 { + rval |= POPCNT + } + if c&(1<<30) != 0 { + rval |= RDRAND + } + if c&(1<<29) != 0 { + rval |= F16C + } + if c&(1<<13) != 0 { + rval |= CX16 + } + if vend == Intel && (d&(1<<28)) != 0 && mfi >= 4 { + if threadsPerCore() > 1 { + rval |= HTT + } + } + + // Check XGETBV, OXSAVE and AVX bits + if c&(1<<26) != 0 && c&(1<<27) != 0 && c&(1<<28) != 0 { + // Check for OS support + eax, _ := xgetbv(0) + if (eax & 0x6) == 0x6 { + rval |= AVX + if (c & 0x00001000) != 0 { + rval |= FMA3 + } + } + } + + // Check AVX2, AVX2 requires OS support, but BMI1/2 don't. + if mfi >= 7 { + _, ebx, ecx, _ := cpuidex(7, 0) + if (rval&AVX) != 0 && (ebx&0x00000020) != 0 { + rval |= AVX2 + } + if (ebx & 0x00000008) != 0 { + rval |= BMI1 + if (ebx & 0x00000100) != 0 { + rval |= BMI2 + } + } + if ebx&(1<<2) != 0 { + rval |= SGX + } + if ebx&(1<<4) != 0 { + rval |= HLE + } + if ebx&(1<<9) != 0 { + rval |= ERMS + } + if ebx&(1<<11) != 0 { + rval |= RTM + } + if ebx&(1<<14) != 0 { + rval |= MPX + } + if ebx&(1<<18) != 0 { + rval |= RDSEED + } + if ebx&(1<<19) != 0 { + rval |= ADX + } + if ebx&(1<<29) != 0 { + rval |= SHA + } + + // Only detect AVX-512 features if XGETBV is supported + if c&((1<<26)|(1<<27)) == (1<<26)|(1<<27) { + // Check for OS support + eax, _ := xgetbv(0) + + // Verify that XCR0[7:5] = ‘111b’ (OPMASK state, upper 256-bit of ZMM0-ZMM15 and + // ZMM16-ZMM31 state are enabled by OS) + /// and that XCR0[2:1] = ‘11b’ (XMM state and YMM state are enabled by OS). + if (eax>>5)&7 == 7 && (eax>>1)&3 == 3 { + if ebx&(1<<16) != 0 { + rval |= AVX512F + } + if ebx&(1<<17) != 0 { + rval |= AVX512DQ + } + if ebx&(1<<21) != 0 { + rval |= AVX512IFMA + } + if ebx&(1<<26) != 0 { + rval |= AVX512PF + } + if ebx&(1<<27) != 0 { + rval |= AVX512ER + } + if ebx&(1<<28) != 0 { + rval |= AVX512CD + } + if ebx&(1<<30) != 0 { + rval |= AVX512BW + } + if ebx&(1<<31) != 0 { + rval |= AVX512VL + } + // ecx + if ecx&(1<<1) != 0 { + rval |= AVX512VBMI + } + } + } + } + + if maxExtendedFunction() >= 0x80000001 { + _, _, c, d := cpuid(0x80000001) + if (c & (1 << 5)) != 0 { + rval |= LZCNT + rval |= POPCNT + } + if (d & (1 << 31)) != 0 { + rval |= AMD3DNOW + } + if (d & (1 << 30)) != 0 { + rval |= AMD3DNOWEXT + } + if (d & (1 << 23)) != 0 { + rval |= MMX + } + if (d & (1 << 22)) != 0 { + rval |= MMXEXT + } + if (c & (1 << 6)) != 0 { + rval |= SSE4A + } + if d&(1<<20) != 0 { + rval |= NX + } + if d&(1<<27) != 0 { + rval |= RDTSCP + } + + /* Allow for selectively disabling SSE2 functions on AMD processors + with SSE2 support but not SSE4a. This includes Athlon64, some + Opteron, and some Sempron processors. MMX, SSE, or 3DNow! are faster + than SSE2 often enough to utilize this special-case flag. + AV_CPU_FLAG_SSE2 and AV_CPU_FLAG_SSE2SLOW are both set in this case + so that SSE2 is used unless explicitly disabled by checking + AV_CPU_FLAG_SSE2SLOW. */ + if vendorID() != Intel && + rval&SSE2 != 0 && (c&0x00000040) == 0 { + rval |= SSE2SLOW + } + + /* XOP and FMA4 use the AVX instruction coding scheme, so they can't be + * used unless the OS has AVX support. */ + if (rval & AVX) != 0 { + if (c & 0x00000800) != 0 { + rval |= XOP + } + if (c & 0x00010000) != 0 { + rval |= FMA4 + } + } + + if vendorID() == Intel { + family, model := familyModel() + if family == 6 && (model == 9 || model == 13 || model == 14) { + /* 6/9 (pentium-m "banias"), 6/13 (pentium-m "dothan"), and + * 6/14 (core1 "yonah") theoretically support sse2, but it's + * usually slower than mmx. */ + if (rval & SSE2) != 0 { + rval |= SSE2SLOW + } + if (rval & SSE3) != 0 { + rval |= SSE3SLOW + } + } + /* The Atom processor has SSSE3 support, which is useful in many cases, + * but sometimes the SSSE3 version is slower than the SSE2 equivalent + * on the Atom, but is generally faster on other processors supporting + * SSSE3. This flag allows for selectively disabling certain SSSE3 + * functions on the Atom. */ + if family == 6 && model == 28 { + rval |= ATOM + } + } + } + return Flags(rval) +} + +func valAsString(values ...uint32) []byte { + r := make([]byte, 4*len(values)) + for i, v := range values { + dst := r[i*4:] + dst[0] = byte(v & 0xff) + dst[1] = byte((v >> 8) & 0xff) + dst[2] = byte((v >> 16) & 0xff) + dst[3] = byte((v >> 24) & 0xff) + switch { + case dst[0] == 0: + return r[:i*4] + case dst[1] == 0: + return r[:i*4+1] + case dst[2] == 0: + return r[:i*4+2] + case dst[3] == 0: + return r[:i*4+3] + } + } + return r +} diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_386.s b/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_386.s new file mode 100644 index 00000000..4d731711 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_386.s @@ -0,0 +1,42 @@ +// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. + +// +build 386,!gccgo + +// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) +TEXT ·asmCpuid(SB), 7, $0 + XORL CX, CX + MOVL op+0(FP), AX + CPUID + MOVL AX, eax+4(FP) + MOVL BX, ebx+8(FP) + MOVL CX, ecx+12(FP) + MOVL DX, edx+16(FP) + RET + +// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) +TEXT ·asmCpuidex(SB), 7, $0 + MOVL op+0(FP), AX + MOVL op2+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func xgetbv(index uint32) (eax, edx uint32) +TEXT ·asmXgetbv(SB), 7, $0 + MOVL index+0(FP), CX + BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV + MOVL AX, eax+4(FP) + MOVL DX, edx+8(FP) + RET + +// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) +TEXT ·asmRdtscpAsm(SB), 7, $0 + BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP + MOVL AX, eax+0(FP) + MOVL BX, ebx+4(FP) + MOVL CX, ecx+8(FP) + MOVL DX, edx+12(FP) + RET diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_amd64.s b/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_amd64.s new file mode 100644 index 00000000..3c1d60e4 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/cpuid_amd64.s @@ -0,0 +1,42 @@ +// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. + +//+build amd64,!gccgo + +// func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) +TEXT ·asmCpuid(SB), 7, $0 + XORQ CX, CX + MOVL op+0(FP), AX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) +TEXT ·asmCpuidex(SB), 7, $0 + MOVL op+0(FP), AX + MOVL op2+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func asmXgetbv(index uint32) (eax, edx uint32) +TEXT ·asmXgetbv(SB), 7, $0 + MOVL index+0(FP), CX + BYTE $0x0f; BYTE $0x01; BYTE $0xd0 // XGETBV + MOVL AX, eax+8(FP) + MOVL DX, edx+12(FP) + RET + +// func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) +TEXT ·asmRdtscpAsm(SB), 7, $0 + BYTE $0x0F; BYTE $0x01; BYTE $0xF9 // RDTSCP + MOVL AX, eax+0(FP) + MOVL BX, ebx+4(FP) + MOVL CX, ecx+8(FP) + MOVL DX, edx+12(FP) + RET diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/detect_intel.go b/Godeps/_workspace/src/github.com/klauspost/cpuid/detect_intel.go new file mode 100644 index 00000000..a5f04dd6 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/detect_intel.go @@ -0,0 +1,17 @@ +// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. + +// +build 386,!gccgo amd64,!gccgo + +package cpuid + +func asmCpuid(op uint32) (eax, ebx, ecx, edx uint32) +func asmCpuidex(op, op2 uint32) (eax, ebx, ecx, edx uint32) +func asmXgetbv(index uint32) (eax, edx uint32) +func asmRdtscpAsm() (eax, ebx, ecx, edx uint32) + +func initCPU() { + cpuid = asmCpuid + cpuidex = asmCpuidex + xgetbv = asmXgetbv + rdtscpAsm = asmRdtscpAsm +} diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/detect_ref.go b/Godeps/_workspace/src/github.com/klauspost/cpuid/detect_ref.go new file mode 100644 index 00000000..909c5d9a --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/detect_ref.go @@ -0,0 +1,23 @@ +// Copyright (c) 2015 Klaus Post, released under MIT License. See LICENSE file. + +// +build !amd64,!386 gccgo + +package cpuid + +func initCPU() { + cpuid = func(op uint32) (eax, ebx, ecx, edx uint32) { + return 0, 0, 0, 0 + } + + cpuidex = func(op, op2 uint32) (eax, ebx, ecx, edx uint32) { + return 0, 0, 0, 0 + } + + xgetbv = func(index uint32) (eax, edx uint32) { + return 0, 0 + } + + rdtscpAsm = func() (eax, ebx, ecx, edx uint32) { + return 0, 0, 0, 0 + } +} diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/generate.go b/Godeps/_workspace/src/github.com/klauspost/cpuid/generate.go new file mode 100644 index 00000000..c060b816 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/generate.go @@ -0,0 +1,3 @@ +package cpuid + +//go:generate go run private-gen.go diff --git a/Godeps/_workspace/src/github.com/klauspost/cpuid/private-gen.go b/Godeps/_workspace/src/github.com/klauspost/cpuid/private-gen.go new file mode 100644 index 00000000..437333d2 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/cpuid/private-gen.go @@ -0,0 +1,476 @@ +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "io" + "io/ioutil" + "log" + "os" + "reflect" + "strings" + "unicode" + "unicode/utf8" +) + +var inFiles = []string{"cpuid.go", "cpuid_test.go"} +var copyFiles = []string{"cpuid_amd64.s", "cpuid_386.s", "detect_ref.go", "detect_intel.go"} +var fileSet = token.NewFileSet() +var reWrites = []rewrite{ + initRewrite("CPUInfo -> cpuInfo"), + initRewrite("Vendor -> vendor"), + initRewrite("Flags -> flags"), + initRewrite("Detect -> detect"), + initRewrite("CPU -> cpu"), +} +var excludeNames = map[string]bool{"string": true, "join": true, "trim": true, + // cpuid_test.go + "t": true, "println": true, "logf": true, "log": true, "fatalf": true, "fatal": true, +} + +var excludePrefixes = []string{"test", "benchmark"} + +func main() { + Package := "private" + parserMode := parser.ParseComments + exported := make(map[string]rewrite) + for _, file := range inFiles { + in, err := os.Open(file) + if err != nil { + log.Fatalf("opening input", err) + } + + src, err := ioutil.ReadAll(in) + if err != nil { + log.Fatalf("reading input", err) + } + + astfile, err := parser.ParseFile(fileSet, file, src, parserMode) + if err != nil { + log.Fatalf("parsing input", err) + } + + for _, rw := range reWrites { + astfile = rw(astfile) + } + + // Inspect the AST and print all identifiers and literals. + var startDecl token.Pos + var endDecl token.Pos + ast.Inspect(astfile, func(n ast.Node) bool { + var s string + switch x := n.(type) { + case *ast.Ident: + if x.IsExported() { + t := strings.ToLower(x.Name) + for _, pre := range excludePrefixes { + if strings.HasPrefix(t, pre) { + return true + } + } + if excludeNames[t] != true { + //if x.Pos() > startDecl && x.Pos() < endDecl { + exported[x.Name] = initRewrite(x.Name + " -> " + t) + } + } + + case *ast.GenDecl: + if x.Tok == token.CONST && x.Lparen > 0 { + startDecl = x.Lparen + endDecl = x.Rparen + // fmt.Printf("Decl:%s -> %s\n", fileSet.Position(startDecl), fileSet.Position(endDecl)) + } + } + if s != "" { + fmt.Printf("%s:\t%s\n", fileSet.Position(n.Pos()), s) + } + return true + }) + + for _, rw := range exported { + astfile = rw(astfile) + } + + var buf bytes.Buffer + + printer.Fprint(&buf, fileSet, astfile) + + // Remove package documentation and insert information + s := buf.String() + ind := strings.Index(buf.String(), "\npackage cpuid") + s = s[ind:] + s = "// Generated, DO NOT EDIT,\n" + + "// but copy it to your own project and rename the package.\n" + + "// See more at http://github.com/klauspost/cpuid\n" + + s + + outputName := Package + string(os.PathSeparator) + file + + err = ioutil.WriteFile(outputName, []byte(s), 0644) + if err != nil { + log.Fatalf("writing output: %s", err) + } + log.Println("Generated", outputName) + } + + for _, file := range copyFiles { + dst := "" + if strings.HasPrefix(file, "cpuid") { + dst = Package + string(os.PathSeparator) + file + } else { + dst = Package + string(os.PathSeparator) + "cpuid_" + file + } + err := copyFile(file, dst) + if err != nil { + log.Fatalf("copying file: %s", err) + } + log.Println("Copied", dst) + } +} + +// CopyFile copies a file from src to dst. If src and dst files exist, and are +// the same, then return success. Copy the file contents from src to dst. +func copyFile(src, dst string) (err error) { + sfi, err := os.Stat(src) + if err != nil { + return + } + if !sfi.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String()) + } + dfi, err := os.Stat(dst) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(dfi.Mode().IsRegular()) { + return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String()) + } + if os.SameFile(sfi, dfi) { + return + } + } + err = copyFileContents(src, dst) + return +} + +// copyFileContents copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +type rewrite func(*ast.File) *ast.File + +// Mostly copied from gofmt +func initRewrite(rewriteRule string) rewrite { + f := strings.Split(rewriteRule, "->") + if len(f) != 2 { + fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n") + os.Exit(2) + } + pattern := parseExpr(f[0], "pattern") + replace := parseExpr(f[1], "replacement") + return func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) } +} + +// parseExpr parses s as an expression. +// It might make sense to expand this to allow statement patterns, +// but there are problems with preserving formatting and also +// with what a wildcard for a statement looks like. +func parseExpr(s, what string) ast.Expr { + x, err := parser.ParseExpr(s) + if err != nil { + fmt.Fprintf(os.Stderr, "parsing %s %s at %s\n", what, s, err) + os.Exit(2) + } + return x +} + +// Keep this function for debugging. +/* +func dump(msg string, val reflect.Value) { + fmt.Printf("%s:\n", msg) + ast.Print(fileSet, val.Interface()) + fmt.Println() +} +*/ + +// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file. +func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File { + cmap := ast.NewCommentMap(fileSet, p, p.Comments) + m := make(map[string]reflect.Value) + pat := reflect.ValueOf(pattern) + repl := reflect.ValueOf(replace) + + var rewriteVal func(val reflect.Value) reflect.Value + rewriteVal = func(val reflect.Value) reflect.Value { + // don't bother if val is invalid to start with + if !val.IsValid() { + return reflect.Value{} + } + for k := range m { + delete(m, k) + } + val = apply(rewriteVal, val) + if match(m, pat, val) { + val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos())) + } + return val + } + + r := apply(rewriteVal, reflect.ValueOf(p)).Interface().(*ast.File) + r.Comments = cmap.Filter(r).Comments() // recreate comments list + return r +} + +// set is a wrapper for x.Set(y); it protects the caller from panics if x cannot be changed to y. +func set(x, y reflect.Value) { + // don't bother if x cannot be set or y is invalid + if !x.CanSet() || !y.IsValid() { + return + } + defer func() { + if x := recover(); x != nil { + if s, ok := x.(string); ok && + (strings.Contains(s, "type mismatch") || strings.Contains(s, "not assignable")) { + // x cannot be set to y - ignore this rewrite + return + } + panic(x) + } + }() + x.Set(y) +} + +// Values/types for special cases. +var ( + objectPtrNil = reflect.ValueOf((*ast.Object)(nil)) + scopePtrNil = reflect.ValueOf((*ast.Scope)(nil)) + + identType = reflect.TypeOf((*ast.Ident)(nil)) + objectPtrType = reflect.TypeOf((*ast.Object)(nil)) + positionType = reflect.TypeOf(token.NoPos) + callExprType = reflect.TypeOf((*ast.CallExpr)(nil)) + scopePtrType = reflect.TypeOf((*ast.Scope)(nil)) +) + +// apply replaces each AST field x in val with f(x), returning val. +// To avoid extra conversions, f operates on the reflect.Value form. +func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value { + if !val.IsValid() { + return reflect.Value{} + } + + // *ast.Objects introduce cycles and are likely incorrect after + // rewrite; don't follow them but replace with nil instead + if val.Type() == objectPtrType { + return objectPtrNil + } + + // similarly for scopes: they are likely incorrect after a rewrite; + // replace them with nil + if val.Type() == scopePtrType { + return scopePtrNil + } + + switch v := reflect.Indirect(val); v.Kind() { + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + e := v.Index(i) + set(e, f(e)) + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + e := v.Field(i) + set(e, f(e)) + } + case reflect.Interface: + e := v.Elem() + set(v, f(e)) + } + return val +} + +func isWildcard(s string) bool { + rune, size := utf8.DecodeRuneInString(s) + return size == len(s) && unicode.IsLower(rune) +} + +// match returns true if pattern matches val, +// recording wildcard submatches in m. +// If m == nil, match checks whether pattern == val. +func match(m map[string]reflect.Value, pattern, val reflect.Value) bool { + // Wildcard matches any expression. If it appears multiple + // times in the pattern, it must match the same expression + // each time. + if m != nil && pattern.IsValid() && pattern.Type() == identType { + name := pattern.Interface().(*ast.Ident).Name + if isWildcard(name) && val.IsValid() { + // wildcards only match valid (non-nil) expressions. + if _, ok := val.Interface().(ast.Expr); ok && !val.IsNil() { + if old, ok := m[name]; ok { + return match(nil, old, val) + } + m[name] = val + return true + } + } + } + + // Otherwise, pattern and val must match recursively. + if !pattern.IsValid() || !val.IsValid() { + return !pattern.IsValid() && !val.IsValid() + } + if pattern.Type() != val.Type() { + return false + } + + // Special cases. + switch pattern.Type() { + case identType: + // For identifiers, only the names need to match + // (and none of the other *ast.Object information). + // This is a common case, handle it all here instead + // of recursing down any further via reflection. + p := pattern.Interface().(*ast.Ident) + v := val.Interface().(*ast.Ident) + return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name + case objectPtrType, positionType: + // object pointers and token positions always match + return true + case callExprType: + // For calls, the Ellipsis fields (token.Position) must + // match since that is how f(x) and f(x...) are different. + // Check them here but fall through for the remaining fields. + p := pattern.Interface().(*ast.CallExpr) + v := val.Interface().(*ast.CallExpr) + if p.Ellipsis.IsValid() != v.Ellipsis.IsValid() { + return false + } + } + + p := reflect.Indirect(pattern) + v := reflect.Indirect(val) + if !p.IsValid() || !v.IsValid() { + return !p.IsValid() && !v.IsValid() + } + + switch p.Kind() { + case reflect.Slice: + if p.Len() != v.Len() { + return false + } + for i := 0; i < p.Len(); i++ { + if !match(m, p.Index(i), v.Index(i)) { + return false + } + } + return true + + case reflect.Struct: + for i := 0; i < p.NumField(); i++ { + if !match(m, p.Field(i), v.Field(i)) { + return false + } + } + return true + + case reflect.Interface: + return match(m, p.Elem(), v.Elem()) + } + + // Handle token integers, etc. + return p.Interface() == v.Interface() +} + +// subst returns a copy of pattern with values from m substituted in place +// of wildcards and pos used as the position of tokens from the pattern. +// if m == nil, subst returns a copy of pattern and doesn't change the line +// number information. +func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value { + if !pattern.IsValid() { + return reflect.Value{} + } + + // Wildcard gets replaced with map value. + if m != nil && pattern.Type() == identType { + name := pattern.Interface().(*ast.Ident).Name + if isWildcard(name) { + if old, ok := m[name]; ok { + return subst(nil, old, reflect.Value{}) + } + } + } + + if pos.IsValid() && pattern.Type() == positionType { + // use new position only if old position was valid in the first place + if old := pattern.Interface().(token.Pos); !old.IsValid() { + return pattern + } + return pos + } + + // Otherwise copy. + switch p := pattern; p.Kind() { + case reflect.Slice: + v := reflect.MakeSlice(p.Type(), p.Len(), p.Len()) + for i := 0; i < p.Len(); i++ { + v.Index(i).Set(subst(m, p.Index(i), pos)) + } + return v + + case reflect.Struct: + v := reflect.New(p.Type()).Elem() + for i := 0; i < p.NumField(); i++ { + v.Field(i).Set(subst(m, p.Field(i), pos)) + } + return v + + case reflect.Ptr: + v := reflect.New(p.Type()).Elem() + if elem := p.Elem(); elem.IsValid() { + v.Set(subst(m, elem, pos).Addr()) + } + return v + + case reflect.Interface: + v := reflect.New(p.Type()).Elem() + if elem := p.Elem(); elem.IsValid() { + v.Set(subst(m, elem, pos)) + } + return v + } + + return pattern +} diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/.gitignore b/Godeps/_workspace/src/github.com/klauspost/crc32/.gitignore new file mode 100644 index 00000000..daf913b1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/.travis.yml b/Godeps/_workspace/src/github.com/klauspost/crc32/.travis.yml new file mode 100644 index 00000000..97717995 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/.travis.yml @@ -0,0 +1,12 @@ +language: go + +go: + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - tip + +script: + - go test -v . + - go test -v -race . diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/LICENSE b/Godeps/_workspace/src/github.com/klauspost/crc32/LICENSE new file mode 100644 index 00000000..4fd5963e --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/LICENSE @@ -0,0 +1,28 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2015 Klaus Post + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/README.md b/Godeps/_workspace/src/github.com/klauspost/crc32/README.md new file mode 100644 index 00000000..440541c7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/README.md @@ -0,0 +1,84 @@ +# crc32 +CRC32 hash with x64 optimizations + +This package is a drop-in replacement for the standard library `hash/crc32` package, that features SSE 4.2 optimizations on x64 platforms, for a 10x speedup. + +[![Build Status](https://travis-ci.org/klauspost/crc32.svg?branch=master)](https://travis-ci.org/klauspost/crc32) + +# usage + +Install using `go get github.com/klauspost/crc32`. This library is based on Go 1.5 code and requires Go 1.3 or newer. + +Replace `import "hash/crc32"` with `import "github.com/klauspost/crc32"` and you are good to go. + +# changes + +* Dec 4, 2015: Uses the "slice-by-8" trick more extensively, which gives a 1.5 to 2.5x speedup if assembler is unavailable. + + +# performance + +For IEEE tables (the most common), there is approximately a factor 10 speedup with "CLMUL" (Carryless multiplication) instruction: +``` +benchmark old ns/op new ns/op delta +BenchmarkCrc32KB 99955 10258 -89.74% + +benchmark old MB/s new MB/s speedup +BenchmarkCrc32KB 327.83 3194.20 9.74x +``` + +For other tables and "CLMUL" capable machines the performance is the same as the standard library. + +Here are some detailed benchmarks, comparing to go 1.5 standard library with and without assembler enabled. + +``` +Std: Standard Go 1.5 library +Crc: Indicates IEEE type CRC. +40B: Size of each slice encoded. +NoAsm: Assembler was disabled (ie. not an AMD64 or SSE 4.2+ capable machine). +Castagnoli: Castagnoli CRC type. + +BenchmarkStdCrc40B-4 10000000 158 ns/op 252.88 MB/s +BenchmarkCrc40BNoAsm-4 20000000 105 ns/op 377.38 MB/s (slice8) +BenchmarkCrc40B-4 20000000 105 ns/op 378.77 MB/s (slice8) + +BenchmarkStdCrc1KB-4 500000 3604 ns/op 284.10 MB/s +BenchmarkCrc1KBNoAsm-4 1000000 1463 ns/op 699.79 MB/s (slice8) +BenchmarkCrc1KB-4 3000000 396 ns/op 2583.69 MB/s (asm) + +BenchmarkStdCrc8KB-4 200000 11417 ns/op 717.48 MB/s (slice8) +BenchmarkCrc8KBNoAsm-4 200000 11317 ns/op 723.85 MB/s (slice8) +BenchmarkCrc8KB-4 500000 2919 ns/op 2805.73 MB/s (asm) + +BenchmarkStdCrc32KB-4 30000 45749 ns/op 716.24 MB/s (slice8) +BenchmarkCrc32KBNoAsm-4 30000 45109 ns/op 726.42 MB/s (slice8) +BenchmarkCrc32KB-4 100000 11497 ns/op 2850.09 MB/s (asm) + +BenchmarkStdNoAsmCastagnol40B-4 10000000 161 ns/op 246.94 MB/s +BenchmarkStdCastagnoli40B-4 50000000 28.4 ns/op 1410.69 MB/s (asm) +BenchmarkCastagnoli40BNoAsm-4 20000000 100 ns/op 398.01 MB/s (slice8) +BenchmarkCastagnoli40B-4 50000000 28.2 ns/op 1419.54 MB/s (asm) + +BenchmarkStdNoAsmCastagnoli1KB-4 500000 3622 ns/op 282.67 MB/s +BenchmarkStdCastagnoli1KB-4 10000000 144 ns/op 7099.78 MB/s (asm) +BenchmarkCastagnoli1KBNoAsm-4 1000000 1475 ns/op 694.14 MB/s (slice8) +BenchmarkCastagnoli1KB-4 10000000 146 ns/op 6993.35 MB/s (asm) + +BenchmarkStdNoAsmCastagnoli8KB-4 50000 28781 ns/op 284.63 MB/s +BenchmarkStdCastagnoli8KB-4 1000000 1029 ns/op 7957.89 MB/s (asm) +BenchmarkCastagnoli8KBNoAsm-4 200000 11410 ns/op 717.94 MB/s (slice8) +BenchmarkCastagnoli8KB-4 1000000 1000 ns/op 8188.71 MB/s (asm) + +BenchmarkStdNoAsmCastagnoli32KB-4 10000 115426 ns/op 283.89 MB/s +BenchmarkStdCastagnoli32KB-4 300000 4065 ns/op 8059.13 MB/s (asm) +BenchmarkCastagnoli32KBNoAsm-4 30000 45171 ns/op 725.41 MB/s (slice8) +BenchmarkCastagnoli32KB-4 500000 4077 ns/op 8035.89 MB/s (asm) +``` + +The IEEE assembler optimizations has been submitted and will be part of the Go 1.6 standard library. + +However, the improved use of slice-by-8 has not, but will probably be submitted for Go 1.7. + +# license + +Standard Go license. Changes are Copyright (c) 2015 Klaus Post under same conditions. diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/crc32.go b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32.go new file mode 100644 index 00000000..8d6ba5d3 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32.go @@ -0,0 +1,186 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32, +// checksum. See http://en.wikipedia.org/wiki/Cyclic_redundancy_check for +// information. +// +// Polynomials are represented in LSB-first form also known as reversed representation. +// +// See http://en.wikipedia.org/wiki/Mathematics_of_cyclic_redundancy_checks#Reversed_representations_and_reciprocal_polynomials +// for information. +package crc32 + +import ( + "hash" + "sync" +) + +// The size of a CRC-32 checksum in bytes. +const Size = 4 + +// Predefined polynomials. +const ( + // IEEE is by far and away the most common CRC-32 polynomial. + // Used by ethernet (IEEE 802.3), v.42, fddi, gzip, zip, png, ... + IEEE = 0xedb88320 + + // Castagnoli's polynomial, used in iSCSI. + // Has better error detection characteristics than IEEE. + // http://dx.doi.org/10.1109/26.231911 + Castagnoli = 0x82f63b78 + + // Koopman's polynomial. + // Also has better error detection characteristics than IEEE. + // http://dx.doi.org/10.1109/DSN.2002.1028931 + Koopman = 0xeb31d82e +) + +// Table is a 256-word table representing the polynomial for efficient processing. +type Table [256]uint32 + +// castagnoliTable points to a lazily initialized Table for the Castagnoli +// polynomial. MakeTable will always return this value when asked to make a +// Castagnoli table so we can compare against it to find when the caller is +// using this polynomial. +var castagnoliTable *Table +var castagnoliTable8 *slicing8Table +var castagnoliOnce sync.Once + +func castagnoliInit() { + castagnoliTable = makeTable(Castagnoli) + castagnoliTable8 = makeTable8(Castagnoli) +} + +// IEEETable is the table for the IEEE polynomial. +var IEEETable = makeTable(IEEE) + +// slicing8Table is array of 8 Tables +type slicing8Table [8]Table + +// ieeeTable8 is the slicing8Table for IEEE +var ieeeTable8 *slicing8Table +var ieeeTable8Once sync.Once + +// MakeTable returns a Table constructed from the specified polynomial. +// The contents of this Table must not be modified. +func MakeTable(poly uint32) *Table { + switch poly { + case IEEE: + return IEEETable + case Castagnoli: + castagnoliOnce.Do(castagnoliInit) + return castagnoliTable + } + return makeTable(poly) +} + +// makeTable returns the Table constructed from the specified polynomial. +func makeTable(poly uint32) *Table { + t := new(Table) + for i := 0; i < 256; i++ { + crc := uint32(i) + for j := 0; j < 8; j++ { + if crc&1 == 1 { + crc = (crc >> 1) ^ poly + } else { + crc >>= 1 + } + } + t[i] = crc + } + return t +} + +// makeTable8 returns slicing8Table constructed from the specified polynomial. +func makeTable8(poly uint32) *slicing8Table { + t := new(slicing8Table) + t[0] = *makeTable(poly) + for i := 0; i < 256; i++ { + crc := t[0][i] + for j := 1; j < 8; j++ { + crc = t[0][crc&0xFF] ^ (crc >> 8) + t[j][i] = crc + } + } + return t +} + +// digest represents the partial evaluation of a checksum. +type digest struct { + crc uint32 + tab *Table +} + +// New creates a new hash.Hash32 computing the CRC-32 checksum +// using the polynomial represented by the Table. +// Its Sum method will lay the value out in big-endian byte order. +func New(tab *Table) hash.Hash32 { return &digest{0, tab} } + +// NewIEEE creates a new hash.Hash32 computing the CRC-32 checksum +// using the IEEE polynomial. +// Its Sum method will lay the value out in big-endian byte order. +func NewIEEE() hash.Hash32 { return New(IEEETable) } + +func (d *digest) Size() int { return Size } + +func (d *digest) BlockSize() int { return 1 } + +func (d *digest) Reset() { d.crc = 0 } + +func update(crc uint32, tab *Table, p []byte) uint32 { + crc = ^crc + for _, v := range p { + crc = tab[byte(crc)^v] ^ (crc >> 8) + } + return ^crc +} + +// updateSlicingBy8 updates CRC using Slicing-by-8 +func updateSlicingBy8(crc uint32, tab *slicing8Table, p []byte) uint32 { + crc = ^crc + for len(p) > 8 { + crc ^= uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 + crc = tab[0][p[7]] ^ tab[1][p[6]] ^ tab[2][p[5]] ^ tab[3][p[4]] ^ + tab[4][crc>>24] ^ tab[5][(crc>>16)&0xFF] ^ + tab[6][(crc>>8)&0xFF] ^ tab[7][crc&0xFF] + p = p[8:] + } + crc = ^crc + if len(p) == 0 { + return crc + } + return update(crc, &tab[0], p) +} + +// Update returns the result of adding the bytes in p to the crc. +func Update(crc uint32, tab *Table, p []byte) uint32 { + if tab == castagnoliTable { + return updateCastagnoli(crc, p) + } + if tab == IEEETable { + return updateIEEE(crc, p) + } + return update(crc, tab, p) +} + +func (d *digest) Write(p []byte) (n int, err error) { + d.crc = Update(d.crc, d.tab, p) + return len(p), nil +} + +func (d *digest) Sum32() uint32 { return d.crc } + +func (d *digest) Sum(in []byte) []byte { + s := d.Sum32() + return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) +} + +// Checksum returns the CRC-32 checksum of data +// using the polynomial represented by the Table. +func Checksum(data []byte, tab *Table) uint32 { return Update(0, tab, data) } + +// ChecksumIEEE returns the CRC-32 checksum of data +// using the IEEE polynomial. +func ChecksumIEEE(data []byte) uint32 { return updateIEEE(0, data) } diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.go b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.go new file mode 100644 index 00000000..4827128e --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.go @@ -0,0 +1,62 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine,!gccgo + +package crc32 + +// This file contains the code to call the SSE 4.2 version of the Castagnoli +// and IEEE CRC. + +// haveSSE41/haveSSE42/haveCLMUL are defined in crc_amd64.s and use +// CPUID to test for SSE 4.1, 4.2 and CLMUL support. +func haveSSE41() bool +func haveSSE42() bool +func haveCLMUL() bool + +// castagnoliSSE42 is defined in crc_amd64.s and uses the SSE4.2 CRC32 +// instruction. +//go:noescape +func castagnoliSSE42(crc uint32, p []byte) uint32 + +// ieeeCLMUL is defined in crc_amd64.s and uses the PCLMULQDQ +// instruction as well as SSE 4.1. +//go:noescape +func ieeeCLMUL(crc uint32, p []byte) uint32 + +var sse42 = haveSSE42() +var useFastIEEE = haveCLMUL() && haveSSE41() + +func updateCastagnoli(crc uint32, p []byte) uint32 { + if sse42 { + return castagnoliSSE42(crc, p) + } + // only use slicing-by-8 when input is >= 16 Bytes + if len(p) >= 16 { + return updateSlicingBy8(crc, castagnoliTable8, p) + } + return update(crc, castagnoliTable, p) +} + +func updateIEEE(crc uint32, p []byte) uint32 { + if useFastIEEE && len(p) >= 64 { + left := len(p) & 15 + do := len(p) - left + crc = ^ieeeCLMUL(^crc, p[:do]) + if left > 0 { + crc = update(crc, IEEETable, p[do:]) + } + return crc + } + + // only use slicing-by-8 when input is >= 16 Bytes + if len(p) >= 16 { + ieeeTable8Once.Do(func() { + ieeeTable8 = makeTable8(IEEE) + }) + return updateSlicingBy8(crc, ieeeTable8, p) + } + + return update(crc, IEEETable, p) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.s b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.s new file mode 100644 index 00000000..9bf05d89 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64.s @@ -0,0 +1,237 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build gc + +#define NOSPLIT 4 +#define RODATA 8 + +// func castagnoliSSE42(crc uint32, p []byte) uint32 +TEXT ·castagnoliSSE42(SB), NOSPLIT, $0 + MOVL crc+0(FP), AX // CRC value + MOVQ p+8(FP), SI // data pointer + MOVQ p_len+16(FP), CX // len(p) + + NOTL AX + + // If there's less than 8 bytes to process, we do it byte-by-byte. + CMPQ CX, $8 + JL cleanup + + // Process individual bytes until the input is 8-byte aligned. +startup: + MOVQ SI, BX + ANDQ $7, BX + JZ aligned + + CRC32B (SI), AX + DECQ CX + INCQ SI + JMP startup + +aligned: + // The input is now 8-byte aligned and we can process 8-byte chunks. + CMPQ CX, $8 + JL cleanup + + CRC32Q (SI), AX + ADDQ $8, SI + SUBQ $8, CX + JMP aligned + +cleanup: + // We may have some bytes left over that we process one at a time. + CMPQ CX, $0 + JE done + + CRC32B (SI), AX + INCQ SI + DECQ CX + JMP cleanup + +done: + NOTL AX + MOVL AX, ret+32(FP) + RET + +// func haveSSE42() bool +TEXT ·haveSSE42(SB), NOSPLIT, $0 + XORQ AX, AX + INCL AX + CPUID + SHRQ $20, CX + ANDQ $1, CX + MOVB CX, ret+0(FP) + RET + +// func haveCLMUL() bool +TEXT ·haveCLMUL(SB), NOSPLIT, $0 + XORQ AX, AX + INCL AX + CPUID + SHRQ $1, CX + ANDQ $1, CX + MOVB CX, ret+0(FP) + RET + +// func haveSSE41() bool +TEXT ·haveSSE41(SB), NOSPLIT, $0 + XORQ AX, AX + INCL AX + CPUID + SHRQ $19, CX + ANDQ $1, CX + MOVB CX, ret+0(FP) + RET + +// CRC32 polynomial data +// +// These constants are lifted from the +// Linux kernel, since they avoid the costly +// PSHUFB 16 byte reversal proposed in the +// original Intel paper. +DATA r2r1kp<>+0(SB)/8, $0x154442bd4 +DATA r2r1kp<>+8(SB)/8, $0x1c6e41596 +DATA r4r3kp<>+0(SB)/8, $0x1751997d0 +DATA r4r3kp<>+8(SB)/8, $0x0ccaa009e +DATA rupolykp<>+0(SB)/8, $0x1db710641 +DATA rupolykp<>+8(SB)/8, $0x1f7011641 +DATA r5kp<>+0(SB)/8, $0x163cd6124 + +GLOBL r2r1kp<>(SB), RODATA, $16 +GLOBL r4r3kp<>(SB), RODATA, $16 +GLOBL rupolykp<>(SB), RODATA, $16 +GLOBL r5kp<>(SB), RODATA, $8 + +// Based on http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf +// len(p) must be at least 64, and must be a multiple of 16. + +// func ieeeCLMUL(crc uint32, p []byte) uint32 +TEXT ·ieeeCLMUL(SB), NOSPLIT, $0 + MOVL crc+0(FP), X0 // Initial CRC value + MOVQ p+8(FP), SI // data pointer + MOVQ p_len+16(FP), CX // len(p) + + MOVOU (SI), X1 + MOVOU 16(SI), X2 + MOVOU 32(SI), X3 + MOVOU 48(SI), X4 + PXOR X0, X1 + ADDQ $64, SI // buf+=64 + SUBQ $64, CX // len-=64 + CMPQ CX, $64 // Less than 64 bytes left + JB remain64 + + MOVOA r2r1kp<>+0(SB), X0 + +loopback64: + MOVOA X1, X5 + MOVOA X2, X6 + MOVOA X3, X7 + MOVOA X4, X8 + + PCLMULQDQ $0, X0, X1 + PCLMULQDQ $0, X0, X2 + PCLMULQDQ $0, X0, X3 + PCLMULQDQ $0, X0, X4 + + // Load next early + MOVOU (SI), X11 + MOVOU 16(SI), X12 + MOVOU 32(SI), X13 + MOVOU 48(SI), X14 + + PCLMULQDQ $0x11, X0, X5 + PCLMULQDQ $0x11, X0, X6 + PCLMULQDQ $0x11, X0, X7 + PCLMULQDQ $0x11, X0, X8 + + PXOR X5, X1 + PXOR X6, X2 + PXOR X7, X3 + PXOR X8, X4 + + PXOR X11, X1 + PXOR X12, X2 + PXOR X13, X3 + PXOR X14, X4 + + ADDQ $0x40, DI + ADDQ $64, SI // buf+=64 + SUBQ $64, CX // len-=64 + CMPQ CX, $64 // Less than 64 bytes left? + JGE loopback64 + + // Fold result into a single register (X1) +remain64: + MOVOA r4r3kp<>+0(SB), X0 + + MOVOA X1, X5 + PCLMULQDQ $0, X0, X1 + PCLMULQDQ $0x11, X0, X5 + PXOR X5, X1 + PXOR X2, X1 + + MOVOA X1, X5 + PCLMULQDQ $0, X0, X1 + PCLMULQDQ $0x11, X0, X5 + PXOR X5, X1 + PXOR X3, X1 + + MOVOA X1, X5 + PCLMULQDQ $0, X0, X1 + PCLMULQDQ $0x11, X0, X5 + PXOR X5, X1 + PXOR X4, X1 + + // More than 16 bytes left? + CMPQ CX, $16 + JB finish + + // Encode 16 bytes +remain16: + MOVOU (SI), X10 + MOVOA X1, X5 + PCLMULQDQ $0, X0, X1 + PCLMULQDQ $0x11, X0, X5 + PXOR X5, X1 + PXOR X10, X1 + SUBQ $16, CX + ADDQ $16, SI + CMPQ CX, $16 + JGE remain16 + +finish: + // Fold final result into 32 bits and return it + PCMPEQB X3, X3 + PCLMULQDQ $1, X1, X0 + PSRLDQ $8, X1 + PXOR X0, X1 + + MOVOA X1, X2 + MOVQ r5kp<>+0(SB), X0 + + // Creates 32 bit mask. Note that we don't care about upper half. + PSRLQ $32, X3 + + PSRLDQ $4, X2 + PAND X3, X1 + PCLMULQDQ $0, X0, X1 + PXOR X2, X1 + + MOVOA rupolykp<>+0(SB), X0 + + MOVOA X1, X2 + PAND X3, X1 + PCLMULQDQ $0x10, X0, X1 + PAND X3, X1 + PCLMULQDQ $0, X0, X1 + PXOR X2, X1 + + // PEXTRD $1, X1, AX (SSE 4.1) + BYTE $0x66; BYTE $0x0f; BYTE $0x3a + BYTE $0x16; BYTE $0xc8; BYTE $0x01 + MOVL AX, ret+32(FP) + + RET diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.go b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.go new file mode 100644 index 00000000..926473e7 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.go @@ -0,0 +1,40 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine,!gccgo + +package crc32 + +// This file contains the code to call the SSE 4.2 version of the Castagnoli +// CRC. + +// haveSSE42 is defined in crc_amd64p32.s and uses CPUID to test for SSE 4.2 +// support. +func haveSSE42() bool + +// castagnoliSSE42 is defined in crc_amd64.s and uses the SSE4.2 CRC32 +// instruction. +//go:noescape +func castagnoliSSE42(crc uint32, p []byte) uint32 + +var sse42 = haveSSE42() + +func updateCastagnoli(crc uint32, p []byte) uint32 { + if sse42 { + return castagnoliSSE42(crc, p) + } + return update(crc, castagnoliTable, p) +} + +func updateIEEE(crc uint32, p []byte) uint32 { + // only use slicing-by-8 when input is >= 4KB + if len(p) >= 4096 { + ieeeTable8Once.Do(func() { + ieeeTable8 = makeTable8(IEEE) + }) + return updateSlicingBy8(crc, ieeeTable8, p) + } + + return update(crc, IEEETable, p) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.s b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.s new file mode 100644 index 00000000..a578d685 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_amd64p32.s @@ -0,0 +1,67 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build gc + +#define NOSPLIT 4 +#define RODATA 8 + +// func castagnoliSSE42(crc uint32, p []byte) uint32 +TEXT ·castagnoliSSE42(SB), NOSPLIT, $0 + MOVL crc+0(FP), AX // CRC value + MOVL p+4(FP), SI // data pointer + MOVL p_len+8(FP), CX // len(p) + + NOTL AX + + // If there's less than 8 bytes to process, we do it byte-by-byte. + CMPQ CX, $8 + JL cleanup + + // Process individual bytes until the input is 8-byte aligned. +startup: + MOVQ SI, BX + ANDQ $7, BX + JZ aligned + + CRC32B (SI), AX + DECQ CX + INCQ SI + JMP startup + +aligned: + // The input is now 8-byte aligned and we can process 8-byte chunks. + CMPQ CX, $8 + JL cleanup + + CRC32Q (SI), AX + ADDQ $8, SI + SUBQ $8, CX + JMP aligned + +cleanup: + // We may have some bytes left over that we process one at a time. + CMPQ CX, $0 + JE done + + CRC32B (SI), AX + INCQ SI + DECQ CX + JMP cleanup + +done: + NOTL AX + MOVL AX, ret+16(FP) + RET + +// func haveSSE42() bool +TEXT ·haveSSE42(SB), NOSPLIT, $0 + XORQ AX, AX + INCL AX + CPUID + SHRQ $20, CX + ANDQ $1, CX + MOVB CX, ret+0(FP) + RET + diff --git a/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_generic.go b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_generic.go new file mode 100644 index 00000000..a53cf96a --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/crc32/crc32_generic.go @@ -0,0 +1,29 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !amd64,!amd64p32 appengine gccgo + +package crc32 + +// This file contains the generic version of updateCastagnoli which does +// slicing-by-8, or uses the fallback for very small sizes. + +func updateCastagnoli(crc uint32, p []byte) uint32 { + // only use slicing-by-8 when input is >= 16 Bytes + if len(p) >= 16 { + return updateSlicingBy8(crc, castagnoliTable8, p) + } + return update(crc, castagnoliTable, p) +} + +func updateIEEE(crc uint32, p []byte) uint32 { + // only use slicing-by-8 when input is >= 16 Bytes + if len(p) >= 16 { + ieeeTable8Once.Do(func() { + ieeeTable8 = makeTable8(IEEE) + }) + return updateSlicingBy8(crc, ieeeTable8, p) + } + return update(crc, IEEETable, p) +} diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/.gitignore b/Godeps/_workspace/src/github.com/klauspost/pgzip/.gitignore new file mode 100644 index 00000000..daf913b1 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/.travis.yml b/Godeps/_workspace/src/github.com/klauspost/pgzip/.travis.yml new file mode 100644 index 00000000..62841685 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/.travis.yml @@ -0,0 +1,18 @@ +language: go + +sudo: false + +os: + - linux + - osx + +go: + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - tip + +script: + - go test -v -cpu=1,2,4 . + - go test -v -cpu=2 -race -short . diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/GO_LICENSE b/Godeps/_workspace/src/github.com/klauspost/pgzip/GO_LICENSE new file mode 100644 index 00000000..74487567 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/GO_LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/LICENSE b/Godeps/_workspace/src/github.com/klauspost/pgzip/LICENSE new file mode 100644 index 00000000..2bdc0d75 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Klaus Post + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/README.md b/Godeps/_workspace/src/github.com/klauspost/pgzip/README.md new file mode 100644 index 00000000..c312f53f --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/README.md @@ -0,0 +1,115 @@ +pgzip +===== + +Go parallel gzip compression/decompression. This is a fully gzip compatible drop in replacement for "compress/gzip". + +This will split compression into blocks that are compressed in parallel. This can be useful for compressing big amounts of data. The output is a standard gzip file. + +The gzip decompression is modified so it decompresses ahead of the current reader. This means that reads will be non-blocking if the decompressor can keep ahead of your code reading from it. CRC calculation also takes place in a separate goroutine. + +You should only use this if you are (de)compressing big amounts of data, say **more than 1MB** at the time, otherwise you will not see any benefit, and it will likely be faster to use the internal gzip library. + +It is important to note that this library creates and reads *standard gzip files*. You do not have to match the compressor/decompressor to get the described speedups, and the gzip files are fully compatible with other gzip readers/writers. + +A golang variant of this is [bgzf](https://godoc.org/github.com/biogo/hts/bgzf), which has the same feature, as well as seeking in the resulting file. The only drawback is a slightly bigger overhead compared to this and pure gzip. See a comparison below. + +[![GoDoc][1]][2] [![Build Status][3]][4] + +[1]: https://godoc.org/github.com/klauspost/pgzip?status.svg +[2]: https://godoc.org/github.com/klauspost/pgzip +[3]: https://travis-ci.org/klauspost/pgzip.svg +[4]: https://travis-ci.org/klauspost/pgzip + +Installation +==== +```go get github.com/klauspost/pgzip``` + +Usage +==== +[Godoc Doumentation](https://godoc.org/github.com/klauspost/pgzip) + +To use as a replacement for gzip, exchange + +```import "compress/gzip"``` +with +```import gzip "github.com/klauspost/pgzip"```. + +# Changes + +* Dec 8, 2015: Decoder now supports the io.WriterTo interface, giving a speedup and less GC pressure. +* Oct 9, 2015: Reduced allocations by ~35 by using sync.Pool. ~15% overall speedup. + +Changes in [github.com/klauspost/compress](https://github.com/klauspost/compress#changelog) are also carried over, so see that for more changes. + +## Compression +The simplest way to use this is to simply do the same as you would when using [compress/gzip](http://golang.org/pkg/compress/gzip). + +To change the block size, use the added (*pgzip.Writer).SetConcurrency(blockSize, blocks int) function. With this you can control the approximate size of your blocks, as well as how many you want to be processing in parallel. Default values for this is SetConcurrency(250000, 16), meaning blocks are split at 250000 bytes and up to 16 blocks can be processing at once before the writer blocks. + + +Example: +``` +var b bytes.Buffer +w := gzip.NewWriter(&b) +w.SetConcurrency(100000, 10) +w.Write([]byte("hello, world\n")) +w.Close() +``` + +To get any performance gains, you should at least be compressing more than 1 megabyte of data at the time. + +You should at least have a block size of 100k and at least a number of blocks that match the number of cores your would like to utilize, but about twice the number of blocks would be the best. + +Another side effect of this is, that it is likely to speed up your other code, since writes to the compressor only blocks if the compressor is already compressing the number of blocks you have specified. This also means you don't have worry about buffering input to the compressor. + +## Decompression + +Decompression works similar to compression. That means that you simply call pgzip the same way as you would call [compress/gzip](http://golang.org/pkg/compress/gzip). + +The only difference is that if you want to specify your own readahead, you have to use `pgzip.NewReaderN(r io.Reader, blockSize, blocks int)` to get a reader with your custom blocksizes. The `blockSize` is the size of each block decoded, and `blocks` is the maximum number of blocks that is decoded ahead. + +See [Example on playground](http://play.golang.org/p/uHv1B5NbDh) + +Performance +==== +## Compression + +See my blog post in [Benchmarks of Golang Gzip](https://blog.klauspost.com/go-gzipdeflate-benchmarks/). + +Compression cost is usually about 0.2% with default settings with a block size of 250k. + +Example with GOMAXPROC set to 8 (quad core with 8 hyperthreads) + +Content is [Matt Mahoneys 10GB corpus](http://mattmahoney.net/dc/10gb.html). Compression level 6. + +Compressor | MB/sec | speedup | size | size overhead (lower=better) +------------|----------|---------|------|--------- +[gzip](http://golang.org/pkg/compress/gzip) (golang) | 7.21MB/s | 1.0x | 4786608902 | 0% +[gzip](http://github.com/klauspost/compress/gzip) (klauspost) | 10.98MB/s | 1.52x | 4781331645 | -0.11% +[pgzip](https://github.com/klauspost/pgzip) (klauspost) | 50.76MB/s|7.04x | 4784121440 | -0.052% +[bgzf](https://godoc.org/github.com/biogo/hts/bgzf) (biogo) | 38.65MB/s | 5.36x | 4924899484 | 2.889% +[pargzip](https://godoc.org/github.com/golang/build/pargzip) (builder) | 32.00MB/s | 4.44x | 4791226567 | 0.096% + +pgzip also contains a [linear time compression](https://github.com/klauspost/compress#linear-time-compression) mode, that will allow compression at ~150MB per core per second, independent of the content. + +See the [complete sheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) for different content types and compression settings. + +## Decompression + +The decompression speedup is there because it allows you to do other work while the decompression is taking place. + +In the example above, the numbers are as follows on a 4 CPU machine: + +Decompressor | Time | Speedup +-------------|------|-------- +[gzip](http://golang.org/pkg/compress/gzip) (golang) | 1m28.85s | 0% +[pgzip](https://github.com/klauspost/pgzip) (golang) | 43.48s | 104% + +But wait, since gzip decompression is inherently singlethreaded (aside from CRC calculation) how can it be more than 100% faster? Because pgzip due to its design also acts as a buffer. When using ubuffered gzip, you are also waiting for io when you are decompressing. If the gzip decoder can keep up, it will always have data ready for your reader, and you will not be waiting for input to the gzip decompressor to complete. + +This is pretty much an optimal situation for pgzip, but it reflects most common usecases for CPU intensive gzip usage. + +I haven't included [bgzf](https://godoc.org/github.com/biogo/hts/bgzf) in this comparision, since it only can decompress files created by a compatible encoder, and therefore cannot be considered a generic gzip decompressor. But if you are able to compress your files with a bgzf compatible program, you can expect it to scale beyond 100%. + +#License +This contains large portions of code from the go repository - see GO_LICENSE for more information. The changes are released under MIT License. See LICENSE for more information. diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/circle.yml b/Godeps/_workspace/src/github.com/klauspost/pgzip/circle.yml new file mode 100644 index 00000000..67b2b162 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/circle.yml @@ -0,0 +1,7 @@ +test: + pre: + - go vet ./... + + override: + - go test -v -cpu=1,2,4 . + - go test -v -cpu=2 -race -short . \ No newline at end of file diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/gunzip.go b/Godeps/_workspace/src/github.com/klauspost/pgzip/gunzip.go new file mode 100644 index 00000000..45b846e0 --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/gunzip.go @@ -0,0 +1,564 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package pgzip implements reading and writing of gzip format compressed files, +// as specified in RFC 1952. +// +// This is a drop in replacement for "compress/gzip". +// This will split compression into blocks that are compressed in parallel. +// This can be useful for compressing big amounts of data. +// The gzip decompression has not been modified, but remains in the package, +// so you can use it as a complete replacement for "compress/gzip". +// +// See more at https://github.com/klauspost/pgzip +package pgzip + +import ( + "bufio" + "errors" + "hash" + "io" + "sync" + "time" + + "github.com/klauspost/compress/flate" + "github.com/klauspost/crc32" +) + +const ( + gzipID1 = 0x1f + gzipID2 = 0x8b + gzipDeflate = 8 + flagText = 1 << 0 + flagHdrCrc = 1 << 1 + flagExtra = 1 << 2 + flagName = 1 << 3 + flagComment = 1 << 4 +) + +func makeReader(r io.Reader) flate.Reader { + if rr, ok := r.(flate.Reader); ok { + return rr + } + return bufio.NewReader(r) +} + +var ( + // ErrChecksum is returned when reading GZIP data that has an invalid checksum. + ErrChecksum = errors.New("gzip: invalid checksum") + // ErrHeader is returned when reading GZIP data that has an invalid header. + ErrHeader = errors.New("gzip: invalid header") +) + +// The gzip file stores a header giving metadata about the compressed file. +// That header is exposed as the fields of the Writer and Reader structs. +type Header struct { + Comment string // comment + Extra []byte // "extra data" + ModTime time.Time // modification time + Name string // file name + OS byte // operating system type +} + +// A Reader is an io.Reader that can be read to retrieve +// uncompressed data from a gzip-format compressed file. +// +// In general, a gzip file can be a concatenation of gzip files, +// each with its own header. Reads from the Reader +// return the concatenation of the uncompressed data of each. +// Only the first header is recorded in the Reader fields. +// +// Gzip files store a length and checksum of the uncompressed data. +// The Reader will return a ErrChecksum when Read +// reaches the end of the uncompressed data if it does not +// have the expected length or checksum. Clients should treat data +// returned by Read as tentative until they receive the io.EOF +// marking the end of the data. +type Reader struct { + Header + r flate.Reader + decompressor io.ReadCloser + digest hash.Hash32 + size uint32 + flg byte + buf [512]byte + err error + closeErr chan error + multistream bool + + readAhead chan read + roff int // read offset + current []byte + closeReader chan struct{} + lastBlock bool + blockSize int + blocks int + + activeRA bool // Indication if readahead is active + mu sync.Mutex // Lock for above + + blockPool chan []byte +} + +type read struct { + b []byte + err error +} + +// NewReader creates a new Reader reading the given reader. +// The implementation buffers input and may read more data than necessary from r. +// It is the caller's responsibility to call Close on the Reader when done. +func NewReader(r io.Reader) (*Reader, error) { + z := new(Reader) + z.blocks = defaultBlocks + z.blockSize = defaultBlockSize + z.r = makeReader(r) + z.digest = crc32.NewIEEE() + z.multistream = true + z.blockPool = make(chan []byte, z.blocks) + for i := 0; i < z.blocks; i++ { + z.blockPool <- make([]byte, z.blockSize) + } + if err := z.readHeader(true); err != nil { + return nil, err + } + return z, nil +} + +// NewReaderN creates a new Reader reading the given reader. +// The implementation buffers input and may read more data than necessary from r. +// It is the caller's responsibility to call Close on the Reader when done. +// +// With this you can control the approximate size of your blocks, +// as well as how many blocks you want to have prefetched. +// +// Default values for this is blockSize = 250000, blocks = 16, +// meaning up to 16 blocks of maximum 250000 bytes will be +// prefetched. +func NewReaderN(r io.Reader, blockSize, blocks int) (*Reader, error) { + z := new(Reader) + z.blocks = blocks + z.blockSize = blockSize + z.r = makeReader(r) + z.digest = crc32.NewIEEE() + z.multistream = true + + // Account for too small values + if z.blocks <= 0 { + z.blocks = defaultBlocks + } + if z.blockSize <= 512 { + z.blockSize = defaultBlockSize + } + z.blockPool = make(chan []byte, z.blocks) + for i := 0; i < z.blocks; i++ { + z.blockPool <- make([]byte, z.blockSize) + } + if err := z.readHeader(true); err != nil { + return nil, err + } + return z, nil +} + +// Reset discards the Reader z's state and makes it equivalent to the +// result of its original state from NewReader, but reading from r instead. +// This permits reusing a Reader rather than allocating a new one. +func (z *Reader) Reset(r io.Reader) error { + z.killReadAhead() + z.r = makeReader(r) + z.digest = crc32.NewIEEE() + z.size = 0 + z.err = nil + z.multistream = true + + // Account for uninitialized values + if z.blocks <= 0 { + z.blocks = defaultBlocks + } + if z.blockSize <= 512 { + z.blockSize = defaultBlockSize + } + + if z.blockPool == nil { + z.blockPool = make(chan []byte, z.blocks) + for i := 0; i < z.blocks; i++ { + z.blockPool <- make([]byte, z.blockSize) + } + } + + return z.readHeader(true) +} + +// Multistream controls whether the reader supports multistream files. +// +// If enabled (the default), the Reader expects the input to be a sequence +// of individually gzipped data streams, each with its own header and +// trailer, ending at EOF. The effect is that the concatenation of a sequence +// of gzipped files is treated as equivalent to the gzip of the concatenation +// of the sequence. This is standard behavior for gzip readers. +// +// Calling Multistream(false) disables this behavior; disabling the behavior +// can be useful when reading file formats that distinguish individual gzip +// data streams or mix gzip data streams with other data streams. +// In this mode, when the Reader reaches the end of the data stream, +// Read returns io.EOF. If the underlying reader implements io.ByteReader, +// it will be left positioned just after the gzip stream. +// To start the next stream, call z.Reset(r) followed by z.Multistream(false). +// If there is no next stream, z.Reset(r) will return io.EOF. +func (z *Reader) Multistream(ok bool) { + z.multistream = ok +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func get4(p []byte) uint32 { + return uint32(p[0]) | uint32(p[1])<<8 | uint32(p[2])<<16 | uint32(p[3])<<24 +} + +func (z *Reader) readString() (string, error) { + var err error + needconv := false + for i := 0; ; i++ { + if i >= len(z.buf) { + return "", ErrHeader + } + z.buf[i], err = z.r.ReadByte() + if err != nil { + return "", err + } + if z.buf[i] > 0x7f { + needconv = true + } + if z.buf[i] == 0 { + // GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). + if needconv { + s := make([]rune, 0, i) + for _, v := range z.buf[0:i] { + s = append(s, rune(v)) + } + return string(s), nil + } + return string(z.buf[0:i]), nil + } + } +} + +func (z *Reader) read2() (uint32, error) { + _, err := io.ReadFull(z.r, z.buf[0:2]) + if err != nil { + return 0, err + } + return uint32(z.buf[0]) | uint32(z.buf[1])<<8, nil +} + +func (z *Reader) readHeader(save bool) error { + z.killReadAhead() + + _, err := io.ReadFull(z.r, z.buf[0:10]) + if err != nil { + return err + } + if z.buf[0] != gzipID1 || z.buf[1] != gzipID2 || z.buf[2] != gzipDeflate { + return ErrHeader + } + z.flg = z.buf[3] + if save { + z.ModTime = time.Unix(int64(get4(z.buf[4:8])), 0) + // z.buf[8] is xfl, ignored + z.OS = z.buf[9] + } + z.digest.Reset() + z.digest.Write(z.buf[0:10]) + + if z.flg&flagExtra != 0 { + n, err := z.read2() + if err != nil { + return err + } + data := make([]byte, n) + if _, err = io.ReadFull(z.r, data); err != nil { + return err + } + if save { + z.Extra = data + } + } + + var s string + if z.flg&flagName != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Name = s + } + } + + if z.flg&flagComment != 0 { + if s, err = z.readString(); err != nil { + return err + } + if save { + z.Comment = s + } + } + + if z.flg&flagHdrCrc != 0 { + n, err := z.read2() + if err != nil { + return err + } + sum := z.digest.Sum32() & 0xFFFF + if n != sum { + return ErrHeader + } + } + + z.digest.Reset() + z.decompressor = flate.NewReader(z.r) + z.doReadAhead() + return nil +} + +func (z *Reader) killReadAhead() error { + z.mu.Lock() + defer z.mu.Unlock() + if z.activeRA { + if z.closeReader != nil { + close(z.closeReader) + } + + // Wait for decompressor to be closed and return error, if any. + e, ok := <-z.closeErr + z.activeRA = false + if !ok { + // Channel is closed, so if there was any error it has already been returned. + return nil + } + return e + } + return nil +} + +// Starts readahead. +// Will return on error (including io.EOF) +// or when z.closeReader is closed. +func (z *Reader) doReadAhead() { + z.mu.Lock() + defer z.mu.Unlock() + z.activeRA = true + + if z.blocks <= 0 { + z.blocks = defaultBlocks + } + if z.blockSize <= 512 { + z.blockSize = defaultBlockSize + } + ra := make(chan read, z.blocks) + z.readAhead = ra + closeReader := make(chan struct{}, 0) + z.closeReader = closeReader + z.lastBlock = false + closeErr := make(chan error, 1) + z.closeErr = closeErr + z.size = 0 + z.roff = 0 + z.current = nil + decomp := z.decompressor + + go func() { + defer func() { + closeErr <- decomp.Close() + close(closeErr) + close(ra) + }() + + // We hold a local reference to digest, since + // it way be changed by reset. + digest := z.digest + var wg sync.WaitGroup + for { + var buf []byte + select { + case buf = <-z.blockPool: + case <-closeReader: + return + } + buf = buf[0:z.blockSize] + // Try to fill the buffer + n, err := io.ReadFull(decomp, buf) + if err == io.ErrUnexpectedEOF { + err = nil + } + if n < len(buf) { + buf = buf[0:n] + } + wg.Wait() + wg.Add(1) + go func() { + digest.Write(buf) + wg.Done() + }() + z.size += uint32(n) + + // If we return any error, out digest must be ready + if err != nil { + wg.Wait() + } + select { + case z.readAhead <- read{b: buf, err: err}: + case <-closeReader: + // Sent on close, we don't care about the next results + return + } + if err != nil { + return + } + } + }() +} + +func (z *Reader) Read(p []byte) (n int, err error) { + if z.err != nil { + return 0, z.err + } + if len(p) == 0 { + return 0, nil + } + + for { + if len(z.current) == 0 && !z.lastBlock { + read := <-z.readAhead + + if read.err != nil { + // If not nil, the reader will have exited + z.closeReader = nil + + if read.err != io.EOF { + z.err = read.err + return + } + if read.err == io.EOF { + z.lastBlock = true + err = nil + } + } + z.current = read.b + z.roff = 0 + } + avail := z.current[z.roff:] + if len(p) >= len(avail) { + // If len(p) >= len(current), return all content of current + n = copy(p, avail) + z.blockPool <- z.current + z.current = nil + if z.lastBlock { + err = io.EOF + break + } + } else { + // We copy as much as there is space for + n = copy(p, avail) + z.roff += n + } + return + } + + // Finished file; check checksum + size. + if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { + z.err = err + return 0, err + } + crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8]) + sum := z.digest.Sum32() + if sum != crc32 || isize != z.size { + z.err = ErrChecksum + return 0, z.err + } + + // File is ok; should we attempt reading one more? + if !z.multistream { + return 0, io.EOF + } + + // Is there another? + if err = z.readHeader(false); err != nil { + z.err = err + return + } + + // Yes. Reset and read from it. + return z.Read(p) +} + +func (z *Reader) WriteTo(w io.Writer) (n int64, err error) { + total := int64(0) + for { + if z.err != nil { + return total, z.err + } + // We write both to output and digest. + for { + // Read from input + read := <-z.readAhead + if read.err != nil { + // If not nil, the reader will have exited + z.closeReader = nil + + if read.err != io.EOF { + z.err = read.err + return total, z.err + } + if read.err == io.EOF { + z.lastBlock = true + err = nil + } + } + // Write what we got + n, err := w.Write(read.b) + if n != len(read.b) { + return total, io.ErrShortWrite + } + total += int64(n) + if err != nil { + return total, err + } + // Put block back + z.blockPool <- read.b + if z.lastBlock { + break + } + } + + // Finished file; check checksum + size. + if _, err := io.ReadFull(z.r, z.buf[0:8]); err != nil { + z.err = err + return total, err + } + crc32, isize := get4(z.buf[0:4]), get4(z.buf[4:8]) + sum := z.digest.Sum32() + if sum != crc32 || isize != z.size { + z.err = ErrChecksum + return total, z.err + } + // File is ok; should we attempt reading one more? + if !z.multistream { + return total, nil + } + + // Is there another? + err = z.readHeader(false) + if err == io.EOF { + return total, nil + } + if err != nil { + z.err = err + return total, err + } + } +} + +// Close closes the Reader. It does not close the underlying io.Reader. +func (z *Reader) Close() error { + return z.killReadAhead() +} diff --git a/Godeps/_workspace/src/github.com/klauspost/pgzip/gzip.go b/Godeps/_workspace/src/github.com/klauspost/pgzip/gzip.go new file mode 100644 index 00000000..872afe7e --- /dev/null +++ b/Godeps/_workspace/src/github.com/klauspost/pgzip/gzip.go @@ -0,0 +1,485 @@ +// Copyright 2010 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pgzip + +import ( + "bytes" + "errors" + "fmt" + "hash" + "io" + "sync" + + "github.com/klauspost/compress/flate" + "github.com/klauspost/crc32" +) + +const ( + defaultBlockSize = 250000 + tailSize = 16384 + defaultBlocks = 16 +) + +// These constants are copied from the flate package, so that code that imports +// "compress/gzip" does not also have to import "compress/flate". +const ( + NoCompression = flate.NoCompression + BestSpeed = flate.BestSpeed + BestCompression = flate.BestCompression + DefaultCompression = flate.DefaultCompression + ConstantCompression = flate.ConstantCompression +) + +// A Writer is an io.WriteCloser. +// Writes to a Writer are compressed and written to w. +type Writer struct { + Header + w io.Writer + level int + wroteHeader bool + blockSize int + blocks int + currentBuffer []byte + prevTail []byte + digest hash.Hash32 + size int + closed bool + buf [10]byte + err error + pushedErr chan error + results chan result + dictFlatePool *sync.Pool + dstPool *sync.Pool +} + +type result struct { + result chan []byte + notifyWritten chan struct{} +} + +// Use SetConcurrency to finetune the concurrency level if needed. +// +// With this you can control the approximate size of your blocks, +// as well as how many you want to be processing in parallel. +// +// Default values for this is SetConcurrency(250000, 16), +// meaning blocks are split at 250000 bytes and up to 16 blocks +// can be processing at once before the writer blocks. +func (z *Writer) SetConcurrency(blockSize, blocks int) error { + if blockSize <= tailSize { + return fmt.Errorf("gzip: block size cannot be less than or equal to %d", tailSize) + } + if blocks <= 0 { + return errors.New("gzip: blocks cannot be zero or less") + } + z.blockSize = blockSize + z.results = make(chan result, blocks) + z.blocks = blocks + return nil +} + +// NewWriter returns a new Writer. +// Writes to the returned writer are compressed and written to w. +// +// It is the caller's responsibility to call Close on the WriteCloser when done. +// Writes may be buffered and not flushed until Close. +// +// Callers that wish to set the fields in Writer.Header must do so before +// the first call to Write or Close. The Comment and Name header fields are +// UTF-8 strings in Go, but the underlying format requires NUL-terminated ISO +// 8859-1 (Latin-1). NUL or non-Latin-1 runes in those strings will lead to an +// error on Write. +func NewWriter(w io.Writer) *Writer { + z, _ := NewWriterLevel(w, DefaultCompression) + return z +} + +// NewWriterLevel is like NewWriter but specifies the compression level instead +// of assuming DefaultCompression. +// +// The compression level can be DefaultCompression, NoCompression, or any +// integer value between BestSpeed and BestCompression inclusive. The error +// returned will be nil if the level is valid. +func NewWriterLevel(w io.Writer, level int) (*Writer, error) { + if level < ConstantCompression || level > BestCompression { + return nil, fmt.Errorf("gzip: invalid compression level: %d", level) + } + z := new(Writer) + z.SetConcurrency(defaultBlockSize, defaultBlocks) + z.init(w, level) + return z, nil +} + +// This function must be used by goroutines to set an +// error condition, since z.err access is restricted +// to the callers goruotine. +func (z *Writer) pushError(err error) { + z.pushedErr <- err + close(z.pushedErr) +} + +func (z *Writer) init(w io.Writer, level int) { + digest := z.digest + if digest != nil { + digest.Reset() + } else { + digest = crc32.NewIEEE() + } + + *z = Writer{ + Header: Header{ + OS: 255, // unknown + }, + w: w, + level: level, + digest: digest, + pushedErr: make(chan error, 1), + results: make(chan result, z.blocks), + blockSize: z.blockSize, + blocks: z.blocks, + } + z.dictFlatePool = &sync.Pool{ + New: func() interface{} { + f, _ := flate.NewWriterDict(w, level, nil) + return f + }, + } + z.dstPool = &sync.Pool{New: func() interface{} { return make([]byte, 0, z.blockSize) }} + +} + +// Reset discards the Writer z's state and makes it equivalent to the +// result of its original state from NewWriter or NewWriterLevel, but +// writing to w instead. This permits reusing a Writer rather than +// allocating a new one. +func (z *Writer) Reset(w io.Writer) { + if z.results != nil && !z.closed { + close(z.results) + } + z.SetConcurrency(defaultBlockSize, defaultBlocks) + z.init(w, z.level) +} + +// GZIP (RFC 1952) is little-endian, unlike ZLIB (RFC 1950). +func put2(p []byte, v uint16) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) +} + +func put4(p []byte, v uint32) { + p[0] = uint8(v >> 0) + p[1] = uint8(v >> 8) + p[2] = uint8(v >> 16) + p[3] = uint8(v >> 24) +} + +// writeBytes writes a length-prefixed byte slice to z.w. +func (z *Writer) writeBytes(b []byte) error { + if len(b) > 0xffff { + return errors.New("gzip.Write: Extra data is too large") + } + put2(z.buf[0:2], uint16(len(b))) + _, err := z.w.Write(z.buf[0:2]) + if err != nil { + return err + } + _, err = z.w.Write(b) + return err +} + +// writeString writes a UTF-8 string s in GZIP's format to z.w. +// GZIP (RFC 1952) specifies that strings are NUL-terminated ISO 8859-1 (Latin-1). +func (z *Writer) writeString(s string) (err error) { + // GZIP stores Latin-1 strings; error if non-Latin-1; convert if non-ASCII. + needconv := false + for _, v := range s { + if v == 0 || v > 0xff { + return errors.New("gzip.Write: non-Latin-1 header string") + } + if v > 0x7f { + needconv = true + } + } + if needconv { + b := make([]byte, 0, len(s)) + for _, v := range s { + b = append(b, byte(v)) + } + _, err = z.w.Write(b) + } else { + _, err = io.WriteString(z.w, s) + } + if err != nil { + return err + } + // GZIP strings are NUL-terminated. + z.buf[0] = 0 + _, err = z.w.Write(z.buf[0:1]) + return err +} + +// compressCurrent will compress the data currently buffered +// This should only be called from the main writer/flush/closer +func (z *Writer) compressCurrent(flush bool) { + r := result{} + r.result = make(chan []byte, 1) + r.notifyWritten = make(chan struct{}, 0) + z.results <- r + + // If block given is more than twice the block size, split it. + c := z.currentBuffer + if len(c) > z.blockSize*2 { + c = c[:z.blockSize] + go compressBlock(c, z.prevTail, *z, r) + z.prevTail = c[len(c)-tailSize:] + z.currentBuffer = z.currentBuffer[z.blockSize:] + z.compressCurrent(flush) + // Last one flushes if needed + return + } + + go compressBlock(c, z.prevTail, *z, r) + if len(c) > tailSize { + z.prevTail = c[len(c)-tailSize:] + } else { + z.prevTail = nil + } + z.currentBuffer = make([]byte, 0, z.blockSize+(z.blockSize/4)) + + // Wait if flushing + if flush { + _ = <-r.notifyWritten + } +} + +// Returns an error if it has been set. +// Cannot be used by functions that are from internal goroutines. +func (z *Writer) checkError() error { + if z.err != nil { + return z.err + } + select { + case err := <-z.pushedErr: + z.err = err + default: + } + return z.err +} + +// Write writes a compressed form of p to the underlying io.Writer. The +// compressed bytes are not necessarily flushed to output until +// the Writer is closed or Flush() is called. +// +// The function will return quickly, if there are unused buffers. +// The sent slice (p) is copied, and the caller is free to re-use the buffer +// when the function returns. +// +// Errors that occur during compression will be reported later, and a nil error +// does not signify that the compression succeeded (since it is most likely still running) +// That means that the call that returns an error may not be the call that caused it. +// Only Flush and Close functions are guaranteed to return any errors up to that point. +func (z *Writer) Write(p []byte) (int, error) { + if z.checkError() != nil { + return 0, z.err + } + // Write the GZIP header lazily. + if !z.wroteHeader { + z.wroteHeader = true + z.buf[0] = gzipID1 + z.buf[1] = gzipID2 + z.buf[2] = gzipDeflate + z.buf[3] = 0 + if z.Extra != nil { + z.buf[3] |= 0x04 + } + if z.Name != "" { + z.buf[3] |= 0x08 + } + if z.Comment != "" { + z.buf[3] |= 0x10 + } + put4(z.buf[4:8], uint32(z.ModTime.Unix())) + if z.level == BestCompression { + z.buf[8] = 2 + } else if z.level == BestSpeed { + z.buf[8] = 4 + } else { + z.buf[8] = 0 + } + z.buf[9] = z.OS + var n int + n, z.err = z.w.Write(z.buf[0:10]) + if z.err != nil { + return n, z.err + } + if z.Extra != nil { + z.err = z.writeBytes(z.Extra) + if z.err != nil { + return n, z.err + } + } + if z.Name != "" { + z.err = z.writeString(z.Name) + if z.err != nil { + return n, z.err + } + } + if z.Comment != "" { + z.err = z.writeString(z.Comment) + if z.err != nil { + return n, z.err + } + } + // Start receiving data from compressors + go func() { + listen := z.results + for { + r, ok := <-listen + // If closed, we are finished. + if !ok { + return + } + buf := <-r.result + n, err := z.w.Write(buf) + if err != nil { + z.pushError(err) + close(r.notifyWritten) + return + } + if n != len(buf) { + z.pushError(fmt.Errorf("gzip: short write %d should be %d", n, len(buf))) + close(r.notifyWritten) + return + } + z.dstPool.Put(buf) + close(r.notifyWritten) + } + }() + z.currentBuffer = make([]byte, 0, z.blockSize+(z.blockSize/4)) + } + // Handle very large writes in a loop + if len(p) > z.blockSize*z.blocks { + q := p + for len(q) > 0 { + length := len(q) + if length > z.blockSize { + length = z.blockSize + } + z.digest.Write(q[:length]) + z.currentBuffer = append(z.currentBuffer, q[:length]...) + if len(z.currentBuffer) >= z.blockSize { + z.compressCurrent(false) + if z.err != nil { + return len(p) - len(q) - length, z.err + } + } + z.size += length + q = q[length:] + } + return len(p), z.err + } else { + z.size += len(p) + z.digest.Write(p) + z.currentBuffer = append(z.currentBuffer, p...) + if len(z.currentBuffer) >= z.blockSize { + z.compressCurrent(false) + } + return len(p), z.err + } +} + +// Step 1: compresses buffer to buffer +// Step 2: send writer to channel +// Step 3: Close result channel to indicate we are done +func compressBlock(p, prevTail []byte, z Writer, r result) { + defer close(r.result) + buf := z.dstPool.Get().([]byte) + dest := bytes.NewBuffer(buf[:0]) + + compressor := z.dictFlatePool.Get().(*flate.Writer) + compressor.ResetDict(dest, prevTail) + compressor.Write(p) + + err := compressor.Flush() + if err != nil { + z.pushError(err) + return + } + if z.closed { + err = compressor.Close() + if err != nil { + z.pushError(err) + return + } + } + z.dictFlatePool.Put(compressor) + // Read back buffer + buf = dest.Bytes() + r.result <- buf +} + +// Flush flushes any pending compressed data to the underlying writer. +// +// It is useful mainly in compressed network protocols, to ensure that +// a remote reader has enough data to reconstruct a packet. Flush does +// not return until the data has been written. If the underlying +// writer returns an error, Flush returns that error. +// +// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH. +func (z *Writer) Flush() error { + if z.checkError() != nil { + return z.err + } + if z.closed { + return nil + } + if !z.wroteHeader { + _, err := z.Write(nil) + if err != nil { + return err + } + } + // We send current block to compression + z.compressCurrent(true) + if z.checkError() != nil { + return z.err + } + + return nil +} + +// UncompressedSize will return the number of bytes written. +// pgzip only, not a function in the official gzip package. +func (z Writer) UncompressedSize() int { + return z.size +} + +// Close closes the Writer, flushing any unwritten data to the underlying +// io.Writer, but does not close the underlying io.Writer. +func (z *Writer) Close() error { + if z.checkError() != nil { + return z.err + } + if z.closed { + return nil + } + + z.closed = true + if !z.wroteHeader { + z.Write(nil) + if z.err != nil { + return z.err + } + } + z.compressCurrent(true) + if z.checkError() != nil { + return z.err + } + close(z.results) + put4(z.buf[0:4], z.digest.Sum32()) + put4(z.buf[4:8], uint32(z.size)) + _, z.err = z.w.Write(z.buf[0:8]) + return z.err +}