From e7a28c381a933483957e19f94d75d65c948b826a Mon Sep 17 00:00:00 2001 From: akutz Date: Tue, 22 Mar 2016 09:22:30 -0500 Subject: [PATCH] libStorage Refactor (DADD) This patch is the basis for libStorage moving forward -- providing a framework more consistent with an evolutionary take on the Docker HTTP platform. Features include: - Contextual logging - Centralized types - Generated server names - Executor generation based on enforced Go interface types - Librarification of the REX-Ray CLI - Using `make run|run-debug|run-tls|run-tls-debug` to start a server directly from source, from the command line. --- .build/test.sh | 32 - .gitignore | 5 +- .gomk/config.mk | 137 + .gomk/go.mk | 375 +++ .gomk/include/arch.mk | 51 + .gomk/include/cross.mk | 81 + .gomk/include/deps.mk | 76 + .gomk/include/gnixutils.mk | 80 + .gomk/include/tools/deploy/bintray.mk | 8 + .gomk/include/tools/pkg/tgz.mk | 30 + .gomk/include/tools/source/gocyclo.mk | 36 + .gomk/include/tools/source/gofmt.mk | 29 + .gomk/include/tools/source/golint.mk | 42 + .gomk/include/tools/source/govet.mk | 28 + .gomk/include/tools/test/codecov.mk | 50 + .gomk/include/tools/test/coveralls.mk | 50 + .gomk/include/version.mk | 131 + .hound.yml | 3 + .tls/client.extensions | 7 + .tls/libstorage-ca.crt | 37 + .tls/libstorage-ca.key | 54 + .tls/libstorage-client.crt | 31 + .tls/libstorage-client.csr | 18 + .tls/libstorage-client.key | 27 + .tls/libstorage-client.pem | 58 + .tls/libstorage-server.crt | 34 + .tls/libstorage-server.csr | 18 + .tls/libstorage-server.key | 27 + .tls/server.extensions | 7 + .travis.yml | 14 +- Makefile | 48 + api/api.go | 121 +- api/api_args.go | 346 --- api/api_replies.go | 76 - api/client/client.go | 277 ++ api/registry/registry.go | 175 ++ api/server/handlers/handlers_errors.go | 61 + api/server/handlers/handlers_instanceid.go | 87 + .../handlers/handlers_instanceid_validator.go | 53 + .../server/handlers/handlers_logging.go | 172 +- api/server/handlers/handlers_post_args.go | 85 + api/server/handlers/handlers_query_params.go | 66 + .../handlers/handlers_schema_validator.go | 114 + .../handlers/handlers_service_validator.go | 69 + .../handlers/handlers_snapshot_validator.go | 64 + .../handlers/handlers_volume_validator.go | 68 + api/server/httputils/httputils.go | 64 + api/server/httputils/httputils_middleware.go | 30 + api/server/httputils/httputils_route.go | 187 ++ api/server/httputils/httputils_types.go | 36 + api/server/router/driver/driver.go | 58 + api/server/router/driver/driver_ignore.go | 1 + api/server/router/driver/driver_routes.go | 154 ++ api/server/router/root/root.go | 41 + api/server/router/root/root_router.go | 38 + api/server/router/router.go | 9 + api/server/router/service/service.go | 63 + api/server/router/service/service_routes.go | 70 + api/server/router/snapshot/snapshot.go | 113 + api/server/router/snapshot/snapshot_routes.go | 171 ++ api/server/router/volume/volume.go | 202 ++ api/server/router/volume/volume_routes.go | 352 +++ api/server/server.go | 498 ++++ api/server/server_middleware.go | 96 + api/server/server_names.go | 1176 ++++++++ api/types/context/context.go | 266 ++ api/types/drivers/drivers.go | 15 + api/types/drivers/drivers_integration.go | 101 + api/types/drivers/drivers_os.go | 59 + api/types/drivers/drivers_storage.go | 160 ++ api/types/http/http_requests.go | 53 + api/types/http/http_responses.go | 117 + api/types/types_errors.go | 38 + api/{api_model.go => types/types_model.go} | 209 +- api/types/types_store.go | 66 + api/utils/schema/gomk.properties | 10 + api/utils/schema/schema.go | 184 ++ api/utils/schema/schema_generated.go | 452 ++++ api/utils/schema/schema_test.go | 258 ++ api/utils/utils.go | 21 + api/utils/utils_context.go | 41 + api/utils/utils_errors.go | 49 + api/utils/utils_store.go | 245 ++ api/utils/utils_store_test.go | 114 + api/utils/utils_tls.go | 96 + client/client.go | 651 ----- context/context.go | 128 - driver/driver.go | 111 - driver/driver_args.go | 102 - drivers/drivers.go | 6 + drivers/storage/mock/mock.go | 178 ++ drivers/storage/mock/mock_driver.go | 290 ++ drivers/storage/mock/mock_executor.go | 34 + drivers/storage/mock/mock_nodriver.go | 105 + drivers/storage/mock/mock_noop.go | 3 + drivers/storage/storage.go | 6 + glide.yaml | 17 +- libstorage.apib | 2359 +++++++++++++++++ libstorage.go | 48 +- libstorage.json | 446 ++++ libstorage_mock_test.go | 109 + libstorage_nomock_test.go | 13 + libstorage_run_test.go | 21 + libstorage_test.go | 533 +--- libstorage_tests_test.go | 41 + service/server/server.go | 230 -- service/service.go | 398 --- 107 files changed, 12395 insertions(+), 2805 deletions(-) delete mode 100755 .build/test.sh create mode 100644 .gomk/config.mk create mode 100644 .gomk/go.mk create mode 100644 .gomk/include/arch.mk create mode 100644 .gomk/include/cross.mk create mode 100644 .gomk/include/deps.mk create mode 100644 .gomk/include/gnixutils.mk create mode 100644 .gomk/include/tools/deploy/bintray.mk create mode 100644 .gomk/include/tools/pkg/tgz.mk create mode 100644 .gomk/include/tools/source/gocyclo.mk create mode 100644 .gomk/include/tools/source/gofmt.mk create mode 100644 .gomk/include/tools/source/golint.mk create mode 100644 .gomk/include/tools/source/govet.mk create mode 100644 .gomk/include/tools/test/codecov.mk create mode 100644 .gomk/include/tools/test/coveralls.mk create mode 100644 .gomk/include/version.mk create mode 100644 .hound.yml create mode 100644 .tls/client.extensions create mode 100644 .tls/libstorage-ca.crt create mode 100644 .tls/libstorage-ca.key create mode 100644 .tls/libstorage-client.crt create mode 100644 .tls/libstorage-client.csr create mode 100644 .tls/libstorage-client.key create mode 100644 .tls/libstorage-client.pem create mode 100644 .tls/libstorage-server.crt create mode 100644 .tls/libstorage-server.csr create mode 100644 .tls/libstorage-server.key create mode 100644 .tls/server.extensions create mode 100644 Makefile delete mode 100644 api/api_args.go delete mode 100644 api/api_replies.go create mode 100644 api/client/client.go create mode 100644 api/registry/registry.go create mode 100644 api/server/handlers/handlers_errors.go create mode 100644 api/server/handlers/handlers_instanceid.go create mode 100644 api/server/handlers/handlers_instanceid_validator.go rename service/server/handlers/logging.go => api/server/handlers/handlers_logging.go (61%) create mode 100644 api/server/handlers/handlers_post_args.go create mode 100644 api/server/handlers/handlers_query_params.go create mode 100644 api/server/handlers/handlers_schema_validator.go create mode 100644 api/server/handlers/handlers_service_validator.go create mode 100644 api/server/handlers/handlers_snapshot_validator.go create mode 100644 api/server/handlers/handlers_volume_validator.go create mode 100644 api/server/httputils/httputils.go create mode 100644 api/server/httputils/httputils_middleware.go create mode 100644 api/server/httputils/httputils_route.go create mode 100644 api/server/httputils/httputils_types.go create mode 100644 api/server/router/driver/driver.go create mode 100644 api/server/router/driver/driver_ignore.go create mode 100644 api/server/router/driver/driver_routes.go create mode 100644 api/server/router/root/root.go create mode 100644 api/server/router/root/root_router.go create mode 100644 api/server/router/router.go create mode 100644 api/server/router/service/service.go create mode 100644 api/server/router/service/service_routes.go create mode 100644 api/server/router/snapshot/snapshot.go create mode 100644 api/server/router/snapshot/snapshot_routes.go create mode 100644 api/server/router/volume/volume.go create mode 100644 api/server/router/volume/volume_routes.go create mode 100644 api/server/server.go create mode 100644 api/server/server_middleware.go create mode 100644 api/server/server_names.go create mode 100644 api/types/context/context.go create mode 100644 api/types/drivers/drivers.go create mode 100644 api/types/drivers/drivers_integration.go create mode 100644 api/types/drivers/drivers_os.go create mode 100644 api/types/drivers/drivers_storage.go create mode 100644 api/types/http/http_requests.go create mode 100644 api/types/http/http_responses.go create mode 100644 api/types/types_errors.go rename api/{api_model.go => types/types_model.go} (59%) create mode 100644 api/types/types_store.go create mode 100644 api/utils/schema/gomk.properties create mode 100644 api/utils/schema/schema.go create mode 100644 api/utils/schema/schema_generated.go create mode 100644 api/utils/schema/schema_test.go create mode 100644 api/utils/utils.go create mode 100644 api/utils/utils_context.go create mode 100644 api/utils/utils_errors.go create mode 100644 api/utils/utils_store.go create mode 100644 api/utils/utils_store_test.go create mode 100644 api/utils/utils_tls.go delete mode 100644 client/client.go delete mode 100644 context/context.go delete mode 100644 driver/driver.go delete mode 100644 driver/driver_args.go create mode 100644 drivers/drivers.go create mode 100644 drivers/storage/mock/mock.go create mode 100644 drivers/storage/mock/mock_driver.go create mode 100644 drivers/storage/mock/mock_executor.go create mode 100644 drivers/storage/mock/mock_nodriver.go create mode 100644 drivers/storage/mock/mock_noop.go create mode 100644 drivers/storage/storage.go create mode 100644 libstorage.apib create mode 100644 libstorage.json create mode 100644 libstorage_mock_test.go create mode 100644 libstorage_nomock_test.go create mode 100644 libstorage_run_test.go create mode 100644 libstorage_tests_test.go delete mode 100644 service/server/server.go delete mode 100644 service/service.go diff --git a/.build/test.sh b/.build/test.sh deleted file mode 100755 index d3893057..00000000 --- a/.build/test.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash - -echo "mode: set" > acc.out -FAIL=0 - -ROOT_PKG="github.com/emccode/libstorage" -COVER_PKG="$ROOT_PKG" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/api" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/client" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/context" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/driver" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/service" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/service/server" -COVER_PKG=$COVER_PKG,"$ROOT_PKG/service/server/handlers" - -go test -coverpkg=$COVER_PKG -coverprofile=profile.out || FAIL=1 -if [ -f profile.out ]; then - cat profile.out | grep -v "mode: set" >> acc.out - rm -f profile.out -fi - -if [ "$FAIL" -ne 0 ]; then - rm -f profile.out acc.out - exit 1 -fi - -if [ -n "$COVERALLS" -a "$FAIL" -eq "0" ]; then - goveralls -v -coverprofile=acc.out -fi - -rm -f profile.out acc.out -exit $FAIL diff --git a/.gitignore b/.gitignore index 0df9a2df..34d32904 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ +libstorage.paw .site/ site/ -.build/*/ -.build/*-filtered.json vendor/ .project +glide.lock +.gomk/tmp # Created by https://www.gitignore.io diff --git a/.gomk/config.mk b/.gomk/config.mk new file mode 100644 index 00000000..df5dda68 --- /dev/null +++ b/.gomk/config.mk @@ -0,0 +1,137 @@ +ifneq (1,$(IS_GO_MK_CONFIG_LOADED)) + +# note that the file is loaded +IS_GO_MK_CONFIG_LOADED := 1 + +EMPTY := +SPACE := $(EMPTY) $(EMPTY) + +# set the make flags +RECURSIVE := 0 +ifneq (,$(filter %-all,$(MAKECMDGOALS))) + RECURSIVE := 1 +else +ifneq (,$(word 2,$(filter %_amd64,$(MAKECMDGOALS)))) + RECURSIVE := 1 +else +ifneq (,$(word 2,$(filter %_386,$(MAKECMDGOALS)))) + RECURSIVE := 1 +endif +endif +endif + +MAKEFLAGS := --no-print-directory +ifeq (1,$(RECURSIVE)) +MAKEFLAGS += --output-sync=recurse +endif + +ifeq (,$(CURDIR)) +CURDIR := $(shell pwd) +endif + +# the build platforms for which to perform the cross-* goals +BUILD_PLATFORMS ?= Linux-x86_64 Darwin-x86_64 + +# an ordered, space-delimited list of the go package directories to build and +# install. the root package should be a '.', and all the remaining packages +# should be in the form of './subdir1', './subdir1/subdir1a', './subdir2' +# +# if this variable is empty then it will be populated automatically by using +# the find command to find all directories not beginning with a leading '.' or +# '_' and named 'vendor' or 'doc'. +GO_PKG_DIRS ?= + +# an ordered, space-delimited list of directory patterns that should be ignored. +# for more information on the pattern matching scheme used, search for help on +# the makefile text functions, specifically the filter-out function. +GO_PKG_DIRS_IGNORE_PATTS ?= ./vendor% ./docs% + +# the suffix to append to the related make targets for a package that has the +# same name as its parent package +GO_DUPLICATE_PKG_SUFFIX ?= -cli + +# flags indicating whether or not the following tools are executed against +# the project's source files. valid values are 1 (enabled) and 0 (disabled). +GO_FMT_ENABLED ?= 1 +GO_LINT_ENABLED ?= 1 +GO_CYCLO_ENABLED ?= 1 +GO_VET_ENABLED ?= 1 + +# flags indicating whether or not the following tools are used to package +# artifacts after a successful build. valid values are 1 (enabled) and 0 +# (diabled). +PKG_TGZ_ENABLED ?= 1 +PKG_TGZ_EXTENSION ?= .tar.gz + +# flag indicating whether or not test coverage results are submitted to +# coveralls. valid values are 1 (enabled) and 0 (disabled). please note that +# even if coveralls is enabled, it will be disabled if the build is not +# running on the travis-ci build system. coveralls is also disabled if no +# tests are detected or all tests are excluded +COVERALLS_ENABLED ?= 0 + +# flag indicating whether or not test coverage results are submitted to +# codecov. valid values are 1 (enabled) and 0 (disabled). please note that +# even if codecov is enabled, it will be disabled if the build is not +# running on the travis-ci build system. codecov is also disabled if no +# tests are detected or all tests are excluded +CODECOV_ENABLED ?= 1 + +# a space-delimited list of coverage profile files to exclude when submitting +# results to coverage providers +COVERAGE_EXCLUDE ?= + +# a flag indicating whether or not to use glide for dependency management. +# if a glide.yaml file is detected glide is automatically used unless this +# flag indicates it should be disabled. valid values are 1 (enabled) and 0 +# (disabled) +GLIDE_ENABLED ?= 1 + +# the version of glide to install. glide releases can be found at +# https://github.com/Masterminds/glide/releases. this variable is only used +# if glide is used, which is determined by the presence of the file glide.yaml. +GLIDE_VERSION ?= 0.9.1 + +# the indent to print prior to echoing commands +INDENT_LEN ?= + +# the tags for building go code +ifeq ($(origin GO_TAGS),undefined) +GO_TAGS ?= mock driver +endif +STRIP_GO_TAGS = $(subst $(COMMA)_,$(COMMA)$(SPACE),$(patsubst -tags%,,$(subst -tags$(SPACE),-tags_,$(subst $(COMMA)$(SPACE),$(COMMA)_,$1)))) + +# flags to use with go build +ifeq ($(origin GO_BUILD_FLAGS),undefined) +GO_BUILD_FLAGS ?= +endif +GO_BUILD_FLAGS := $(call STRIP_GO_TAGS,$(GO_BUILD_FLAGS)) + +# flags to use with go install +ifeq ($(origin GO_INSTALL_FLAGS),undefined) +GO_INSTALL_FLAGS ?= +endif +GO_INSTALL_FLAGS := $(call STRIP_GO_TAGS,$(GO_INSTALL_FLAGS)) + +# flags to use with go clean +ifeq ($(origin GO_CLEAN_FLAGS),undefined) +GO_CLEAN_FLAGS ?= -i +endif + +# flags to use when building tests +ifeq ($(origin GO_TEST_BUILD_FLAGS),undefined) +GO_TEST_BUILD_FLAGS ?= +endif +GO_TEST_BUILD_FLAGS := $(call STRIP_GO_TAGS,$(GO_TEST_BUILD_FLAGS)) + +# flags to use when executing tests +ifeq ($(origin GO_TEST_RUN_FLAGS),undefined) +GO_TEST_RUN_FLAGS ?= -test.v +endif +GO_TEST_RUN_FLAGS := $(call STRIP_GO_TAGS,$(GO_TEST_RUN_FLAGS)) + +# a flag indicating whether or not to execute go get during a dependency goal. +# valid values are 1 (enabled) and 0 (disabled) +GO_GET_ENABLED ?= 1 + +endif diff --git a/.gomk/go.mk b/.gomk/go.mk new file mode 100644 index 00000000..3d0280b5 --- /dev/null +++ b/.gomk/go.mk @@ -0,0 +1,375 @@ +ifneq (1,$(IS_GO_MK_LOADED)) + +# note that the file is loaded +IS_GO_MK_LOADED := 1 + +# enable go 1.5 vendoring +export GO15VENDOREXPERIMENT := 1 + +# include the main configuration file. this is where modules will put settings +# that should be readily apparent to users +include .gomk/config.mk + +GO_BIN ?= $(GOPATH)/bin +GO_SRC := $(GOPATH)/src +GO_PKG := $(GOPATH)/pkg + +# the INDENT "function" enables indenting commands prior to them being +# echoed +ifeq (,$(INDENT_LEN)) +INDENT_CHARS := "" +else +INDENT_CHARS := "$(foreach i,$(INDENT_LEN), )" +endif +ifeq (1,$(RECURSIVE)) +INDENT_LEN := "$(strip . . $(INDENT_LEN))" +endif +INDENT := printf $(INDENT_CHARS) + +GO_VENDOR_DIR := vendor +$(GO_VENDOR_DIR)-clean: + @$(INDENT) + $(RM) -f -d $(GO_VENDOR_DIR) +GO_CLOBBER += $(GO_VENDOR_DIR)-clean + +# note that GO_CLEAN should be invoked as part of GO_CLOBBER +GO_CLOBBER += $(GO_CLEAN) + +# the path to the directory that contains the modular gomk makefiles +GOMK_I := .gomk/include + +################################################################################ +## STD *NIX UTILS ## +################################################################################ +RM := rm +MKDIR := mkdir +TOUCH := touch +ENV := env +MV := mv +CP := cp +TAR := tar +CURL := curl +CAT := cat +GREP := grep +SED := sed +DATE := date + +################################################################################ +## PRE-BUILD RULES INCLUDES ## +################################################################################ + +# include the basic makefile pieces +include $(GOMK_I)/arch.mk \ + $(GOMK_I)/gnixutils.mk + +################################################################################ +## GOMK TEMP DIRS ## +################################################################################ +GO_MK_TMP_DIR := .gomk/tmp +$(GO_MK_TMP_DIR)-clobber: + @$(INDENT) + $(RM) -f -d $(GO_MK_TMP_DIR) +GO_CLOBBER += $(GO_MK_TMP_DIR)-clobber + +# indicate where marker files are stored +GO_MARKERS_DIR := $(GO_MK_TMP_DIR)/markers + +# indicate where tests and test coverage output is stored +GO_TESTS_DIR := $(GO_MK_TMP_DIR)/tests + +# a temporary build area +GO_TMP_BUILD_DIR := $(GO_MK_TMP_DIR)/build +GO_TMP_BUILD_BIN_DIR := $(GO_TMP_BUILD_DIR)/bin/$(GOOS)_$(GOARCH) +GO_TMP_BUILD_PKG_DIR := $(GO_TMP_BUILD_DIR)/pkg/$(GOOS)_$(GOARCH) + +################################################################################ +## POST-GOMK TEMP DIRS INCLUDES ## +################################################################################ +include $(GOMK_I)/deps.mk \ + $(GOMK_I)/version.mk + +# include all the tools used on the source files +include $(GOMK_I)/tools/source/*.mk + +################################################################################ +## PACKAGE INCLUDES ## +################################################################################ +include $(GOMK_I)/tools/pkg/*.mk + +# pkg area +GO_TMP_PKG_DIR := $(GO_MK_TMP_DIR)/pkg/$(V_OS)_$(V_ARCH) + +################################################################################ +## FUNCS ## +################################################################################ + +# this function returns the path to a go source file's related marker file +GO_TOOL_MARKER = $(GO_MARKERS_DIR)/$(subst ./,,$(dir $(1))$(notdir $(1)).$(2)) + +# this function updates a marker file +GO_TOUCH_MARKER = $(MKDIR) -p $(dir $(1)) && $(TOUCH) $(1) + +# recursively list the contents of a directory +RLSDIR = $(wildcard $1) $(foreach d,$(wildcard $1*),$(call RLSDIR,$d/)) + +################################################################################ +## GO PROJ VARS ## +################################################################################ + +# for examples of how the following varibles are initialized, assume the +# project is github.com/akutz/gomk + +# gomk +GO_PROJ_NAME := $(notdir $(CURDIR)) + +# github.com/akutz/gomk +GO_PROJ_IMPORT_PATH := $(subst $(GO_SRC)/,,$(CURDIR)) + +# $GOPATH/pkg/$GOOS_$GOARCH/github.com/akutz/gomk +GO_PROJ_PKG_PATH := $(GO_PKG)/$(GOOS)_$(GOARCH)/$(GO_PROJ_IMPORT_PATH) + +# $GOPATH/pkg/$GOOS_$GOARCH/github.com/akutz/ +GO_PROJ_PKG_PARENT_PATH := $(dir $(GO_PROJ_PKG_PATH)) + +# if the GO_PKG_DIRS variable is empty then discover all sub-directories except +# the ones that should be specifically excluded +ifeq (,$(strip $(GO_PKG_DIRS))) +GO_PKG_DIRS := $(filter-out $(GO_PKG_DIRS_IGNORE_PATTS),$(sort $(dir $(call RLSDIR,./)))) +GO_PKG_DIRS := $(addsuffix ...,$(GO_PKG_DIRS)) +GO_PKG_DIRS := $(subst /...,,$(GO_PKG_DIRS)) +endif + +PATHS_MANAGED_BY_MAKEFILE := + +################################################################################ +## GO BUILD / INSTALL ## +################################################################################ + +define GO_TOOL_DEF +ifeq (1,$$($2_ENABLED)) +$$(foreach sf,$3,$$(eval $$(call $2,$$(sf),$1$4))) +GO_SRC_TOOL_MARKERS_$1$4 += $$($2_MARKER_PATHS_$1$4) +GO_SRC_TOOL_MARKERS += $$(GO_SRC_TOOL_MARKERS_$1$4) +endif +endef + +define GO_PROJ_BUILD_ARCHIVE + +ifeq ($1,.) +GO_PKG_NAME_$1 := $$(GO_PROJ_NAME) +GO_PKG_NAME_FULL_$1 := $$(GO_PROJ_NAME) +else +GO_PKG_NAME_$1 := $$(subst ./,,$1) +GO_PKG_NAME_FULL_$1 := $$(GO_PROJ_NAME)/$$(GO_PKG_NAME_$1) +endif + +ifneq (,$$(wildcard $1/gomk.properties)) +include $1/gomk.properties +endif + +ifeq (,$$(GO_BUILD_OUTPUT_FILE_$1)) +GO_PKG_ARCHIVE_PATH_$1 := $$(GO_PKG_NAME_$1).a +GO_PKG_ARCHIVE_PATH_FULL_$1 := $$(GO_PKG_NAME_FULL_$1).a +GO_BUILD_OUTPUT_FILE_$1 := $$(GO_PROJ_PKG_PARENT_PATH)$$(GO_PKG_ARCHIVE_PATH_FULL_$1) +endif + +ifeq (,$$(GO_BUILD_OUTPUT_TMP_FILE_$1)) +GO_BUILD_OUTPUT_TMP_FILE_$1 := $$(GO_TMP_BUILD_PKG_DIR)/$$(GO_PKG_ARCHIVE_PATH_FULL_$1) +endif + +ifeq (,$$(GO_INSTALL_FLAGS_$1)) +GO_INSTALL_FLAGS_$1 := $$(GO_INSTALL_FLAGS) +endif +GO_INSTALL_FLAGS_$1 := $$(call STRIP_GO_TAGS,$$(GO_INSTALL_FLAGS_$1)) + +ifeq (,$$(GO_BUILD_FLAGS_$1)) +GO_BUILD_FLAGS_$1 := $$(GO_BUILD_FLAGS) +endif +GO_BUILD_FLAGS_$1 := $$(call STRIP_GO_TAGS,$$(GO_BUILD_FLAGS_$1)) + +ifneq (,$$(GO_TAGS)) +GO_INSTALL_FLAGS_$1 += -tags '$$(GO_TAGS)' +GO_BUILD_FLAGS_$1 += -tags '$$(GO_TAGS)' +endif + +GO_PKG_ALL_SOURCE_FILES_$1 += $$(wildcard $1/*.go) +GO_PKG_ALL_SOURCE_FILES_$1 := $$(sort $$(GO_PKG_ALL_SOURCE_FILES_$1)) +GO_PKG_SOURCE_FILES_$1 := $$(filter-out %_test.go,$$(GO_PKG_ALL_SOURCE_FILES_$1)) +GO_PKG_TEST_FILES_$1 := $$(filter %_test.go,$$(GO_PKG_ALL_SOURCE_FILES_$1)) + +# handle possible, duplicate target names +ifneq ($1,./$$(GO_PROJ_NAME)) +GO_PKG_NAME_$1_TARGET_NAME := $$(GO_PKG_NAME_$1) +else +GO_PKG_NAME_$1_TARGET_NAME := $$(GO_PKG_NAME_$1)$$(GO_DUPLICATE_PKG_SUFFIX) +endif + +# check to see if the build target is an archive, shared-object, or executable +# binary +ifeq (,$$(filter-out %.a,$$(GO_BUILD_OUTPUT_FILE_$1))) + GO_PKG_IS_ARCHIVE_$1 := 1 +else + ifeq (,$$(filter-out %.so,$$(GO_BUILD_OUTPUT_FILE_$1))) + GO_PKG_IS_SHARED_OBJ_$1 := 1 + else + GO_PKG_IS_EXE_FILE_$1 := 1 + endif +endif + +# indicate which tools should be executed against the source files +$$(foreach t,$$(GO_BUILD_DEP_RULES),$$(eval $$(call GO_TOOL_DEF,$1,$$(t),$$(GO_PKG_SOURCE_FILES_$1)))) + +$$(GO_PKG_SOURCE_FILES_$1):: $$(GO_GET_MARKERS) + @$(TOUCH) $$@ + +# go install +$$(GO_PKG_NAME_$1_TARGET_NAME): $$(GO_BUILD_OUTPUT_FILE_$1) +ifeq (0,$$(MAKELEVEL)) +$$(GO_BUILD_OUTPUT_FILE_$1): $$(GO_SRC_TOOL_MARKERS_$1) +endif +$$(GO_BUILD_OUTPUT_FILE_$1): $$(GO_PKG_SOURCE_FILES_$1) +ifneq (1,$$(GO_PKG_IS_EXE_FILE_$1)) + @$(INDENT) + go build $$(GO_INSTALL_FLAGS_$1) -o $$@ $1 +else +ifeq (,$$(wildcard $$(dir $$(GO_BUILD_OUTPUT_FILE_$1)))) + @$(MKDIR) -p $$(@D) +endif + @$(INDENT) + go build $$(GO_INSTALL_FLAGS_$1) -o $$@ $1 +endif +GO_INSTALL += $$(GO_PKG_NAME_$1_TARGET_NAME) + +# go build +$$(GO_PKG_NAME_$1_TARGET_NAME)-build: $$(GO_BUILD_OUTPUT_TMP_FILE_$1) +ifeq (0,$$(MAKELEVEL)) +$$(GO_BUILD_OUTPUT_TMP_FILE_$1): $$(GO_SRC_TOOL_MARKERS_$1) +endif +$$(GO_BUILD_OUTPUT_TMP_FILE_$1): $$(GO_PKG_SOURCE_FILES_$1) +ifeq (,$$(wildcard $$(@D))) + @$(MKDIR) -p $$(@D) +endif + @$(INDENT) + go build $$(GO_BUILD_FLAGS_$1) -o $$@ $1 +GO_BUILD += $$(GO_PKG_NAME_$1_TARGET_NAME)-build + +# go clean +$$(GO_PKG_NAME_$1_TARGET_NAME)-clean: + @$(INDENT) + go clean $(GO_CLEAN_FLAGS) $1 +ifneq (,$$(strip $$(GO_BUILD_OUTPUT_FILE_$1))) + @$(INDENT) + $(RM) -f $$(GO_BUILD_OUTPUT_FILE_$1) +endif +ifneq (,$$(strip $$(GO_BUILD_OUTPUT_TMP_FILE_$1))) + @$(INDENT) + $(RM) -f $$(GO_BUILD_OUTPUT_TMP_FILE_$1) +endif +GO_CLEAN += $$(GO_PKG_NAME_$1_TARGET_NAME)-clean + +ifeq (1,$$(GO_PKG_TGZ_$1)) +$$(eval $$(call GO_PKG_TGZ_RULE,$$(GO_BUILD_OUTPUT_FILE_$1),$$(GO_PKG_NAME_$1_TARGET_NAME))) +endif + +# create the cross-install, cross-build, and cross-clean goals for this pkg +#$$(foreach bp,$$(BUILD_PLATFORMS),$$(eval $$(call CROSS_RULES,$$(bp),$$(GO_PKG_NAME_$1),$$(GO_PKG_NAME_$1_TARGET_NAME)))) + +################################################################################ +## GO TEST ## +################################################################################ +ifneq (,$$(GO_PKG_TEST_FILES_$1)) + +# indicate which tools should be executed against the test source files +$$(foreach t,$$(GO_BUILD_DEP_RULES),$$(eval $$(call GO_TOOL_DEF,$1,$$(t),$$(GO_PKG_TEST_FILES_$1),_TEST))) + +ifeq (.,$1) +GO_PKG_TEST_PATH_$1 := $$(GO_PKG_NAME_$1).test +GO_PKG_TEST_$1 := $$(GO_PKG_NAME_$1)-test +else +GO_PKG_TEST_PATH_$1 := $$(GO_PKG_NAME_$1)/$$(notdir $$(GO_PKG_NAME_$1)).test +GO_PKG_TEST_$1 := $$(GO_PKG_NAME_$1)/$$(notdir $$(GO_PKG_NAME_$1))-test +endif +GO_PKG_TEST_PATH_FULL_$1 := $$(GO_TESTS_DIR)/$$(GO_PKG_TEST_PATH_$1) + +GO_PKG_COVER_PROFILE_$1 := $$(GO_PKG_TEST_PATH_FULL_$1).out + +ifeq (,$$(findstring $$(GO_PKG_COVER_PROFILE_$1),$$(COVERAGE_EXCLUDE))) +GO_COVER_PROFILES += $$(GO_PKG_COVER_PROFILE_$1) +endif + +ifneq (,$$(GO_TEST_COVER_PKGS_$1)) +GO_TEST_COVER_PKGS_$1 := -coverpkg $$(GO_TEST_COVER_PKGS_$1) +endif + +$$(GO_PKG_TEST_PATH_$1): $$(GO_PKG_TEST_PATH_FULL_$1) +$$(GO_PKG_TEST_PATH_FULL_$1): $$(GO_SRC_TOOL_MARKERS_$1_TEST) \ + $$(GO_PKG_TEST_FILES_$1) \ + $$(GO_BUILD_OUTPUT_FILE_$1) + @$(INDENT) +ifeq (,$$(GO_TAGS)) + go test $$(GO_TEST_BUILD_FLAGS) -cover -c $1 -o $$@ $$(GO_TEST_COVER_PKGS_$1) +else + go test -tags '$$(GO_TAGS)' $$(GO_TEST_BUILD_FLAGS) -cover -c $1 -o $$@ $$(GO_TEST_COVER_PKGS_$1) +endif + +$$(GO_PKG_TEST_$1): $$(GO_PKG_COVER_PROFILE_$1) +$$(GO_PKG_COVER_PROFILE_$1): $$(GO_PKG_TEST_PATH_FULL_$1) + @$(INDENT) + $$? $(GO_TEST_RUN_FLAGS) -test.coverprofile $$@ + +$$(GO_PKG_TEST_FILES_$1): $$(GO_PKG_SOURCE_FILES_$1) + @$(TOUCH) $$@ + +$$(GO_PKG_TEST_PATH_$1)-clean: + @$(INDENT) + $(RM) -f $$(GO_PKG_TEST_PATH_FULL_$1) $$(GO_PKG_COVER_PROFILE_$1) +ifneq (,$$(strip $$(GO_SRC_TOOL_MARKERS_$1))) + @$(INDENT) + $(RM) -f $$(GO_SRC_TOOL_MARKERS_$1_TEST) +endif + +GO_TEST_BUILD += $$(GO_PKG_TEST_PATH_$1) +GO_TEST += $$(GO_PKG_TEST_$1) +GO_TEST_CLEAN += $$(GO_PKG_TEST_PATH_$1)-clean +GO_CLEAN += $$(GO_PKG_TEST_PATH_$1)-clean + +endif +endef + +define GO_PROJ_BUILD_RULES + +# if the current path has a makefile in it, then we should note that +ifneq (.,$1) +ifneq (,$$(wildcard $1/Makefile)) +PATHS_MANAGED_BY_MAKEFILE += $1% +endif +endif + +# do not build the current path if it or any parent path has a Makefile present +ifeq ($1,$$(filter-out $$(PATHS_MANAGED_BY_MAKEFILE),$1)) +$$(eval $$(call GO_PROJ_BUILD_ARCHIVE,$(1))) +endif + +endef + +# execute the build rules +$(foreach gpp,$(GO_PKG_DIRS),$(eval $(call GO_PROJ_BUILD_RULES,$(gpp)))) + +# only clean the targets in GO_CLEAN_ONCE at the initial make call +ifeq (0,$(MAKELEVEL)) +GO_CLEAN += $(GO_CLEAN_ONCE) +endif + +################################################################################ +## POST-BUILD RULES INCLUDES ## +################################################################################ + +# include all the tools used during test processing +include $(GOMK_I)/tools/test/*.mk + +################################################################################ +## CROSS-BUILD INCLUDES ## +################################################################################ +include $(GOMK_I)/cross.mk + +endif diff --git a/.gomk/include/arch.mk b/.gomk/include/arch.mk new file mode 100644 index 00000000..ea8461bd --- /dev/null +++ b/.gomk/include/arch.mk @@ -0,0 +1,51 @@ +ifneq (1,$(IS_GOMK_ARCH_LOADED)) + +# note that the file is loaded +IS_GOMK_ARCH_LOADED := 1 + +# set the go os and architecture types as well the sed command to use based on +# the os and architecture types +ifeq ($(OS),Windows_NT) + SYS_GOOS := windows + export GOOS ?= windows + ifeq ($(PROCESSOR_ARCHITECTURE),AMD64) + SYS_GOARCH := amd64 + export GOARCH ?= amd64 + endif + ifeq ($(PROCESSOR_ARCHITECTURE),x86) + SYS_GOARCH := 386 + export GOARCH ?= 386 + endif +else + UNAME_S := $(shell uname -s) + ifeq ($(UNAME_S),Linux) + SYS_GOOS := linux + export GOOS ?= linux + endif + ifeq ($(UNAME_S),Darwin) + SYS_GOOS := darwin + export GOOS ?= darwin + SYS_GOARCH := amd64 + export GOARCH ?= amd64 + endif + + UNAME_P := $(shell uname -p) + ifeq (,$(SYS_GOARCH)) + ifeq ($(UNAME_P),x86_64) + SYS_GOARCH := amd64 + endif + ifneq ($(filter %86,$(UNAME_P)),) + SYS_GOARCH := 386 + endif + endif + + ifeq ($(origin GOARCH), undefined) + ifeq ($(UNAME_P),x86_64) + export GOARCH = amd64 + endif + ifneq ($(filter %86,$(UNAME_P)),) + export GOARCH = 386 + endif + endif +endif +endif diff --git a/.gomk/include/cross.mk b/.gomk/include/cross.mk new file mode 100644 index 00000000..d50ebebd --- /dev/null +++ b/.gomk/include/cross.mk @@ -0,0 +1,81 @@ +ifneq (1,$(IS_GOMK_CROSS_LOADED)) + +# note that the file is loaded +IS_GOMK_CROSS_LOADED := 1 + +define CROSS_RULES +X_BP_$1 := $$(subst -, ,$1) +X_BP_OS_$1 := $$(firstword $$(X_BP_$1)) +X_BP_ARCH_$1 := $$(lastword $$(X_BP_$1)) + +ifeq (Linux,$$(X_BP_OS_$1)) + X_GOOS_$1 := linux +else + ifeq (Darwin,$$(X_BP_OS_$1)) + X_GOOS_$1 := darwin + else + ifeq (Windows,$$(X_BP_OS_$1)) + X_GOOS_$1 := windows + endif + endif +endif + +ifeq (i386,$$(X_BP_ARCH_$1)) + X_GOARCH_$1 := 386 +else + ifeq (x86_64,$$(X_BP_ARCH_$1)) + X_GOARCH_$1 := amd64 + endif +endif + +ifeq (1,$(RECURSIVE)) + RECURSIVE_INDENT := INDENT_LEN=$$(INDENT_LEN) +endif + +ifeq (,$2) + +install-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_DEPS) $$(GO_SRC_TOOL_MARKERS) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) install +GO_CROSS_INSTALL += install-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +build-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_DEPS) $$(GO_SRC_TOOL_MARKERS) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) build +GO_CROSS_BUILD += build-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +package-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_DEPS) $$(GO_SRC_TOOL_MARKERS) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) package +GO_CROSS_PACKAGE += package-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +clean-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_CLEAN_ONCE) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) clean +GO_CROSS_CLEAN += clean-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +else + +$3-install-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_DEPS) $$(GO_SRC_TOOL_MARKERS) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) $2 install +GO_CROSS_INSTALL += $3-install-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +$3-build-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_DEPS) $$(GO_SRC_TOOL_MARKERS) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) $2 build +GO_CROSS_BUILD += $3-build-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +$3-clean-$$(X_GOOS_$1)_$$(X_GOARCH_$1): $$(GO_CLEAN_ONCE) + $$(ENV) $$(RECURSIVE_INDENT) GOOS=$$(X_GOOS_$1) GOARCH=$$(X_GOARCH_$1) $$(MAKE) $2 clean +GO_CROSS_CLEAN += $3-clean-$$(X_GOOS_$1)_$$(X_GOARCH_$1) + +endif + +endef + +$(foreach bp,$(BUILD_PLATFORMS),$(eval $(call CROSS_RULES,$(bp)))) +GO_PHONY += $(GO_CROSS_INSTALL) $(GO_CROSS_BUILD) \ + $(GO_CROSS_PACKAGE) $(GO_CROSS_CLEAN) + +install-all: $(GO_CROSS_INSTALL) +build-all: $(GO_CROSS_BUILD) +package-all: $(GO_CROSS_PACKAGE) +clean-all: $(GO_CROSS_CLEAN) +GO_PHONY += install-all build-all package-all clean-all + +endif diff --git a/.gomk/include/deps.mk b/.gomk/include/deps.mk new file mode 100644 index 00000000..2904dfa8 --- /dev/null +++ b/.gomk/include/deps.mk @@ -0,0 +1,76 @@ +ifneq (1,$(IS_GO_MK_DEPS_LOADED)) + +# note that the file is loaded +IS_GO_MK_DEPS_LOADED := 1 + +################################################################################ +## GLIDE ## +################################################################################ +ifeq (1,$(GLIDE_ENABLED)) +ifneq (,$(wildcard glide.yaml)) + +ifeq (,$(strip $(GLIDE_HOME))) +ifeq (windows,$(SYS_GOOS)) +export GLIDE_HOME=$(USERPROFILE) +else +export GLIDE_HOME=$(HOME) +endif +endif + +GLIDE_BIN := $(GO_BIN)/glide +GLIDE_URL_BASE := https://github.com/Masterminds/glide/releases/download +GLIDE_URL_FILE := glide-$(GLIDE_VERSION)-$(SYS_GOOS)-$(SYS_GOARCH).tar.gz +GLIDE_URL := $(GLIDE_URL_BASE)/$(GLIDE_VERSION)/$(GLIDE_URL_FILE) + +GLIDE_LOCK := glide.lock +GLIDE_YAML := glide.yaml + +$(GLIDE_LOCK): $(GLIDE_YAML) + $(GLIDE_BIN) up + +$(GLIDE_LOCK)-clean: + $(RM) -f $(GLIDE_LOCK) +GO_CLOBBER += $(GLIDE_LOCK)-clean + +$(GLIDE_YAML): $(GLIDE_BIN) + +$(GLIDE_BIN): | $(GNIX_UTILS) + $(MKDIR) -p .glide && \ + cd .glide && \ + $(CURL) -S -L -o $(GLIDE_URL_FILE) $(GLIDE_URL) && \ + $(TAR) -x -v -z -f $(GLIDE_URL_FILE) && \ + $(MKDIR) -p $(GO_BIN) && \ + $(MV) $(SYS_GOOS)-$(SYS_GOARCH)/glide $(GLIDE_BIN) && \ + cd .. && \ + $(RM) -f -d .glide + +GO_GET_MARKERS += $(GLIDE_LOCK) +GO_BUILD_DEPS += $(GLIDE_LOCK) +GO_DEPS += $(GLIDE_LOCK) + +endif +endif + +################################################################################ +## GO GET ## +################################################################################ +ifeq (1,$(GO_GET_ENABLED)) +GO_GET := $(GO_MARKERS_DIR)/go.get + +ifneq (,$(GLIDE_LOCK)) +$(GO_GET): $(GLIDE_LOCK) +endif +$(GO_GET): | $(GNIX_UTILS) + go get -v -d -t $(GO_PKG_DIRS) + @$(call GO_TOUCH_MARKER,$@) + +$(GO_GET)-clean: + $(RM) -f $(GO_GET) +GO_CLOBBER += $(GO_GET)-clean + +GO_GET_MARKERS += $(GO_GET) +GO_BUILD_DEPS += $(GO_GET) +GO_DEPS += $(GO_GET) + +endif +endif diff --git a/.gomk/include/gnixutils.mk b/.gomk/include/gnixutils.mk new file mode 100644 index 00000000..abb4b6a4 --- /dev/null +++ b/.gomk/include/gnixutils.mk @@ -0,0 +1,80 @@ +ifneq (1,$(IS_GOMK_GNIXUTILS_LOADED)) + +# note that the file is loaded +IS_GOMK_GNIXUTILS_LOADED := 1 + +GNIX_SUFFIX := gnix + +RM := $(GO_BIN)/rm-$(GNIX_SUFFIX) +MKDIR := $(GO_BIN)/mkdir-$(GNIX_SUFFIX) +TOUCH := $(GO_BIN)/touch-$(GNIX_SUFFIX) +ENV := $(GO_BIN)/env-$(GNIX_SUFFIX) +MV := $(GO_BIN)/mv-$(GNIX_SUFFIX) +CP := $(GO_BIN)/cp-$(GNIX_SUFFIX) +TAR := $(GO_BIN)/tar-$(GNIX_SUFFIX) +CURL := $(GO_BIN)/curl-$(GNIX_SUFFIX) +CAT := $(GO_BIN)/cat-$(GNIX_SUFFIX) +GREP := $(GO_BIN)/grep-$(GNIX_SUFFIX) +SED := $(GO_BIN)/sed-$(GNIX_SUFFIX) +DATE := $(GO_BIN)/date-$(GNIX_SUFFIX) +PRINTF := $(GO_BIN)/printf-$(GNIX_SUFFIX) +ECHO := $(PRINTF) + +GNIX_UTILS := $(GO_SRC)/github.com/akutz/gnixutils/LICENSE + +$(GNIX_UTILS): + go get -v -d -u github.com/akutz/gnixutils + +$(RM): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/rm +GO_DEPS += $(RM) + +$(MKDIR): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/mkdir +GO_DEPS += $(MKDIR) + +$(TOUCH): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/touch +GO_DEPS += $(TOUCH) + +$(ENV): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/env +GO_DEPS += $(ENV) + +$(MV): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/mv +GO_DEPS += $(MV) + +$(CP): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/cp +GO_DEPS += $(CP) + +$(TAR): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/tar +GO_DEPS += $(TAR) + +$(CURL): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/curl +GO_DEPS += $(CURL) + +$(CAT): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/cat +GO_DEPS += $(CAT) + +$(GREP): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/grep +GO_DEPS += $(GREP) + +$(SED): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/sed +GO_DEPS += $(SED) + +$(DATE): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/date +GO_DEPS += $(DATE) + +$(PRINTF): | $(GNIX_UTILS) + go build -o $@ github.com/akutz/gnixutils/cli/printf +GO_DEPS += $(PRINTF) + +endif diff --git a/.gomk/include/tools/deploy/bintray.mk b/.gomk/include/tools/deploy/bintray.mk new file mode 100644 index 00000000..8638266d --- /dev/null +++ b/.gomk/include/tools/deploy/bintray.mk @@ -0,0 +1,8 @@ +ifneq (1,$(IS_GOMK_BINTRAY_LOADED)) + +# note that the file is loaded +IS_GOMK_BINTRAY_LOADED := 1 + + + +endif diff --git a/.gomk/include/tools/pkg/tgz.mk b/.gomk/include/tools/pkg/tgz.mk new file mode 100644 index 00000000..97465910 --- /dev/null +++ b/.gomk/include/tools/pkg/tgz.mk @@ -0,0 +1,30 @@ +ifneq (1,$(IS_GOMK_TGZ_LOADED)) + +# note that the file is loaded +IS_GOMK_TGZ_LOADED := 1 + +ifeq (1,$(PKG_TGZ_ENABLED)) + +define GO_PKG_TGZ_RULE +GO_PKG_TGZ_NAME_$1 := $2-tgz +GO_PKG_TGZ_FILE_$1 := $$(GO_TMP_PKG_DIR)/$$(notdir $1)-$$(V_OS)-$$(V_ARCH)-$$(V_SEMVER)$$(PKG_TGZ_EXTENSION) + +$$(GO_PKG_TGZ_NAME_$1): $$(GO_PKG_TGZ_FILE_$1) +$$(GO_PKG_TGZ_FILE_$1): $1 | $$(TAR) + @$$(MKDIR) -p $$(@D) + @$$(INDENT) + $$(TAR) -C $$( $@ + $(foreach f,$?,$(GREP) -v "mode: set" $(f) >> $@ &&) true + +$(CODECOV_PROFILE)-clean: + $(RM) -f $(CODECOV_PROFILE) + +GO_COVER := $(CODECOV_MARKER) +$(CODECOV_MARKER): $(CODECOV_PROFILE) + curl -sSL https://codecov.io/bash | bash -s -- -f $? + @$(call GO_TOUCH_MARKER,$@) + +$(CODECOV_MARKER)-clean: + $(RM) -f $(CODECOV_MARKER) + +GO_COVER_CLEAN := $(CODECOV_PROFILE)-clean $(CODECOV_MARKER)-clean +GO_CLEAN += $(GO_COVER_CLEAN) + +endif +endif +endif + +endif diff --git a/.gomk/include/tools/test/coveralls.mk b/.gomk/include/tools/test/coveralls.mk new file mode 100644 index 00000000..dc6ef473 --- /dev/null +++ b/.gomk/include/tools/test/coveralls.mk @@ -0,0 +1,50 @@ +ifneq (1,$(IS_GOMK_COVERALLS_LOADED)) + +IS_GOMK_COVERALLS_LOADED := 1 + +ifeq (true,$(TRAVIS)) +ifeq (1,$(COVERALLS_ENABLED)) +ifneq (,$(GO_COVER_PROFILES)) + +GOCOV_BIN := $(GO_BIN)/gocov +$(GOCOV_BIN): + go get -v github.com/axw/gocov/gocov + +GOVERALLS_BIN := $(GO_BIN)/goveralls +$(GOVERALLS_BIN): + go get -v github.com/mattn/goveralls + +COVER_BIN := $(GO_BIN)/cover +$(COVER_BIN): + go get -v golang.org/x/tools/cmd/cover + +GO_COVER_DEPS := $(GOCOV_BIN) $(GOVERALLS_BIN) $(COVER_BIN) +GO_TEST_DEPS += $(GO_COVER_DEPS) +GO_DEPS += $(GO_COVER_DEPS) + +COVERALLS_PROFILE := $(GO_TESTS_DIR)/coveralls.out +COVERALLS_MARKER := $(GO_MARKERS_DIR)/go.coveralls + +$(COVERALLS_PROFILE): $(GO_COVER_PROFILES) + echo "mode: set" > $@ + $(foreach f,$?,$(GREP) -v "mode: set" $(f) >> $@ &&) true + +$(COVERALLS_PROFILE)-clean: + $(RM) -f $(COVERALLS_PROFILE) + +GO_COVER := $(COVERALLS_MARKER) +$(COVERALLS_MARKER): $(COVERALLS_PROFILE) + goveralls -coverprofile=$? + @$(call GO_TOUCH_MARKER,$@) + +$(COVERALLS_MARKER)-clean: + $(RM) -f $(COVERALLS_MARKER) + +GO_COVER_CLEAN := $(COVERALLS_PROFILE)-clean $(COVERALLS_MARKER)-clean +GO_CLEAN += $(GO_COVER_CLEAN) + +endif +endif +endif + +endif diff --git a/.gomk/include/version.mk b/.gomk/include/version.mk new file mode 100644 index 00000000..c8dbad33 --- /dev/null +++ b/.gomk/include/version.mk @@ -0,0 +1,131 @@ +ifneq (1,$(IS_GOMK_VERSION_LOADED)) + +# note that the file is loaded +IS_GOMK_VERSION_LOADED := 1 + +VERSION_DEPS := $(SED) $(GREP) $(CAT) + +ifneq (,$(wildcard $(VERSION_DEPS))) +ifneq (,$(wildcard .git)) + +include $(GOMK_I)/arch.mk + +# the version's binary os and architecture type +ifeq ($(GOOS),windows) + V_OS := Windows_NT +endif +ifeq ($(GOOS),linux) + V_OS := Linux +endif +ifeq ($(GOOS),darwin) + V_OS := Darwin +endif +ifeq ($(GOARCH),386) + V_ARCH := i386 +endif +ifeq ($(GOARCH),amd64) + V_ARCH := x86_64 +endif +V_OS_ARCH := $(V_OS)-$(V_ARCH) + +# parse a semver +SEMVER_PATT := ^[^\d]*(\d+)\.(\d+)\.(\d+)(?:-([a-zA-Z].+?))?(?:-(\d+)-g(.+?)(?:-(dirty))?)?$$ +PARSE_SEMVER = $(shell echo $(1) | $(SED) -e 's/$(SEMVER_PATT)/$(2)/gim') + +# describe the git information and create a parsing function for it +GIT_DESCRIBE := $(shell git describe --long --dirty 2> /dev/null) +ifeq (,$(GIT_DESCRIBE)) + GIT_DESCRIBE := v0.0.1-dev-0-g1234567-dirty +endif +PARSE_GIT_DESCRIBE = $(call PARSE_SEMVER,$(GIT_DESCRIBE),$(1)) + +# parse the version components from the git information +V_MAJOR := $(call PARSE_GIT_DESCRIBE,$$1) +V_MINOR := $(call PARSE_GIT_DESCRIBE,$$2) +V_PATCH := $(call PARSE_GIT_DESCRIBE,$$3) +V_NOTES := $(call PARSE_GIT_DESCRIBE,$$4) +V_BUILD := $(call PARSE_GIT_DESCRIBE,$$5) +V_SHA_SHORT := $(call PARSE_GIT_DESCRIBE,$$6) +V_DIRTY := $(call PARSE_GIT_DESCRIBE,$$7) + +# the long commit hash +V_SHA_LONG := $(shell git show HEAD -s --format=%H) + +# the branch name, possibly from travis-ci +ifeq ($(origin TRAVIS_BRANCH), undefined) + TRAVIS_BRANCH := $(shell git branch | $(GREP) -F '*' | $(SED) 's/\* //') +else + ifeq ($(strip $(TRAVIS_BRANCH)),) + TRAVIS_BRANCH := $(shell git branch | $(GREP) -F '*' | $(SED) 's/\* //') + endif +endif +ifeq ($(origin TRAVIS_TAG), undefined) + TRAVIS_TAG := $(TRAVIS_BRANCH) +else + ifeq ($(strip $(TRAVIS_TAG)),) + TRAVIS_TAG := $(TRAVIS_BRANCH) + endif +endif +V_BRANCH := $(TRAVIS_TAG) + +# the build date as an epoch +V_EPOCH = $(shell $(DATE) +%s) + +# the build date +V_BUILD_DATE = $(shell $(DATE) -d $(V_EPOCH) +"%a, %d %b %Y %H:%M:%S %Z") + +# the release date as required by bintray +V_RELEASE_DATE = $(shell $(DATE) -d $(V_EPOCH) +"%Y-%m-%d") + +# init the semver +V_SEMVER := $(V_MAJOR).$(V_MINOR).$(V_PATCH) +ifneq ($(V_NOTES),) + V_SEMVER := $(V_SEMVER)-$(V_NOTES) +endif + +# get the version file's version +ifneq (,$(wildcard VERSION)) +V_FILE = $(strip $(shell $(CAT) VERSION 2> /dev/null)) +endif + +# append the build number and dirty values to the semver if appropriate +ifneq (,$(V_BUILD)) + ifneq (0,$(V_BUILD)) + # if the version file's version is different than the version parsed + # from the git describe information then use the version file's + # version + ifneq (,$(wildcard VERSION)) + ifneq ($(V_SEMVER),$(V_FILE)) + V_MAJOR := $(call PARSE_SEMVER,$(V_FILE),$$1) + V_MINOR := $(call PARSE_SEMVER,$(V_FILE),$$2) + V_PATCH := $(call PARSE_SEMVER,$(V_FILE),$$3) + V_NOTES := $(call PARSE_SEMVER,$(V_FILE),$$4) + V_SEMVER := $(V_MAJOR).$(V_MINOR).$(V_PATCH) + ifneq ($(V_NOTES),) + V_SEMVER := $(V_SEMVER)-$(V_NOTES) + endif + endif + endif + V_SEMVER := $(V_SEMVER)+$(V_BUILD) + endif +endif +ifeq ($(V_DIRTY),dirty) + V_SEMVER := $(V_SEMVER)+$(V_DIRTY) +endif + +# the rpm version cannot have any dashes +V_RPM_SEMVER := $(subst -,+,$(V_SEMVER)) + +version: + @echo SemVer: $(V_SEMVER) + @echo RpmVer: $(V_RPM_SEMVER) + @echo Binary: $(V_OS_ARCH) + @echo Branch: $(V_BRANCH) + @echo Commit: $(V_SHA_LONG) + @echo Formed: $(V_BUILD_DATE) +GO_PHONY += version + +endif +endif + +endif diff --git a/.hound.yml b/.hound.yml new file mode 100644 index 00000000..44964e91 --- /dev/null +++ b/.hound.yml @@ -0,0 +1,3 @@ +go: + enabled: true +fail_on_violations: true diff --git a/.tls/client.extensions b/.tls/client.extensions new file mode 100644 index 00000000..bc73c045 --- /dev/null +++ b/.tls/client.extensions @@ -0,0 +1,7 @@ +basicConstraints = CA:FALSE +nsCertType = client, email +nsComment = "OpenSSL Generated Client Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +keyUsage = critical, nonRepudiation, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, emailProtection diff --git a/.tls/libstorage-ca.crt b/.tls/libstorage-ca.crt new file mode 100644 index 00000000..0ff63d6d --- /dev/null +++ b/.tls/libstorage-ca.crt @@ -0,0 +1,37 @@ +-----BEGIN CERTIFICATE----- +MIIGiTCCBHGgAwIBAgIJANAQ2YHcXD/EMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzEOMAwGA1UECBMFVGV4YXMxDzANBgNVBAcTBkF1c3RpbjEMMAoGA1UE +ChMDRU1DMRIwEAYDVQQLFAlFTUN7Y29kZX0xFjAUBgNVBAMTDWxpYnN0b3JhZ2Ut +Y2ExHzAdBgkqhkiG9w0BCQEWEHNha3V0ekBnbWFpbC5jb20wHhcNMTYwMzE2MTAy +ODU4WhcNMjYwMzE0MTAyODU4WjCBiTELMAkGA1UEBhMCVVMxDjAMBgNVBAgTBVRl +eGFzMQ8wDQYDVQQHEwZBdXN0aW4xDDAKBgNVBAoTA0VNQzESMBAGA1UECxQJRU1D +e2NvZGV9MRYwFAYDVQQDEw1saWJzdG9yYWdlLWNhMR8wHQYJKoZIhvcNAQkBFhBz +YWt1dHpAZ21haWwuY29tMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA +oQxXtfcdtaEv9CI0mMK0p14Mg+QIKrvndpPHwEZrx1NVIohQDWHFuVCaTcxpagAN +C8FYB3nV8ZXXFMWioTGQ57Py7ip3EB8GPIjPFr2cGt8obo3fHNlXMdZWp+N6Z2S9 +JFdKbGyUWoexKNebWh9WsEhLWlzMJg7zP7FwuJFjvc9XtdAIib7eZRNN2rwmU4Hc +478RoeGY5VlJcuyuB/UTQDLAy/rj8RvsMrMp8OkaavcbvVodK5W/l/rTkXhObl+V +wTBPiAWB6R0vJ9DptBwmtRH2MC64W6OThPGj04UHVMwUvYFFNG1qMYsiJMTN2Kxb +D9JvldwvC3ZAouI9PjwugCfqxsHL87yuFchEQKgts58Mx7uXo68pqQuxijgTdSH2 +cjux6fnhb2Cz6jl3VNd6cYxstIUR5Q4YdXJNm/V5Mr1/clSwJ+1QsuCyIHHzygrc +2ST8gzpQzhjHPHI9yMRzIbwAhQKCh2Bs+UiisULN4XZi1hRT3I4tFfLm9OKpO71+ +YXK7BdBuf7cAYm5uRqz8CikkM8XFirgG8SWThNkCpqbYBvj22Czcz4OgQgvVacaF +I+BCJaOfTHD+yyNAlWVRiMLNpdDiwRw2CSBdJCNwd6n8TjbzapeeoxBCb/DGSWCP +FSgkO1oENNJVxxN/Pq9A7ZnwK3dm4YCM7KVAOpzV2JMCAwEAAaOB8TCB7jAdBgNV +HQ4EFgQUDt3bDhWHZ8ACc2lzcjDjvRgmJxkwgb4GA1UdIwSBtjCBs4AUDt3bDhWH +Z8ACc2lzcjDjvRgmJxmhgY+kgYwwgYkxCzAJBgNVBAYTAlVTMQ4wDAYDVQQIEwVU +ZXhhczEPMA0GA1UEBxMGQXVzdGluMQwwCgYDVQQKEwNFTUMxEjAQBgNVBAsUCUVN +Q3tjb2RlfTEWMBQGA1UEAxMNbGlic3RvcmFnZS1jYTEfMB0GCSqGSIb3DQEJARYQ +c2FrdXR6QGdtYWlsLmNvbYIJANAQ2YHcXD/EMAwGA1UdEwQFMAMBAf8wDQYJKoZI +hvcNAQEFBQADggIBAEUGVPw4ROG9MROGQhlZAIszWOKfBSvQcw43nodd5G1ljJNP +PKZr7hM+BbRgO2kWDf42MganAaHDnJ70vvpmF8XiRIwUoZwshO44efCUN5GlU9q1 +5qjINvMZWNstN9Ls/B4GrmxvgqM3sPc/ykeYqY9S9tNvtophGbgWc9r2HOCaEmN8 +pT2Nz40LDHW/FnJxb23bR7lnS7cuvngCjSYsBSYyeYigDKUhgkKrH8HT+QzsoJTG +gK/11G2yiCmWdoGdTh06P+aG2n0WmnpvfT0EIZk1yPnj0ewAv5hfT8d030A/6jKH +BNPxmEYe0TpaZi5zPACjqWumiqQaoR9e0RLdVttYSWTvAhGPlvLWJvfZyYqyxlpZ +CXKmbbL4xQM2FKU/lGkWu1/uAlBcB+CgTXWPkJ3Suqbaw0MPKZNnXo2xK+Cap3G2 +1tcb6vPgLMCg3qRkfb2GTvEC2/glc0nnqCrlb98FH1LIXgGAgUTEHi+LGs1sJzfo +ud4IoENw2VKT7FeIUts5FNaaLgdoKDmnH+TiXlNypXuCM8/LHGBKq+sgFygAhHK+ +kW7bmO5Y2MRR3JUlGNZOupkiJiRyFpBueOTIlZ3IXQ1f6nhcPGZozW6Xi1B3++qc +5+9W+wetsqzDL2P3l22QsspfHQ4PB02UOJ4xVuxyKAIyQFMqHcNqJa717USp +-----END CERTIFICATE----- diff --git a/.tls/libstorage-ca.key b/.tls/libstorage-ca.key new file mode 100644 index 00000000..2bf21a53 --- /dev/null +++ b/.tls/libstorage-ca.key @@ -0,0 +1,54 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,C601A1EEF5FBDF83 + +tHGlvnopZYWRUoYYDankU83aoPGZyrQrZBNkSAb0GdXxrU79sE/k3U7scqmLynmc +BsBwoaOdXlmzgTcG/haofj0jTHdn7a2wyG7bwkjnwXG27/QtaPdicIBtr3NIxH1E +CliNYgtp35EpeAOTobB/MfaPs/cNBA08ymVnvcFzvcVIF8bQyHwiel8E6Qaw6pUy +07DTMm0ha67bDcxMp/E35DZQ2echtUO2TYUgeMQzV1E2pxS/aUYBNCaeC2iYFa0J +z/LszlbAVbkzAbOpM8+63Z5TdPvFRMbvcZ1FvHZDPhavHaP6fQGEP4isegnqf4LO +BC8UHOWBmd+7NSHHFjQJ1ff4pmgluRTasp9Btx+z6o2uoW/Bz5J5mSNOB4DFHlHK +XvCrWjbOsTHsLEjZEenEGBIjqiWHGFGlB0n8qCfEQixAa7A622SKg6WTmnEdmEVz +APo2Oe1Ih3DzQOMCkCW8t8lV5Ybcp6PBxUuNC32Jx2Jq2Ln8+hEozJ24exYtFzqR +CHzZRFqT5tYfSiVeVOFvYH9X+0Q/ZoX9J/lU3lkS7WECA/SyN8dSX9qiexeqR80O +E2Wy+I0o93ZIQf52cPaawoclHEwKbG5gYwrzAuxElgTx9lFpB/5k/JZeHvTTsoEu +420wURKjco/+UIxqnBsRoCycCXuzx/9FGM6kCn/qzPKbLwu8LA4yiZYxHmhfWUbO +Oka81MS2/WIgtEa3pSlm0XYf1pXIYGZnMhOrgwGxfJjvFlBZL6wmw+1BnoMFTy/z +BRYgZHRRQ1pr/kiYG5zNGGZZBK1GVPO0DUP3WXfttAAYfBvtfHVOAO9MirmZqC7p +YH3b1RiejHnwgXLnoDFdQUOMLTkk5X2+0yGFJBOQk2Aw8FMsuhpTrJXT++uM4J6L +pgsFwfNWBWP+/4qGod9F3bl4vEdmwNlcwIJyvctpVMVax6nIdQUq85vntfog692K +k1T4FG1lowhXMU5Coc2r7Jg1rspK9WoNXHokUx9nfo/8CVY3YEC73CiBZ8EZ8c7W +wrQ89MkTSpr2Id8GbMA6ZAxZ15WXnBJmqLHrWOHTwh/58+NiqDh7qhiNzA7rpbsj +5nkFqMtCd8kiA3PU6VAsjb+moMjvaUvUFHmlZSJJaFLEv7ANUX2V00KSDWcwkzsP +bZjxNhMQ5IopNZy3Guul81lGNQtq0s6+Y0QeOgaaeHplP0TkwkKPFhcarlhyxzak +5wVcPoxKtoOj68QypLEg5QnirNhXzjSytNAhD79E3wXh7MiAyCcDe0an8Stvrk6x +8F9/Jy93dox94ztz0CmhvFaeYUbjhzjWi4GrdsxUWtghwkL0c5klIsaUWeA17+dA +TEmjFqwXNk+8LFxJd6PckPQ4OL+8T0/5Qmah3N/YTEZ86Cho7JDIFNjDx2uvl1mQ +Y3M83I6nR8D5nA1nJkAeS1Jp+iJM1DS4lSlgkSN8aocmOkpNXjNEk1CZd0k7pQYC +sNmCvL61+gkhHXkfvUeM6YYsVYk45jV8/YaiHNdNLDQ44seaiHZ94vOHP/3IBd1p +EmmvJHcPxZIR2ibmq9TPDAOjLNVzA4AwtRfcLGRjvJaaXdG/3OqnwtZoIZpN8NOv +ueL2KNc9Fen8bTnhgD+NzFQD12Cw4IH5pVkGVAEK3/4hxlsSbNsmQSP1YGUp4VEP +xr4l9X8XVKu34adt439qlqi4bCQsyOPzoNXP+8IZBxZ8wsKsavkWyd+dnd+X6VUV +6sBh/Gv0LS05fPoDaek7I20w4P5C2feaRbpAYFSlFaqw6/0LxRj4OnBSRigfTI7e +ErNidjXz/+GShPmzRe1Kk31Ej3cje1iFojetSmj/4LPy4wt1oD6sqza7eTkI3QC2 +HRMOApyoMZ9HraDJoMsTezNrtt4RS2NOJJifA+mqzLoW8E6LQiJTGsgfkYNdBavD +l1rrlADxEpCUdu7QzUVTKoorN6svmYlf4Z+7CfDRZLSmqj5/78ZW1/9+g7fFP1j0 +40rmefXLJMDVlxFd7JaaOvNqO8586JxRtqPzfBdAY89scza6vrRbb635I2Wb9K7A +FRpa8G0NgWuAaNvkpOa0alJz/rp+cU+6OfPSw49ZZWkvNMeDMF2fu5fTcPsaiNpM +8jZ1CKzI44U+gTODkS4Mc9HuzOGk+qGOLjSRCaXs1dE7UoowjIiCOOoC5sxc5AkL +7HZX+LQTip3aRNRuCB+qMbiwew3NMoIyYuGPX9k42HkJD2B0ziqjK9nfNau1qrYI +OqW9zYQNSLs8vvo2MmI0D3t/FCcKHeIUT1u+m5M11rTNr157mNh0HSre40I90KPy +q1u4hJ1GHxOWkydqUL6Mdt9z/AadMmkzDipXSkdtNedQvb4XvIMKDAQ266lCaZUk +EqV61EP5qNciyT4kgi6dTCGLkIAc5iQD8GzTXAR1UnmLHX/nhAgDHytj4q04ufL6 +sQNoun1oMDmR0XtoQ/eGLU4kk2v1bWZoGSIr6bZZQPWj1pJsrNOKIgHa6To39KlC +T7DylazAiv5szt0y50kPsI0aliZ/CYkmSqp8ynkllpU3G5Oh4A1//UtL1FACngfo +GYUVEAX8Sd8j0Pnl29Ejd8MB4jmH2FcfwvcYM0Vk3H4HGcfXrYB5/bG5QFAwguDq +gDajfWkhlc9LGR/7aJ9VLboN0ZZEFRxwNonliz8H0encugO3ZqfRGubpyiH6+CiU +XKyuCyqykz/7SogdGuDF9Ld6aFIxMH6w+9FbLLkbwyxxhgBB8TEIQ0X5t+0owssw +dklJ9faY94sP+LNwoVuZhtIOusdLJuY0+0zL0vRKcn0Dq5tH/PnT9UP3cCsiAhDT +LCQqAhakhuLIVwMKuiqpVLkkQuV0oFsSCc5I0hxH615EPBYHkw7jA6+FP6PkfSoy +nmm6L59qDtYHIeVhm1CD+PzTZhX9VwG6yph4qrOTFYd36qFPrhqNeU75DGOh0P6t +U4mBe/vDDCvnx/Ywed3rLHalrOInyiveaxb5/JfUpJx9PZ4D+2GoZbYMUjBvoHjZ +0KCdiW+S1GZjRFLgl4EQENyA9I+Um+EcqZTrvrYqaWrRjI+4VdPnWQURJ6QPl344 +7H5OuunPDGIxtPzapq3s1t4YKGQJ8HD80pyXXDFzIoBLBegO2v63RrdsUX8zfNxi +-----END RSA PRIVATE KEY----- diff --git a/.tls/libstorage-client.crt b/.tls/libstorage-client.crt new file mode 100644 index 00000000..741d357f --- /dev/null +++ b/.tls/libstorage-client.crt @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgICA+kwDQYJKoZIhvcNAQEFBQAwgYkxCzAJBgNVBAYTAlVT +MQ4wDAYDVQQIEwVUZXhhczEPMA0GA1UEBxMGQXVzdGluMQwwCgYDVQQKEwNFTUMx +EjAQBgNVBAsUCUVNQ3tjb2RlfTEWMBQGA1UEAxMNbGlic3RvcmFnZS1jYTEfMB0G +CSqGSIb3DQEJARYQc2FrdXR6QGdtYWlsLmNvbTAeFw0xNjAzMTYxMTAyMDNaFw0y +NjAzMTQxMTAyMDNaMIGNMQswCQYDVQQGEwJVUzEOMAwGA1UECBMFVGV4YXMxDzAN +BgNVBAcTBkF1c3RpbjEMMAoGA1UEChMDRU1DMRIwEAYDVQQLFAlFTUN7Y29kZX0x +GjAYBgNVBAMTEWxpYnN0b3JhZ2UtY2xpZW50MR8wHQYJKoZIhvcNAQkBFhBzYWt1 +dHpAZ21haWwuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqaM9 +2z4S2kCsnbH2eWfvSNu5g29WnCUBxHdDQNouoMSXLGjqyRSuDzufhy9PXRZ2M/7f +cTThjm/qONTdck6XdcnjYmNBMgDGCU5EBHRCL7SM2lAtXCq0+5GIc5Gg+Mt987E8 +xVJYMKE695s/HsS134A+uIB3OAa6+8cHW2GT70k/lkcOIrVk72flQptEDP5Ny9fU +mTnx2C1R/YEF+r/xXvnKvTQyty6elURYedZ1gQqgeQb1JR+7w8bp4CslFS95XEVl +ITM7TGeCfUUIf05UO5I6//MYgfuhc07/5s2zBdy36z4VvIXDPJv/AmRrAbgZw3wH +2gin5qzf5Fl0xzTuZwIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB +BAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg +Q2VydGlmaWNhdGUwHQYDVR0OBBYEFMyHBcRxHdU1xBn+niLHkeo1D123MB8GA1Ud +IwQYMBaAFA7d2w4Vh2fAAnNpc3Iw470YJicZMA4GA1UdDwEB/wQEAwIF4DAdBgNV +HSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQEFBQADggIBAEYm +HJQ+HtGPWs9MaW/b++VrkhSbtvY3FG2ZTouMgQcXmy7xNXvrAu9FemlH3IjHQy8N +SyvjpS2VbSo74qJduvOeNkO/Cr8BLcUxNG4d3B6BsZ2EqKBZLlQrn6L43GREYuqQ +OfWlc3coJVDsKkLFqcamOvLw9wwmKxKIC+OmRYIESAC5ktZYcnkB+hrFt8SJreFY +PZRFNl9CrzKQCtW3xK/qO0JNbL5/74oGoxMTxpxtgMORp/uRjdW6h8H7Wf89V/CY +usdjdRNE4ohOFCela6tCpc14a4ZG85Op6z5N4SRC2SbigBgkUf+3OkF9/iirpV5N +GTlcPIlsD8UXVhhXll2XXki4CJekO2urSNd71R2FJISXlLzw7bVrrkSXXPOY51m0 +/+nhzNwTFBaNlJICOOM/3piVcpphkJgiai4N6lSkOnd5OVUUYMZKfTAl77ANzZsJ +6YRyosbXzalKHdAsrkhZ5TdRKAcUDH5pMmOH+ZgCybuTYMMVhsdEYMU9deUAR+rr +GnKSTG5AIlxpgOlPINsdhIimP4Uj0QbmvCGSi5Q1ljRwQgzIOH6LHJeeJcPBTVUC +qN6vrjhvwg6ngQPmHMKb3Wz273Mdox/AGCoBaiwqiaNAIKoUDldttpPM34h0A1og +qx5+VdZiX0yp4q0FYVzOXQkpIwO0HnujfZW1tPq5 +-----END CERTIFICATE----- diff --git a/.tls/libstorage-client.csr b/.tls/libstorage-client.csr new file mode 100644 index 00000000..6651446b --- /dev/null +++ b/.tls/libstorage-client.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC0zCCAbsCAQAwgY0xCzAJBgNVBAYTAlVTMQ4wDAYDVQQIEwVUZXhhczEPMA0G +A1UEBxMGQXVzdGluMQwwCgYDVQQKEwNFTUMxEjAQBgNVBAsUCUVNQ3tjb2RlfTEa +MBgGA1UEAxMRbGlic3RvcmFnZS1jbGllbnQxHzAdBgkqhkiG9w0BCQEWEHNha3V0 +ekBnbWFpbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCpoz3b +PhLaQKydsfZ5Z+9I27mDb1acJQHEd0NA2i6gxJcsaOrJFK4PO5+HL09dFnYz/t9x +NOGOb+o41N1yTpd1yeNiY0EyAMYJTkQEdEIvtIzaUC1cKrT7kYhzkaD4y33zsTzF +UlgwoTr3mz8exLXfgD64gHc4Brr7xwdbYZPvST+WRw4itWTvZ+VCm0QM/k3L19SZ +OfHYLVH9gQX6v/Fe+cq9NDK3Lp6VRFh51nWBCqB5BvUlH7vDxungKyUVL3lcRWUh +MztMZ4J9RQh/TlQ7kjr/8xiB+6FzTv/mzbMF3LfrPhW8hcM8m/8CZGsBuBnDfAfa +CKfmrN/kWXTHNO5nAgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAbKEgCGRXNmA6 +5SN+ulp068Tti3YZSwx4hT3Zoe81LXydR0m3kwoUiD8Z0VGG9NDtk0CVIS7mgmDP +oMJJvEZcnX+8iBfN67CzBoF85JvqEXLZ7q4vX4K4EUR/weMQzwQL8bY6Ud5dDIeN +/P8cX29bia4UCyq7C4WTHxNMRNcq6asMgKkyCC3DOrYVvx66IYx9e4Nu6sPPDbNy +rVAD9C87k5yelOXmOPgWLqjSr/rUOVNz/lB5+Su+jp53YeY8H1HQTk7M7tRjs/rQ +BkRU8VC+BQSCn+yobVoM9NYUTGHr/OAL7nadefkq96tR9Q8bqBNp32XQQFZX9DKy +NJKxO8953w== +-----END CERTIFICATE REQUEST----- diff --git a/.tls/libstorage-client.key b/.tls/libstorage-client.key new file mode 100644 index 00000000..13c349fa --- /dev/null +++ b/.tls/libstorage-client.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAqaM92z4S2kCsnbH2eWfvSNu5g29WnCUBxHdDQNouoMSXLGjq +yRSuDzufhy9PXRZ2M/7fcTThjm/qONTdck6XdcnjYmNBMgDGCU5EBHRCL7SM2lAt +XCq0+5GIc5Gg+Mt987E8xVJYMKE695s/HsS134A+uIB3OAa6+8cHW2GT70k/lkcO +IrVk72flQptEDP5Ny9fUmTnx2C1R/YEF+r/xXvnKvTQyty6elURYedZ1gQqgeQb1 +JR+7w8bp4CslFS95XEVlITM7TGeCfUUIf05UO5I6//MYgfuhc07/5s2zBdy36z4V +vIXDPJv/AmRrAbgZw3wH2gin5qzf5Fl0xzTuZwIDAQABAoIBAEtWtngSX2kS5Ba9 +HMvafwkKnJ9k6UA7p0dL+FBrFd5MsR8GEY/wYUVeo4D0X/jlquV4wl+SrsIFri5D +S34irV2gph3iGuhMM6eaXNblkyeAtjWW+i4+wVYg0ksRWD5mka+S/XxdD9G5KWki +R6wwjN2QJUrnswrs4kpDoF7TPVI9qnfmIaR8kFoMjHy3KUqRDBDEqjrGoL+2126J +W8tBikS0YVZJRZVpGp3qwxo/WMhumbRXCIP3E9H39lA/bxcSZLNfvA5679HU9e3h +y3weloHfxGtJs8LIMRlgCWppIHwFX6nXfI1iJnUIdmNyVBK5xK4O0S/3NXRo1/Vk +MI9j7vECgYEA2fFbKQ6kNM7sKNH1QT0FwWUVogICeXFTLZSPAkSdvSyO7KvRV1Fw +Te1RKnNSOo+996+PxExrSVz7Li8CFiaFIJZOKhHnv/6zdJDxEw9/aKIsNCKG0HS4 +cerUcDDgCCSyBZtBI/n4bCF1OWRattNYWDJktTEgc6IAXCfUOqYP7LMCgYEAx0KD +STNBAp0o19pj7AjbV49elt0UgPusrWVDDYERWyQXwA1RO2uGVzVeURxBQLJdXpSU +t+aEbPlvbm5aOwu2ZgVyh7gJDk2/zqQwsdF0X+NvpX17NPYhgAzkkxoNBi8Gdkye +O0y9ZmHS0NdTOGcJIFSQuWI0scPRb0IxMiWbuX0CgYEAmYgFiC7gTrNWeosuEv9C +BrOHQdHYnGTRC9MTy6060gGJzgBcQP8F9l+wRg9nZgnM8aejx81t5wixih9jgOcx +8XrxJYHnmMF8+ikBK0RHpRZZvYB1KHSRWu7rKP3FVPdE0d5FTWrTLpmDUyL3JMRO +ABPADkTFb7A4QLNuNsK1hTECgYEAwW5sDC9ZOLRf5cr3nTar10AyTrmnId8vLdRT +V1SoCJXtCF7lMI2dCCOiv5JyNd/wi9Qo2q1IeJFb/xPZO+CO3FMFb4LUJ7KKNItn +SfdOxZb/1uMNK8iTlL7vVW3AnvQfjc+q+13vruffwySEgQhk76GoLI6NINQ7B4p2 +P3wKw/ECgYBPwvuhS6AquRbh5UcoNzcUgYiK/2HMKsAJ0f/awfIQzOmNSl8aj1RJ +AjmoNq954X4FNML8i0SpCOJTlY2pndyWaMrFSrsUkGzB/9AgoUwCL7PjUp+l3s3E +vVV7o8hY7AL7PN0BtFfUXGnD2rVc42SzHh10DmpGcvkw5TRpsabzYg== +-----END RSA PRIVATE KEY----- diff --git a/.tls/libstorage-client.pem b/.tls/libstorage-client.pem new file mode 100644 index 00000000..d7ed5746 --- /dev/null +++ b/.tls/libstorage-client.pem @@ -0,0 +1,58 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAqaM92z4S2kCsnbH2eWfvSNu5g29WnCUBxHdDQNouoMSXLGjq +yRSuDzufhy9PXRZ2M/7fcTThjm/qONTdck6XdcnjYmNBMgDGCU5EBHRCL7SM2lAt +XCq0+5GIc5Gg+Mt987E8xVJYMKE695s/HsS134A+uIB3OAa6+8cHW2GT70k/lkcO +IrVk72flQptEDP5Ny9fUmTnx2C1R/YEF+r/xXvnKvTQyty6elURYedZ1gQqgeQb1 +JR+7w8bp4CslFS95XEVlITM7TGeCfUUIf05UO5I6//MYgfuhc07/5s2zBdy36z4V +vIXDPJv/AmRrAbgZw3wH2gin5qzf5Fl0xzTuZwIDAQABAoIBAEtWtngSX2kS5Ba9 +HMvafwkKnJ9k6UA7p0dL+FBrFd5MsR8GEY/wYUVeo4D0X/jlquV4wl+SrsIFri5D +S34irV2gph3iGuhMM6eaXNblkyeAtjWW+i4+wVYg0ksRWD5mka+S/XxdD9G5KWki +R6wwjN2QJUrnswrs4kpDoF7TPVI9qnfmIaR8kFoMjHy3KUqRDBDEqjrGoL+2126J +W8tBikS0YVZJRZVpGp3qwxo/WMhumbRXCIP3E9H39lA/bxcSZLNfvA5679HU9e3h +y3weloHfxGtJs8LIMRlgCWppIHwFX6nXfI1iJnUIdmNyVBK5xK4O0S/3NXRo1/Vk +MI9j7vECgYEA2fFbKQ6kNM7sKNH1QT0FwWUVogICeXFTLZSPAkSdvSyO7KvRV1Fw +Te1RKnNSOo+996+PxExrSVz7Li8CFiaFIJZOKhHnv/6zdJDxEw9/aKIsNCKG0HS4 +cerUcDDgCCSyBZtBI/n4bCF1OWRattNYWDJktTEgc6IAXCfUOqYP7LMCgYEAx0KD +STNBAp0o19pj7AjbV49elt0UgPusrWVDDYERWyQXwA1RO2uGVzVeURxBQLJdXpSU +t+aEbPlvbm5aOwu2ZgVyh7gJDk2/zqQwsdF0X+NvpX17NPYhgAzkkxoNBi8Gdkye +O0y9ZmHS0NdTOGcJIFSQuWI0scPRb0IxMiWbuX0CgYEAmYgFiC7gTrNWeosuEv9C +BrOHQdHYnGTRC9MTy6060gGJzgBcQP8F9l+wRg9nZgnM8aejx81t5wixih9jgOcx +8XrxJYHnmMF8+ikBK0RHpRZZvYB1KHSRWu7rKP3FVPdE0d5FTWrTLpmDUyL3JMRO +ABPADkTFb7A4QLNuNsK1hTECgYEAwW5sDC9ZOLRf5cr3nTar10AyTrmnId8vLdRT +V1SoCJXtCF7lMI2dCCOiv5JyNd/wi9Qo2q1IeJFb/xPZO+CO3FMFb4LUJ7KKNItn +SfdOxZb/1uMNK8iTlL7vVW3AnvQfjc+q+13vruffwySEgQhk76GoLI6NINQ7B4p2 +P3wKw/ECgYBPwvuhS6AquRbh5UcoNzcUgYiK/2HMKsAJ0f/awfIQzOmNSl8aj1RJ +AjmoNq954X4FNML8i0SpCOJTlY2pndyWaMrFSrsUkGzB/9AgoUwCL7PjUp+l3s3E +vVV7o8hY7AL7PN0BtFfUXGnD2rVc42SzHh10DmpGcvkw5TRpsabzYg== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgICA+kwDQYJKoZIhvcNAQEFBQAwgYkxCzAJBgNVBAYTAlVT +MQ4wDAYDVQQIEwVUZXhhczEPMA0GA1UEBxMGQXVzdGluMQwwCgYDVQQKEwNFTUMx +EjAQBgNVBAsUCUVNQ3tjb2RlfTEWMBQGA1UEAxMNbGlic3RvcmFnZS1jYTEfMB0G +CSqGSIb3DQEJARYQc2FrdXR6QGdtYWlsLmNvbTAeFw0xNjAzMTYxMTAyMDNaFw0y +NjAzMTQxMTAyMDNaMIGNMQswCQYDVQQGEwJVUzEOMAwGA1UECBMFVGV4YXMxDzAN +BgNVBAcTBkF1c3RpbjEMMAoGA1UEChMDRU1DMRIwEAYDVQQLFAlFTUN7Y29kZX0x +GjAYBgNVBAMTEWxpYnN0b3JhZ2UtY2xpZW50MR8wHQYJKoZIhvcNAQkBFhBzYWt1 +dHpAZ21haWwuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqaM9 +2z4S2kCsnbH2eWfvSNu5g29WnCUBxHdDQNouoMSXLGjqyRSuDzufhy9PXRZ2M/7f +cTThjm/qONTdck6XdcnjYmNBMgDGCU5EBHRCL7SM2lAtXCq0+5GIc5Gg+Mt987E8 +xVJYMKE695s/HsS134A+uIB3OAa6+8cHW2GT70k/lkcOIrVk72flQptEDP5Ny9fU +mTnx2C1R/YEF+r/xXvnKvTQyty6elURYedZ1gQqgeQb1JR+7w8bp4CslFS95XEVl +ITM7TGeCfUUIf05UO5I6//MYgfuhc07/5s2zBdy36z4VvIXDPJv/AmRrAbgZw3wH +2gin5qzf5Fl0xzTuZwIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEB +BAQDAgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQg +Q2VydGlmaWNhdGUwHQYDVR0OBBYEFMyHBcRxHdU1xBn+niLHkeo1D123MB8GA1Ud +IwQYMBaAFA7d2w4Vh2fAAnNpc3Iw470YJicZMA4GA1UdDwEB/wQEAwIF4DAdBgNV +HSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQEFBQADggIBAEYm +HJQ+HtGPWs9MaW/b++VrkhSbtvY3FG2ZTouMgQcXmy7xNXvrAu9FemlH3IjHQy8N +SyvjpS2VbSo74qJduvOeNkO/Cr8BLcUxNG4d3B6BsZ2EqKBZLlQrn6L43GREYuqQ +OfWlc3coJVDsKkLFqcamOvLw9wwmKxKIC+OmRYIESAC5ktZYcnkB+hrFt8SJreFY +PZRFNl9CrzKQCtW3xK/qO0JNbL5/74oGoxMTxpxtgMORp/uRjdW6h8H7Wf89V/CY +usdjdRNE4ohOFCela6tCpc14a4ZG85Op6z5N4SRC2SbigBgkUf+3OkF9/iirpV5N +GTlcPIlsD8UXVhhXll2XXki4CJekO2urSNd71R2FJISXlLzw7bVrrkSXXPOY51m0 +/+nhzNwTFBaNlJICOOM/3piVcpphkJgiai4N6lSkOnd5OVUUYMZKfTAl77ANzZsJ +6YRyosbXzalKHdAsrkhZ5TdRKAcUDH5pMmOH+ZgCybuTYMMVhsdEYMU9deUAR+rr +GnKSTG5AIlxpgOlPINsdhIimP4Uj0QbmvCGSi5Q1ljRwQgzIOH6LHJeeJcPBTVUC +qN6vrjhvwg6ngQPmHMKb3Wz273Mdox/AGCoBaiwqiaNAIKoUDldttpPM34h0A1og +qx5+VdZiX0yp4q0FYVzOXQkpIwO0HnujfZW1tPq5 +-----END CERTIFICATE----- diff --git a/.tls/libstorage-server.crt b/.tls/libstorage-server.crt new file mode 100644 index 00000000..84835a34 --- /dev/null +++ b/.tls/libstorage-server.crt @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF8jCCA9qgAwIBAgICA+gwDQYJKoZIhvcNAQEFBQAwgYkxCzAJBgNVBAYTAlVT +MQ4wDAYDVQQIEwVUZXhhczEPMA0GA1UEBxMGQXVzdGluMQwwCgYDVQQKEwNFTUMx +EjAQBgNVBAsUCUVNQ3tjb2RlfTEWMBQGA1UEAxMNbGlic3RvcmFnZS1jYTEfMB0G +CSqGSIb3DQEJARYQc2FrdXR6QGdtYWlsLmNvbTAeFw0xNjAzMTYxMTAxMTZaFw0y +NjAzMTQxMTAxMTZaMIGNMQswCQYDVQQGEwJVUzEOMAwGA1UECBMFVGV4YXMxDzAN +BgNVBAcTBkF1c3RpbjEMMAoGA1UEChMDRU1DMRIwEAYDVQQLFAlFTUN7Y29kZX0x +GjAYBgNVBAMTEWxpYnN0b3JhZ2Utc2VydmVyMR8wHQYJKoZIhvcNAQkBFhBzYWt1 +dHpAZ21haWwuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAqHYs +sa1RYfdLhuMczur9DAHjN+sX7KHs9HoBAThZN0gtDgGNjX/ffvtB+SO+KQV43VIg +b1xy4dCSCpWpKLGie6GeaJ54g8JFZV02251p5X4VFb0jTS7dXvVfaed+eOk/TRMU +ASlZOVLNd556usMIQL7BCeF+yK+JPIaYBFrYXwg6vtUmsAyN4Dxo2aPvmA2bly7P +M0PIwAlCOof64mgi5OH/jUpCHC8XGYA0FVyFq6aURhzkwefc7yzDaS3LXbgVH/Oo +KQJXLQ8m77+aHAB+OYG84beZl2KkGE4B/aO0+MVWm6X4h27gAfcAncBo28lWQPDb +R4yV2M94xl5N4m7IeQIDAQABo4IBXDCCAVgwCQYDVR0TBAIwADARBglghkgBhvhC +AQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIFNlcnZl +ciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUR1oERbiB0eVemnCPH5XY2Wbyy6wwgb4G +A1UdIwSBtjCBs4AUDt3bDhWHZ8ACc2lzcjDjvRgmJxmhgY+kgYwwgYkxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIEwVUZXhhczEPMA0GA1UEBxMGQXVzdGluMQwwCgYDVQQK +EwNFTUMxEjAQBgNVBAsUCUVNQ3tjb2RlfTEWMBQGA1UEAxMNbGlic3RvcmFnZS1j +YTEfMB0GCSqGSIb3DQEJARYQc2FrdXR6QGdtYWlsLmNvbYIJANAQ2YHcXD/EMA4G +A1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggrBgEFBQcDATANBgkqhkiG9w0BAQUF +AAOCAgEATzR15dlZIHwDDh/+YL5fhrLKZX7TPJKACdvSHjszs7Sf0EsBcVLirBeQ +sUnBndIU+N1crbxHX0ISlAm4y6RBE+kH8AimHRsIECFszO0yQvrqnmSEVg3rJCDW +QRqRU3OAfKrkfp/LpQvnkIo9TEzCaQJMyVr/00xkM8bP9HI0CoJaXEN14MUKSIUJ +IH+YkeoG3m6sVxqshzeTGl6tq4TuFsQJYJcDwJ5uHtvluVuDysBayufL+X7IPqGO +p5aOY+Q+f5MHgr/ayjfiD58riMbH9G4fFDNUALUqKgJlpm6oriVryJD2xT3Itrkk +U8n6F8OJpMguqXFldbJsFb9QrpQYx5xVcQDcCGAnxanoQKPBtXT0U+jVVYT40JPg +tI0fjy3k4Y3s8IpY8p23GWVAnnL4LkDPJa2dvFTCbEPxOw8uDqtUO3SLOMOwhWA/ +TOAio381/g+Zwmfxbsfn9FhW7GmLyYu9+srdF1P1MpV2KTH4u7dd9p2OwtdYbFsA +ssVVg84raQozNarj0grTcYyXfcGMZ/BO1mXQtUCxA/D+PRnEloUesrGWNyu4LDbF +6f7GWMiPj2dls3IfY+szcTw6oRB+Y7zUxgoQVXlg8Hs0i1r1bFo5vRjEOARmoQr+ +E7ptnZvwPbN/nluLOOdlkBaZ6l8q0P9Ogf01KsA/UFq9Y5XMRN4= +-----END CERTIFICATE----- diff --git a/.tls/libstorage-server.csr b/.tls/libstorage-server.csr new file mode 100644 index 00000000..0a621af5 --- /dev/null +++ b/.tls/libstorage-server.csr @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIIC0zCCAbsCAQAwgY0xCzAJBgNVBAYTAlVTMQ4wDAYDVQQIEwVUZXhhczEPMA0G +A1UEBxMGQXVzdGluMQwwCgYDVQQKEwNFTUMxEjAQBgNVBAsUCUVNQ3tjb2RlfTEa +MBgGA1UEAxMRbGlic3RvcmFnZS1zZXJ2ZXIxHzAdBgkqhkiG9w0BCQEWEHNha3V0 +ekBnbWFpbC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCodiyx +rVFh90uG4xzO6v0MAeM36xfsoez0egEBOFk3SC0OAY2Nf99++0H5I74pBXjdUiBv +XHLh0JIKlakosaJ7oZ5onniDwkVlXTbbnWnlfhUVvSNNLt1e9V9p53546T9NExQB +KVk5Us13nnq6wwhAvsEJ4X7Ir4k8hpgEWthfCDq+1SawDI3gPGjZo++YDZuXLs8z +Q8jACUI6h/riaCLk4f+NSkIcLxcZgDQVXIWrppRGHOTB59zvLMNpLctduBUf86gp +AlctDybvv5ocAH45gbzht5mXYqQYTgH9o7T4xVabpfiHbuAB9wCdwGjbyVZA8NtH +jJXYz3jGXk3ibsh5AgMBAAGgADANBgkqhkiG9w0BAQsFAAOCAQEAMycgiBqG1gsm +DCf49sxY6csnQtsx8gGuuVWeLyYeIKsFTVVX6dOmG+L6dJEvmnMYDnRi+WKGuxAX +7UKGw1i5WzRucF8Cq4pzZNZrdJxXxb1gQy1TGCUUCMYDwYgQn2tyRuBT6AJHjIFG +LtDFmPpXfCknA/nw1SlIyPc9oGUXWQwU+ljmMb7Ag0TuCEUl8LKQtmI6T3HyJ4Cz +dLWpUuVtSG2vhFx1rIJIZ8kSJvL7l5bN3pqMPja15mOEHCJ6KAl1yJ9E/sOOW/A0 +r5MWfwr7Z7RP29E5CWp9b0Vrpj8QANwAwXghIY6XTF/GhJ510Sac+QAMnvWbLpgJ +yOmlJd2D4g== +-----END CERTIFICATE REQUEST----- diff --git a/.tls/libstorage-server.key b/.tls/libstorage-server.key new file mode 100644 index 00000000..fd215dad --- /dev/null +++ b/.tls/libstorage-server.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAqHYssa1RYfdLhuMczur9DAHjN+sX7KHs9HoBAThZN0gtDgGN +jX/ffvtB+SO+KQV43VIgb1xy4dCSCpWpKLGie6GeaJ54g8JFZV02251p5X4VFb0j +TS7dXvVfaed+eOk/TRMUASlZOVLNd556usMIQL7BCeF+yK+JPIaYBFrYXwg6vtUm +sAyN4Dxo2aPvmA2bly7PM0PIwAlCOof64mgi5OH/jUpCHC8XGYA0FVyFq6aURhzk +wefc7yzDaS3LXbgVH/OoKQJXLQ8m77+aHAB+OYG84beZl2KkGE4B/aO0+MVWm6X4 +h27gAfcAncBo28lWQPDbR4yV2M94xl5N4m7IeQIDAQABAoIBAQCRyEKBN+V3i9P2 +VM/3WG/HVlLVb0Ly6mXuYy4/ag36wyEKP9nJm+FDOBgti/rh8PRZQtsCw9Q/Col3 +U3Bh4OclagV1r73G9/Wp9HKmtqCPkv6YI2dLQcUciecZ9NUDuYWoI1xqbOfjrX5V +h/XZbTHVJb5T2Koo7Y8rq6YeDqe0BDmFm/rJHgHNakD2FpNFKUBYHoUfGPU0zuip +1p6ZKSX/pWRxinrSPJH54Zl3WeoN15pKbtXdwh94fkrygcwYcehapYN9ujcHZcUJ +tXgOZJ0XalBk4WytwM+QkwVe0baIXERDnqCQS7av9MSeHibnLVmMxpI+fR84Evz7 +z6LFq/EhAoGBANZuK6LOZhMv85eBUoDsYbV0eebbklE7J9neclnSBS2bHuSk/nIR ++trtHFv9l4k5K+Oop63nbsHX5aEbLrlp7BZuTBQID2BZ6q55XWSJKK08sUMlulLw +NwPRjtbROvVofj3Q6Amdlf66zpJ0fR3nds2iMgfE+sI+8qYsh0l6NbW9AoGBAMke +ppIBQw1tpnzvF7rUdN42SLIBmF5L7IOhEZweJ6+LjyQVnH33H0M7Jv2wbCx8Jkeh +lBZ8eeUMSsry0lyqOHyg7UETqVuDV7ErCSR8FL861nPn8kFNoq2apeZ0drw2xeE6 +BeSEsvDhIPPnQep8oZ0O8Rjf39Pib44ToWq0JvNtAoGAJKeyl+MWeeMxjc2Sj+1Y +io89o2QXcAFfv5OSEp6fOfuRXV6DDHbcXf44YdVIyTFXulQDTewI9+PzIgYmh5V+ +wRrbsHTsQ/k679ZZS61SocKFPsg9QJ5FmUaCV2Bu5rKVGfYTJEmm8WN3mnuFQ85k +daRrTv6yKvBdxGBKRBo7AjUCgYANvGYr+qIVvLNuPPYl8HS66II2hh1d81mH8+w7 ++WNEfgecs00o3UPpV5TmJrJ8p04F/mca0g2RMzG4grUTVxzchjEuDKW4dlP66bGK +KF9SYDZdXC4Tf7XonXNPNg0V9be2FjxoyxddlEKn5dd+qFxxWZ/lzwR+eCyeS4Du +xLcUUQKBgChMF5AuwW/So3bNq3tjdbbjw14fFlFFZrPYYhm3IjypqVzNRk9v9Gun +fpp58Sk+xfLGhFhh5b/E5fBz/0hzMpr2eKmfju0WTzO/pspygacdQeAcRHVEePuK +iH6U+wS/TIS4FWir+hW874b1Q6qochoi/8F6ks/xE1b40paJoHy4 +-----END RSA PRIVATE KEY----- diff --git a/.tls/server.extensions b/.tls/server.extensions new file mode 100644 index 00000000..32843767 --- /dev/null +++ b/.tls/server.extensions @@ -0,0 +1,7 @@ +basicConstraints = CA:FALSE +nsCertType = server +nsComment = "OpenSSL Generated Server Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = serverAuth diff --git a/.travis.yml b/.travis.yml index 89acb7b4..aa77762b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,10 @@ language: go go: - 1.5.1 -before_install: - - go get github.com/axw/gocov/gocov - - go get github.com/mattn/goveralls - - if ! go get code.google.com/p/go.tools/cmd/cover; then go get golang.org/x/tools/cmd/cover; fi - - go get github.com/Masterminds/glide +go_import_path: github.com/emccode/libstorage install: - - go get -d $(glide nv) - - glide up + - make deps script: - - go install $(glide nv) - - .build/test.sh + - make install + - env LIBSTORAGE_DEBUG=true make test + - make cover diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..bd984734 --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +all: install + +include .gomk/go.mk + +deps: $(GO_DEPS) + +build: deps $(GO_SRC_TOOL_MARKERS) $(GO_BUILD) + +install: $(GO_INSTALL) + +test: $(GO_TEST) + +test-build: $(GO_TEST_BUILD) + +test-clean: $(GO_TEST_CLEAN) + +cover: $(GO_COVER) + +cover-clean: $(GO_COVER_CLEAN) + +package: $(GO_PACKAGE) + +package-clean: $(GO_PACKAGE_CLEAN) + +clean: $(GO_CLEAN) + +clobber: $(GO_CLOBBER) + +run: + go test -tags 'run mock driver' -v + +run-debug: + env LIBSTORAGE_DEBUG=true $(MAKE) run + +run-tls: + env LIBSTORAGE_TESTRUN_TLS=true $(MAKE) run + +run-tls-debug: + env LIBSTORAGE_DEBUG=true LIBSTORAGE_TESTRUN_TLS=true $(MAKE) run + +.PHONY: all install build deps \ + test test-build test-clean \ + test-mock test-build-mock \ + cover cover-clean \ + package package-clean \ + clean clobber \ + run run-debug run-tls run-tls-debug \ + $(GO_PHONY) diff --git a/api/api.go b/api/api.go index 1669a2fa..2613801a 100644 --- a/api/api.go +++ b/api/api.go @@ -1,120 +1,13 @@ package api import ( - "net/http" + "github.com/blang/semver" ) -// ServiceEndpoint is the interface for the libStorage service/API. -type ServiceEndpoint interface { +var ( + // Version of the current REST API + Version = semver.MustParse("1.0.0") - // GetServiceInfo returns information about the service. - GetServiceInfo( - req *http.Request, - args *GetServiceInfoArgs, - reply *GetServiceInfoReply) error - - // GetVolumeMapping lists the block devices that are attached to the - // instance. - GetVolumeMapping( - req *http.Request, - args *GetVolumeMappingArgs, - reply *GetVolumeMappingReply) error - - // GetInstance retrieves the local instance. - GetInstance( - req *http.Request, - args *GetInstanceArgs, - reply *GetInstanceReply) error - - // GetNextAvailableDeviceName gets the driver's NextAvailableDeviceName - // information. - GetNextAvailableDeviceName( - req *http.Request, - args *GetNextAvailableDeviceNameArgs, - reply *GetNextAvailableDeviceNameReply) error - - // GetVolume returns all volumes for the instance based on either volumeID - // or volumeName that are available to the instance. - GetVolume( - req *http.Request, - args *GetVolumeArgs, - reply *GetVolumeReply) error - - // GetVolumeAttach returns the attachment details based on volumeID or - // volumeName where the volume is currently attached. - GetVolumeAttach( - req *http.Request, - args *GetVolumeAttachArgs, - reply *GetVolumeAttachReply) error - - // CreateSnapshot is a synch/async operation that returns snapshots that - // have been performed based on supplying a snapshotName, source volumeID, - // and optional description. - CreateSnapshot( - req *http.Request, - args *CreateSnapshotArgs, - reply *CreateSnapshotReply) error - - // GetSnapshot returns a list of snapshots for a volume based on volumeID, - // snapshotID, or snapshotName. - GetSnapshot( - req *http.Request, - args *GetSnapshotArgs, - reply *GetSnapshotReply) error - - // RemoveSnapshot will remove a snapshot based on the snapshotID. - RemoveSnapshot( - req *http.Request, - args *RemoveSnapshotArgs, - reply *RemoveSnapshotReply) error - - // CreateVolume is sync/async and will create an return a new/existing - // Volume based on volumeID/snapshotID with a name of volumeName and a size - // in GB. Optionally based on the storage driver, a volumeType, IOPS, and - // availabilityZone could be defined. - CreateVolume( - req *http.Request, - args *CreateVolumeArgs, - reply *CreateVolumeReply) error - - // RemoveVolume will remove a volume based on volumeID. - RemoveVolume( - req *http.Request, - args *RemoveVolumeArgs, - reply *RemoveVolumeReply) error - - // AttachVolume returns a list of VolumeAttachments is sync/async that will - // attach a volume to an instance based on volumeID and instanceID. - AttachVolume( - req *http.Request, - args *AttachVolumeArgs, - reply *AttachVolumeReply) error - - // DetachVolume is sync/async that will detach the volumeID from the local - // instance or the instanceID. - DetachVolume( - req *http.Request, - args *DetachVolumeArgs, - reply *DetachVolumeReply) error - - // CopySnapshot is a sync/async and returns a snapshot that will copy a - // snapshot based on volumeID/snapshotID/snapshotName and create a new - // snapshot of desinationSnapshotName in the destinationRegion location. - CopySnapshot( - req *http.Request, - args *CopySnapshotArgs, - reply *CopySnapshotReply) error - - // GetClientTool gets the client tool provided by the driver. This tool is - // executed on the client-side of the connection in order to discover - // information only available to the client, such as the client's instance - // ID or a local device map. - // - // The client tool is returned as a byte array that's either a binary file - // or a unicode-encoded, plain-text script file. Use the file extension - // of the client tool's file name to determine the file type. - GetClientTool( - req *http.Request, - args *GetClientToolArgs, - reply *GetClientToolReply) error -} + // MinVersion represents Minimun REST API version supported + MinVersion = semver.MustParse("1.0.0") +) diff --git a/api/api_args.go b/api/api_args.go deleted file mode 100644 index 9459c9ab..00000000 --- a/api/api_args.go +++ /dev/null @@ -1,346 +0,0 @@ -package api - -/******************************************************************************* -** GetServiceInfo -*******************************************************************************/ - -// GetServiceInfoArgs are the arguments expected by the GetServiceInfo function. -type GetServiceInfoArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetServiceInfoArgsOptional `json:"optional"` - Required GetServiceInfoArgsRequired `json:"required"` -} - -// GetServiceInfoArgsRequired are the required arguments expected by the -// GetServiceInfoArgs function. -type GetServiceInfoArgsRequired struct { -} - -// GetServiceInfoArgsOptional are the optional arguments expected by the -// GetServiceInfoArgs function. -type GetServiceInfoArgsOptional struct { -} - -/******************************************************************************* -** GetNextAvailableDeviceName -*******************************************************************************/ - -// GetNextAvailableDeviceNameArgs are the arguments expected by the -// GetNextAvailableDeviceName function. -type GetNextAvailableDeviceNameArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetNextAvailableDeviceNameArgsOptional `json:"optional"` - Required GetNextAvailableDeviceNameArgsRequired `json:"required"` -} - -// GetNextAvailableDeviceNameArgsRequired are the required arguments -// expected by the GetNextAvailableDeviceName function. -type GetNextAvailableDeviceNameArgsRequired struct { -} - -// GetNextAvailableDeviceNameArgsOptional are the optional arguments -// expected by the GetNextAvailableDeviceName function. -type GetNextAvailableDeviceNameArgsOptional struct { -} - -/******************************************************************************* -** GetVolumeMapping -*******************************************************************************/ - -// GetVolumeMappingArgs are the arguments expected by the GetVolumeMapping -// function. -type GetVolumeMappingArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetVolumeMappingArgsOptional `json:"optional"` - Required GetVolumeMappingArgsRequired `json:"required"` -} - -// GetVolumeMappingArgsRequired are the required arguments expected by the -// GetVolumeMapping function. -type GetVolumeMappingArgsRequired struct { -} - -// GetVolumeMappingArgsOptional are the optional arguments expected by the -// GetVolumeMapping function. -type GetVolumeMappingArgsOptional struct { -} - -/******************************************************************************* -** GetInstance -*******************************************************************************/ - -// GetInstanceArgs are the arguments expected by the GetInstance function. -type GetInstanceArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetInstanceArgsOptional `json:"optional"` - Required GetInstanceArgsRequired `json:"required"` -} - -// GetInstanceArgsRequired are the required arguments expected by the -// GetInstance function. -type GetInstanceArgsRequired struct { -} - -// GetInstanceArgsOptional are the optional arguments expected by the -// GetInstance function. -type GetInstanceArgsOptional struct { -} - -/******************************************************************************* -** GetVolume -*******************************************************************************/ - -// GetVolumeArgs are the arguments expected by the GetVolume function. -type GetVolumeArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetVolumeArgsOptional `json:"optional"` - Required GetVolumeArgsRequired `json:"required"` -} - -// GetVolumeArgsRequired are the required arguments expected by the -// GetVolume function. -type GetVolumeArgsRequired struct { -} - -// GetVolumeArgsOptional are the optional arguments expected by the -// GetVolume function. -type GetVolumeArgsOptional struct { - VolumeID string `json:"volumeID"` - VolumeName string `json:"volumeName"` -} - -/******************************************************************************* -** GetVolumeAttach -*******************************************************************************/ - -// GetVolumeAttachArgs are the arguments expected by the GetVolumeAttach -// function. -type GetVolumeAttachArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetVolumeAttachArgsOptional `json:"optional"` - Required GetVolumeAttachArgsRequired `json:"required"` -} - -// GetVolumeAttachArgsRequired are the required arguments expected by the -// GetVolumeAttach function. -type GetVolumeAttachArgsRequired struct { -} - -// GetVolumeAttachArgsOptional are the optional arguments expected by the -// GetVolumeAttach function. -type GetVolumeAttachArgsOptional struct { - VolumeID string `json:"volumeID"` -} - -/******************************************************************************* -** CreateSnapshot -*******************************************************************************/ - -// CreateSnapshotArgs are the arguments expected by the CreateSnapshot function. -type CreateSnapshotArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional CreateSnapshotArgsOptional `json:"optional"` - Required CreateSnapshotArgsRequired `json:"required"` -} - -// CreateSnapshotArgsRequired are the required arguments expected by the -// CreateSnapshot function. -type CreateSnapshotArgsRequired struct { -} - -// CreateSnapshotArgsOptional are the optional arguments expected by the -// CreateSnapshot function. -type CreateSnapshotArgsOptional struct { - Description string `json:"description"` - SnapshotName string `json:"snapshotName"` - VolumeID string `json:"volumeID"` -} - -/******************************************************************************* -** GetSnapshot -*******************************************************************************/ - -// GetSnapshotArgs are the arguments expected by the GetSnapshot function. -type GetSnapshotArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetSnapshotArgsOptional `json:"optional"` - Required GetSnapshotArgsRequired `json:"required"` -} - -// GetSnapshotArgsRequired are the required arguments expected by the -// GetSnapshot function. -type GetSnapshotArgsRequired struct { -} - -// GetSnapshotArgsOptional are the optional arguments expected by the -// GetSnapshot function. -type GetSnapshotArgsOptional struct { - SnapshotID string `json:"snapshotID"` - SnapshotName string `json:"snapshotName"` - VolumeID string `json:"volumeID"` -} - -/******************************************************************************* -** RemoveSnapshot -*******************************************************************************/ - -// RemoveSnapshotArgs are the arguments expected by the RemoveSnapshot function. -type RemoveSnapshotArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional RemoveSnapshotArgsOptional `json:"optional"` - Required RemoveSnapshotArgsRequired `json:"required"` -} - -// RemoveSnapshotArgsRequired are the required arguments expected by the -// RemoveSnapshot function. -type RemoveSnapshotArgsRequired struct { - SnapshotID string `json:"snapshotID"` -} - -// RemoveSnapshotArgsOptional are the optional arguments expected by the -// RemoveSnapshot function. -type RemoveSnapshotArgsOptional struct { -} - -/******************************************************************************* -** CreateVolume -*******************************************************************************/ - -// CreateVolumeArgs are the arguments expected by the CreateVolume function. -type CreateVolumeArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional CreateVolumeArgsOptional `json:"optional"` - Required CreateVolumeArgsRequired `json:"required"` -} - -// CreateVolumeArgsRequired are the required arguments expected by the -// CreateVolume function. -type CreateVolumeArgsRequired struct { -} - -// CreateVolumeArgsOptional are the optional arguments expected by the -// CreateVolume function. -type CreateVolumeArgsOptional struct { - AvailabilityZone string `json:"availabilityZone"` - IOPS int64 `json:"iops"` - Size int64 `json:"size"` - SnapshotID string `json:"snapshotID"` - VolumeID string `json:"volumeID"` - VolumeName string `json:"volumeName"` - VolumeType string `json:"volumeType"` -} - -/******************************************************************************* -** RemoveVolume -*******************************************************************************/ - -// RemoveVolumeArgs are the arguments expected by the RemoveVolume function. -type RemoveVolumeArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional RemoveVolumeArgsOptional `json:"optional"` - Required RemoveVolumeArgsRequired `json:"required"` -} - -// RemoveVolumeArgsRequired are the required arguments expected by the -// RemoveVolume function. -type RemoveVolumeArgsRequired struct { - VolumeID string `json:"volumeID"` -} - -// RemoveVolumeArgsOptional are the optional arguments expected by the -// RemoveVolume function. -type RemoveVolumeArgsOptional struct { -} - -/******************************************************************************* -** AttachVolume -*******************************************************************************/ - -// AttachVolumeArgs are the arguments expected by the AttachVolume function. -type AttachVolumeArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional AttachVolumeArgsOptional `json:"optional"` - Required AttachVolumeArgsRequired `json:"required"` -} - -// AttachVolumeArgsRequired are the required arguments expected by the -// AttachVolume function. -type AttachVolumeArgsRequired struct { - NextDeviceName string `json:"nextDeviceName"` - VolumeID string `json:"volumeID"` -} - -// AttachVolumeArgsOptional are the optional arguments expected by the -// AttachVolume function. -type AttachVolumeArgsOptional struct { -} - -/******************************************************************************* -** DetachVolume -*******************************************************************************/ - -// DetachVolumeArgs are the arguments expected by the DetachVolume function. -type DetachVolumeArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional DetachVolumeArgsOptional `json:"optional"` - Required DetachVolumeArgsRequired `json:"required"` -} - -// DetachVolumeArgsRequired are the required arguments expected by the -// DetachVolume function. -type DetachVolumeArgsRequired struct { - VolumeID string `json:"volumeID"` -} - -// DetachVolumeArgsOptional are the optional arguments expected by the -// DetachVolume function. -type DetachVolumeArgsOptional struct { -} - -/******************************************************************************* -** CopySnapshot -*******************************************************************************/ - -// CopySnapshotArgs are the arguments expected by the CopySnapshot function. -type CopySnapshotArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional CopySnapshotArgsOptional `json:"optional"` - Required CopySnapshotArgsRequired `json:"required"` -} - -// CopySnapshotArgsRequired are the required arguments expected by the -// CopySnapshot function. -type CopySnapshotArgsRequired struct { -} - -// CopySnapshotArgsOptional are the optional arguments expected by the -// CopySnapshot function. -type CopySnapshotArgsOptional struct { - DestinationRegion string `json:"destinationRegion"` - DestinationSnapshotName string `json:"destinationSnapshotName"` - SnapshotID string `json:"snapshotID"` - SnapshotName string `json:"snapshotName"` - VolumeID string `json:"volumeID"` -} - -/******************************************************************************* -** GetClientTool -*******************************************************************************/ - -// GetClientToolArgs are the arguments expected by the GetClientTool function. -type GetClientToolArgs struct { - Extensions map[string]interface{} `json:"extensions"` - Optional GetClientToolArgsOptional `json:"optional"` - Required GetClientToolArgsRequired `json:"required"` -} - -// GetClientToolArgsRequired are the required arguments expected by the -// GetClientTool function. -type GetClientToolArgsRequired struct { -} - -// GetClientToolArgsOptional are the optional arguments expected by the -// GetClientTool function. -type GetClientToolArgsOptional struct { - OmitBinary bool `json:"omitBinary"` -} diff --git a/api/api_replies.go b/api/api_replies.go deleted file mode 100644 index 39b88a72..00000000 --- a/api/api_replies.go +++ /dev/null @@ -1,76 +0,0 @@ -package api - -// GetServiceInfoReply is the reply from the GetServiceInfo function. -type GetServiceInfoReply struct { - Name string `json:"name"` - Driver string `json:"driver"` - RegisteredDrivers []string `json:"registeredDrivers"` -} - -// GetNextAvailableDeviceNameReply is the reply from the -// GetNextAvailableDeviceName function. -type GetNextAvailableDeviceNameReply struct { - Next *NextAvailableDeviceName `json:"next"` -} - -// GetVolumeMappingReply is the reply from the GetVolumeMapping function. -type GetVolumeMappingReply struct { - BlockDevices []*BlockDevice `json:"blockDevices"` -} - -// GetInstanceReply is the reply from the GetInstance function. -type GetInstanceReply struct { - Instance *Instance `json:"instance"` -} - -// GetVolumeReply is the reply from the GetVolume function. -type GetVolumeReply struct { - Volumes []*Volume `json:"volumes"` -} - -// GetVolumeAttachReply is the reply from the GetVolumeAttach function. -type GetVolumeAttachReply struct { - Attachments []*VolumeAttachment `json:"attachments"` -} - -// CreateSnapshotReply is the reply from the CreateSnapshot function. -type CreateSnapshotReply struct { - Snapshots []*Snapshot `json:"snapshots"` -} - -// GetSnapshotReply is the reply from the GetSnapshot function. -type GetSnapshotReply struct { - Snapshots []*Snapshot `json:"snapshots"` -} - -// RemoveSnapshotReply is the reply from the RemoveSnapshot function. -type RemoveSnapshotReply struct { -} - -// CreateVolumeReply is the reply from the CreateVolume function. -type CreateVolumeReply struct { - Volume *Volume `json:"volume"` -} - -// RemoveVolumeReply is the reply from the RemoveVolume function. -type RemoveVolumeReply struct { -} - -// AttachVolumeReply is the reply from the AttachVolume function. -type AttachVolumeReply struct { - Attachments []*VolumeAttachment `json:"attachments"` -} - -// DetachVolumeReply is the reply from the DetachVolume function. -type DetachVolumeReply struct { -} - -// CopySnapshotReply is the reply from the CopySnapshot function. -type CopySnapshotReply struct { - Snapshot *Snapshot `json:"snapshot"` -} - -// GetClientToolReply is the reply from the GetClientTool function. -type GetClientToolReply struct { - ClientTool *ClientTool `json:"clientTool"` -} diff --git a/api/client/client.go b/api/client/client.go new file mode 100644 index 00000000..f0c5f251 --- /dev/null +++ b/api/client/client.go @@ -0,0 +1,277 @@ +package client + +import ( + "bufio" + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httputil" + "regexp" + + log "github.com/Sirupsen/logrus" + "github.com/akutz/gofig" + "github.com/akutz/goof" + "github.com/akutz/gotil" + //gjson "github.com/gorilla/rpc/json" + "golang.org/x/net/context/ctxhttp" + + "github.com/emccode/libstorage/api/types/context" + httptypes "github.com/emccode/libstorage/api/types/http" + "github.com/emccode/libstorage/api/utils" +) + +// Client is the interface for Golang libStorage clients. +type Client interface { + + // Root returns a list of root resources. + Root() ([]string, error) + + // Volumes returns a list of all Volumes for all Services. + Volumes() (httptypes.ServiceVolumeMap, error) +} + +type client struct { + config gofig.Config + httpClient *http.Client + proto string + laddr string + tlsConfig *tls.Config + logRequests bool + logResponses bool + ctx context.Context +} + +// Dial opens a connection to a remote libStorage serice and returns the client +// that can be used to communicate with said endpoint. +// +// If the config parameter is nil a default instance is created. The +// function dials the libStorage service specified by the configuration +// property libstorage.host. +func Dial( + ctx context.Context, + config gofig.Config) (Client, error) { + + c := &client{config: config} + c.logRequests = c.config.GetBool( + "libstorage.client.http.logging.logrequest") + c.logResponses = c.config.GetBool( + "libstorage.client.http.logging.logresponse") + + logFields := log.Fields{} + + host := config.GetString("libstorage.host") + if host == "" { + return nil, goof.New("libstorage.host is required") + } + + tlsConfig, tlsFields, err := + utils.ParseTLSConfig(config.Scope("libstorage.client")) + if err != nil { + return nil, err + } + c.tlsConfig = tlsConfig + for k, v := range tlsFields { + logFields[k] = v + } + + cProto, cLaddr, err := gotil.ParseAddress(host) + if err != nil { + return nil, err + } + c.proto = cProto + c.laddr = cLaddr + + if ctx == nil { + log.Debug("created empty context for client") + ctx = context.Background() + } + ctx = ctx.WithContextID("host", host) + + c.httpClient = &http.Client{ + Transport: &http.Transport{ + Dial: func(proto, addr string) (conn net.Conn, err error) { + if tlsConfig == nil { + return net.Dial(cProto, cLaddr) + } + return tls.Dial(cProto, cLaddr, tlsConfig) + }, + }, + } + + ctx.Log().WithFields(logFields).Info("configured client") + + c.ctx = ctx + return c, nil +} + +func (c *client) Root() ([]string, error) { + reply := httptypes.RootResponse{} + if err := c.httpGet("/", &reply); err != nil { + return nil, err + } + return reply, nil +} + +func (c *client) Volumes() (httptypes.ServiceVolumeMap, error) { + reply := httptypes.ServiceVolumeMap{} + if err := c.httpGet("/volumes", &reply); err != nil { + return nil, err + } + return reply, nil +} + +func (c *client) httpDo(method, path string, payload, reply interface{}) error { + + reqBody, err := encPayload(payload) + if err != nil { + return err + } + + host := c.laddr + if c.proto == "unix" { + host = "libstorage-server" + } + if c.tlsConfig != nil && c.tlsConfig.ServerName != "" { + host = c.tlsConfig.ServerName + } + + url := fmt.Sprintf("http://%s%s", host, path) + c.ctx.Log().WithField("url", url).Debug("built request url") + req, err := http.NewRequest(method, url, reqBody) + if err != nil { + return err + } + c.logRequest(req) + + res, err := ctxhttp.Do(c.ctx, c.httpClient, req) + if err != nil { + return err + } + c.logResponse(res) + + if err := decRes(res.Body, reply); err != nil { + return err + } + + return nil +} + +func (c *client) httpGet(path string, reply interface{}) error { + return c.httpDo("GET", path, nil, reply) +} + +func (c *client) httpPost( + path string, + payload interface{}, reply interface{}) error { + + return c.httpDo("POST", path, payload, reply) +} + +func (c *client) httpDelete(path string, reply interface{}) error { + + return c.httpDo("DELETE", path, nil, reply) +} + +func encPayload(payload interface{}) (io.Reader, error) { + if payload == nil { + return nil, nil + } + + buf, err := json.Marshal(payload) + if err != nil { + return nil, err + } + + return bytes.NewReader(buf), nil +} + +func decRes(body io.Reader, reply interface{}) error { + buf, err := ioutil.ReadAll(body) + if err != nil { + return err + } + if err := json.Unmarshal(buf, reply); err != nil { + return err + } + return nil +} + +func (c *client) logRequest(req *http.Request) { + + if !c.logRequests { + return + } + + w := log.StandardLogger().Writer() + + fmt.Fprintln(w, "") + fmt.Fprint(w, " -------------------------- ") + fmt.Fprint(w, "HTTP REQUEST (CLIENT)") + fmt.Fprintln(w, " -------------------------") + + buf, err := httputil.DumpRequest(req, true) + if err != nil { + return + } + + gotil.WriteIndented(w, buf) + fmt.Fprintln(w) +} + +func (c *client) logResponse(res *http.Response) { + + if !c.logResponses { + return + } + + w := log.StandardLogger().Writer() + + fmt.Fprintln(w) + fmt.Fprint(w, " -------------------------- ") + fmt.Fprint(w, "HTTP RESPONSE (CLIENT)") + fmt.Fprintln(w, " -------------------------") + + buf, err := httputil.DumpResponse(res, true) + if err != nil { + return + } + + bw := &bytes.Buffer{} + gotil.WriteIndented(bw, buf) + + scanner := bufio.NewScanner(bw) + for { + if !scanner.Scan() { + break + } + fmt.Fprintln(w, scanner.Text()) + } +} + +func (c *client) getLocalDevices(prefix string) ([]string, error) { + + path := c.config.GetString("libstorage.client.localdevicesfile") + buf, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + + deviceNames := []string{} + scan := bufio.NewScanner(bytes.NewReader(buf)) + + rx := regexp.MustCompile(fmt.Sprintf(`^.+?\s(%s\w+)$`, prefix)) + for scan.Scan() { + l := scan.Text() + m := rx.FindStringSubmatch(l) + if len(m) > 0 { + deviceNames = append(deviceNames, fmt.Sprintf("/dev/%s", m[1])) + } + } + + return deviceNames, nil +} diff --git a/api/registry/registry.go b/api/registry/registry.go new file mode 100644 index 00000000..e4f28e3f --- /dev/null +++ b/api/registry/registry.go @@ -0,0 +1,175 @@ +// Package registry is the central hub for Drivers and other types that +// follow the init-time registration. +package registry + +import ( + "strings" + "sync" + + "github.com/akutz/goof" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types/drivers" +) + +var ( + storDriverCtors = map[string]drivers.NewStorageDriver{} + storDriverCtorsRWL = &sync.RWMutex{} + + osDriverCtors = map[string]drivers.NewOSDriver{} + osDriverCtorsRWL = &sync.RWMutex{} + + intDriverCtors = map[string]drivers.NewIntegrationDriver{} + intDriverCtorsRWL = &sync.RWMutex{} + + routers = []httputils.Router{} + routersRWL = &sync.RWMutex{} +) + +// RegisterRouter registers a Router. +func RegisterRouter(router httputils.Router) { + routersRWL.Lock() + defer routersRWL.Unlock() + routers = append(routers, router) +} + +// RegisterStorageDriver registers a StorageDriver. +func RegisterStorageDriver(name string, ctor drivers.NewStorageDriver) { + storDriverCtorsRWL.Lock() + defer storDriverCtorsRWL.Unlock() + storDriverCtors[strings.ToLower(name)] = ctor +} + +// RegisterOSDriver registers a OSDriver. +func RegisterOSDriver(name string, ctor drivers.NewOSDriver) { + osDriverCtorsRWL.Lock() + defer osDriverCtorsRWL.Unlock() + osDriverCtors[strings.ToLower(name)] = ctor +} + +// RegisterIntegrationDriver registers a IntegrationDriver. +func RegisterIntegrationDriver(name string, ctor drivers.NewIntegrationDriver) { + intDriverCtorsRWL.Lock() + defer intDriverCtorsRWL.Unlock() + intDriverCtors[strings.ToLower(name)] = ctor +} + +// NewStorageDriver returns a new instance of the driver specified by the +// driver name. +func NewStorageDriver(name string) (drivers.StorageDriver, error) { + + var ok bool + var ctor drivers.NewStorageDriver + + func() { + storDriverCtorsRWL.RLock() + defer storDriverCtorsRWL.RUnlock() + ctor, ok = storDriverCtors[strings.ToLower(name)] + }() + + if !ok { + return nil, goof.WithField("driver", name, "invalid driver name") + } + + return ctor(), nil +} + +// NewOSDriver returns a new instance of the driver specified by the +// driver name. +func NewOSDriver(name string) (drivers.OSDriver, error) { + + var ok bool + var ctor drivers.NewOSDriver + + func() { + osDriverCtorsRWL.RLock() + defer osDriverCtorsRWL.RUnlock() + ctor, ok = osDriverCtors[strings.ToLower(name)] + }() + + if !ok { + return nil, goof.WithField("driver", name, "invalid driver name") + } + + return ctor(), nil +} + +// NewIntegrationDriver returns a new instance of the driver specified by the +// driver name. +func NewIntegrationDriver(name string) (drivers.IntegrationDriver, error) { + + var ok bool + var ctor drivers.NewIntegrationDriver + + func() { + intDriverCtorsRWL.RLock() + defer intDriverCtorsRWL.RUnlock() + ctor, ok = intDriverCtors[strings.ToLower(name)] + }() + + if !ok { + return nil, goof.WithField("driver", name, "invalid driver name") + } + + return ctor(), nil +} + +// StorageDrivers returns a channel on which new instances of all registered +// storage drivers can be received. +func StorageDrivers() <-chan drivers.StorageDriver { + c := make(chan drivers.StorageDriver) + go func() { + storDriverCtorsRWL.RLock() + defer storDriverCtorsRWL.RUnlock() + for _, ctor := range storDriverCtors { + c <- ctor() + } + close(c) + }() + return c +} + +// OSDrivers returns a channel on which new instances of all registered +// OS drivers can be received. +func OSDrivers() <-chan drivers.OSDriver { + c := make(chan drivers.OSDriver) + go func() { + osDriverCtorsRWL.RLock() + defer osDriverCtorsRWL.RUnlock() + for _, ctor := range osDriverCtors { + c <- ctor() + } + close(c) + }() + return c +} + +// IntegrationDrivers returns a channel on which new instances of all registered +// integration drivers can be received. +func IntegrationDrivers() <-chan drivers.IntegrationDriver { + c := make(chan drivers.IntegrationDriver) + go func() { + intDriverCtorsRWL.RLock() + defer intDriverCtorsRWL.RUnlock() + for _, ctor := range intDriverCtors { + c <- ctor() + } + close(c) + }() + return c +} + +// Routers returns a channel on which new instances of all registered routers +// can be received. +func Routers() <-chan httputils.Router { + c := make(chan httputils.Router) + go func() { + routersRWL.RLock() + defer routersRWL.RUnlock() + for _, r := range routers { + c <- r + } + close(c) + }() + return c +} diff --git a/api/server/handlers/handlers_errors.go b/api/server/handlers/handlers_errors.go new file mode 100644 index 00000000..05f31ce5 --- /dev/null +++ b/api/server/handlers/handlers_errors.go @@ -0,0 +1,61 @@ +package handlers + +import ( + "net/http" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// errorHandler is a global HTTP filter for handlling errors +type errorHandler struct { + handler httputils.APIFunc +} + +// NewErrorHandler returns a new global HTTP filter for handling errors. +func NewErrorHandler() httputils.Middleware { + return &errorHandler{} +} + +func (h *errorHandler) Name() string { + return "error-handler" +} + +func (h *errorHandler) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *errorHandler) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + err := h.handler(ctx, w, req, store) + if err == nil { + return nil + } + + ctx.Log().Error(err) + + jsonError := types.JSONError{ + Status: getStatus(err), + Message: err.Error(), + Error: err, + } + + httputils.WriteJSON(w, jsonError.Status, jsonError) + return nil +} + +func getStatus(err error) int { + switch err.(type) { + case *types.ErrNotFound: + return http.StatusNotFound + default: + return http.StatusInternalServerError + } +} diff --git a/api/server/handlers/handlers_instanceid.go b/api/server/handlers/handlers_instanceid.go new file mode 100644 index 00000000..df1b25dc --- /dev/null +++ b/api/server/handlers/handlers_instanceid.go @@ -0,0 +1,87 @@ +package handlers + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "strings" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +const ( + libStorageInstanceIDHeaderKey = "libstorage-instanceid" +) + +// instanceIDHandler is a global HTTP filter for grokking the InstanceIDs +// from the headers +type instanceIDHandler struct { + handler httputils.APIFunc +} + +// NewInstanceIDHandler returns a new global HTTP filter for grokking the +// InstanceIDs from the headers +func NewInstanceIDHandler() httputils.Middleware { + return &instanceIDHandler{} +} + +func (h *instanceIDHandler) Name() string { + return "instanceIDs-handler" +} + +func (h *instanceIDHandler) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *instanceIDHandler) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + var iidHeader string + for k, v := range req.Header { + if strings.ToLower(k) == libStorageInstanceIDHeaderKey { + iidHeader = v[0] + break + } + } + + ctx.Log().WithField( + libStorageInstanceIDHeaderKey, iidHeader).Debug("http header") + + var iidPairs []string + if len(iidHeader) > 0 { + iidPairs = strings.Split(iidHeader, ",") + } + + iidMap := map[string]*types.InstanceID{} + + for _, iidPair := range iidPairs { + iidParts := strings.Split(iidPair, ":") + iidService := iidParts[0] + iidBase64 := iidParts[1] + + iid := &types.InstanceID{} + decoded, err := base64.StdEncoding.DecodeString(iidBase64) + if err != nil { + return err + } + + if err := json.Unmarshal(decoded, iid); err != nil { + return err + } + + iidMap[strings.ToLower(iidService)] = iid + } + + if len(iidMap) > 0 { + ctx = ctx.WithValue("instanceIDs", iidMap) + } + + return h.handler(ctx, w, req, store) +} diff --git a/api/server/handlers/handlers_instanceid_validator.go b/api/server/handlers/handlers_instanceid_validator.go new file mode 100644 index 00000000..8d4b5ecc --- /dev/null +++ b/api/server/handlers/handlers_instanceid_validator.go @@ -0,0 +1,53 @@ +package handlers + +import ( + "net/http" + + "github.com/akutz/goof" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// instanceIDValidator is a global HTTP filter for validating that the +// InstanceID for a context is present when it's required +type instanceIDValidator struct { + required bool + handler httputils.APIFunc +} + +// NewInstanceIDValidator returns a new global HTTP filter for validating that +// the InstanceID for a context is present when it's required +func NewInstanceIDValidator(required bool) httputils.Middleware { + return &instanceIDValidator{required: required} +} + +func (h *instanceIDValidator) Name() string { + return "instanceID-validator" +} + +func (h *instanceIDValidator) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *instanceIDValidator) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + iid := ctx.InstanceID() + + if h.required && iid == nil { + return goof.New("instanceID required") + } + + if store.GetBool("attachments") && iid == nil { + return goof.New("cannot get attachments without instance ID") + } + + return h.handler(ctx, w, req, store) +} diff --git a/service/server/handlers/logging.go b/api/server/handlers/handlers_logging.go similarity index 61% rename from service/server/handlers/logging.go rename to api/server/handlers/handlers_logging.go index b693b312..766714a5 100644 --- a/service/server/handlers/logging.go +++ b/api/server/handlers/handlers_logging.go @@ -1,6 +1,7 @@ package handlers import ( + "bytes" "fmt" "io" "net" @@ -8,92 +9,85 @@ import ( "net/http/httptest" "net/http/httputil" "net/url" - "os" "strconv" "strings" "time" "unicode/utf8" - log "github.com/Sirupsen/logrus" - "github.com/akutz/gofig" "github.com/akutz/gotil" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" ) const ( lowerhex = "0123456789abcdef" ) -// LoggingHandler is an HTTP logging handler for the libStorage service +// loggingHandler is an HTTP logging handler for the libStorage service // endpoint. -type LoggingHandler struct { +type loggingHandler struct { logRequests bool logResponses bool - stdOut io.WriteCloser - stdErr io.WriteCloser - handler http.Handler - config gofig.Config + writer io.Writer + handler httputils.APIFunc } -// NewLoggingHandler instantiates a new instance of the LoggingHandler type. +// NewLoggingHandler instantiates a new instance of the loggingHandler type. func NewLoggingHandler( - stdOut, stdErr io.WriteCloser, - handler http.Handler, - config gofig.Config) *LoggingHandler { - - h := &LoggingHandler{ - handler: handler, - config: config, - stdOut: stdOut, - stdErr: stdErr, - } + w io.Writer, + logHTTPRequests, logHTTPResponses bool) httputils.Middleware { - h.logRequests = config.GetBool( - "libstorage.server.http.logging.logrequest") - h.logResponses = config.GetBool( - "libstorage.server.http.logging.logresponse") + return &loggingHandler{ + writer: w, + logRequests: logHTTPRequests, + logResponses: logHTTPResponses, + } +} - return h +func (h *loggingHandler) Name() string { + return "logging-handler" } -// GetLogIO gets an io.WriteCloser using a given property name from a -// configuration instance. -func GetLogIO( - propName string, - config gofig.Config) io.WriteCloser { - - if path := config.GetString(propName); path != "" { - logio, err := os.OpenFile( - path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - log.Error(err) - } - log.WithFields(log.Fields{ - "logType": propName, - "logPath": path, - }).Debug("using log file") - return logio - } - return log.StandardLogger().Writer() +func (h *loggingHandler) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle } -// ServeHTTP serves the HTTP request and writes the response. -func (h *LoggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { +// Handle is the type's Handler function. +func (h *loggingHandler) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + bw := &bytes.Buffer{} + defer func(w io.Writer) { + h.writer.Write(bw.Bytes()) + }(bw) + var err error var reqDump []byte if h.logRequests { if reqDump, err = httputil.DumpRequest(req, true); err != nil { - log.Error(err) + return err } } rec := httptest.NewRecorder() - h.handler.ServeHTTP(rec, req) + reqErr := h.handler(ctx, rec, req, store) + + logRequest(h.logRequests, bw, rec, req, reqDump) + + if reqErr != nil { + return reqErr + } - logRequest(h.logRequests, h.stdOut, rec, req, reqDump) if h.logResponses { - fmt.Fprintln(h.stdOut, "") - logResponse(h.stdOut, rec, req) - fmt.Fprintln(h.stdOut, "") + fmt.Fprintln(bw, "") + logResponse(bw, rec, req) + fmt.Fprintln(bw, "") } w.WriteHeader(rec.Code) @@ -101,6 +95,8 @@ func (h *LoggingHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) { w.Header()[k] = v } w.Write(rec.Body.Bytes()) + + return nil } func logRequest( @@ -222,43 +218,47 @@ func appendQuoted(buf []byte, s string) []byte { buf = append(buf, runeTmp[:n]...) continue } - switch r { - case '\a': - buf = append(buf, `\a`...) - case '\b': - buf = append(buf, `\b`...) - case '\f': - buf = append(buf, `\f`...) - case '\n': - buf = append(buf, `\n`...) - case '\r': - buf = append(buf, `\r`...) - case '\t': - buf = append(buf, `\t`...) - case '\v': - buf = append(buf, `\v`...) + buf = appendRune(buf, s, r) + } + return buf +} + +func appendRune(buf []byte, s string, r rune) []byte { + switch r { + case '\a': + buf = append(buf, `\a`...) + case '\b': + buf = append(buf, `\b`...) + case '\f': + buf = append(buf, `\f`...) + case '\n': + buf = append(buf, `\n`...) + case '\r': + buf = append(buf, `\r`...) + case '\t': + buf = append(buf, `\t`...) + case '\v': + buf = append(buf, `\v`...) + default: + switch { + case r < ' ': + buf = append(buf, `\x`...) + buf = append(buf, lowerhex[s[0]>>4]) + buf = append(buf, lowerhex[s[0]&0xF]) + case r > utf8.MaxRune: + r = 0xFFFD + fallthrough + case r < 0x10000: + buf = append(buf, `\u`...) + for s := 12; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) + } default: - switch { - case r < ' ': - buf = append(buf, `\x`...) - buf = append(buf, lowerhex[s[0]>>4]) - buf = append(buf, lowerhex[s[0]&0xF]) - case r > utf8.MaxRune: - r = 0xFFFD - fallthrough - case r < 0x10000: - buf = append(buf, `\u`...) - for s := 12; s >= 0; s -= 4 { - buf = append(buf, lowerhex[r>>uint(s)&0xF]) - } - default: - buf = append(buf, `\U`...) - for s := 28; s >= 0; s -= 4 { - buf = append(buf, lowerhex[r>>uint(s)&0xF]) - } + buf = append(buf, `\U`...) + for s := 28; s >= 0; s -= 4 { + buf = append(buf, lowerhex[r>>uint(s)&0xF]) } } } return buf - } diff --git a/api/server/handlers/handlers_post_args.go b/api/server/handlers/handlers_post_args.go new file mode 100644 index 00000000..3feb1fe3 --- /dev/null +++ b/api/server/handlers/handlers_post_args.go @@ -0,0 +1,85 @@ +package handlers + +import ( + "fmt" + "net/http" + "reflect" + "strings" + + //log "github.com/Sirupsen/logrus" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// postArgsHandler is an HTTP filter for injecting the store with the POST +// object's fields and additional options +type postArgsHandler struct { + handler httputils.APIFunc +} + +// NewPostArgsHandler returns a new filter for injecting the store with the +// POST object's fields and additional options. +func NewPostArgsHandler() httputils.Middleware { + return &postArgsHandler{} +} + +func (h *postArgsHandler) Name() string { + return "post-args-handler" +} + +func (h *postArgsHandler) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *postArgsHandler) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + reqObj := ctx.Value("reqObj") + if reqObj == nil { + return fmt.Errorf("missing request object") + } + + v := reflect.ValueOf(reqObj).Elem() + t := v.Type() + + for i := 0; i < v.NumField(); i++ { + ft := t.Field(i) + fv := v.Field(i).Interface() + + switch tfv := fv.(type) { + case nil: + // do nothing + case map[string]interface{}: + // it's the Opts field; iterate Opts and add them to the store + for k, v := range tfv { + store.Set(k, v) + } + default: + // add it to the store + store.Set(getFieldName(ft), fv) + } + + } + + return h.handler(ctx, w, req, store) +} + +func getFieldName(ft reflect.StructField) string { + fn := ft.Name + if tag := ft.Tag.Get("json"); tag != "" { + if tag != "-" { + tagParts := strings.Split(tag, ",") + if tagParts[0] != "" { + fn = tagParts[0] + } + } + } + return fn +} diff --git a/api/server/handlers/handlers_query_params.go b/api/server/handlers/handlers_query_params.go new file mode 100644 index 00000000..f5c76319 --- /dev/null +++ b/api/server/handlers/handlers_query_params.go @@ -0,0 +1,66 @@ +package handlers + +import ( + "net/http" + "strconv" + + log "github.com/Sirupsen/logrus" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// queryParamsHandler is an HTTP filter for injecting the store with query +// parameters +type queryParamsHandler struct { + handler httputils.APIFunc +} + +func (h *queryParamsHandler) Name() string { + return "query-params-handler" +} + +// NewQueryParamsHandler returns a new filter for injecting the store with query +// parameters +func NewQueryParamsHandler() httputils.Middleware { + return &queryParamsHandler{} +} + +func (h *queryParamsHandler) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *queryParamsHandler) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + for k, v := range req.URL.Query() { + ctx.Log().WithFields(log.Fields{ + "key": k, + "value": v, + "len(value)": len(v), + }).Debug("query param") + switch len(v) { + case 0: + store.Set(k, true) + case 1: + if len(v[0]) == 0 { + store.Set(k, true) + } else { + if b, err := strconv.ParseBool(v[0]); err != nil { + store.Set(k, b) + } else { + store.Set(k, v[0]) + } + } + default: + store.Set(k, v) + } + } + return h.handler(ctx, w, req, store) +} diff --git a/api/server/handlers/handlers_schema_validator.go b/api/server/handlers/handlers_schema_validator.go new file mode 100644 index 00000000..75d9a782 --- /dev/null +++ b/api/server/handlers/handlers_schema_validator.go @@ -0,0 +1,114 @@ +package handlers + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + + //log "github.com/Sirupsen/logrus" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/utils/schema" +) + +// schemaValidator is an HTTP filter for validating incoming request payloads +type schemaValidator struct { + reqSchema []byte + resSchema []byte + newReqObjFunc func() interface{} + handler httputils.APIFunc +} + +// NewSchemaValidator returns a new filter for validating request payloads and +// response payloads against defined JSON schemas. +func NewSchemaValidator( + reqSchema, resSchema []byte, + newReqObjFunc func() interface{}) httputils.Middleware { + + return &schemaValidator{ + reqSchema: reqSchema, + resSchema: resSchema, + newReqObjFunc: newReqObjFunc, + } +} + +func (h *schemaValidator) Name() string { + return "schmea-validator" +} + +func (h *schemaValidator) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *schemaValidator) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + reqBody, err := ioutil.ReadAll(req.Body) + if err != nil { + return fmt.Errorf("validate req schema: read req error: %v", err) + } + + // do the request validation + if h.reqSchema != nil { + err = schema.Validate(ctx, h.reqSchema, reqBody) + if err != nil { + return fmt.Errorf("validate req schema: validation error: %v", err) + } + } + + // create the object for the request payload if there is a function for it + if h.newReqObjFunc != nil { + reqObj := h.newReqObjFunc() + if len(reqBody) > 0 { + if err = json.Unmarshal(reqBody, reqObj); err != nil { + return fmt.Errorf( + "validate req schema: unmarshal error: %v", err) + } + } + ctx = context.WithValue(ctx, "reqObj", reqObj) + } + + // if there's not response schema then just return the result of the next + // handler + if h.resSchema == nil { + return h.handler(ctx, w, req, store) + } + + // at this point we know there's going to be response validation, so + // we need to record the result of the next handler in order to intercept + // the response payload to validate it + rec := httptest.NewRecorder() + + // invoke the next handler with a recorder + err = h.handler(ctx, rec, req, store) + if err != nil { + return err + } + + // do the response validation + resBody := rec.Body.Bytes() + err = schema.Validate(ctx, h.resSchema, resBody) + if err != nil { + return err + } + + // write the recorded result of the next handler to the resposne writer + w.WriteHeader(rec.Code) + for k, v := range rec.HeaderMap { + w.Header()[k] = v + } + if _, err = w.Write(resBody); err != nil { + return err + } + + return nil +} diff --git a/api/server/handlers/handlers_service_validator.go b/api/server/handlers/handlers_service_validator.go new file mode 100644 index 00000000..9f090e78 --- /dev/null +++ b/api/server/handlers/handlers_service_validator.go @@ -0,0 +1,69 @@ +package handlers + +import ( + "net/http" + "strings" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/utils" +) + +// serviceValidator is an HTTP filter for validating that the service +// specified as part of the path is valid. +type serviceValidator struct { + services map[string]httputils.Service + handler httputils.APIFunc +} + +// NewServiceValidator returns a new filter for validating that the service +// specified as part of the path is valid. +func NewServiceValidator( + services map[string]httputils.Service) httputils.Middleware { + return &serviceValidator{services: services} +} + +func (h *serviceValidator) Name() string { + return "service-validator" +} + +func (h *serviceValidator) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *serviceValidator) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + if !store.IsSet("service") { + return utils.NewStoreKeyErr("service") + } + + serviceName := store.GetString("service") + service, ok := h.services[strings.ToLower(serviceName)] + if !ok { + return utils.NewNotFoundError(serviceName) + } + + instanceIDs, ok := ctx.Value("instanceIDs").(map[string]*types.InstanceID) + if ok { + if iid, ok := instanceIDs[strings.ToLower(serviceName)]; ok { + ctx = ctx.WithInstanceID(iid) + ctx = ctx.WithContextID("instanceID", iid.ID) + ctx.Log().Debug("set instanceID") + } + } + + ctx = ctx.WithValue("service", service) + ctx = ctx.WithValue("serviceID", service.Name()) + ctx = ctx.WithContextID("service", service.Name()) + ctx = ctx.WithContextID("driver", service.Driver().Name()) + + ctx.Log().Debug("set service context") + return h.handler(ctx, w, req, store) +} diff --git a/api/server/handlers/handlers_snapshot_validator.go b/api/server/handlers/handlers_snapshot_validator.go new file mode 100644 index 00000000..a0a77c48 --- /dev/null +++ b/api/server/handlers/handlers_snapshot_validator.go @@ -0,0 +1,64 @@ +package handlers + +import ( + "net/http" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/utils" +) + +// snapshotValidator is an HTTP filter for validating that the snapshot +// specified as part of the path is valid. +type snapshotValidator struct { + handler httputils.APIFunc +} + +// NewSnapshotValidator returns a new filter for validating that the snapshot +// specified as part of the path is valid. +func NewSnapshotValidator() httputils.Middleware { + return &snapshotValidator{} +} + +func (h *snapshotValidator) Name() string { + return "snapshot-validator" +} + +func (h *snapshotValidator) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *snapshotValidator) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + if !store.IsSet("snapshotID") { + return utils.NewStoreKeyErr("snapshotID") + } + + service, err := httputils.GetService(ctx) + if err != nil { + return err + } + + snapshotID := store.GetString("snapshotID") + + snapshot, err := service.Driver().SnapshotInspect( + ctx, + snapshotID, + store) + if err != nil { + return err + } + + ctx = ctx.WithValue("snapshot", snapshot) + ctx = ctx.WithContextID("snapshot", snapshot.ID) + ctx.Log().Debug("set snapshot context") + + return h.handler(ctx, w, req, store) +} diff --git a/api/server/handlers/handlers_volume_validator.go b/api/server/handlers/handlers_volume_validator.go new file mode 100644 index 00000000..d61552fe --- /dev/null +++ b/api/server/handlers/handlers_volume_validator.go @@ -0,0 +1,68 @@ +package handlers + +import ( + "net/http" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" + "github.com/emccode/libstorage/api/utils" +) + +// volumeValidator is an HTTP filter for validating that the volume +// specified as part of the path is valid. +type volumeValidator struct { + handler httputils.APIFunc +} + +// NewVolumeValidator returns a new filter for validating that the volume +// specified as part of the path is valid. +func NewVolumeValidator() httputils.Middleware { + return &volumeValidator{} +} + +func (h *volumeValidator) Name() string { + return "volume-validator" +} + +func (h *volumeValidator) Handler(m httputils.APIFunc) httputils.APIFunc { + h.handler = m + return h.Handle +} + +// Handle is the type's Handler function. +func (h *volumeValidator) Handle( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + if !store.IsSet("volumeID") { + return utils.NewStoreKeyErr("volumeID") + } + + service, err := httputils.GetService(ctx) + if err != nil { + return err + } + + volumeID := store.GetString("volumeID") + + volume, err := service.Driver().VolumeInspect( + ctx, + volumeID, + &drivers.VolumeInspectOpts{ + Attachments: store.GetBool("attachments"), + Opts: store, + }) + if err != nil { + return err + } + + ctx = ctx.WithValue("volume", volume) + ctx = ctx.WithContextID("volume", volume.ID) + ctx.Log().Debug("set volume context") + + return h.handler(ctx, w, req, store) +} diff --git a/api/server/httputils/httputils.go b/api/server/httputils/httputils.go new file mode 100644 index 00000000..3538c160 --- /dev/null +++ b/api/server/httputils/httputils.go @@ -0,0 +1,64 @@ +package httputils + +import ( + "encoding/json" + "net/http" + + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" + "github.com/emccode/libstorage/api/utils" +) + +var ( + serviceTypeName = utils.GetTypePkgPathAndName((*Service)(nil)) + storageDriverTypeName = utils.GetTypePkgPathAndName((*drivers.StorageDriver)(nil)) +) + +// GetService gets the Service instance from a context. +func GetService(ctx context.Context) (Service, error) { + serviceObj := ctx.Value("service") + if serviceObj == nil { + return nil, utils.NewContextKeyErr("service") + } + service, ok := serviceObj.(Service) + if !ok { + return nil, utils.NewContextTypeErr( + "service", serviceTypeName, utils.GetTypePkgPathAndName(serviceObj)) + } + return service, nil +} + +// GetStorageDriver gets the StorageDriver instance from a context. +func GetStorageDriver(ctx context.Context) (drivers.StorageDriver, error) { + service, err := GetService(ctx) + if err != nil { + return nil, err + } + return service.Driver(), nil +} + +// WriteJSON writes the value v to the http response stream as json with +// standard json encoding. +func WriteJSON(w http.ResponseWriter, code int, v interface{}) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + buf, err := json.MarshalIndent(v, "", " ") + if err != nil { + return err + } + if _, err := w.Write(buf); err != nil { + return err + } + return nil + //return json.NewEncoder(w).Encode(v) +} + +// WriteData writes the value v to the http response stream as binary. +func WriteData(w http.ResponseWriter, code int, v []byte) error { + w.Header().Set("Content-Type", "application/octet-stream") + w.WriteHeader(code) + if _, err := w.Write(v); err != nil { + return err + } + return nil +} diff --git a/api/server/httputils/httputils_middleware.go b/api/server/httputils/httputils_middleware.go new file mode 100644 index 00000000..f6ca4964 --- /dev/null +++ b/api/server/httputils/httputils_middleware.go @@ -0,0 +1,30 @@ +package httputils + +import ( + "net/http" + + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// MiddlewareFunc is an adapter to allow the use of ordinary functions as API +// filters. Any function that has the appropriate signature can be register as +// a middleware. +type MiddlewareFunc func(handler APIFunc) APIFunc + +// Middleware is middleware for a route. +type Middleware interface { + + // Name returns the name of the middlware. + Name() string + + // Handler enables the chaining of middlware. + Handler(handler APIFunc) APIFunc + + // Handle is for processing an incoming request. + Handle( + ctx context.Context, + w http.ResponseWriter, + r *http.Request, + store types.Store) error +} diff --git a/api/server/httputils/httputils_route.go b/api/server/httputils/httputils_route.go new file mode 100644 index 00000000..3e976d7f --- /dev/null +++ b/api/server/httputils/httputils_route.go @@ -0,0 +1,187 @@ +package httputils + +import ( + "github.com/akutz/gofig" +) + +// Router defines an interface to specify a group of routes to add the the +// server. +type Router interface { + + // Routes returns all of the router's routes. + Routes() []Route + + // Name returns the name of the router. + Name() string + + // Init initializes the router. + Init(config gofig.Config, services map[string]Service) +} + +// NewRouteFunc returns a new route. +type NewRouteFunc func(config gofig.Config) Route + +// Route defines an individual API route in the server. +type Route interface { + + // Queries add query strings that must match for a route. + Queries(queries ...string) Route + + // Middlewares adds middleware to the route. + Middlewares(middlewares ...Middleware) Route + + // Name returns the name of the route. + GetName() string + + // GetHandler returns the raw function to create the http handler. + GetHandler() APIFunc + + // GetMethod returns the http method that the route responds to. + GetMethod() string + + // GetPath returns the subpath where the route responds to. + GetPath() string + + // GetQueries returns the query strings for which the route should respond. + GetQueries() []string + + // GetMiddlewares returns a list of route-specific middleware. + GetMiddlewares() []Middleware +} + +// route defines an individual API route. +type route struct { + name string + method string + path string + queries []string + handler APIFunc + middlewares []Middleware +} + +// Method specifies the method for the route. +func (r *route) Method(method string) Route { + r.method = method + return r +} + +// Path specifies the path for the route. +func (r *route) Path(path string) Route { + r.path = path + return r +} + +// Queries add query strings that must match for a route. +func (r *route) Queries(queries ...string) Route { + r.queries = append(r.queries, queries...) + if len(queries) == 1 { + r.queries = append(r.queries, "") + } + return r +} + +// Handler specifies the handler for the route. +func (r *route) Handler(handler APIFunc) Route { + r.handler = handler + return r +} + +// Middlewares adds middleware to the route. +func (r *route) Middlewares( + middlewares ...Middleware) Route { + r.middlewares = append(r.middlewares, middlewares...) + return r +} + +// Name returns the name of the route. +func (r *route) GetName() string { + return r.name +} + +// GetHandler returns the APIFunc to let the server wrap it in middlewares +func (r *route) GetHandler() APIFunc { + return r.handler +} + +// GetMethod returns the http method that the route responds to. +func (r *route) GetMethod() string { + return r.method +} + +// GetPath returns the subpath where the route responds to. +func (r *route) GetPath() string { + return r.path +} + +// GetQueries returns the query strings for which the route should respond. +func (r *route) GetQueries() []string { + return r.queries +} + +// GetMiddlwares returns a list of route-specific middleware. +func (r *route) GetMiddlewares() []Middleware { + return r.middlewares +} + +// NewRoute initialies a new local route for the reouter +func NewRoute( + name, method, path string, + handler APIFunc, + middlewares ...Middleware) Route { + + return &route{ + name: name, + method: method, + path: path, + handler: handler, + middlewares: middlewares, + } +} + +// NewGetRoute initializes a new route with the http method GET. +func NewGetRoute( + name, path string, + handler APIFunc, + middlewares ...Middleware) Route { + return NewRoute(name, "GET", path, handler, middlewares...) +} + +// NewPostRoute initializes a new route with the http method POST. +func NewPostRoute( + name, path string, + handler APIFunc, + middlewares ...Middleware) Route { + return NewRoute(name, "POST", path, handler, middlewares...) +} + +// NewPutRoute initializes a new route with the http method PUT. +func NewPutRoute( + name, path string, + handler APIFunc, + middlewares ...Middleware) Route { + return NewRoute(name, "PUT", path, handler, middlewares...) +} + +// NewDeleteRoute initializes a new route with the http method DELETE. +func NewDeleteRoute( + name, path string, + handler APIFunc, + middlewares ...Middleware) Route { + return NewRoute(name, "DELETE", path, handler, middlewares...) +} + +// NewOptionsRoute initializes a new route with the http method OPTIONS +func NewOptionsRoute( + name, path string, + handler APIFunc, + middlewares ...Middleware) Route { + return NewRoute(name, "OPTIONS", path, handler, middlewares...) +} + +// NewHeadRoute initializes a new route with the http method HEAD. +func NewHeadRoute( + name, path string, + handler APIFunc, + middlewares ...Middleware) Route { + return NewRoute(name, "HEAD", path, handler, middlewares...) +} diff --git a/api/server/httputils/httputils_types.go b/api/server/httputils/httputils_types.go new file mode 100644 index 00000000..40741d11 --- /dev/null +++ b/api/server/httputils/httputils_types.go @@ -0,0 +1,36 @@ +package httputils + +import ( + "net/http" + + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" +) + +// APIFunc is an adapter to allow the use of ordinary functions as API +// endpoints. Any function that has the appropriate signature can be register +// as an API endpoint. +type APIFunc func( + ctx context.Context, + w http.ResponseWriter, + r *http.Request, + store types.Store) error + +// NewRequestObjFunc is a function that creates a new instance of the type to +// which the request body is serialized. +type NewRequestObjFunc func() interface{} + +// Service is information about a service. +type Service interface { + // Name gets the name of the service. + Name() string + + // Driver gets the service's driver. + Driver() drivers.StorageDriver + + // Config gets the service's configuration. + Config() gofig.Config +} diff --git a/api/server/router/driver/driver.go b/api/server/router/driver/driver.go new file mode 100644 index 00000000..07adf48a --- /dev/null +++ b/api/server/router/driver/driver.go @@ -0,0 +1,58 @@ +// +build ignore + +package driver + +import ( + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/server/router" + "github.com/emccode/libstorage/api/types/drivers" +) + +// driverRouter is a router to talk with the driver controller +type driverRouter struct { + config gofig.Config + drivers map[string]drivers.StorageDriver + routes []router.Route +} + +// NewRouter initializes a new driverRouter +func NewRouter( + config gofig.Config, + drivers map[string]drivers.StorageDriver) router.Router { + + r := &driverRouter{ + config: config, + drivers: drivers, + } + + r.initRoutes() + return r +} + +// Routes returns the available routers to the driver controller +func (r *driverRouter) Routes() []router.Route { + return r.routes +} + +func (r *driverRouter) initRoutes() { + r.routes = []router.Route{ + // GET + router.NewGetRoute( + "drivers", + "/drivers", + r.driversList), + router.NewGetRoute( + "driverInspect", + "/drivers/{driver}", + r.driverInspect), + router.NewGetRoute( + "executors", + "/drivers/{driver}/executors", + r.executorsList), + router.NewGetRoute( + "executorInspect", + "/drivers/{driver}/executors/{name}", + r.executorDownload), + } +} diff --git a/api/server/router/driver/driver_ignore.go b/api/server/router/driver/driver_ignore.go new file mode 100644 index 00000000..bce7c468 --- /dev/null +++ b/api/server/router/driver/driver_ignore.go @@ -0,0 +1 @@ +package driver diff --git a/api/server/router/driver/driver_routes.go b/api/server/router/driver/driver_routes.go new file mode 100644 index 00000000..9edfe161 --- /dev/null +++ b/api/server/router/driver/driver_routes.go @@ -0,0 +1,154 @@ +// +build ignore + +package driver + +import ( + "crypto/md5" + "fmt" + "net/http" + "strings" + + "github.com/akutz/goof" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" + httptypes "github.com/emccode/libstorage/api/types/http" +) + +func (r *driverRouter) driversList( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + var reply httptypes.DriversResponse = map[string]*types.DriverInfo{} + + for _, d := range r.drivers { + reply[d.Name()] = &types.DriverInfo{ + Name: d.Name(), + Type: d.Type(), + NextDevice: d.NextDevice(), + Executors: getExecutors(ctx, d), + } + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *driverRouter) driverInspect( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + dname := store.GetString("driver") + + d, ok := r.drivers[strings.ToLower(dname)] + if !ok { + return goof.New("invalid driver") + } + + reply := &types.DriverInfo{ + Name: d.Name(), + Type: d.Type(), + NextDevice: d.NextDevice(), + Executors: getExecutors(ctx, d), + } + + return httputils.WriteJSON(w, http.StatusOK, reply) +} + +func getExecutors( + ctx context.Context, d drivers.StorageDriver) []*types.ExecutorInfo { + + executors := d.Executors() + hashExecutors(ctx, d, executors) + return executors +} + +func (r *driverRouter) executorsList( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + dname := store.GetString("driver") + + d, ok := r.drivers[strings.ToLower(dname)] + if !ok { + return goof.New("invalid driver") + } + + var reply types.ExecutorsListResponse = getExecutors(ctx, d) + + return httputils.WriteJSON(w, http.StatusOK, reply) +} + +func (r *driverRouter) executorDownload( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + dname := store.GetString("driver") + + d, ok := r.drivers[strings.ToLower(dname)] + if !ok { + return goof.New("invalid driver") + } + + executor, err := getExecutor(ctx, d, store.GetString("name")) + if err != nil { + return err + } + + return httputils.WriteData(w, http.StatusOK, executor) +} + +func getExecutor( + ctx context.Context, + d drivers.StorageDriver, + goos, goarch string, + store types.Store) ([]byte, error) { + + tool, err := d.ExecutorInspect(ctx, goos, goarch, store) + if err != nil { + return nil, err + } + + return tool, nil +} + +func hashExecutors( + ctx context.Context, + d drivers.StorageDriver, + tinfos []*types.ExecutorInfo) error { + + for _, ti := range tinfos { + if err := hashExecutor(ctx, d, ti); err != nil { + return err + } + } + return nil +} + +func hashExecutor( + ctx context.Context, + d drivers.StorageDriver, + tinfo *types.ExecutorInfo) error { + + if tinfo.MD5Checksum == "" { + return nil + } + + tool, err := getExecutor(ctx, d, tinfo.Name) + if err != nil { + return err + } + + tinfo.MD5Checksum = fmt.Sprintf("%x", md5.Sum(tool)) + return nil +} diff --git a/api/server/router/root/root.go b/api/server/router/root/root.go new file mode 100644 index 00000000..309d100d --- /dev/null +++ b/api/server/router/root/root.go @@ -0,0 +1,41 @@ +package root + +import ( + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/server/httputils" +) + +func init() { + registry.RegisterRouter(&router{}) +} + +type router struct { + config gofig.Config + services map[string]httputils.Service + routes []httputils.Route +} + +func (r *router) Name() string { + return "root-router" +} + +func (r *router) Init( + config gofig.Config, services map[string]httputils.Service) { + r.config = config + r.services = services + r.initRoutes() +} + +// Routes returns the available routes. +func (r *router) Routes() []httputils.Route { + return r.routes +} + +func (r *router) initRoutes() { + r.routes = []httputils.Route{ + // GET + httputils.NewGetRoute("root", "/", r.root), + } +} diff --git a/api/server/router/root/root_router.go b/api/server/router/root/root_router.go new file mode 100644 index 00000000..d13acd3f --- /dev/null +++ b/api/server/router/root/root_router.go @@ -0,0 +1,38 @@ +package root + +import ( + "fmt" + "net/http" + "regexp" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + httptypes "github.com/emccode/libstorage/api/types/http" +) + +var ( + rootURLRx = regexp.MustCompile(`^(.+)/[^/]*$`) +) + +func (r *router) root( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + proto := "http" + if req.TLS != nil { + proto = "https" + } + rootURL := fmt.Sprintf("%s://%s", proto, req.Host) + + var reply httptypes.RootResponse = []string{ + fmt.Sprintf("%s/services", rootURL), + fmt.Sprintf("%s/snapshots", rootURL), + fmt.Sprintf("%s/volumes", rootURL), + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} diff --git a/api/server/router/router.go b/api/server/router/router.go new file mode 100644 index 00000000..97fbd37b --- /dev/null +++ b/api/server/router/router.go @@ -0,0 +1,9 @@ +package router + +import ( + // imports to load packges + _ "github.com/emccode/libstorage/api/server/router/root" + _ "github.com/emccode/libstorage/api/server/router/service" + _ "github.com/emccode/libstorage/api/server/router/snapshot" + _ "github.com/emccode/libstorage/api/server/router/volume" +) diff --git a/api/server/router/service/service.go b/api/server/router/service/service.go new file mode 100644 index 00000000..0f1b2fbd --- /dev/null +++ b/api/server/router/service/service.go @@ -0,0 +1,63 @@ +package service + +import ( + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/server/handlers" + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/utils/schema" +) + +func init() { + registry.RegisterRouter(&router{}) +} + +type router struct { + config gofig.Config + services map[string]httputils.Service + routes []httputils.Route +} + +func (r *router) Name() string { + return "service-router" +} + +func (r *router) Init( + config gofig.Config, services map[string]httputils.Service) { + r.config = config + r.services = services + r.initRoutes() +} + +// Routes returns the available routes. +func (r *router) Routes() []httputils.Route { + return r.routes +} + +func (r *router) initRoutes() { + + r.routes = []httputils.Route{ + + // GET + httputils.NewGetRoute( + "services", + "/services", + r.servicesList, + handlers.NewSchemaValidator(nil, schema.ServiceInfoMapSchema, nil)), + + httputils.NewGetRoute( + "serviceInspect", + "/services/{service}", + r.serviceInspect, + handlers.NewServiceValidator(r.services), + handlers.NewSchemaValidator(nil, schema.ServiceInfoSchema, nil)), + + httputils.NewGetRoute( + "serviceInspect", + "/services/{service}/executors", + r.serviceInspect, + handlers.NewServiceValidator(r.services), + handlers.NewSchemaValidator(nil, schema.ServiceInfoSchema, nil)), + } +} diff --git a/api/server/router/service/service_routes.go b/api/server/router/service/service_routes.go new file mode 100644 index 00000000..09416416 --- /dev/null +++ b/api/server/router/service/service_routes.go @@ -0,0 +1,70 @@ +package service + +import ( + "net/http" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + httptypes "github.com/emccode/libstorage/api/types/http" +) + +func (r *router) servicesList( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + var reply httptypes.ServicesResponse = map[string]*types.ServiceInfo{} + for _, service := range r.services { + si := toServiceInfo(service) + reply[si.Name] = si + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) serviceInspect( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + service, err := httputils.GetService(ctx) + if err != nil { + return err + } + reply := toServiceInfo(service) + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) executorsList( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + service, err := httputils.GetService(ctx) + if err != nil { + return err + } + + //driverName := service.Driver().Name() + + reply := toServiceInfo(service) + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func toServiceInfo(service httputils.Service) *types.ServiceInfo { + return &types.ServiceInfo{ + Name: service.Name(), + Driver: &types.DriverInfo{ + Name: service.Driver().Name(), + Type: service.Driver().Type(), + NextDevice: service.Driver().NextDeviceInfo(), + }, + } +} diff --git a/api/server/router/snapshot/snapshot.go b/api/server/router/snapshot/snapshot.go new file mode 100644 index 00000000..ea30409b --- /dev/null +++ b/api/server/router/snapshot/snapshot.go @@ -0,0 +1,113 @@ +package snapshot + +import ( + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/server/handlers" + "github.com/emccode/libstorage/api/server/httputils" + httptypes "github.com/emccode/libstorage/api/types/http" + "github.com/emccode/libstorage/api/utils/schema" +) + +func init() { + registry.RegisterRouter(&router{}) +} + +type router struct { + config gofig.Config + services map[string]httputils.Service + routes []httputils.Route +} + +func (r *router) Name() string { + return "snapshot-router" +} + +func (r *router) Init( + config gofig.Config, services map[string]httputils.Service) { + r.config = config + r.services = services + r.initRoutes() +} + +// Routes returns the available routes. +func (r *router) Routes() []httputils.Route { + return r.routes +} + +func (r *router) initRoutes() { + r.routes = []httputils.Route{ + + // GET + + // get all snapshots from all services + httputils.NewGetRoute( + "snapshots", + "/snapshots", + r.snapshots, + handlers.NewSchemaValidator( + nil, schema.ServiceSnapshotMapSchema, nil), + ), + + // get all snapshots from a specific service + httputils.NewGetRoute( + "snapshotsForService", + "/snapshots/{service}", + r.snapshotsForService, + handlers.NewServiceValidator(r.services), + handlers.NewSchemaValidator( + nil, schema.SnapshotMapSchema, nil), + ), + + // get a specific snapshot from a specific service + httputils.NewGetRoute( + "snapshotInspect", + "/snapshots/{service}/{snapshotID}", + r.snapshotInspect, + handlers.NewServiceValidator(r.services), + handlers.NewSnapshotValidator(), + handlers.NewSchemaValidator(nil, schema.SnapshotSchema, nil), + ), + + // POST + + // create volume from snapshot + httputils.NewPostRoute( + "snapshotCreate", + "/snapshots/{service}/{snapshotID}", + r.volumeCreate, + handlers.NewServiceValidator(r.services), + handlers.NewSnapshotValidator(), + handlers.NewSchemaValidator( + schema.VolumeCreateFromSnapshotRequestSchema, + schema.VolumeSchema, + func() interface{} { + return &httptypes.VolumeCreateFromSnapshotRequest{} + }), + handlers.NewPostArgsHandler(), + ).Queries("create"), + + // copy snapshot + httputils.NewPostRoute( + "snapshotCopy", + "/snapshots/{service}/{snapshotID}", + r.snapshotCopy, + handlers.NewServiceValidator(r.services), + handlers.NewSnapshotValidator(), + handlers.NewSchemaValidator( + schema.SnapshotCopyRequestSchema, + schema.SnapshotSchema, + func() interface{} { + return &httptypes.SnapshotCopyRequest{} + }), + handlers.NewPostArgsHandler(), + ).Queries("copy"), + + // DELETE + httputils.NewDeleteRoute( + "snapshotRemove", + "/snapshots/{service}/{snapshotID}", + r.snapshotRemove), + } +} diff --git a/api/server/router/snapshot/snapshot_routes.go b/api/server/router/snapshot/snapshot_routes.go new file mode 100644 index 00000000..123eccb3 --- /dev/null +++ b/api/server/router/snapshot/snapshot_routes.go @@ -0,0 +1,171 @@ +package snapshot + +import ( + "net/http" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + httptypes "github.com/emccode/libstorage/api/types/http" + "github.com/emccode/libstorage/api/utils" +) + +func (r *router) snapshots( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + var reply httptypes.ServiceSnapshotMap = map[string]httptypes.SnapshotMap{} + + for _, service := range r.services { + + snapshots, err := service.Driver().Snapshots(ctx, store) + if err != nil { + return utils.NewBatchProcessErr(reply, err) + } + + snapshotMap := map[string]*types.Snapshot{} + reply[service.Name()] = snapshotMap + for _, s := range snapshots { + snapshotMap[s.ID] = s + } + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) snapshotsForService( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + var reply httptypes.SnapshotMap = map[string]*types.Snapshot{} + + snapshots, err := d.Snapshots(ctx, store) + if err != nil { + return utils.NewBatchProcessErr(reply, err) + } + + for _, s := range snapshots { + reply[s.ID] = s + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) snapshotInspect( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + reply, err := getSnapshot(ctx) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) snapshotRemove( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + err = d.SnapshotRemove( + ctx, + store.GetString("snapshotID"), + store) + if err != nil { + return err + } + + w.WriteHeader(http.StatusResetContent) + return nil +} + +func (r *router) volumeCreate( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + reply, err := d.VolumeCreateFromSnapshot( + ctx, + store.GetString("snapshotID"), + store.GetString("volumeName"), + store) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) snapshotCopy( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + reply, err := d.SnapshotCopy( + ctx, + store.GetString("snapshotID"), + store.GetString("snapshotName"), + store.GetString("destinationID"), + store) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +var ( + snapshotTypeName = utils.GetTypePkgPathAndName( + &types.Snapshot{}) +) + +func getSnapshot( + ctx context.Context) (*types.Snapshot, error) { + obj := ctx.Value("snapshot") + if obj == nil { + return nil, utils.NewContextKeyErr("snapshot") + } + typedObj, ok := obj.(*types.Snapshot) + if !ok { + return nil, utils.NewContextTypeErr( + "snapshot", + snapshotTypeName, + utils.GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} diff --git a/api/server/router/volume/volume.go b/api/server/router/volume/volume.go new file mode 100644 index 00000000..d1f6d4f3 --- /dev/null +++ b/api/server/router/volume/volume.go @@ -0,0 +1,202 @@ +package volume + +import ( + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/server/handlers" + "github.com/emccode/libstorage/api/server/httputils" + httptypes "github.com/emccode/libstorage/api/types/http" + "github.com/emccode/libstorage/api/utils/schema" +) + +func init() { + registry.RegisterRouter(&router{}) +} + +type router struct { + config gofig.Config + services map[string]httputils.Service + routes []httputils.Route +} + +func (r *router) Name() string { + return "volume-router" +} + +func (r *router) Init( + config gofig.Config, services map[string]httputils.Service) { + r.config = config + r.services = services + r.initRoutes() +} + +// Routes returns the available routes. +func (r *router) Routes() []httputils.Route { + return r.routes +} + +func (r *router) initRoutes() { + r.routes = []httputils.Route{ + // GET + + // get all volumes (and possibly attachments) from all services + httputils.NewGetRoute( + "volumesAndAttachments", + "/volumes", + newVolumesRoute(r.services, true).volumes, + handlers.NewInstanceIDValidator(false), + handlers.NewSchemaValidator(nil, schema.ServiceVolumeMapSchema, nil), + ).Queries("attachments"), + + // get all volumes from all services + httputils.NewGetRoute( + "volumes", + "/volumes", + newVolumesRoute(r.services, false).volumes, + handlers.NewSchemaValidator(nil, schema.ServiceVolumeMapSchema, nil), + ), + + // get all volumes (and possibly attachments) from a specific service + httputils.NewGetRoute( + "volumesAndAttachmentsForService", + "/volumes/{service}", + newVolumesRoute(r.services, true).volumesForService, + handlers.NewServiceValidator(r.services), + handlers.NewInstanceIDValidator(false), + handlers.NewSchemaValidator(nil, schema.VolumeMapSchema, nil), + ).Queries("attachments"), + + // get all volumes from a specific service + httputils.NewGetRoute( + "volumesForService", + "/volumes/{service}", + newVolumesRoute(r.services, false).volumesForService, + handlers.NewServiceValidator(r.services), + handlers.NewSchemaValidator(nil, schema.VolumeMapSchema, nil), + ), + + // get a specific volume (and possibly attachments) from a specific + // service + httputils.NewGetRoute( + "volumeAndAttachmentsInspect", + "/volumes/{service}/{volumeID}", + r.volumeInspect, + handlers.NewServiceValidator(r.services), + handlers.NewInstanceIDValidator(false), + handlers.NewVolumeValidator(), + handlers.NewSchemaValidator(nil, schema.VolumeSchema, nil), + ).Queries("attachments"), + + // get a specific volume from a specific service + httputils.NewGetRoute( + "volumeInspect", + "/volumes/{service}/{volumeID}", + r.volumeInspect, + handlers.NewServiceValidator(r.services), + handlers.NewVolumeValidator(), + handlers.NewSchemaValidator(nil, schema.VolumeSchema, nil), + ), + + // POST + + // detach all volumes for a service + httputils.NewPostRoute( + "volumesDetachForService", + "/volumes/{service}", + r.volumeDetachAllForService, + handlers.NewServiceValidator(r.services), + handlers.NewSchemaValidator( + schema.VolumeDetachRequestSchema, + schema.VolumeMapSchema, + func() interface{} { return &httptypes.VolumeDetachRequest{} }), + ).Queries("detach"), + + // create a new volume + httputils.NewPostRoute( + "volumeCreate", + "/volumes/{service}", + r.volumeCreate, + handlers.NewServiceValidator(r.services), + handlers.NewSchemaValidator( + schema.VolumeCreateRequestSchema, + schema.VolumeSchema, + func() interface{} { return &httptypes.VolumeCreateRequest{} }), + handlers.NewPostArgsHandler(), + ), + + // create a new volume using an existing volume as the baseline + httputils.NewPostRoute( + "volumeCopy", + "/volumes/{service}/{volumeID}", + r.volumeCopy, + handlers.NewServiceValidator(r.services), + handlers.NewVolumeValidator(), + handlers.NewSchemaValidator( + schema.VolumeCopyRequestSchema, + schema.VolumeSchema, + func() interface{} { return &httptypes.VolumeCopyRequest{} }), + handlers.NewPostArgsHandler(), + ).Queries("copy"), + + // snapshot an existing volume + httputils.NewPostRoute( + "volumeSnapshot", + "/volumes/{service}/{volumeID}", + r.volumeSnapshot, + handlers.NewServiceValidator(r.services), + handlers.NewVolumeValidator(), + handlers.NewSchemaValidator( + schema.VolumeSnapshotRequestSchema, + schema.SnapshotSchema, + func() interface{} { return &httptypes.VolumeSnapshotRequest{} }), + ).Queries("snapshot"), + + // attach an existing volume + httputils.NewPostRoute( + "volumeAttach", + "/volumes/{service}/{volumeID}", + r.volumeAttach, + handlers.NewServiceValidator(r.services), + handlers.NewInstanceIDValidator(true), + handlers.NewVolumeValidator(), + handlers.NewSchemaValidator( + schema.VolumeAttachRequestSchema, + schema.VolumeSchema, + func() interface{} { return &httptypes.VolumeAttachRequest{} }), + ).Queries("attach"), + + // detach all volumes for all services + httputils.NewPostRoute( + "volumesDetachAll", + "/volumes", + r.volumeDetachAll, + handlers.NewSchemaValidator( + schema.VolumeDetachRequestSchema, + schema.ServiceVolumeMapSchema, + func() interface{} { return &httptypes.VolumeDetachRequest{} }), + ).Queries("detach"), + + // detach an individual volume + httputils.NewPostRoute( + "volumeDetach", + "/volumes/{service}/{volumeID}", + r.volumeDetach, + handlers.NewServiceValidator(r.services), + handlers.NewVolumeValidator(), + handlers.NewSchemaValidator( + schema.VolumeDetachRequestSchema, + schema.VolumeSchema, + func() interface{} { return &httptypes.VolumeDetachRequest{} }), + ).Queries("detach"), + + // DELETE + httputils.NewDeleteRoute( + "volumeRemove", + "/volumes/{service}/{volumeID}", + r.volumeRemove, + handlers.NewServiceValidator(r.services), + handlers.NewVolumeValidator(), + ), + } +} diff --git a/api/server/router/volume/volume_routes.go b/api/server/router/volume/volume_routes.go new file mode 100644 index 00000000..88eeec32 --- /dev/null +++ b/api/server/router/volume/volume_routes.go @@ -0,0 +1,352 @@ +package volume + +import ( + "net/http" + + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" + httptypes "github.com/emccode/libstorage/api/types/http" + "github.com/emccode/libstorage/api/utils" +) + +func newVolumesRoute( + services map[string]httputils.Service, + queryAttachments bool) *volumesRoute { + return &volumesRoute{services, queryAttachments} +} + +type volumesRoute struct { + services map[string]httputils.Service + queryAttachments bool +} + +func (r *volumesRoute) volumes(ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + attachments := false + if r.queryAttachments { + attachments = store.GetBool("attachments") + } + + var reply httptypes.ServiceVolumeMap = map[string]httptypes.VolumeMap{} + + for _, service := range r.services { + + volumes, err := service.Driver().Volumes( + ctx, + &drivers.VolumesOpts{ + Attachments: attachments, + Opts: store, + }) + if err != nil { + return utils.NewBatchProcessErr(reply, err) + } + + volumeMap := map[string]*types.Volume{} + reply[service.Name()] = volumeMap + for _, v := range volumes { + volumeMap[v.ID] = v + } + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *volumesRoute) volumesForService( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + var reply httptypes.VolumeMap = map[string]*types.Volume{} + + volumes, err := d.Volumes( + ctx, + &drivers.VolumesOpts{ + Attachments: store.GetBool("attachments"), + Opts: store, + }) + if err != nil { + return utils.NewBatchProcessErr(reply, err) + } + + for _, v := range volumes { + reply[v.ID] = v + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) volumeInspect( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + reply, err := getVolume(ctx) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) volumeCreate( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + reply, err := d.VolumeCreate( + ctx, + store.GetString("name"), + &drivers.VolumeCreateOpts{ + AvailabilityZone: store.GetStringPtr("availabilityZone"), + IOPS: store.GetInt64Ptr("iops"), + Size: store.GetInt64Ptr("size"), + Type: store.GetStringPtr("type"), + Opts: store, + }) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) volumeCopy( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + reply, err := d.VolumeCopy( + ctx, + store.GetString("volumeID"), + store.GetString("volumeName"), + store) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) volumeSnapshot( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + reply, err := d.VolumeSnapshot( + ctx, + store.GetString("volumeID"), + store.GetString("snapshotName"), + store) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) volumeAttach( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + reply, err := d.VolumeAttach( + ctx, + store.GetString("volumeID"), + &drivers.VolumeAttachByIDOpts{ + NextDevice: store.GetStringPtr("nextDeviceName"), + Opts: store, + }) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusOK, reply) + return nil +} + +func (r *router) volumeDetach( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + volume, err := getVolume(ctx) + if err != nil { + return err + } + + err = d.VolumeDetach( + ctx, + volume.ID, + store) + if err != nil { + return err + } + + httputils.WriteJSON(w, http.StatusResetContent, volume) + return nil +} + +func (r *router) volumeDetachAll( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + var reply httptypes.ServiceVolumeMap = map[string]httptypes.VolumeMap{} + + for _, service := range r.services { + + svcCtx := ctx.WithValue("serviceID", service.Name()) + + d := service.Driver() + + volumes, err := d.Volumes( + svcCtx, + &drivers.VolumesOpts{ + Attachments: store.GetBool("attachments"), + Opts: store, + }) + if err != nil { + return utils.NewBatchProcessErr(reply, err) + } + + volumeMap := map[string]*types.Volume{} + reply[service.Name()] = volumeMap + + for _, volume := range volumes { + if err := d.VolumeDetach(svcCtx, volume.ID, store); err != nil { + return utils.NewBatchProcessErr(reply, err) + } + volumeMap[volume.ID] = volume + } + } + + httputils.WriteJSON(w, http.StatusResetContent, reply) + return nil +} + +func (r *router) volumeDetachAllForService( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + var reply httptypes.VolumeMap = map[string]*types.Volume{} + + volumes, err := d.Volumes( + ctx, + &drivers.VolumesOpts{ + Attachments: store.GetBool("attachments"), + Opts: store, + }) + if err != nil { + return err + } + + for _, volume := range volumes { + if err := d.VolumeDetach(ctx, volume.ID, store); err != nil { + return utils.NewBatchProcessErr(reply, err) + } + reply[volume.ID] = volume + } + + httputils.WriteJSON(w, http.StatusResetContent, reply) + return nil +} + +func (r *router) volumeRemove( + ctx context.Context, + w http.ResponseWriter, + req *http.Request, + store types.Store) error { + + d, err := httputils.GetStorageDriver(ctx) + if err != nil { + return err + } + + err = d.VolumeRemove( + ctx, + store.GetString("volumeID"), + store) + if err != nil { + return err + } + + w.WriteHeader(http.StatusResetContent) + return nil +} + +var ( + volumeTypeName = utils.GetTypePkgPathAndName( + &types.Volume{}) +) + +func getVolume( + ctx context.Context) (*types.Volume, error) { + obj := ctx.Value("volume") + if obj == nil { + return nil, utils.NewContextKeyErr("volume") + } + typedObj, ok := obj.(*types.Volume) + if !ok { + return nil, utils.NewContextTypeErr( + "volume", + volumeTypeName, + utils.GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} diff --git a/api/server/server.go b/api/server/server.go new file mode 100644 index 00000000..47a9fc64 --- /dev/null +++ b/api/server/server.go @@ -0,0 +1,498 @@ +package server + +import ( + "crypto/tls" + "fmt" + "io" + golog "log" + "net" + "net/http" + "os" + "strings" + "sync" + "time" + + log "github.com/Sirupsen/logrus" + "github.com/akutz/gofig" + "github.com/akutz/goof" + "github.com/akutz/gotil" + "github.com/gorilla/mux" + + // imported to load routers + _ "github.com/emccode/libstorage/api/server/router" + + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" + "github.com/emccode/libstorage/api/utils" +) + +type server struct { + name string + ctx context.Context + addrs []string + config gofig.Config + servers []*HTTPServer + services map[string]httputils.Service + closeSignal chan int + closedSignal chan int + closeOnce *sync.Once + + routers []httputils.Router + routeHandlers map[string][]httputils.Middleware + globalHandlers []httputils.Middleware + + logHTTPEnabled bool + logHTTPRequests bool + logHTTPResponses bool + + stdOut io.WriteCloser + stdErr io.WriteCloser +} + +func newServer(config gofig.Config) (*server, error) { + + s := &server{ + name: randomServerName(), + ctx: context.Background(), + config: config, + closeSignal: make(chan int), + closedSignal: make(chan int), + closeOnce: &sync.Once{}, + } + + s.ctx = s.ctx.WithContextID("server", s.name) + s.ctx.Log().Debug("initializing server") + + if err := s.initEndpoints(); err != nil { + return nil, err + } + s.ctx.Log().Debug("initialized endpoints") + + if err := s.initServices(); err != nil { + return nil, err + } + s.ctx.Log().Debug("initialized services") + + s.logHTTPEnabled = config.GetBool("libstorage.server.http.logging.enabled") + if s.logHTTPEnabled { + + s.logHTTPRequests = config.GetBool( + "libstorage.server.http.logging.logrequest") + s.logHTTPResponses = config.GetBool( + "libstorage.server.http.logging.logresponse") + + s.stdOut = getLogIO( + "libstorage.server.http.logging.out", config) + s.stdErr = getLogIO( + "libstorage.server.http.logging.err", config) + } + + s.initGlobalMiddleware() + + if err := s.initRouters(); err != nil { + return nil, err + } + + return s, nil +} + +// Serve starts serving the configured libStorage endpoints. This function +// returns a channel on which errors are received. Reading this channel is +// also the prescribed manner for clients wishing to block until the server is +// shutdown as the error channel will be closed when the server is stopped. +func Serve(config gofig.Config) (io.Closer, <-chan error) { + + s, err := newServer(config) + if err != nil { + errs := make(chan error) + go func() { + errs <- err + close(errs) + }() + return nil, errs + } + + errs := make(chan error, len(s.servers)) + srvErrs := make(chan error, len(s.servers)) + + for _, srv := range s.servers { + srv.srv.Handler = s.createMux(srv.Context()) + go func(srv *HTTPServer) { + srv.Context().Log().Info("api listening") + if err := srv.Serve(); err != nil { + if !strings.Contains( + err.Error(), "use of closed network connection") { + srvErrs <- err + } + } + }(srv) + } + + go func() { + s.ctx.Log().Debugln("waiting for err or close signal") + select { + case err := <-srvErrs: + errs <- err + s.ctx.Log().Debug("received server error") + case <-s.closeSignal: + s.ctx.Log().Debug("received close signal") + } + close(errs) + s.ctx.Log().Debugln("closed server error channel") + s.closedSignal <- 1 + }() + + // wait a second for all the configured endpoints to start. this isn't + // pretty, but the underlying golang http package doesn't really provide + // a better option + timeout := time.NewTimer(time.Second * 1) + <-timeout.C + + s.ctx.Log().Info("server started") + return s, errs +} + +// Close closes servers and thus stop receiving requests +func (s *server) Close() (err error) { + s.closeOnce.Do( + func() { + err = s.close() + s.closeSignal <- 1 + <-s.closedSignal + }) + return +} + +func (s *server) close() error { + s.ctx.Log().Info("shutting down server") + + for _, srv := range s.servers { + srv.ctx.Log().Info("shutting down endpoint") + if err := srv.Close(); err != nil { + srv.Context().Log().Error(err) + } + if srv.l.Addr().Network() == "unix" { + laddr := srv.l.Addr().String() + srv.Context().Log().WithField( + "path", laddr).Debug("removed unix socket") + os.RemoveAll(laddr) + } + srv.Context().Log().Debug("shutdown endpoint complete") + } + + if s.stdOut != nil { + if err := s.stdOut.Close(); err != nil { + log.Error(err) + } + } + + if s.stdErr != nil { + if err := s.stdErr.Close(); err != nil { + log.Error(err) + } + } + + s.ctx.Log().Debug("shutdown server complete") + + return nil +} + +func (s *server) initEndpoints() error { + + endpointsObj := s.config.Get("libstorage.server.endpoints") + if endpointsObj == nil { + return goof.New("no endpoints defined") + } + + endpoints, ok := endpointsObj.(map[string]interface{}) + if !ok { + return goof.New("endpoints invalid type") + } + + for endpointName := range endpoints { + + endpoint := fmt.Sprintf("libstorage.server.endpoints.%s", endpointName) + address := fmt.Sprintf("%s.address", endpoint) + + laddr := s.config.GetString(address) + if laddr == "" { + return goof.WithField("endpoint", endpoint, "missing address") + } + + proto, addr, err := gotil.ParseAddress(laddr) + if err != nil { + return err + } + + logFields := map[string]interface{}{ + "endpoint": endpointName, + "address": laddr, + } + + tlsConfig, tlsFields, err := + utils.ParseTLSConfig(s.config.Scope(endpoint)) + if err != nil { + return err + } + for k, v := range tlsFields { + logFields[k] = v + } + + log.WithFields(logFields).Info("configured endpoint") + + srv, err := s.newHTTPServer(proto, addr, tlsConfig) + if err != nil { + return err + } + + srv.Context().Log().Info("server created") + s.servers = append(s.servers, srv) + } + + return nil +} + +type service struct { + driver drivers.StorageDriver + config gofig.Config + name string +} + +func (s *service) Config() gofig.Config { + return s.config +} + +func (s *service) Driver() drivers.StorageDriver { + return s.driver +} + +func (s *service) Name() string { + return s.name +} + +func (s *server) initServices() error { + + cfgSvcs := s.config.Get("libstorage.server.services") + cfgSvcsMap, ok := cfgSvcs.(map[string]interface{}) + if !ok { + return goof.New("invalid format libstorage.server.services") + } + log.WithField("count", len(cfgSvcsMap)).Debug("got services map") + + s.services = map[string]httputils.Service{} + + for name := range cfgSvcsMap { + name = strings.ToLower(name) + + log.WithField("name", name).Debug("processing service config") + + scope := fmt.Sprintf("libstorage.server.services.%s", name) + log.WithField("scope", scope).Debug("getting scoped config for service") + sc := s.config.Scope(scope) + + driverName := sc.GetString("driver") + if driverName == "" { + driverName = sc.GetString("libstorage.driver") + if driverName == "" { + return goof.WithField( + "service", name, "error getting driver name") + } + } + + driver, err := registry.NewStorageDriver(driverName) + if err != nil { + return err + } + + if err := driver.Init(sc); err != nil { + return err + } + + svc := &service{ + name: name, + driver: driver, + config: sc, + } + + log.WithFields(log.Fields{ + "service": svc.Name(), + "driver": driver.Name(), + }).Info("created new service") + + s.services[name] = svc + } + + return nil +} + +func (s *server) initRouters() error { + for r := range registry.Routers() { + r.Init(s.config, s.services) + s.addRouter(r) + log.WithFields(log.Fields{ + "router": r.Name(), + "len(routes)": len(r.Routes()), + }).Info("initialized router") + } + + // now that the routers are initialized, initialize the router middleware + s.initRouteMiddleware() + + return nil +} + +func (s *server) addRouter(r httputils.Router) { + s.routers = append(s.routers, r) +} + +func (s *server) makeHTTPHandler( + ctx context.Context, + route httputils.Route) http.HandlerFunc { + + return func(w http.ResponseWriter, req *http.Request) { + + fctx := context.NewContext(ctx, req) + fctx = ctx.WithContextID("route", route.GetName()) + fctx = ctx.WithRoute(route.GetName()) + + if req.TLS != nil { + if len(req.TLS.PeerCertificates) > 0 { + fctx = ctx.WithContextID("user", + req.TLS.PeerCertificates[0].Subject.CommonName) + } + } + + ctx.Log().Debug("http request") + + vars := mux.Vars(req) + if vars == nil { + vars = map[string]string{} + } + store := utils.NewStoreWithVars(vars) + + handlerFunc := s.handleWithMiddleware(fctx, route) + if err := handlerFunc(fctx, w, req, store); err != nil { + fctx.Log().Error(err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + } +} + +func (s *server) createMux(ctx context.Context) *mux.Router { + m := mux.NewRouter() + for _, apiRouter := range s.routers { + for _, r := range apiRouter.Routes() { + f := s.makeHTTPHandler(ctx, r) + mr := m.Path(r.GetPath()) + mr = mr.Name(r.GetName()) + mr = mr.Methods(r.GetMethod()) + mr = mr.Queries(r.GetQueries()...) + mr.Handler(f) + if ctx.Log().Level >= log.DebugLevel { + ctx.Log().WithFields(log.Fields{ + "route": r.GetName(), + "path": r.GetPath(), + "method": r.GetMethod(), + "queries": r.GetQueries(), + "len(queries)": len(r.GetQueries()), + }).Debug("registered route") + } else { + ctx.Log().WithField( + "route", r.GetName()).Info("registered route") + } + } + } + return m +} + +func (s *server) newHTTPServer( + proto, laddr string, tlsConfig *tls.Config) (*HTTPServer, error) { + + var ( + l net.Listener + err error + ) + + if tlsConfig != nil { + l, err = tls.Listen(proto, laddr, tlsConfig) + } else { + l, err = net.Listen(proto, laddr) + } + + if err != nil { + return nil, err + } + + host := fmt.Sprintf("%s://%s", proto, laddr) + ctx := s.ctx + ctx = ctx.WithContextID("host", host) + ctx = ctx.WithContextID("tls", fmt.Sprintf("%v", tlsConfig != nil)) + errLogger := &httpServerErrLogger{ctx.Log()} + + srv := &http.Server{Addr: l.Addr().String()} + srv.ErrorLog = golog.New(errLogger, "", 0) + + return &HTTPServer{ + srv: srv, + l: l, + ctx: ctx, + }, nil +} + +// HTTPServer contains an instance of http server and the listener. +// +// srv *http.Server, contains configuration to create a http server and a mux +// router with all api end points. +// +// l net.Listener, is a TCP or Socket listener that dispatches incoming +// request to the router. +type HTTPServer struct { + srv *http.Server + l net.Listener + ctx context.Context +} + +// Serve starts listening for inbound requests. +func (s *HTTPServer) Serve() error { + return s.srv.Serve(s.l) +} + +// Close closes the HTTPServer from listening for the inbound requests. +func (s *HTTPServer) Close() error { + return s.l.Close() +} + +// Context returns this server's context. +func (s *HTTPServer) Context() context.Context { + return s.ctx +} + +func getLogIO( + propName string, + config gofig.Config) io.WriteCloser { + + if path := config.GetString(propName); path != "" { + logio, err := os.OpenFile( + path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + log.Error(err) + } + log.WithFields(log.Fields{ + "logType": propName, + "logPath": path, + }).Debug("using log file") + return logio + } + return log.StandardLogger().Writer() +} + +type httpServerErrLogger struct { + logger *log.Logger +} + +func (l *httpServerErrLogger) Write(p []byte) (n int, err error) { + l.logger.Error(string(p)) + return len(p), nil +} diff --git a/api/server/server_middleware.go b/api/server/server_middleware.go new file mode 100644 index 00000000..a06dd70d --- /dev/null +++ b/api/server/server_middleware.go @@ -0,0 +1,96 @@ +package server + +import ( + "github.com/emccode/libstorage/api/server/handlers" + "github.com/emccode/libstorage/api/server/httputils" + "github.com/emccode/libstorage/api/types/context" +) + +func (s *server) initGlobalMiddleware() { + + s.addGlobalMiddleware(handlers.NewQueryParamsHandler()) + + if s.logHTTPEnabled { + s.addGlobalMiddleware(handlers.NewLoggingHandler( + s.stdOut, + s.logHTTPRequests, + s.logHTTPResponses)) + } + + s.addGlobalMiddleware(handlers.NewErrorHandler()) + s.addGlobalMiddleware(handlers.NewInstanceIDHandler()) +} + +func (s *server) initRouteMiddleware() { + // add the route-specific middleware for all the existing routes. it's + // also possible to add route-specific middleware that is not defined as + // part of a route's Middlewares collection. + s.routeHandlers = map[string][]httputils.Middleware{} + for _, router := range s.routers { + for _, r := range router.Routes() { + s.addRouterMiddleware(r, r.GetMiddlewares()...) + } + } +} + +func (s *server) addRouterMiddleware( + r httputils.Route, middlewares ...httputils.Middleware) { + + middlewaresForRouteName, ok := s.routeHandlers[r.GetName()] + if !ok { + middlewaresForRouteName = []httputils.Middleware{} + } + middlewaresForRouteName = append(middlewaresForRouteName, middlewares...) + s.routeHandlers[r.GetName()] = middlewaresForRouteName +} + +func (s *server) addGlobalMiddleware(m httputils.Middleware) { + s.globalHandlers = append(s.globalHandlers, m) +} + +func (s *server) handleWithMiddleware( + ctx context.Context, + route httputils.Route) httputils.APIFunc { + + handler := route.GetHandler() + + // add the route handlers + /*for h := range reverse(route.GetMiddlewares()) { + handler = h.Handler(handler) + log.WithFields(log.Fields{ + "route": route.GetName(), + "middleware": h.Name(), + }).Debug("added route middlware") + }*/ + + middlewaresForRouteName, ok := s.routeHandlers[route.GetName()] + if !ok { + ctx.Log().Warn("no middlewares for route") + } else { + for h := range reverse(middlewaresForRouteName) { + handler = h.Handler(handler) + ctx.Log().WithField( + "middleware", h.Name()).Debug("added route middlware") + } + } + + // add the global handlers + for h := range reverse(s.globalHandlers) { + handler = h.Handler(handler) + ctx.Log().WithField( + "middleware", h.Name()).Debug("added global middlware") + } + + return handler +} + +func reverse(middlewares []httputils.Middleware) chan httputils.Middleware { + ret := make(chan httputils.Middleware) + go func() { + for i := range middlewares { + ret <- middlewares[len(middlewares)-1-i] + } + close(ret) + }() + return ret +} diff --git a/api/server/server_names.go b/api/server/server_names.go new file mode 100644 index 00000000..4a8f9ada --- /dev/null +++ b/api/server/server_names.go @@ -0,0 +1,1176 @@ +package server + +import ( + "fmt" + "math/rand" + "time" +) + +func randomServerName() string { + x := serverNameRandomizer.Intn(maxAdjectives) + y := serverNameRandomizer.Intn(maxNouns) + z := serverNameRandomizer.Intn(maxCountries) + + return fmt.Sprintf("%s-%s-%s", + possibleServerNames["adjectives"][x], + possibleServerNames["nouns"][y], + possibleServerNames["countries"][z]) +} + +const ( + maxAdjectives = 527 - 1 + maxNouns = 364 - 1 + maxCountries = 249 - 1 +) + +var ( + serverNameRandomizer = rand.New(rand.NewSource(time.Now().UnixNano())) + possibleServerNames = map[string][]string{ + "adjectives": []string{ + "black", + "white", + "gray", + "brown", + "red", + "pink", + "crimson", + "carnelian", + "orange", + "yellow", + "ivory", + "cream", + "green", + "viridian", + "aquamarine", + "cyan", + "blue", + "cerulean", + "azure", + "indigo", + "navy", + "violet", + "purple", + "lavender", + "magenta", + "rainbow", + "iridescent", + "spectrum", + "prism", + "bold", + "vivid", + "pale", + "clear", + "glass", + "translucent", + "misty", + "dark", + "light", + "gold", + "silver", + "copper", + "bronze", + "steel", + "iron", + "brass", + "mercury", + "zinc", + "chrome", + "platinum", + "titanium", + "nickel", + "lead", + "pewter", + "rust", + "metal", + "stone", + "quartz", + "granite", + "marble", + "alabaster", + "agate", + "jasper", + "pebble", + "pyrite", + "crystal", + "geode", + "obsidian", + "mica", + "flint", + "sand", + "gravel", + "boulder", + "basalt", + "ruby", + "beryl", + "scarlet", + "citrine", + "sulpher", + "topaz", + "amber", + "emerald", + "malachite", + "jade", + "abalone", + "lapis", + "sapphire", + "diamond", + "peridot", + "gem", + "jewel", + "bevel", + "coral", + "jet", + "ebony", + "wood", + "tree", + "cherry", + "maple", + "cedar", + "branch", + "bramble", + "rowan", + "ash", + "fir", + "pine", + "cactus", + "alder", + "grove", + "forest", + "jungle", + "palm", + "bush", + "mulberry", + "juniper", + "vine", + "ivy", + "rose", + "lily", + "tulip", + "daffodil", + "honeysuckle", + "fuschia", + "hazel", + "walnut", + "almond", + "lime", + "lemon", + "apple", + "blossom", + "bloom", + "crocus", + "rose", + "buttercup", + "dandelion", + "iris", + "carnation", + "fern", + "root", + "branch", + "leaf", + "seed", + "flower", + "petal", + "pollen", + "orchid", + "mangrove", + "cypress", + "sequoia", + "sage", + "heather", + "snapdragon", + "daisy", + "mountain", + "hill", + "alpine", + "chestnut", + "valley", + "glacier", + "forest", + "grove", + "glen", + "tree", + "thorn", + "stump", + "desert", + "canyon", + "dune", + "oasis", + "mirage", + "well", + "spring", + "meadow", + "field", + "prairie", + "grass", + "tundra", + "island", + "shore", + "sand", + "shell", + "surf", + "wave", + "foam", + "tide", + "lake", + "river", + "brook", + "stream", + "pool", + "pond", + "sun", + "sprinkle", + "shade", + "shadow", + "rain", + "cloud", + "storm", + "hail", + "snow", + "sleet", + "thunder", + "lightning", + "wind", + "hurricane", + "typhoon", + "dawn", + "sunrise", + "morning", + "noon", + "twilight", + "evening", + "sunset", + "midnight", + "night", + "sky", + "star", + "stellar", + "comet", + "nebula", + "quasar", + "solar", + "lunar", + "planet", + "meteor", + "sprout", + "pear", + "plum", + "kiwi", + "berry", + "apricot", + "peach", + "mango", + "pineapple", + "coconut", + "olive", + "ginger", + "root", + "plain", + "fancy", + "stripe", + "spot", + "speckle", + "spangle", + "ring", + "band", + "blaze", + "paint", + "pinto", + "shade", + "tabby", + "brindle", + "patch", + "calico", + "checker", + "dot", + "pattern", + "glitter", + "glimmer", + "shimmer", + "dull", + "dust", + "dirt", + "glaze", + "scratch", + "quick", + "swift", + "fast", + "slow", + "clever", + "fire", + "flicker", + "flash", + "spark", + "ember", + "coal", + "flame", + "chocolate", + "vanilla", + "sugar", + "spice", + "cake", + "pie", + "cookie", + "candy", + "caramel", + "spiral", + "round", + "jelly", + "square", + "narrow", + "long", + "short", + "small", + "tiny", + "big", + "giant", + "great", + "atom", + "peppermint", + "mint", + "butter", + "fringe", + "rag", + "quilt", + "truth", + "lie", + "holy", + "curse", + "noble", + "sly", + "brave", + "shy", + "lava", + "foul", + "leather", + "fantasy", + "keen", + "luminous", + "feather", + "sticky", + "gossamer", + "cotton", + "rattle", + "silk", + "satin", + "cord", + "denim", + "flannel", + "plaid", + "wool", + "linen", + "silent", + "flax", + "weak", + "valiant", + "fierce", + "gentle", + "rhinestone", + "splash", + "north", + "south", + "east", + "west", + "summer", + "winter", + "autumn", + "spring", + "season", + "equinox", + "solstice", + "paper", + "motley", + "torch", + "ballistic", + "rampant", + "shag", + "freckle", + "wild", + "free", + "chain", + "sheer", + "crazy", + "mad", + "candle", + "ribbon", + "lace", + "notch", + "wax", + "shine", + "shallow", + "deep", + "bubble", + "harvest", + "fluff", + "venom", + "boom", + "slash", + "rune", + "cold", + "quill", + "love", + "hate", + "garnet", + "zircon", + "power", + "bone", + "void", + "horn", + "glory", + "cyber", + "nova", + "hot", + "helix", + "cosmic", + "quark", + "quiver", + "holly", + "clover", + "polar", + "regal", + "ripple", + "ebony", + "wheat", + "phantom", + "dew", + "chisel", + "crack", + "chatter", + "laser", + "foil", + "tin", + "clever", + "treasure", + "maze", + "twisty", + "curly", + "fortune", + "fate", + "destiny", + "cute", + "slime", + "ink", + "disco", + "plume", + "time", + "psychadelic", + "relic", + "fossil", + "water", + "savage", + "ancient", + "rapid", + "road", + "trail", + "stitch", + "button", + "bow", + "nimble", + "zest", + "sour", + "bitter", + "phase", + "fan", + "frill", + "plump", + "pickle", + "mud", + "puddle", + "pond", + "river", + "spring", + "stream", + "battle", + "arrow", + "plume", + "roan", + "pitch", + "tar", + "cat", + "dog", + "horse", + "lizard", + "bird", + "fish", + "saber", + "scythe", + "sharp", + "soft", + "razor", + "neon", + "dandy", + "weed", + "swamp", + "marsh", + "bog", + "peat", + "moor", + "muck", + "mire", + "grave", + "fair", + "just", + "brick", + "puzzle", + "skitter", + "prong", + "fork", + "dent", + "dour", + "warp", + "luck", + "coffee", + "split", + "chip", + "hollow", + "heavy", + "legend", + "hickory", + "mesquite", + "nettle", + "rogue", + "charm", + "prickle", + "bead", + "sponge", + "whip", + "bald", + "frost", + "fog", + "oil", + "veil", + "cliff", + "volcano", + "rift", + "maze", + "proud", + "dew", + "mirror", + "shard", + "salt", + "pepper", + "honey", + "thread", + "bristle", + "ripple", + "glow", + "zenith", + }, + "nouns": []string{ + "head", + "crest", + "crown", + "tooth", + "fang", + "horn", + "frill", + "skull", + "bone", + "tongue", + "throat", + "voice", + "nose", + "snout", + "chin", + "eye", + "sight", + "seer", + "speaker", + "singer", + "song", + "chanter", + "howler", + "chatter", + "shrieker", + "shriek", + "jaw", + "bite", + "biter", + "neck", + "shoulder", + "fin", + "wing", + "arm", + "lifter", + "grasp", + "grabber", + "hand", + "paw", + "foot", + "finger", + "toe", + "thumb", + "talon", + "palm", + "touch", + "racer", + "runner", + "hoof", + "fly", + "flier", + "swoop", + "roar", + "hiss", + "hisser", + "snarl", + "dive", + "diver", + "rib", + "chest", + "back", + "ridge", + "leg", + "legs", + "tail", + "beak", + "walker", + "lasher", + "swisher", + "carver", + "kicker", + "roarer", + "crusher", + "spike", + "shaker", + "charger", + "hunter", + "weaver", + "crafter", + "binder", + "scribe", + "muse", + "snap", + "snapper", + "slayer", + "stalker", + "track", + "tracker", + "scar", + "scarer", + "fright", + "killer", + "death", + "doom", + "healer", + "saver", + "friend", + "foe", + "guardian", + "thunder", + "lightning", + "cloud", + "storm", + "forger", + "scale", + "hair", + "braid", + "nape", + "belly", + "thief", + "stealer", + "reaper", + "giver", + "taker", + "dancer", + "player", + "gambler", + "twister", + "turner", + "painter", + "dart", + "drifter", + "sting", + "stinger", + "venom", + "spur", + "ripper", + "swallow", + "devourer", + "knight", + "lady", + "lord", + "queen", + "king", + "master", + "mistress", + "prince", + "princess", + "duke", + "dutchess", + "samurai", + "ninja", + "knave", + "slave", + "servant", + "sage", + "wizard", + "witch", + "warlock", + "warrior", + "jester", + "paladin", + "bard", + "trader", + "sword", + "shield", + "knife", + "dagger", + "arrow", + "bow", + "fighter", + "bane", + "follower", + "leader", + "scourge", + "watcher", + "cat", + "panther", + "tiger", + "cougar", + "puma", + "jaguar", + "ocelot", + "lynx", + "lion", + "leopard", + "ferret", + "weasel", + "wolverine", + "bear", + "raccoon", + "dog", + "wolf", + "kitten", + "puppy", + "cub", + "fox", + "hound", + "terrier", + "coyote", + "hyena", + "jackal", + "pig", + "horse", + "donkey", + "stallion", + "mare", + "zebra", + "antelope", + "gazelle", + "deer", + "buffalo", + "bison", + "boar", + "elk", + "whale", + "dolphin", + "shark", + "fish", + "minnow", + "salmon", + "ray", + "fisher", + "otter", + "gull", + "duck", + "goose", + "crow", + "raven", + "bird", + "eagle", + "raptor", + "hawk", + "falcon", + "moose", + "heron", + "owl", + "stork", + "crane", + "sparrow", + "robin", + "parrot", + "cockatoo", + "carp", + "lizard", + "gecko", + "iguana", + "snake", + "python", + "viper", + "boa", + "condor", + "vulture", + "spider", + "fly", + "scorpion", + "heron", + "oriole", + "toucan", + "bee", + "wasp", + "hornet", + "rabbit", + "bunny", + "hare", + "brow", + "mustang", + "ox", + "piper", + "soarer", + "flasher", + "moth", + "mask", + "hide", + "hero", + "antler", + "chill", + "chiller", + "gem", + "ogre", + "myth", + "elf", + "fairy", + "pixie", + "dragon", + "griffin", + "unicorn", + "pegasus", + "sprite", + "fancier", + "chopper", + "slicer", + "skinner", + "butterfly", + "legend", + "wanderer", + "rover", + "raver", + "loon", + "lancer", + "glass", + "glazer", + "flame", + "crystal", + "lantern", + "lighter", + "cloak", + "bell", + "ringer", + "keeper", + "centaur", + "bolt", + "catcher", + "whimsey", + "quester", + "rat", + "mouse", + "serpent", + "wyrm", + "gargoyle", + "thorn", + "whip", + "rider", + "spirit", + "sentry", + "bat", + "beetle", + "burn", + "cowl", + "stone", + "gem", + "collar", + "mark", + "grin", + "scowl", + "spear", + "razor", + "edge", + "seeker", + "jay", + "ape", + "monkey", + "gorilla", + "koala", + "kangaroo", + "yak", + "sloth", + "ant", + "roach", + "weed", + "seed", + "eater", + "razor", + "shirt", + "face", + "goat", + "mind", + "shift", + "rider", + "face", + "mole", + "vole", + "pirate", + "llama", + "stag", + "bug", + "cap", + "boot", + "drop", + "hugger", + "sargent", + "snagglefoot", + "carpet", + "curtain", + }, + "countries": []string{ + "af", + "ax", + "al", + "dz", + "as", + "ad", + "ao", + "ai", + "aq", + "ag", + "ar", + "am", + "aw", + "au", + "at", + "az", + "bs", + "bh", + "bd", + "bb", + "by", + "be", + "bz", + "bj", + "bm", + "bt", + "bo", + "bq", + "ba", + "bw", + "bv", + "br", + "io", + "bn", + "bg", + "bf", + "bi", + "kh", + "cm", + "ca", + "cv", + "ky", + "cf", + "td", + "cl", + "cn", + "cx", + "cc", + "co", + "km", + "cg", + "cd", + "ck", + "cr", + "ci", + "hr", + "cu", + "cw", + "cy", + "cz", + "dk", + "dj", + "dm", + "do", + "ec", + "eg", + "sv", + "gq", + "er", + "ee", + "et", + "fk", + "fo", + "fj", + "fi", + "fr", + "gf", + "pf", + "tf", + "ga", + "gm", + "ge", + "de", + "gh", + "gi", + "gr", + "gl", + "gd", + "gp", + "gu", + "gt", + "gg", + "gn", + "gw", + "gy", + "ht", + "hm", + "va", + "hn", + "hk", + "hu", + "is", + "in", + "id", + "ir", + "iq", + "ie", + "im", + "il", + "it", + "jm", + "jp", + "je", + "jo", + "kz", + "ke", + "ki", + "kp", + "kr", + "kw", + "kg", + "la", + "lv", + "lb", + "ls", + "lr", + "ly", + "li", + "lt", + "lu", + "mo", + "mk", + "mg", + "mw", + "my", + "mv", + "ml", + "mt", + "mh", + "mq", + "mr", + "mu", + "yt", + "mx", + "fm", + "md", + "mc", + "mn", + "me", + "ms", + "ma", + "mz", + "mm", + "na", + "nr", + "np", + "nl", + "nc", + "nz", + "ni", + "ne", + "ng", + "nu", + "nf", + "mp", + "no", + "om", + "pk", + "pw", + "ps", + "pa", + "pg", + "py", + "pe", + "ph", + "pn", + "pl", + "pt", + "pr", + "qa", + "re", + "ro", + "ru", + "rw", + "bl", + "sh", + "kn", + "lc", + "mf", + "pm", + "vc", + "ws", + "sm", + "st", + "sa", + "sn", + "rs", + "sc", + "sl", + "sg", + "sx", + "sk", + "si", + "sb", + "so", + "za", + "gs", + "ss", + "es", + "lk", + "sd", + "sr", + "sj", + "sz", + "se", + "ch", + "sy", + "tw", + "tj", + "tz", + "th", + "tl", + "tg", + "tk", + "to", + "tt", + "tn", + "tr", + "tm", + "tc", + "tv", + "ug", + "ua", + "ae", + "gb", + "us", + "um", + "uy", + "uz", + "vu", + "ve", + "vn", + "vg", + "vi", + "wf", + "eh", + "ye", + "zm", + "zw", + }, + } +) diff --git a/api/types/context/context.go b/api/types/context/context.go new file mode 100644 index 00000000..0b1ccea0 --- /dev/null +++ b/api/types/context/context.go @@ -0,0 +1,266 @@ +package context + +import ( + "net/http" + + log "github.com/Sirupsen/logrus" + gcontext "github.com/gorilla/context" + "golang.org/x/net/context" + + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/utils" +) + +var ( + instanceIDTypeName = utils.GetTypePkgPathAndName(&types.InstanceID{}) + loggerTypeName = utils.GetTypePkgPathAndName(&log.Logger{}) +) + +// Context is a libStorage context. +type Context interface { + context.Context + + // InstanceID returns the context's instance ID. + InstanceID() *types.InstanceID + + // Profile returns the context's profile name. + Profile() string + + // Route returns the name of context's route. + Route() string + + // Log returns the context's logger. + Log() *log.Logger + + // WithInstanceID returns a context with the provided instance ID. + WithInstanceID(instanceID *types.InstanceID) Context + + // WithProfile returns a context with the provided profile. + WithProfile(profile string) Context + + // WithRoute returns a contex with the provided route name. + WithRoute(routeName string) Context + + // WithContextID returns a context with the provided context ID information. + // The context ID is often used with logging to identify a log statement's + // origin. + WithContextID(id, value string) Context + + // WithValue returns a context with the provided value. + WithValue(key interface{}, val interface{}) Context +} + +type libstorContext struct { + context.Context + req *http.Request + logger *log.Logger + contextIDs map[string]string +} + +// Background initializes a new, empty context. +func Background() Context { + return NewContext(context.Background(), nil) +} + +// NewContext initializes a new libStorage context. +func NewContext(parent context.Context, r *http.Request) Context { + + var parentLogger *log.Logger + if parentCtx, ok := parent.(Context); ok { + parentLogger = parentCtx.Log() + } else { + parentLogger = log.StandardLogger() + } + + ctx := &libstorContext{ + Context: parent, + req: r, + } + + logger := &log.Logger{ + Formatter: &fieldFormatter{parentLogger.Formatter, ctx}, + Out: parentLogger.Out, + Hooks: parentLogger.Hooks, + Level: parentLogger.Level, + } + ctx.logger = logger + + return ctx +} + +// WithInstanceID returns a context with the provided instance ID. +func WithInstanceID( + parent context.Context, + instanceID *types.InstanceID) Context { + return WithValue(parent, "instanceID", instanceID) +} + +// WithProfile returns a context with the provided profile. +func WithProfile( + parent context.Context, + profile string) Context { + return WithValue(parent, "profile", profile) +} + +// WithRoute returns a contex with the provided route name. +func WithRoute(parent context.Context, routeName string) Context { + return WithValue(parent, "route", routeName) +} + +// WithContextID returns a context with the provided context ID information. +// The context ID is often used with logging to identify a log statement's +// origin. +func WithContextID( + parent context.Context, + id, value string) Context { + + contextID := map[string]string{id: value} + parentContextID, ok := parent.Value("contextID").(map[string]string) + if ok { + for k, v := range parentContextID { + contextID[k] = v + } + } + return WithValue(parent, "contextID", contextID) +} + +// WithValue returns a context with the provided value. +func WithValue( + parent context.Context, + key interface{}, + val interface{}) Context { + return NewContext(context.WithValue(parent, key, val), HTTPRequest(parent)) +} + +// InstanceID gets the context's instance ID. +func InstanceID(ctx context.Context) (*types.InstanceID, error) { + obj := ctx.Value("instanceID") + if obj == nil { + return nil, utils.NewContextKeyErr("instanceID") + } + typedObj, ok := obj.(*types.InstanceID) + if !ok { + return nil, utils.NewContextTypeErr( + "instanceID", instanceIDTypeName, utils.GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} + +// Profile gets the context's profile. +func Profile(ctx context.Context) (string, error) { + obj := ctx.Value("profile") + if obj == nil { + return "", utils.NewContextKeyErr("profile") + } + typedObj, ok := obj.(string) + if !ok { + return "", utils.NewContextTypeErr( + "profile", "string", utils.GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} + +// Route gets the context's route name. +func Route(ctx context.Context) (string, error) { + obj := ctx.Value("route") + if obj == nil { + return "", utils.NewContextKeyErr("route") + } + typedObj, ok := obj.(string) + if !ok { + return "", utils.NewContextTypeErr( + "route", "string", utils.GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} + +// HTTPRequest returns the *http.Request associated with ctx using +// NewRequestContext, if any. +func HTTPRequest(ctx context.Context) *http.Request { + req, ok := ctx.Value(reqKey).(*http.Request) + if !ok { + return nil + } + return req +} + +type key int + +const reqKey key = 0 + +// Value returns Gorilla's context package's value for this Context's request +// and key. It delegates to the parent Context if there is no such value. +func (ctx *libstorContext) Value(key interface{}) interface{} { + if ctx.req == nil { + ctx.Context.Value(key) + } + + if key == reqKey { + return ctx.req + } + if val, ok := gcontext.GetOk(ctx.req, key); ok { + return val + } + + return ctx.Context.Value(key) +} + +func (ctx *libstorContext) InstanceID() *types.InstanceID { + v, _ := InstanceID(ctx) + return v +} + +func (ctx *libstorContext) Profile() string { + v, _ := Profile(ctx) + return v +} + +func (ctx *libstorContext) Route() string { + v, _ := Route(ctx) + return v +} + +func (ctx *libstorContext) Log() *log.Logger { + return ctx.logger +} + +func (ctx *libstorContext) WithInstanceID( + instanceID *types.InstanceID) Context { + return WithInstanceID(ctx, instanceID) +} + +func (ctx *libstorContext) WithProfile(profile string) Context { + return WithProfile(ctx, profile) +} + +func (ctx *libstorContext) WithRoute(routeName string) Context { + return WithRoute(ctx, routeName) +} + +func (ctx *libstorContext) WithContextID(id, value string) Context { + return WithContextID(ctx, id, value) +} + +func (ctx *libstorContext) WithValue( + key interface{}, value interface{}) Context { + return WithValue(ctx, key, value) +} + +type fieldFormatter struct { + f log.Formatter + ctx Context +} + +func (f *fieldFormatter) Format(entry *log.Entry) ([]byte, error) { + contextID, ok := f.ctx.Value("contextID").(map[string]string) + if !ok { + return f.f.Format(entry) + } + if entry.Data == nil { + entry.Data = log.Fields{} + } + for k, v := range contextID { + entry.Data[k] = v + } + return f.f.Format(entry) +} diff --git a/api/types/drivers/drivers.go b/api/types/drivers/drivers.go new file mode 100644 index 00000000..5f4da78f --- /dev/null +++ b/api/types/drivers/drivers.go @@ -0,0 +1,15 @@ +package drivers + +import ( + "github.com/akutz/gofig" +) + +// Driver is the base interface for a libStorage driver. +type Driver interface { + + // Name returns the name of the driver + Name() string + + // Init initializes the driver. + Init(config gofig.Config) error +} diff --git a/api/types/drivers/drivers_integration.go b/api/types/drivers/drivers_integration.go new file mode 100644 index 00000000..db303081 --- /dev/null +++ b/api/types/drivers/drivers_integration.go @@ -0,0 +1,101 @@ +package drivers + +import ( + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// NewIntegrationDriver is a function that constructs a new IntegrationDriver. +type NewIntegrationDriver func() IntegrationDriver + +// VolumeMountOpts are options for mounting a volume. +type VolumeMountOpts struct { + OverwriteFS bool + NewFSType string + Preempt bool + Opts types.Store +} + +// VolumeAttachByNameOpts are options for attaching a volume by its name. +type VolumeAttachByNameOpts struct { + Force bool + Opts types.Store +} + +// VolumeDetachByNameOpts are options for detaching a volume by its name. +type VolumeDetachByNameOpts struct { + Force bool + Opts types.Store +} + +// IntegrationDriver is the interface implemented to integrate external +// storage consumers, such as Docker, with libStorage. +type IntegrationDriver interface { + Driver + + // Volumes returns all available volumes. + Volumes( + ctx context.Context, + opts types.Store) ([]map[string]string, error) + + // Inspect returns a specific volume as identified by the provided + // volume name. + Inspect( + ctx context.Context, + volumeName string, + opts types.Store) (map[string]string, error) + + // Mount will return a mount point path when specifying either a volumeName + // or volumeID. If a overwriteFs boolean is specified it will overwrite + // the FS based on newFsType if it is detected that there is no FS present. + Mount( + ctx context.Context, + volumeID, volumeName string, + opts *VolumeMountOpts) (string, error) + + // Unmount will unmount the specified volume by volumeName or volumeID. + Unmount( + ctx context.Context, + volumeID, volumeName string, + opts types.Store) error + + // Path will return the mounted path of the volumeName or volumeID. + Path( + ctx context.Context, + volumeID, volumeName string, + opts types.Store) (string, error) + + // Create will create a new volume with the volumeName and opts. + Create( + ctx context.Context, + volumeName string, + opts types.Store) error + + // Remove will remove a volume of volumeName. + Remove( + ctx context.Context, + volumeName string, + opts types.Store) error + + // Attach will attach a volume based on volumeName to the instance of + // instanceID. + Attach( + ctx context.Context, + volumeName string, + opts *VolumeAttachByNameOpts) (string, error) + + // Detach will detach a volume based on volumeName to the instance of + // instanceID. + Detach( + ctx context.Context, + volumeName string, + opts *VolumeDetachByNameOpts) error + + // NetworkName will return an identifier of a volume that is relevant when + // corelating a local device to a device that is the volumeName to the + // local instanceID. + NetworkName( + ctx context.Context, + volumeName string, + opts types.Store) (string, error) +} diff --git a/api/types/drivers/drivers_os.go b/api/types/drivers/drivers_os.go new file mode 100644 index 00000000..36e94fd4 --- /dev/null +++ b/api/types/drivers/drivers_os.go @@ -0,0 +1,59 @@ +package drivers + +import ( + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// NewOSDriver is a function that constructs a new OSDriver. +type NewOSDriver func() OSDriver + +// DeviceMountOpts are options when mounting a device. +type DeviceMountOpts struct { + MountOptions string + MountLabel string + Opts types.Store +} + +// DeviceFormatOpts are options when formatting a device. +type DeviceFormatOpts struct { + NewFSType string + OverwriteFS bool + Opts types.Store +} + +// OSDriver is the interface implemented by types that provide OS introspection +// and management. +type OSDriver interface { + Driver + + // Inspect get a list of mount points for a local device. + Inspect( + ctx context.Context, + deviceName, mountPoint string, + opts types.Store) ([]*types.MountInfo, error) + + // Mount mounts a device to a specified path. + Mount( + ctx context.Context, + deviceName, mountPoint string, + opts *DeviceMountOpts) error + + // Unmount unmounts the underlying device from the specified path. + Unmount( + ctx context.Context, + mountPoint string, + opts types.Store) error + + // IsMounted checks whether a path is mounted or not + IsMounted( + ctx context.Context, + mountPoint string, + opts types.Store) (bool, error) + + // Format formats a device. + Format( + ctx context.Context, + deviceName string, + opts *DeviceFormatOpts) error +} diff --git a/api/types/drivers/drivers_storage.go b/api/types/drivers/drivers_storage.go new file mode 100644 index 00000000..1d96eaac --- /dev/null +++ b/api/types/drivers/drivers_storage.go @@ -0,0 +1,160 @@ +package drivers + +import ( + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +// NewStorageDriver is a function that constructs a new StorageDriver. +type NewStorageDriver func() StorageDriver + +// VolumesOpts are options when inspecting a volume. +type VolumesOpts struct { + Attachments bool + Opts types.Store +} + +// VolumeInspectOpts are options when inspecting a volume. +type VolumeInspectOpts struct { + Attachments bool + Opts types.Store +} + +// VolumeCreateOpts are options when creating a new volume. +type VolumeCreateOpts struct { + AvailabilityZone *string + IOPS *int64 + Size *int64 + Type *string + Opts types.Store +} + +// VolumeAttachByIDOpts are options for attaching a volume by its ID. +type VolumeAttachByIDOpts struct { + NextDevice *string + Opts types.Store +} + +/* +StorageDriver is a libStorage driver used by the routes to implement the backend +functionality. + +Functions that inspect a resource or send an operation to a resource should +always return ErrResourceNotFound if the acted upon resource cannot be found. +*/ +type StorageDriver interface { + Driver + + // Type returns the type of storage the driver provides. + Type() types.StorageType + + // NextDeviceInfo returns the information about the driver's next available + // device workflow. + NextDeviceInfo() *types.NextDeviceInfo + + /*************************************************************************** + ** Executor ** + ***************************************************************************/ + // InstanceID returns the local system's InstanceID. + InstanceID( + ctx context.Context, + opts types.Store) (*types.InstanceID, error) + + // NextDevice returns the next available device. + NextDevice( + ctx context.Context, + opts types.Store) (string, error) + + // LocalDevices returns a map of the system's local devices. + LocalDevices( + ctx context.Context, + opts types.Store) (map[string]string, error) + + /*************************************************************************** + ** Instance ** + ***************************************************************************/ + // InstanceInspect returns an instance. + InstanceInspect( + ctx context.Context, + opts types.Store) (*types.Instance, error) + + /*************************************************************************** + ** Volumes ** + ***************************************************************************/ + // Volumes returns all volumes or a filtered list of volumes. + Volumes( + ctx context.Context, + opts *VolumesOpts) ([]*types.Volume, error) + + // VolumeInspect inspects a single volume. + VolumeInspect( + ctx context.Context, + volumeID string, + opts *VolumeInspectOpts) (*types.Volume, error) + + // VolumeCreate creates a new volume. + VolumeCreate( + ctx context.Context, + name string, + opts *VolumeCreateOpts) (*types.Volume, error) + + // VolumeCreateFromSnapshot creates a new volume from an existing snapshot. + VolumeCreateFromSnapshot( + ctx context.Context, + snapshotID, volumeName string, + opts types.Store) (*types.Volume, error) + + // VolumeCopy copies an existing volume. + VolumeCopy( + ctx context.Context, + volumeID, volumeName string, + opts types.Store) (*types.Volume, error) + + // VolumeSnapshot snapshots a volume. + VolumeSnapshot( + ctx context.Context, + volumeID, snapshotName string, + opts types.Store) (*types.Snapshot, error) + + // VolumeRemove removes a volume. + VolumeRemove( + ctx context.Context, + volumeID string, + opts types.Store) error + + // VolumeAttach attaches a volume. + VolumeAttach( + ctx context.Context, + volumeID string, + opts *VolumeAttachByIDOpts) (*types.Volume, error) + + // VolumeDetach detaches a volume. + VolumeDetach( + ctx context.Context, + volumeID string, + opts types.Store) error + + /*************************************************************************** + ** Snapshots ** + ***************************************************************************/ + // Snapshots returns all volumes or a filtered list of snapshots. + Snapshots(ctx context.Context, opts types.Store) ([]*types.Snapshot, error) + + // SnapshotInspect inspects a single snapshot. + SnapshotInspect( + ctx context.Context, + snapshotID string, + opts types.Store) (*types.Snapshot, error) + + // SnapshotCopy copies an existing snapshot. + SnapshotCopy( + ctx context.Context, + snapshotID, snapshotName, destinationID string, + opts types.Store) (*types.Snapshot, error) + + // SnapshotRemove removes a snapshot. + SnapshotRemove( + ctx context.Context, + snapshotID string, + opts types.Store) error +} diff --git a/api/types/http/http_requests.go b/api/types/http/http_requests.go new file mode 100644 index 00000000..7f4e2d6e --- /dev/null +++ b/api/types/http/http_requests.go @@ -0,0 +1,53 @@ +package http + +// VolumeCreateRequest is the JSON body for creating a new volume. +type VolumeCreateRequest struct { + Name string `json:"name"` + AvailabilityZone *string `json:"availabilityZone"` + IOPS *int64 `json:"iops"` + Size *int64 `json:"size"` + Type *string `json:"type"` + Opts map[string]interface{} `json:"opts"` +} + +// VolumeCreateFromSnapshotRequest is the JSON body for creating a new volume +// from an existing snapshot. +type VolumeCreateFromSnapshotRequest struct { + VolumeName string `json:"volumeName"` + Opts map[string]interface{} `json:"opts"` +} + +// VolumeCopyRequest is the JSON body for copying a volume. +type VolumeCopyRequest struct { + VolumeName string `json:"volumeName"` + Opts map[string]interface{} `json:"opts"` +} + +// VolumeSnapshotRequest is the JSON body for snapshotting a volume. +type VolumeSnapshotRequest struct { + SnapshotName string `json:"snapshotName"` + Opts map[string]interface{} `json:"opts"` +} + +// VolumeAttachRequest is the JSON body for attaching a volume to an instance. +type VolumeAttachRequest struct { + NextDeviceName string `json:"nextDeviceName"` + Opts map[string]interface{} `json:"opts"` +} + +// VolumeDetachRequest is the JSON body for detaching a volume from an instance. +type VolumeDetachRequest struct { + Opts map[string]interface{} `json:"opts"` +} + +// SnapshotCopyRequest is the JSON body for copying a snapshot. +type SnapshotCopyRequest struct { + SnapshotName string `json:"snapshotName"` + DestinationID string `json:"destinationID"` + Opts map[string]interface{} `json:"opts"` +} + +// SnapshotRemoveRequest is the JSON body for removing a snapshot. +type SnapshotRemoveRequest struct { + Opts map[string]interface{} `json:"opts"` +} diff --git a/api/types/http/http_responses.go b/api/types/http/http_responses.go new file mode 100644 index 00000000..4e111ece --- /dev/null +++ b/api/types/http/http_responses.go @@ -0,0 +1,117 @@ +package http + +import ( + "github.com/emccode/libstorage/api/types" +) + +/****************************************************************************** +** Root ** +*******************************************************************************/ + +// RootResponse is the response when getting root information about the service. +type RootResponse []string + +/****************************************************************************** +** Drivers ** +*******************************************************************************/ + +// DriversResponse is the response when getting one to many DriverInfos. +type DriversResponse map[string]*types.DriverInfo + +// DriverInspectResponse is the response when getting a single DriverInfo. +type DriverInspectResponse *types.DriverInfo + +/****************************************************************************** +** Executors ** +*******************************************************************************/ + +// ExecutorsListResponse is the response when getting one to many ExecutorInfos. +type ExecutorsListResponse []*types.ExecutorInfo + +/****************************************************************************** +** Instances ** +*******************************************************************************/ +/* +// InstanceRef is a wrapper for an Instance object along with its location. +type InstanceRef struct { + Service string `json:"service,omitempty"` + Instance *Instance `json:"instance"` +} + +// InstancesResponse is the response when getting one to many Instances. +type InstancesResponse []*InstanceRef + +// InstanceInspectResponse is the response when getting a single Instance. +type InstanceInspectResponse *Instance*/ + +/****************************************************************************** +** Services ** +*******************************************************************************/ + +// ServicesResponse is the response when getting one to many ServiceInfos. +type ServicesResponse map[string]*types.ServiceInfo + +// ServiceInspectResponse is the response when getting a single ServiceInfo. +type ServiceInspectResponse *types.ServiceInfo + +/****************************************************************************** +** Snapshots ** +*******************************************************************************/ + +// ServiceSnapshotMap is the response for listing snapshots for multiple +// services. +type ServiceSnapshotMap map[string]SnapshotMap + +// SnapshotMap is the response for listing snapshots for a single service. +type SnapshotMap map[string]*types.Snapshot + +// SnapshotInspectResponse is the response when getting a single Snapshot. +type SnapshotInspectResponse *types.Snapshot + +// SnapshotCreateResponse is the response when creating a Snapshot. +type SnapshotCreateResponse *types.Snapshot + +// SnapshotCopyResponse is the response when copying a Snapshot. +type SnapshotCopyResponse *types.Snapshot + +// SnapshotRemoveResponse is the response when removing a Snapshot. +type SnapshotRemoveResponse struct { +} + +/****************************************************************************** +** Volumes ** +*******************************************************************************/ + +// ServiceVolumeMap is the response for listing volumes for multiple services. +type ServiceVolumeMap map[string]VolumeMap + +// VolumeMap is the response for listing volumes for a single service. +type VolumeMap map[string]*types.Volume + +// VolumeInspectResponse is the response when getting a single Volume. +type VolumeInspectResponse *types.Volume + +// VolumeCreateResponse is the response when creating a Volume. +type VolumeCreateResponse *types.Volume + +// VolumeCopyResponse is the response when copying a Volume. +type VolumeCopyResponse *types.Volume + +// VolumeCreateFromSnapshotResponse is the response when creating a Volume +// from an existing snapshot. +type VolumeCreateFromSnapshotResponse *types.Volume + +// VolumeRemoveResponse is the response when removing a Volume. +type VolumeRemoveResponse struct { +} + +// VolumeAttachResponse is the response when attaching a Volume. +type VolumeAttachResponse struct { +} + +// VolumeDetachResponse is the response when detching a single Volume. +type VolumeDetachResponse struct { +} + +// VolumeDetachMultipleResponse is the response when detching multiple Volumes. +type VolumeDetachMultipleResponse ServiceVolumeMap diff --git a/api/types/types_errors.go b/api/types/types_errors.go new file mode 100644 index 00000000..9a5d6d30 --- /dev/null +++ b/api/types/types_errors.go @@ -0,0 +1,38 @@ +package types + +import ( + "github.com/akutz/goof" +) + +// JSONError is the base type for errors returned from the REST API. +type JSONError struct { + Message string `json:"message"` + Status int `json:"status"` + Error error `json:"error,omitempty"` +} + +// ErrNotImplemented is the error that Driver implementations should return if +// a function is not implemented. +var ErrNotImplemented = goof.New("not implemented") + +// ErrNotFound occurs when a Driver inspects or sends an operation to a +// resource that cannot be found. +type ErrNotFound struct{ *goof.Goof } + +// ErrStoreKey occurs when no value exists for a specified store key. +type ErrStoreKey struct{ *goof.Goof } + +// ErrContextKey occurs when no value exists for a specified context key. +type ErrContextKey struct{ *goof.Goof } + +// ErrContextType occurs when a value exists in the context but is not the +// expected typed. +type ErrContextType struct{ *goof.Goof } + +// ErrDriverTypeErr occurs when a Driver is constructed with an invalid type. +type ErrDriverTypeErr struct{ *goof.Goof } + +// ErrBatchProcess occurs when a batch process is interrupted by an error +// before the process is complete. This error will contain information about +// the objects for which the process did complete. +type ErrBatchProcess struct{ *goof.Goof } diff --git a/api/api_model.go b/api/types/types_model.go similarity index 59% rename from api/api_model.go rename to api/types/types_model.go index e17d35ac..20a88ac2 100644 --- a/api/api_model.go +++ b/api/types/types_model.go @@ -1,4 +1,18 @@ -package api +package types + +// StorageType is the type of storage a driver provides. +type StorageType string + +const ( + // Block is block storage. + Block StorageType = "block" + + // NAS is network attached storage. + NAS StorageType = "nas" + + // Object is object-backed storage. + Object StorageType = "object" +) // InstanceID identifies a host to a remote storage platform. type InstanceID struct { @@ -6,7 +20,7 @@ type InstanceID struct { ID string `json:"id"` // Metadata is any extra information about the instance ID. - Metadata interface{} `json:"metadata"` + Metadata interface{} `json:"metadata,omitempty"` } // Instance provides information about a storage object. @@ -22,32 +36,9 @@ type Instance struct { // The region from which the object originates. Region string `json:"region"` -} - -// NextAvailableDeviceName assists the libStorage client in determining the -// next available device name by providing the driver's device prefix and -// optional pattern. -// -// For example, the Amazon Web Services (AWS) device prefix is "xvd" and its -// pattern is "[a-z]". These two values would be used to determine on an EC2 -// instance where "/dev/xvda" and "/dev/xvdb" are in use that the next -// available device name is "/dev/xvdc". -// -// If the Ignore field is set to true then the client logic does not invoke the -// GetNextAvailableDeviceName function prior to submitting an AttachVolume -// request to the server. -type NextAvailableDeviceName struct { - // Ignore is a flag that indicates whether the client logic should invoke - // the GetNextAvailableDeviceName function prior to submitting an - // AttachVolume request to the server. - Ignore bool `json:"ignore"` - // Prefix is the first part of a device path's value after the "/dev/" - // porition. For example, the prefix in "/dev/xvda" is "xvd". - Prefix string `json:"prefix"` - - // Pattern is the regex to match the part of a device path after the prefix. - Pattern string `json:"pattern"` + // Fields are additional properties that can be defined for this type. + Fields map[string]string `json:"fields,omitempty"` } // MountInfo reveals information about a particular mounted filesystem. This @@ -88,84 +79,69 @@ type MountInfo struct { // VFSOpts represents per super block options. VFSOpts string `json:"vfsOpts"` -} -// BlockDevice provides information about a block-storage device. -type BlockDevice struct { - // The name of the device. - DeviceName string `json:"deviceName"` - - // The ID of the instance to which the device is connected. - InstanceID *InstanceID `json:"instanceID"` - - // The name of the network on which the device resides. - NetworkName string `json:"networkName"` - - // The name of the provider that owns the block device. - ProviderName string `json:"providerName"` - - // The region from which the device originates. - Region string `json:"region"` - - // The device status. - Status string `json:"status"` - - // The ID of the volume for which the device is mounted. - VolumeID string `json:"volumeID"` + // Fields are additional properties that can be defined for this type. + Fields map[string]string `json:"fields,omitempty"` } // Snapshot provides information about a storage-layer snapshot. type Snapshot struct { // A description of the snapshot. - Description string `json:"description"` + Description string `json:"description,omitempty"` // The name of the snapshot. - Name string `json:"name"` + Name string `json:"name,omitempty"` // The snapshot's ID. - SnapshotID string `json:"snapshotID"` + ID string `json:"id"` - // The time at which the request to create the snapshot was submitted. - StartTime string `json:"startTime"` + // The time (epoch) at which the request to create the snapshot was submitted. + StartTime int64 `json:"startTime,omitempty"` // The status of the snapshot. - Status string `json:"status"` + Status string `json:"status,omitempty"` // The ID of the volume to which the snapshot belongs. - VolumeID string `json:"volumeID"` + VolumeID string `json:"volumeID,omitempty"` // The size of the volume to which the snapshot belongs. - VolumeSize string `json:"volumeSize"` + VolumeSize int64 `json:"volumeSize,omitempty"` + + // Fields are additional properties that can be defined for this type. + Fields map[string]string `json:"fields,omitempty"` } // Volume provides information about a storage volume. type Volume struct { // The volume's attachments. - Attachments []*VolumeAttachment `json:"attachments"` + Attachments []*VolumeAttachment `json:"attachments,omitempty"` // The availability zone for which the volume is available. - AvailabilityZone string `json:"availabilityZone"` + AvailabilityZone string `json:"availabilityZone,omitempty"` // The volume IOPs. - IOPS int64 `json:"iops"` + IOPS int64 `json:"iops,omitempty"` // The name of the volume. Name string `json:"name"` // The name of the network on which the volume resides. - NetworkName string `json:"networkName"` + NetworkName string `json:"networkName,omitempty"` // The size of the volume. - Size string `json:"size"` + Size int64 `json:"size,omitempty"` // The volume status. - Status string `json:"status"` + Status string `json:"status,omitempty"` // The volume ID. - VolumeID string `json:"volumeID"` + ID string `json:"id"` // The volume type. - VolumeType string `json:"volumeType"` + Type string `json:"type"` + + // Fields are additional properties that can be defined for this type. + Fields map[string]string `json:"fields,omitempty"` } // VolumeAttachment provides information about an object attached to a @@ -175,6 +151,10 @@ type VolumeAttachment struct { // attached is mounted. DeviceName string `json:"deviceName"` + // MountPoint is the mount point for the volume. This field is set when a + // volume is retrieved via an integration driver. + MountPoint string `json:"mountPoint,omitempty"` + // The ID of the instance on which the volume to which the attachment // belongs is mounted. InstanceID *InstanceID `json:"instanceID"` @@ -184,20 +164,97 @@ type VolumeAttachment struct { // The ID of the volume to which the attachment belongs. VolumeID string `json:"volumeID"` + + // Fields are additional properties that can be defined for this type. + Fields map[string]string `json:"fields,omitempty"` +} + +// VolumeDevice provides information about a volume's backing storage +// device. This might be a block device, NAS device, object device, etc. +type VolumeDevice struct { + // The name of the device. + Name string `json:"name"` + + // The ID of the instance to which the device is connected. + InstanceID *InstanceID `json:"instanceID"` + + // The name of the network on which the device resides. + NetworkName string `json:"networkName"` + + // The name of the provider that owns the block device. + ProviderName string `json:"providerName"` + + // The region from which the device originates. + Region string `json:"region"` + + // The device status. + Status string `json:"status"` + + // The ID of the volume for which the device is mounted. + VolumeID string `json:"volumeID"` + + // Fields are additional properties that can be defined for this type. + Fields map[string]string `json:"fields,omitempty"` } -// ClientTool contains information about a client tool executor, such as +// ExecutorInfo contains information about a client-side executor, such as // its name and MD5 checksum. -type ClientTool struct { - // Data is a byte array that's either a binary file or a unicode-encoded, - // plain-text script file. Use the file extension of the client tool's file - // name to determine the file type. - Data []byte `json:"data"` - - // MD5Checksum is the MD5 checksum of the tool. This can be used to - // determine if a local copy of the tool needs to be updated. +type ExecutorInfo struct { + + // MD5Checksum is the MD5 checksum of the executor. This can be used to + // determine if a local copy of the executor needs to be updated. MD5Checksum string `json:"md5checksum"` - // Name is the file name of the tool, including the file extension. + // Name is the file name of the executor, including the file extension. Name string `json:"name"` } + +// ServiceInfo is information about a service. +type ServiceInfo struct { + // Name is the service's name. + Name string `json:"name"` + + // Driver is the name of the driver registered for the service. + Driver *DriverInfo `json:"driver"` +} + +// DriverInfo is information about a driver. +type DriverInfo struct { + // Name is the driver's name. + Name string `json:"name"` + + // Type is the type of storage the driver provides: block, nas, object. + Type StorageType `json:"type"` + + // NextDevice is the next available device information for the service. + NextDevice *NextDeviceInfo `json:"nextDevice"` + + // Executors is information about the driver's client-side executors. + Executors []*ExecutorInfo `json:"executors"` +} + +// NextDeviceInfo assists the libStorage client in determining the +// next available device name by providing the driver's device prefix and +// optional pattern. +// +// For example, the Amazon Web Services (AWS) device prefix is "xvd" and its +// pattern is "[a-z]". These two values would be used to determine on an EC2 +// instance where "/dev/xvda" and "/dev/xvdb" are in use that the next +// available device name is "/dev/xvdc". +// +// If the Ignore field is set to true then the client logic does not invoke the +// GetNextAvailableDeviceName function prior to submitting an AttachVolume +// request to the server. +type NextDeviceInfo struct { + // Ignore is a flag that indicates whether the client logic should invoke + // the GetNextAvailableDeviceName function prior to submitting an + // AttachVolume request to the server. + Ignore bool `json:"ignore"` + + // Prefix is the first part of a device path's value after the "/dev/" + // porition. For example, the prefix in "/dev/xvda" is "xvd". + Prefix string `json:"prefix"` + + // Pattern is the regex to match the part of a device path after the prefix. + Pattern string `json:"pattern"` +} diff --git a/api/types/types_store.go b/api/types/types_store.go new file mode 100644 index 00000000..834ef4b2 --- /dev/null +++ b/api/types/types_store.go @@ -0,0 +1,66 @@ +package types + +// Store is a key/value store with case-insensitive keys. +type Store interface { + // Keys returns a list of all the keys in the store. + Keys() []string + + // IsSet returns true if a key exists. + IsSet(k string) bool + + // Get returns a value for the key; a nil value if the key does not exist. + Get(k string) interface{} + + // GetString returns a string value for a key; an empty string if the key + // does not exist. + GetString(k string) string + + // GetStringPtr returns a pointer to a string value for a key; nil if + // the key does not exist. + GetStringPtr(k string) *string + + // GetBool returns a boolean value for the key; false if the key does not + // exist. + GetBool(k string) bool + + // GetBoolPtr returns a pointer to a boolean value for the key; nil if the + // key does not exist. + GetBoolPtr(k string) *bool + + // GetInt return an int value for the key; 0 if the key does not exist. + GetInt(k string) int + + // GetInt return a pointer to an int value for the key; nil if the key does + // not exist. + GetIntPtr(k string) *int + + // GetInt64 return an int64 value for the key; 0 if the key does not exist. + GetInt64(k string) int64 + + // GetInt64Ptr return a pointer to an int64 value for the key; nil if the + // key does not exist. + GetInt64Ptr(k string) *int64 + + // GetIntSlice returns an int slice value for a key; a nil value if + // the key does not exist. + GetIntSlice(k string) []int + + // GetStringSlice returns a string slice value for a key; a nil value if + // the key does not exist. + GetStringSlice(k string) []string + + // GetBoolSlice returns a bool slice value for a key; a nil value if + // the key does not exist. + GetBoolSlice(k string) []bool + + // GetMap returns a map value for a key; a nil value if the key does not + // exist. + GetMap(k string) map[string]interface{} + + // GetStore returns a Store value for a key; a nil value if the key does + // not exist. + GetStore(k string) Store + + // Set sets a key/value in the store. + Set(k string, v interface{}) +} diff --git a/api/utils/schema/gomk.properties b/api/utils/schema/gomk.properties new file mode 100644 index 00000000..240e4ee9 --- /dev/null +++ b/api/utils/schema/gomk.properties @@ -0,0 +1,10 @@ +LIBSTORAGE_JSON := ./libstorage.json +LIBSTORAGE_JSON_EMBEDDED := ./api/utils/schema/schema_generated.go +LIBSTORAGE_JSON_SCHEMA := $(shell $(CAT) $(LIBSTORAGE_JSON)) + +$(LIBSTORAGE_JSON_EMBEDDED):: $(LIBSTORAGE_JSON) | $(PRINTF) + @$(PRINTF) "package schema\n\nconst (\n" >$@ + @$(PRINTF) "\t// JSONSchema is the libStorage API JSON schema\n" >>$@ + @$(PRINTF) "\tJSONSchema = \`" >>$@ + @$(SED) -e 's/^//' $< >>$@ + @$(PRINTF) "\`\n)\n" >>$@ diff --git a/api/utils/schema/schema.go b/api/utils/schema/schema.go new file mode 100644 index 00000000..087237ab --- /dev/null +++ b/api/utils/schema/schema.go @@ -0,0 +1,184 @@ +package schema + +import ( + "bytes" + "encoding/json" + "fmt" + + log "github.com/Sirupsen/logrus" + "github.com/cesanta/ucl" + "github.com/cesanta/validate-json/schema" + + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + httptypes "github.com/emccode/libstorage/api/types/http" +) + +const ( + jsonSchemaID = "https://github.com/emccode/libstorage" +) + +var ( + jsonSchema = []byte(JSONSchema) + + // VolumeSchema is the JSON schema for the Volume resource. + VolumeSchema = buildSchemaVar("volume") + + // VolumeAttachmentSchema is the JSON schema for the VolumeAttachment + // resource. + VolumeAttachmentSchema = buildSchemaVar("volumeAttachment") + + // ServiceVolumeMapSchema is the JSON schema for the ServiceVolumeMap + // resource. + ServiceVolumeMapSchema = buildSchemaVar("serviceVolumeMap") + + // ServiceSnapshotMapSchema is the JSON schema for the ServiceSnapshotMap + // resource. + ServiceSnapshotMapSchema = buildSchemaVar("serviceSnapshotMap") + + // VolumeMapSchema is the JSON schema for the VolumeMap resource. + VolumeMapSchema = buildSchemaVar("volumeMap") + + // SnapshotMapSchema is the JSON schema for the SnapshotMap resource. + SnapshotMapSchema = buildSchemaVar("snapshotMap") + + // SnapshotSchema is the JSON schema for the Snapshot resource. + SnapshotSchema = buildSchemaVar("snapshot") + + // ServiceInfoSchema is the JSON schema for the ServiceInfo resource. + ServiceInfoSchema = buildSchemaVar("serviceInfo") + + // ServiceInfoMapSchema is the JSON schemea for a map[string]*ServiceInfo. + ServiceInfoMapSchema = buildSchemaVar("serviceInfoMap") + + // DriverInfoSchema is the JSON schema for the DriverInfo resource. + DriverInfoSchema = buildSchemaVar("driverInfo") + + // ExecutorInfoSchema is the JSON schema for the ExecutorInfo resource. + ExecutorInfoSchema = buildSchemaVar("executorInfo") + + // VolumeCreateRequestSchema is the JSON schema for a Volume creation + // request. + VolumeCreateRequestSchema = buildSchemaVar("volumeCreateRequest") + + // VolumeCopyRequestSchema is the JSON schema for a Volume copy + // request. + VolumeCopyRequestSchema = buildSchemaVar("volumeCopyRequest") + + // VolumeSnapshotRequestSchema is the JSON schema for a Volume snapshot + // request. + VolumeSnapshotRequestSchema = buildSchemaVar("volumeSnapshotRequest") + + // VolumeAttachRequestSchema is the JSON schema for a Volume attach + // request. + VolumeAttachRequestSchema = buildSchemaVar("volumeAttachRequest") + + // VolumeDetachRequestSchema is the JSON schema for a Volume detach + // request. + VolumeDetachRequestSchema = buildSchemaVar("volumeDetachRequest") + + // SnapshotCopyRequestSchema is the JSON schema for a Snapshot copy + // request. + SnapshotCopyRequestSchema = buildSchemaVar("snapshotCopyRequest") + + // VolumeCreateFromSnapshotRequestSchema is the JSON schema for a + // Volume create from Snapshot request. + VolumeCreateFromSnapshotRequestSchema = buildSchemaVar( + "volumeCreateFromSnapshotRequest") +) + +func buildSchemaVar(name string) []byte { + return []byte(fmt.Sprintf(`{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "%s#/definitions/%s" +}`, jsonSchemaID, name)) +} + +// ValidateVolume validates a Volume object using the JSON schema. If the +// object is valid no error is returned. The first return value, the object +// marshaled to JSON, is returned whether or not the validation is successful. +func ValidateVolume(v *types.Volume) ([]byte, error) { + return validateObject(VolumeSchema, v) +} + +// ValidateSnapshot validates a Snapshot object using the JSON schema. If the +// object is valid no error is returned. The first return value, the object +// marshaled to JSON, is returned whether or not the validation is successful. +func ValidateSnapshot(v *types.Snapshot) ([]byte, error) { + return validateObject(SnapshotSchema, v) +} + +// ValidateVolumeCreateRequest validates a VolumeCreateRequest object using the +// JSON schema. If the object is valid no error is returned. The first return +// value, the object marshaled to JSON, is returned whether or not the +// validation is successful. +func ValidateVolumeCreateRequest( + v *httptypes.VolumeCreateRequest) ([]byte, error) { + return validateObject(VolumeCreateRequestSchema, v) +} + +func validateObject(s []byte, o interface{}) (d []byte, e error) { + if d, e = json.Marshal(o); e != nil { + return + } + if e = Validate(nil, s, d); e != nil { + return + } + return +} + +func getSchemaValidator(s []byte) (*schema.Validator, error) { + volSchema, err := ucl.Parse(bytes.NewReader(s)) + if err != nil { + return nil, err + } + + rootSchema, err := ucl.Parse(bytes.NewReader(jsonSchema)) + if err != nil { + return nil, err + } + + loader := schema.NewLoader() + + if err := loader.Add(rootSchema); err != nil { + return nil, err + } + + validator, err := schema.NewValidator(volSchema, loader) + if err != nil { + return nil, err + } + + return validator, nil +} + +// Validate validates the provided data (d) against the provided schema (s). +func Validate(ctx context.Context, s, d []byte) error { + + var l *log.Logger + if ctx == nil { + l = log.StandardLogger() + } else { + l = ctx.Log() + } + + l.WithFields(log.Fields{ + "schema": string(s), + "body": string(d), + }).Debug("validating schema") + + validator, err := getSchemaValidator(s) + if err != nil { + return err + } + + if len(d) == 0 { + d = []byte("{}") + } + + data, err := ucl.Parse(bytes.NewReader(d)) + if err != nil { + return err + } + return validator.Validate(data) +} diff --git a/api/utils/schema/schema_generated.go b/api/utils/schema/schema_generated.go new file mode 100644 index 00000000..32d145b5 --- /dev/null +++ b/api/utils/schema/schema_generated.go @@ -0,0 +1,452 @@ +package schema + +const ( + // JSONSchema is the libStorage API JSON schema + JSONSchema = `{ + "id": "https://github.com/emccode/libstorage", + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": { + "volume": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The volume ID." + }, + "name": { + "type": "string", + "description": "The volume name." + }, + "type": { + "type": "string", + "description": "The volume type." + }, + "attachments": { + "type": "array", + "description": "The volume's attachments.", + "items": { "$ref": "#/definitions/volumeAttachment" } + }, + "availabilityZone": { + "type": "string", + "description": "The zone for which the volume is available." + }, + "iops": { + "type": "number", + "description": "The volume IOPs." + }, + "networkName": { + "type": "string", + "description": "The name of the network on which the volume resides." + }, + "size": { + "type": "number", + "description": "The volume size (GB)." + }, + "status": { + "type": "string", + "description": "The volume status." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "id", "name", "size" ], + "additionalProperties": false + }, + + + "volumeAttachment": { + "type": "object", + "properties": { + "instanceID": { "$ref": "#/definitions/instanceID" }, + "deviceName": { + "type": "string", + "description": "The name of the device on to which the volume is mounted." + }, + "status": { + "type": "string", + "description": "The status of the attachment." + }, + "volumeID": { + "type": "string", + "description": "The ID of the volume to which the attachment belongs." + }, + "mountPoint": { + "type": "string", + "description": "The file system path to which the volume is mounted." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "instanceID", "deviceName", "volumeID" ], + "additionalProperties": false + }, + + + "instanceID": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The instance ID." + }, + "metadata": { + "type": "object", + "description": "Extra information about the instance ID." + } + }, + "required": [ "id" ], + "additionalProperties": false + }, + + + "instance": { + "type": "object", + "properties": { + "instanceID": { "$ref": "#/definitions/instanceID" }, + "name": { + "type": "string", + "description": "The name of the instance." + }, + "providerName": { + "type": "string", + "description": "The name of the provider that owns the object." + }, + "region": { + "type": "string", + "description": "The region from which the object originates." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "id" ], + "additionalProperties": false + }, + + + "snapshot": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The snapshot's ID." + }, + "name": { + "type": "string", + "description": "The name of the snapshot." + }, + "description": { + "type": "string", + "description": "A description of the snapshot." + }, + "startTime": { + "type": "number", + "description": "The time (epoch) at which the request to create the snapshot was submitted." + }, + "status": { + "type": "string", + "description": "The status of the snapshot." + }, + "volumeID": { + "type": "string", + "description": "The ID of the volume to which the snapshot belongs." + }, + "volumeSize": { + "type": "number", + "description": "The size of the volume to which the snapshot belongs." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "id" ], + "additionalProperties": false + }, + + + "serviceInfo": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name is the service's name." + }, + "driver": { "$ref": "#/definitions/driverInfo" } + }, + "required": [ "name", "driver" ], + "additionalProperties": false + }, + + + "driverInfo": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server." + }, + "type": { + "type": "string", + "description": "Type is the type of storage the driver provides: block, nas, object." + }, + "nextDevice": { "$ref": "#/definitions/nextDeviceInfo" }, + "executors": { + "type": "array", + "description": "Executors is information about the driver's client-side executors.", + "items": { "$ref": "#/definitions/executorInfo" } + } + }, + "required": [ "name", "type" ], + "additionalProperties": false + }, + + + "executorInfo": { + "type": "object", + "properties": { + "md5checksum": { + "type": "string", + "description": "MD5Checksum is the MD5 checksum of the executor. This can be used to determine if a local copy of the executor needs to be updated." + }, + "name": { + "type": "string", + "description": "Name is the file name of the executor, including the file extension." + } + }, + "required": [ "md5checksum", "name" ], + "additionalProperties": false + }, + + + "nextDeviceInfo": { + "type": "object", + "properties": { + "ignore": { + "type": "boolean", + "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server." + }, + "prefix": { + "type": "string", + "description": "Prefix is the first part of a device path's value after the \"/dev/\" portion. For example, the prefix in \"/dev/xvda\" is \"xvd\"." + }, + "pattern": { + "type": "string", + "description": "Pattern is the regex to match the part of a device path after the prefix." + } + }, + "additionalProperties": false + }, + + + "fields": { + "type": "object", + "description": "Fields are additional properties that can be defined for this type.", + "patternProperties": { + ".+": { "type": "string" } + }, + "additionalProperties": true + }, + + + "volumeMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/volume" } + }, + "additionalProperties": false + }, + + + "snapshotMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/snapshot" } + }, + "additionalProperties": false + }, + + + "serviceVolumeMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/volumeMap" } + }, + "additionalProperties": false + }, + + + "serviceSnapshotMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/snapshotMap" } + }, + "additionalProperties": false + }, + + + "serviceInfoMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/serviceInfo" } + }, + "additionalProperties": false + }, + + + "driverInfoMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/driverInfo" } + }, + "additionalProperties": false + }, + + + "opts": { + "type": "object", + "description": "Opts are additional properties that can be defined for POST requests.", + "patternProperties": { + ".+": { + "anyOf": [ + { "type": "array" }, + { "type": "boolean" }, + { "type": "integer" }, + { "type": "number" }, + { "type": "null" }, + { "type": "string" }, + { "$ref": "#/definitions/opts" } + ] + } + }, + "additionalProperties": true + }, + + + "volumeCreateRequest": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "availabilityZone": { + "type": "string" + }, + "iops": { + "type": "number" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "name" ], + "additionalProperties": false + }, + + + "volumeCreateFromSnapshotRequest": { + "type": "object", + "properties": { + "volumeName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "volumeName" ], + "additionalProperties": false + }, + + + "volumeCopyRequest": { + "type": "object", + "properties": { + "volumeName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "volumeName" ], + "additionalProperties": false + }, + + + "volumeSnapshotRequest": { + "type": "object", + "properties": { + "snapshotName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "snapshotName" ], + "additionalProperties": false + }, + + + "volumeAttachRequest": { + "type": "object", + "properties": { + "nextDeviceName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "additionalProperties": false + }, + + + "volumeDetachRequest": { + "type": "object", + "properties": { + "opts": { "$ref" : "#/definitions/opts" } + }, + "additionalProperties": false + }, + + + "snapshotCopyRequest": { + "type": "object", + "properties": { + "snapshotName": { + "type": "string" + }, + "destinationID": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "snapshotName", "destinationID" ], + "additionalProperties": false + }, + + + "snapshotRemoveRequest": { + "type": "object", + "properties": { + "opts": { "$ref" : "#/definitions/opts" } + }, + "additionalProperties": false + }, + + + "error": { + "type": "object", + "properties": { + "message": { + "type": "string", + "pattern": "^.{10,}|.*[Ee]rror$" + }, + "status": { + "type": "number", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ "message", "status" ], + "additionalProperties": false + } + } +} +` +) diff --git a/api/utils/schema/schema_test.go b/api/utils/schema/schema_test.go new file mode 100644 index 00000000..3901692e --- /dev/null +++ b/api/utils/schema/schema_test.go @@ -0,0 +1,258 @@ +package schema + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/emccode/libstorage/api/types" + httptypes "github.com/emccode/libstorage/api/types/http" +) + +func TestVolumeObject(t *testing.T) { + + v := &types.Volume{ + ID: "vol-000", + Name: "Volume 000", + Size: 378, + Attachments: []*types.VolumeAttachment{ + &types.VolumeAttachment{ + InstanceID: &types.InstanceID{ + ID: "iid-000", + }, + VolumeID: "vol-000", + DeviceName: "/dev/xvd000", + }, + }, + Fields: map[string]string{ + "priority": "2", + "owner": "root@example.com", + }, + } + + d, err := ValidateVolume(v) + if d == nil { + assert.NoError(t, err, string(d)) + } else { + assert.NoError(t, err) + } +} + +func TestVolumeSchema(t *testing.T) { + s := VolumeSchema + + d := []byte(`{ + "id": "vol-000", + "name": "Volume 000", + "size": 378, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": "2", + "owner": "root@example.com" + } +}`) + + err := Validate(nil, s, d) + assert.NoError(t, err) + + d = []byte(`{ + "id2": "vol-000", + "name": "Volume 000", + "size": 378, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": "2", + "owner": "root@example.com" + } +}`) + + err = Validate(nil, s, d) + assert.Error(t, err) + assert.EqualError(t, err, `"#" must have property "id"`) + + d = []byte(`{ + "id": "vol-000", + "name": "Volume 000", + "size": 378, + "attachments": [ + { + "instanceID": { + "id": null + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": "2", + "owner": "root@example.com" + } +}`) + + err = Validate(nil, s, d) + assert.Error(t, err) + assert.EqualError(t, err, `"#/attachments/[0]/instanceID/id": must be of type "string"`) + + d = []byte(`{ + "id": "vol-000", + "name": "Volume 000", + "size": 378, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": 2, + "owner": "root@example.com" + } +}`) + + err = Validate(nil, s, d) + assert.Error(t, err) + assert.EqualError(t, err, `"#/fields/priority": must be of type "string"`) +} + +func TestSnapshotObject(t *testing.T) { + + s := &types.Snapshot{ + ID: "snap-000", + Name: "Snapshot 000", + VolumeID: "vol-000", + VolumeSize: 10240, + StartTime: 1455826676, + Fields: map[string]string{ + "sparse": "true", + "region": "US", + }, + } + + d, err := ValidateSnapshot(s) + if d == nil { + assert.NoError(t, err, string(d)) + } else { + assert.NoError(t, err) + } +} + +func TestSnapshotSchema(t *testing.T) { + s := SnapshotSchema + + d := []byte(`{ + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": "true", + "region": "US" + } +}`) + + err := Validate(nil, s, d) + assert.NoError(t, err) + + d = []byte(`{ + "id2": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": "true", + "region": "US" + } +}`) + + err = Validate(nil, s, d) + assert.Error(t, err) + assert.EqualError(t, err, `"#" must have property "id"`) + + d = []byte(`{ + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": "10240", + "fields": { + "sparse": "true", + "region": "US" + } +}`) + + err = Validate(nil, s, d) + assert.Error(t, err) + assert.EqualError(t, err, `"#/volumeSize": must be of type "number"`) + + d = []byte(`{ + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } +}`) + + err = Validate(nil, s, d) + assert.Error(t, err) + assert.EqualError(t, err, `"#/fields/sparse": must be of type "string"`) +} + +func TestVolumeCreateRequestObject(t *testing.T) { + + availabilityZone := "US" + iops := int64(1000) + size := int64(10240) + volType := "gold" + + v := &httptypes.VolumeCreateRequest{ + Name: "Volume 001", + AvailabilityZone: &availabilityZone, + IOPS: &iops, + Size: &size, + Type: &volType, + Opts: map[string]interface{}{ + "priority": 2, + "owner": "root@example.com", + "customData": map[string]int{ + "1": 1, + "2": 2, + }, + }, + } + + d, err := ValidateVolumeCreateRequest(v) + if d == nil { + assert.NoError(t, err, string(d)) + } else { + assert.NoError(t, err) + } +} diff --git a/api/utils/utils.go b/api/utils/utils.go new file mode 100644 index 00000000..5bd3fab0 --- /dev/null +++ b/api/utils/utils.go @@ -0,0 +1,21 @@ +package utils + +import ( + "fmt" + "reflect" +) + +// GetTypePkgPathAndName gets ths type and package path of the provided +// instance. +func GetTypePkgPathAndName(i interface{}) string { + t := reflect.TypeOf(i) + if t.Kind() == reflect.Ptr || t.Kind() == reflect.Interface { + t = t.Elem() + } + pkgPath := t.PkgPath() + typeName := t.Name() + if pkgPath == "" { + return typeName + } + return fmt.Sprintf("%s.%s", pkgPath, typeName) +} diff --git a/api/utils/utils_context.go b/api/utils/utils_context.go new file mode 100644 index 00000000..f10a96cf --- /dev/null +++ b/api/utils/utils_context.go @@ -0,0 +1,41 @@ +package utils + +import ( + log "github.com/Sirupsen/logrus" + "golang.org/x/net/context" + + "github.com/emccode/libstorage/api/types" +) + +var ( + instanceIDTypeName = GetTypePkgPathAndName(&types.InstanceID{}) + loggerTypeName = GetTypePkgPathAndName(&log.Logger{}) +) + +// GetInstanceID gets a pointer to an InstanceID from the context. +func GetInstanceID(ctx context.Context) (*types.InstanceID, error) { + obj := ctx.Value("instanceID") + if obj == nil { + return nil, NewContextKeyErr("instanceID") + } + typedObj, ok := obj.(*types.InstanceID) + if !ok { + return nil, NewContextTypeErr( + "instanceID", instanceIDTypeName, GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} + +// GetLogger gets a pointer to a Logger from the context. +func GetLogger(ctx context.Context) (*log.Logger, error) { + obj := ctx.Value("logger") + if obj == nil { + return nil, NewContextKeyErr("logger") + } + typedObj, ok := obj.(*log.Logger) + if !ok { + return nil, NewContextTypeErr( + "logger", loggerTypeName, GetTypePkgPathAndName(obj)) + } + return typedObj, nil +} diff --git a/api/utils/utils_errors.go b/api/utils/utils_errors.go new file mode 100644 index 00000000..b48557e5 --- /dev/null +++ b/api/utils/utils_errors.go @@ -0,0 +1,49 @@ +package utils + +import ( + "github.com/akutz/goof" + + "github.com/emccode/libstorage/api/types" +) + +// NewNotFoundError returns a new ErrNotFound error. +func NewNotFoundError(resourceID string) error { + return &types.ErrNotFound{ + Goof: goof.WithField("resourceID", resourceID, "resource not found")} +} + +// NewStoreKeyErr returns a new ErrStoreKey error. +func NewStoreKeyErr(storeKey string) error { + return &types.ErrStoreKey{ + Goof: goof.WithField("storeKey", storeKey, "missing store key")} +} + +// NewContextKeyErr returns a new ErrContextKey error. +func NewContextKeyErr(contextKey string) error { + return &types.ErrContextKey{ + Goof: goof.WithField("contextKey", contextKey, "missing context key")} +} + +// NewContextTypeErr returns a new ErrContextType error. +func NewContextTypeErr( + contextKey, expectedType, actualType string) error { + return &types.ErrContextType{Goof: goof.WithFields(goof.Fields{ + "contextKey": contextKey, + "expectedType": expectedType, + "actualType": actualType, + }, "invalid context type")} +} + +// NewDriverTypeErr returns a new ErrDriverTypeErr error. +func NewDriverTypeErr(expectedType, actualType string) error { + return &types.ErrDriverTypeErr{Goof: goof.WithFields(goof.Fields{ + "expectedType": expectedType, + "actualType": actualType, + }, "invalid driver type")} +} + +// NewBatchProcessErr returns a new ErrBatchProcess error. +func NewBatchProcessErr(completed interface{}, err error) error { + return &types.ErrBatchProcess{Goof: goof.WithFieldE( + "completed", completed, "batch processing error", err)} +} diff --git a/api/utils/utils_store.go b/api/utils/utils_store.go new file mode 100644 index 00000000..f0e06ef6 --- /dev/null +++ b/api/utils/utils_store.go @@ -0,0 +1,245 @@ +package utils + +import ( + "fmt" + "reflect" + "strconv" + "strings" + + "github.com/emccode/libstorage/api/types" +) + +type keyValueStore struct { + store map[string]interface{} +} + +// NewStore initializes a new instance of the Store type. +func NewStore() types.Store { + return newKeyValueStore(map[string]interface{}{}) +} + +// NewStoreWithData initializes a new instance of the Store type. +func NewStoreWithData(data map[string]interface{}) types.Store { + return newKeyValueStore(data) +} + +// NewStoreWithVars initializes a new instance of the Store type. +func NewStoreWithVars(vars map[string]string) types.Store { + m := map[string]interface{}{} + for k, v := range vars { + m[k] = v + } + return newKeyValueStore(m) +} + +func newKeyValueStore(m map[string]interface{}) types.Store { + cm := map[string]interface{}{} + for k, v := range m { + cm[strings.ToLower(k)] = v + } + return &keyValueStore{cm} +} + +func (s *keyValueStore) IsSet(k string) bool { + _, ok := s.store[strings.ToLower(k)] + return ok +} + +func (s *keyValueStore) Get(k string) interface{} { + return s.store[strings.ToLower(k)] +} + +func (s *keyValueStore) GetStore(k string) types.Store { + v := s.Get(k) + switch tv := v.(type) { + case types.Store: + return tv + default: + return nil + } +} + +func (s *keyValueStore) GetString(k string) string { + v := s.Get(k) + switch tv := v.(type) { + case string: + return tv + case nil: + return "" + default: + return fmt.Sprintf("%v", tv) + } +} + +func (s *keyValueStore) GetStringPtr(k string) *string { + v := s.Get(k) + switch tv := v.(type) { + case *string: + return tv + case string: + return &tv + case nil: + return nil + default: + str := getStrFromPossiblePtr(v) + return &str + } +} + +func (s *keyValueStore) GetBool(k string) bool { + v := s.Get(k) + switch tv := v.(type) { + case bool: + return tv + case nil: + return false + default: + b, _ := strconv.ParseBool(s.GetString(k)) + return b + } +} + +func (s *keyValueStore) GetBoolPtr(k string) *bool { + v := s.Get(k) + switch tv := v.(type) { + case *bool: + return tv + case bool: + return &tv + case nil: + return nil + default: + str := getStrFromPossiblePtr(v) + b, _ := strconv.ParseBool(str) + return &b + } +} + +func (s *keyValueStore) GetInt(k string) int { + v := s.Get(k) + switch tv := v.(type) { + case int: + return tv + case nil: + return 0 + default: + if iv, err := strconv.ParseInt(s.GetString(k), 10, 64); err == nil { + return int(iv) + } + return 0 + } +} + +func (s *keyValueStore) GetIntPtr(k string) *int { + v := s.Get(k) + switch tv := v.(type) { + case *int: + return tv + case int: + return &tv + case nil: + return nil + default: + str := getStrFromPossiblePtr(v) + var iivp *int + if iv, err := strconv.ParseInt(str, 10, 64); err == nil { + iiv := int(iv) + iivp = &iiv + } + return iivp + } +} + +func (s *keyValueStore) GetInt64(k string) int64 { + v := s.Get(k) + switch tv := v.(type) { + case int64: + return tv + case nil: + return 0 + default: + if iv, err := strconv.ParseInt(s.GetString(k), 10, 64); err == nil { + return iv + } + return 0 + } +} + +func (s *keyValueStore) GetInt64Ptr(k string) *int64 { + v := s.Get(k) + switch tv := v.(type) { + case *int64: + return tv + case int64: + return &tv + case nil: + return nil + default: + str := getStrFromPossiblePtr(v) + var ivp *int64 + if iv, err := strconv.ParseInt(str, 10, 64); err == nil { + ivp = &iv + } + return ivp + } +} + +func (s *keyValueStore) GetStringSlice(k string) []string { + v := s.Get(k) + switch tv := v.(type) { + case []string: + return tv + default: + return nil + } +} + +func (s *keyValueStore) GetIntSlice(k string) []int { + v := s.Get(k) + switch tv := v.(type) { + case []int: + return tv + default: + return nil + } +} + +func (s *keyValueStore) GetBoolSlice(k string) []bool { + v := s.Get(k) + switch tv := v.(type) { + case []bool: + return tv + default: + return nil + } +} + +func (s *keyValueStore) GetMap(k string) map[string]interface{} { + v := s.Get(k) + switch tv := v.(type) { + case map[string]interface{}: + return tv + default: + return nil + } +} + +func (s *keyValueStore) Set(k string, v interface{}) { + s.store[strings.ToLower(k)] = v +} + +func (s *keyValueStore) Keys() []string { + keys := []string{} + for k := range s.store { + keys = append(keys, k) + } + return keys +} + +func getStrFromPossiblePtr(i interface{}) string { + rv := reflect.ValueOf(i) + if rv.Kind() == reflect.Ptr { + return fmt.Sprintf("%v", rv.Elem()) + } + return fmt.Sprintf("%v", i) +} diff --git a/api/utils/utils_store_test.go b/api/utils/utils_store_test.go new file mode 100644 index 00000000..806cf44c --- /dev/null +++ b/api/utils/utils_store_test.go @@ -0,0 +1,114 @@ +package utils + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewStore(t *testing.T) { + s := NewStore() + assert.False(t, s.IsSet("hello")) + assert.Nil(t, s.Get("hello")) +} + +func TestGetStringPtr(t *testing.T) { + s := NewStore() + v := "hello" + fv := fmt.Sprintf("%v", v) + s.Set("myVal", fv) + pv := s.GetStringPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &fv) + pv = s.GetStringPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", v) + pv = s.GetStringPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &v) + pv = s.GetStringPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) +} + +func TestGetBoolPtr(t *testing.T) { + s := NewStore() + v := true + fv := fmt.Sprintf("%v", v) + s.Set("myVal", fv) + pv := s.GetBoolPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &fv) + pv = s.GetBoolPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", v) + pv = s.GetBoolPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &v) + pv = s.GetBoolPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) +} + +func TestGetIntPtr(t *testing.T) { + s := NewStore() + v := 5 + fv := fmt.Sprintf("%v", v) + s.Set("myVal", fv) + pv := s.GetIntPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &fv) + pv = s.GetIntPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", v) + pv = s.GetIntPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &v) + pv = s.GetIntPtr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) +} + +func TestGetInt64Ptr(t *testing.T) { + s := NewStore() + v := 5 + fv := fmt.Sprintf("%v", v) + s.Set("myVal", fv) + pv := s.GetInt64Ptr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &fv) + pv = s.GetInt64Ptr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", v) + pv = s.GetInt64Ptr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) + + s.Set("myVal", &v) + pv = s.GetInt64Ptr("myVal") + assert.NotNil(t, pv) + assert.EqualValues(t, v, *pv) +} diff --git a/api/utils/utils_tls.go b/api/utils/utils_tls.go new file mode 100644 index 00000000..79091f72 --- /dev/null +++ b/api/utils/utils_tls.go @@ -0,0 +1,96 @@ +package utils + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "os" + + log "github.com/Sirupsen/logrus" + "github.com/akutz/gofig" + "github.com/akutz/goof" + "github.com/akutz/gotil" +) + +// ParseTLSConfig returns a new TLS configuration. +func ParseTLSConfig(config gofig.Config) (*tls.Config, log.Fields, error) { + + if !config.IsSet("tls") { + return nil, nil, nil + } + + fields := log.Fields{} + + if !config.IsSet("tls.keyFile") { + return nil, nil, goof.New("keyFile required") + } + keyFile := config.GetString("tls.keyFile") + if !gotil.FileExists(keyFile) { + return nil, nil, goof.WithField("path", keyFile, "invalid key file") + } + fields["tls.keyFile"] = keyFile + + if !config.IsSet("tls.certFile") { + return nil, nil, goof.New("certFile required") + } + certFile := config.GetString("tls.certFile") + if !gotil.FileExists(certFile) { + return nil, nil, goof.WithField("path", certFile, "invalid cert file") + } + fields["tls.certFile"] = keyFile + + cer, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, nil, err + } + + tlsConfig := &tls.Config{Certificates: []tls.Certificate{cer}} + + if config.IsSet("tls.serverName") { + serverName := config.GetString("tls.serverName") + tlsConfig.ServerName = serverName + fields["tls.serverName"] = serverName + } + + if config.IsSet("tls.clientCertRequired") { + clientCertRequired := config.GetBool("tls.clientCertRequired") + if clientCertRequired { + tlsConfig.ClientAuth = tls.RequireAndVerifyClientCert + } + fields["tls.clientCertRequired"] = clientCertRequired + } + + if config.IsSet("tls.trustedCertsFile") { + trustedCertsFile := config.GetString("tls.trustedCertsFile") + + if !gotil.FileExists(trustedCertsFile) { + return nil, nil, goof.WithField( + "path", trustedCertsFile, "invalid trust file") + } + + fields["tls.trustedCertsFile"] = trustedCertsFile + + buf, err := func() ([]byte, error) { + f, err := os.Open(trustedCertsFile) + if err != nil { + return nil, err + } + defer f.Close() + buf, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + return buf, nil + }() + if err != nil { + return nil, nil, err + } + + certPool := x509.NewCertPool() + certPool.AppendCertsFromPEM(buf) + tlsConfig.RootCAs = certPool + tlsConfig.ClientCAs = certPool + } + + return tlsConfig, fields, nil +} diff --git a/client/client.go b/client/client.go deleted file mode 100644 index 8f6de947..00000000 --- a/client/client.go +++ /dev/null @@ -1,651 +0,0 @@ -package client - -import ( - "bufio" - "bytes" - "encoding/base64" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/httputil" - "os" - "os/exec" - "regexp" - - log "github.com/Sirupsen/logrus" - "github.com/akutz/gofig" - "github.com/akutz/goof" - "github.com/akutz/gotil" - gjson "github.com/gorilla/rpc/json" - "golang.org/x/net/context" - "golang.org/x/net/context/ctxhttp" - - "github.com/emccode/libstorage/api" -) - -var ( - netProtoRx = regexp.MustCompile("(?i)tcp") - - letters = []string{ - "a", "b", "c", "d", "e", "f", "g", "h", - "i", "j", "k", "l", "m", "n", "o", "p"} -) - -// Client is the interface for Golang libStorage clients. -type Client interface { - - // GetInstanceID gets the instance ID. - GetInstanceID(ctx context.Context) (*api.InstanceID, error) - - // GetNextAvailableDeviceName gets the name of the next available device. - GetNextAvailableDeviceName( - ctx context.Context, - args *api.GetNextAvailableDeviceNameArgs) (string, error) - - // GetServiceInfo returns information about the service. - GetServiceInfo( - ctx context.Context, - args *api.GetServiceInfoArgs) (*api.GetServiceInfoReply, error) - - // GetVolumeMapping lists the block devices that are attached to the - GetVolumeMapping( - ctx context.Context, - args *api.GetVolumeMappingArgs) ([]*api.BlockDevice, error) - - // GetInstance retrieves the local instance. - GetInstance( - ctx context.Context, - args *api.GetInstanceArgs) (*api.Instance, error) - - // GetVolume returns all volumes for the instance based on either volumeID - // or volumeName that are available to the instance. - GetVolume( - ctx context.Context, - args *api.GetVolumeArgs) ([]*api.Volume, error) - - // GetVolumeAttach returns the attachment details based on volumeID or - // volumeName where the volume is currently attached. - GetVolumeAttach( - ctx context.Context, - args *api.GetVolumeAttachArgs) ([]*api.VolumeAttachment, error) - - // CreateSnapshot is a synch/async operation that returns snapshots that - // have been performed based on supplying a snapshotName, source volumeID, - // and optional description. - CreateSnapshot( - ctx context.Context, - args *api.CreateSnapshotArgs) ([]*api.Snapshot, error) - - // GetSnapshot returns a list of snapshots for a volume based on volumeID, - // snapshotID, or snapshotName. - GetSnapshot( - ctx context.Context, - args *api.GetSnapshotArgs) ([]*api.Snapshot, error) - - // RemoveSnapshot will remove a snapshot based on the snapshotID. - RemoveSnapshot( - ctx context.Context, - args *api.RemoveSnapshotArgs) error - - // CreateVolume is sync/async and will create an return a new/existing - // Volume based on volumeID/snapshotID with a name of volumeName and a size - // in GB. Optionally based on the storage driver, a volumeType, IOPS, and - // availabilityZone could be defined. - CreateVolume( - ctx context.Context, - args *api.CreateVolumeArgs) (*api.Volume, error) - - // RemoveVolume will remove a volume based on volumeID. - RemoveVolume( - ctx context.Context, - args *api.RemoveVolumeArgs) error - - // AttachVolume returns a list of VolumeAttachments is sync/async that will - // attach a volume to an instance based on volumeID and ctx. - AttachVolume( - ctx context.Context, - args *api.AttachVolumeArgs) ([]*api.VolumeAttachment, error) - - // DetachVolume is sync/async that will detach the volumeID from the local - // instance or the ctx. - DetachVolume( - ctx context.Context, - args *api.DetachVolumeArgs) error - - // CopySnapshot is a sync/async and returns a snapshot that will copy a - // snapshot based on volumeID/snapshotID/snapshotName and create a new - // snapshot of desinationSnapshotName in the destinationRegion location. - CopySnapshot( - ctx context.Context, - args *api.CopySnapshotArgs) (*api.Snapshot, error) - - // GetClientTool gets the client tool provided by the driver. This tool is - // executed on the client-side of the connection in order to discover - // information only available to the client, such as the client's instance - // ID or a local device map. - // - // The client tool is returned as a byte array that's either a binary file - // or a unicode-encoded, plain-text script file. Use the file extension - // of the client tool's file name to determine the file type. - GetClientTool( - ctx context.Context, - args *api.GetClientToolArgs) (*api.ClientTool, error) -} - -type client struct { - config gofig.Config - url string - instanceID *api.InstanceID - instanceIDJSON string - instanceIDBase64 string - clientToolPath string - logRequests bool - logResponses bool - nextDevice *api.NextAvailableDeviceName -} - -// Dial opens a connection to a remote libStorage serice and returns the client -// that can be used to communicate with said endpoint. -// -// If the config parameter is nil a default instance is created. The -// function dials the libStorage service specified by the configuration -// property libstorage.host. -func Dial( - ctx context.Context, - config gofig.Config) (Client, error) { - - c := &client{config: config} - c.logRequests = c.config.GetBool( - "libstorage.client.http.logging.logrequest") - c.logResponses = c.config.GetBool( - "libstorage.client.http.logging.logresponse") - - host := config.GetString("libstorage.host") - if host == "" { - return nil, goof.New("libstorage.host is required") - } - log.WithField("host", host).Debug("got libStorage host") - - serviceName := config.GetString("libstorage.service.name") - if serviceName == "" { - return nil, goof.New("libstorage.service.name is required") - } - log.WithField( - "serviceName", serviceName).Debug("got libStorage serviceName name") - - if ctx == nil { - log.Debug("created empty context for dialer") - ctx = context.Background() - } - - netProto, laddr, err := gotil.ParseAddress(host) - if err != nil { - return nil, err - } - - if !netProtoRx.MatchString(netProto) { - return nil, goof.WithField("netProto", netProto, "tcp protocol only") - } - - c.url = fmt.Sprintf("http://%s/libStorage/services/%s", laddr, serviceName) - log.WithField("url", c.url).Debug("got libStorage service URL") - - if err := c.initClientTool(ctx); err != nil { - return nil, err - } - - if err := c.initInstanceID(ctx); err != nil { - return nil, err - } - - if err := c.initNextAvailableDeviceName(ctx); err != nil { - return nil, err - } - - log.WithField("url", c.url).Debug("successfuly dialed libStorage service") - return c, nil -} - -func (c *client) initNextAvailableDeviceName(ctx context.Context) error { - args := &api.GetNextAvailableDeviceNameArgs{} - reply := &api.GetNextAvailableDeviceNameReply{} - if err := c.post( - ctx, "GetNextAvailableDeviceName", args, reply); err != nil { - return err - } - c.nextDevice = reply.Next - return nil -} - -func (c *client) GetNextAvailableDeviceName( - ctx context.Context, - args *api.GetNextAvailableDeviceNameArgs) (string, error) { - - if c.nextDevice.Ignore { - return "", nil - } - - blockDevices, err := c.GetVolumeMapping(ctx, &api.GetVolumeMappingArgs{}) - if err != nil { - return "", err - } - - lettersInUse := map[string]bool{} - - var rx *regexp.Regexp - if c.nextDevice.Pattern == "" { - rx = regexp.MustCompile( - fmt.Sprintf(`^/dev/%s([a-z])$`, c.nextDevice.Prefix)) - } else { - rx = regexp.MustCompile( - fmt.Sprintf( - `^/dev/%s(%s)$`, c.nextDevice.Prefix, c.nextDevice.Pattern)) - } - - for _, d := range blockDevices { - m := rx.FindStringSubmatch(d.DeviceName) - if len(m) > 0 { - lettersInUse[m[1]] = true - } - } - - localDevices, err := c.getLocalDevices(c.nextDevice.Prefix) - if err != nil { - return "", err - } - - for _, d := range localDevices { - m := rx.FindStringSubmatch(d) - if len(m) > 0 { - lettersInUse[m[1]] = true - } - } - - for _, l := range letters { - if !lettersInUse[l] { - n := fmt.Sprintf("/dev/%s%s", c.nextDevice.Prefix, l) - log.WithField("name", n).Debug("got next available device name") - return n, nil - } - } - - return "", nil -} - -func (c *client) GetInstanceID(ctx context.Context) (*api.InstanceID, error) { - return c.instanceID, nil -} - -func (c *client) GetVolumeMapping( - ctx context.Context, - args *api.GetVolumeMappingArgs) ([]*api.BlockDevice, error) { - reply := &api.GetVolumeMappingReply{} - if err := c.post(ctx, "GetVolumeMapping", args, reply); err != nil { - return nil, err - } - return reply.BlockDevices, nil -} - -func (c *client) GetInstance( - ctx context.Context, - args *api.GetInstanceArgs) (*api.Instance, error) { - reply := &api.GetInstanceReply{} - if err := c.post(ctx, "GetInstance", args, reply); err != nil { - return nil, err - } - return reply.Instance, nil -} - -func (c *client) GetVolume( - ctx context.Context, - args *api.GetVolumeArgs) ([]*api.Volume, error) { - reply := &api.GetVolumeReply{} - if err := c.post(ctx, "GetVolume", args, reply); err != nil { - return nil, err - } - return reply.Volumes, nil -} - -func (c *client) GetVolumeAttach( - ctx context.Context, - args *api.GetVolumeAttachArgs) ([]*api.VolumeAttachment, error) { - reply := &api.GetVolumeAttachReply{} - if err := c.post(ctx, "GetVolumeAttach", args, reply); err != nil { - return nil, err - } - return reply.Attachments, nil -} - -func (c *client) CreateSnapshot( - ctx context.Context, - args *api.CreateSnapshotArgs) ([]*api.Snapshot, error) { - - reply := &api.CreateSnapshotReply{} - if err := c.post(ctx, "CreateSnapshot", args, reply); err != nil { - return nil, err - } - return reply.Snapshots, nil -} - -func (c *client) GetSnapshot( - ctx context.Context, - args *api.GetSnapshotArgs) ([]*api.Snapshot, error) { - reply := &api.GetSnapshotReply{} - if err := c.post(ctx, "GetSnapshot", args, reply); err != nil { - return nil, err - } - return reply.Snapshots, nil -} - -func (c *client) RemoveSnapshot( - ctx context.Context, - args *api.RemoveSnapshotArgs) error { - reply := &api.RemoveSnapshotReply{} - if err := c.post(ctx, "RemoveSnapshot", args, reply); err != nil { - return err - } - return nil -} - -func (c *client) CreateVolume( - ctx context.Context, - args *api.CreateVolumeArgs) (*api.Volume, error) { - reply := &api.CreateVolumeReply{} - if err := c.post(ctx, "CreateVolume", args, reply); err != nil { - return nil, err - } - return reply.Volume, nil -} - -func (c *client) RemoveVolume( - ctx context.Context, - args *api.RemoveVolumeArgs) error { - reply := &api.RemoveVolumeReply{} - if err := c.post(ctx, "RemoveVolume", args, reply); err != nil { - return err - } - return nil -} - -func (c *client) AttachVolume( - ctx context.Context, - args *api.AttachVolumeArgs) ([]*api.VolumeAttachment, error) { - if args.Required.NextDeviceName == "" { - - } - reply := &api.AttachVolumeReply{} - if err := c.post(ctx, "AttachVolume", args, reply); err != nil { - return nil, err - } - return reply.Attachments, nil -} - -func (c *client) DetachVolume( - ctx context.Context, - args *api.DetachVolumeArgs) error { - reply := &api.DetachVolumeReply{} - if err := c.post(ctx, "DetachVolume", args, reply); err != nil { - return err - } - return nil -} - -func (c *client) CopySnapshot( - ctx context.Context, - args *api.CopySnapshotArgs) (*api.Snapshot, error) { - reply := &api.CopySnapshotReply{} - if err := c.post(ctx, "CopySnapshot", args, reply); err != nil { - return nil, err - } - return reply.Snapshot, nil -} - -func (c *client) GetServiceInfo( - ctx context.Context, - args *api.GetServiceInfoArgs) (*api.GetServiceInfoReply, error) { - reply := &api.GetServiceInfoReply{} - if err := c.post(ctx, "GetServiceInfo", args, reply); err != nil { - return nil, err - } - return reply, nil -} - -func (c *client) GetClientTool( - ctx context.Context, - args *api.GetClientToolArgs) (*api.ClientTool, error) { - reply := &api.GetClientToolReply{} - if err := c.post(ctx, "GetClientTool", args, reply); err != nil { - return nil, err - } - return reply.ClientTool, nil -} - -func (c *client) post( - ctx context.Context, - method string, - args interface{}, - reply interface{}) error { - - method = fmt.Sprintf("libStorage.%s", method) - - log.WithFields(log.Fields{ - "url": c.url, - "method": method, - }).Debug("begin libStorage method") - - m, err := encReq(method, args) - if err != nil { - return err - } - - req, err := http.NewRequest("POST", c.url, bytes.NewReader(m)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - - if c.instanceIDBase64 != "" { - req.Header.Set("libStorage-InstanceID", c.instanceIDBase64) - } - - c.logRequest(log.StandardLogger().Writer(), req) - - res, err := ctxhttp.Do(ctx, nil, req) - if err != nil { - return err - } - - c.logResponse(log.StandardLogger().Writer(), res) - - if err := gjson.DecodeClientResponse(res.Body, reply); err != nil { - return err - } - - log.WithFields(log.Fields{ - "url": c.url, - "method": method, - }).Debug("end libStorage method") - - return nil -} - -func encReq(method string, args interface{}) ([]byte, error) { - enc, err := gjson.EncodeClientRequest(method, args) - if err != nil { - return nil, err - } - return enc, nil -} - -func decRes(body io.Reader, reply interface{}) error { - buf, err := ioutil.ReadAll(body) - if err != nil { - return err - } - log.WithField("json", string(buf)).Debug("response body") - b := bufio.NewReader(bytes.NewBuffer(buf)) - if err := gjson.DecodeClientResponse(b, reply); err != nil { - return err - } - return nil -} - -func (c *client) logRequest( - w io.Writer, - req *http.Request) { - - if !c.logRequests { - return - } - - fmt.Fprintln(w, "") - fmt.Fprint(w, " -------------------------- ") - fmt.Fprint(w, "HTTP REQUEST (CLIENT)") - fmt.Fprintln(w, " -------------------------") - - buf, err := httputil.DumpRequest(req, true) - if err != nil { - return - } - - gotil.WriteIndented(w, buf) -} - -func (c *client) logResponse( - w io.Writer, - res *http.Response) { - - if !c.logResponses { - return - } - - fmt.Fprint(w, " -------------------------- ") - fmt.Fprint(w, "HTTP RESPONSE (CLIENT)") - fmt.Fprintln(w, " -------------------------") - - buf, err := httputil.DumpResponse(res, true) - if err != nil { - return - } - - gotil.WriteIndented(w, buf) -} - -func (c *client) execClientToolInstanceID( - ctx context.Context) ([]byte, error) { - log.WithField("path", c.clientToolPath).Debug( - "executing client tool GetInstanceID") - return exec.Command(c.clientToolPath, "GetInstanceID").Output() -} - -func (c *client) execClientToolNextDevID(ctx context.Context) ([]byte, error) { - - log.WithField("path", c.clientToolPath).Debug( - "executing client tool GetNextAvailableDeviceName") - - bds, err := c.GetVolumeMapping(ctx, &api.GetVolumeMappingArgs{}) - if err != nil { - return nil, err - } - - bdsJSON, err := json.MarshalIndent(bds, "", " ") - if err != nil { - return nil, err - } - - env := os.Environ() - env = append(env, fmt.Sprintf("BLOCK_DEVICES_JSON=%s", string(bdsJSON))) - - cmd := exec.Command(c.clientToolPath, "GetNextAvailableDeviceName") - cmd.Env = env - return cmd.Output() -} - -func (c *client) initClientTool(ctx context.Context) error { - - args := &api.GetClientToolArgs{ - Optional: api.GetClientToolArgsOptional{ - OmitBinary: true, - }, - } - clientTool, err := c.GetClientTool(ctx, args) - if err != nil { - return err - } - - if gotil.FileExistsInPath(clientTool.Name) { - c.clientToolPath = clientTool.Name - log.WithField("path", c.clientToolPath).Debug( - "client tool exists in path") - return nil - } - - args.Optional.OmitBinary = false - clientTool, err = c.GetClientTool(ctx, args) - if err != nil { - return err - } - - toolDir := c.config.GetString("libstorage.client.tooldir") - toolPath := fmt.Sprintf("%s/%s", toolDir, clientTool.Name) - log.WithField("path", toolPath).Debug("writing client tool") - - if err := ioutil.WriteFile( - toolPath, clientTool.Data, 0755); err != nil { - return err - } - - c.clientToolPath = toolPath - log.WithField("path", c.clientToolPath).Debug( - "client tool path initialized") - return nil -} - -func (c *client) initInstanceID(ctx context.Context) error { - - log.Debug("begin get libStorage instanceID") - - iidJSON, err := c.execClientToolInstanceID(ctx) - if err != nil { - return err - } - - err = json.Unmarshal(iidJSON, &c.instanceID) - if err != nil { - return err - } - c.instanceIDJSON = string(iidJSON) - c.instanceIDBase64 = base64.URLEncoding.EncodeToString(iidJSON) - - log.WithFields(log.Fields{ - "instanceID": c.instanceID, - "instanceIDJSON": c.instanceIDJSON, - "instanceIDBase64": c.instanceIDBase64, - }).Debug("end get libStorage instanceID") - return nil -} - -func (c *client) getLocalDevices(prefix string) ([]string, error) { - - path := c.config.GetString("libstorage.client.localdevicesfile") - buf, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - deviceNames := []string{} - scan := bufio.NewScanner(bytes.NewReader(buf)) - - rx := regexp.MustCompile(fmt.Sprintf(`^.+?\s(%s\w+)$`, prefix)) - for scan.Scan() { - l := scan.Text() - m := rx.FindStringSubmatch(l) - if len(m) > 0 { - deviceNames = append(deviceNames, fmt.Sprintf("/dev/%s", m[1])) - } - } - - return deviceNames, nil -} diff --git a/context/context.go b/context/context.go deleted file mode 100644 index c4cf2458..00000000 --- a/context/context.go +++ /dev/null @@ -1,128 +0,0 @@ -package context - -import ( - "net/http" - - "github.com/akutz/goof" - gcontext "github.com/gorilla/context" - "golang.org/x/net/context" - - "github.com/emccode/libstorage/api" -) - -// Context is the libStorage implementation of the golang context pattern as -// described in the blog post at https://blog.golang.org/context. -type Context interface { - context.Context - - // InstanceID gets the context's instance ID. - InstanceID() *api.InstanceID - - // Profile gets the context's profile. - Profile() string - - // The *http.Request associated with this context, if any. - HTTPRequest() *http.Request -} - -type libstorContext struct { - context.Context - req *http.Request -} - -// Background initializes a new, empty context. -func Background() Context { - return NewContext(context.Background(), nil) -} - -// NewContext initializes a new libStorage context. -func NewContext(parent context.Context, r *http.Request) Context { - return &libstorContext{parent, r} -} - -// WithInstanceID returns a context with the provided instance ID. -func WithInstanceID( - parent context.Context, - instanceID *api.InstanceID) Context { - return WithValue(parent, "instanceID", instanceID) -} - -// WithProfile returns a context with the provided profile. This context type -// is valid on the service-side only and will be ignored on the client-side. -func WithProfile( - parent context.Context, - profile string) Context { - return WithValue(parent, "profile", profile) -} - -// WithValue returns a context with the provided value. -func WithValue( - parent context.Context, - key interface{}, - val interface{}) Context { - - switch tp := parent.(type) { - case Context: - return NewContext(context.WithValue(tp, key, val), tp.HTTPRequest()) - default: - return NewContext(context.WithValue(parent, key, val), nil) - } -} - -// InstanceID gets the context's instance ID. -func (ctx *libstorContext) InstanceID() *api.InstanceID { - v, ok := ctx.Value("instanceID").(*api.InstanceID) - if !ok { - panic(goof.New("invalid instanceID")) - } - return v -} - -// Profile gets the context's profile. -func (ctx *libstorContext) Profile() string { - v, ok := ctx.Value("profile").(string) - if !ok { - panic(goof.New("invalid profile")) - } - return v -} - -// HTTPRequest returns the *http.Request associated with ctx using -// NewRequestContext, if any. -func (ctx *libstorContext) HTTPRequest() *http.Request { - if req, ok := HTTPRequest(ctx); ok { - return req - } - return nil -} - -type key int - -const reqKey key = 0 - -// Value returns Gorilla's context package's value for this Context's request -// and key. It delegates to the parent Context if there is no such value. -func (ctx *libstorContext) Value(key interface{}) interface{} { - if ctx.req == nil { - ctx.Context.Value(key) - } - - if key == reqKey { - return ctx.req - } - if val, ok := gcontext.GetOk(ctx.req, key); ok { - return val - } - - return ctx.Context.Value(key) -} - -// HTTPRequest returns the *http.Request associated with ctx using -// NewRequestContext, if any. -func HTTPRequest(ctx context.Context) (*http.Request, bool) { - // We cannot use ctx.(*wrapper).req to get the request because ctx may - // be a Context derived from a *wrapper. Instead, we use Value to - // access the request if it is anywhere up the Context tree. - req, ok := ctx.Value(reqKey).(*http.Request) - return req, ok -} diff --git a/driver/driver.go b/driver/driver.go deleted file mode 100644 index b3b9daaf..00000000 --- a/driver/driver.go +++ /dev/null @@ -1,111 +0,0 @@ -package driver - -import ( - "github.com/akutz/gofig" - - "github.com/emccode/libstorage/api" - "github.com/emccode/libstorage/context" -) - -// NewDriver is a function that constructs a new driver. -type NewDriver func(c gofig.Config) Driver - -// Driver represents a libStorage driver. -type Driver interface { - // The name of the driver. - Name() string - - // Init initializes the driver. - Init() error - - // GetNextAvailableDeviceName gets the driver's NextAvailableDeviceName - // information. - GetNextAvailableDeviceName( - ctx context.Context, - args *api.GetNextAvailableDeviceNameArgs) ( - *api.NextAvailableDeviceName, error) - - // GetVolumeMapping lists the block devices that are attached to the - GetVolumeMapping( - ctx context.Context, - args *api.GetVolumeMappingArgs) ([]*api.BlockDevice, error) - - // GetInstance retrieves the local instance. - GetInstance( - ctx context.Context, - args *api.GetInstanceArgs) (*api.Instance, error) - - // GetVolume returns all volumes for the instance based on either volumeID - // or volumeName that are available to the instance. - GetVolume( - ctx context.Context, - args *api.GetVolumeArgs) ([]*api.Volume, error) - - // GetVolumeAttach returns the attachment details based on volumeID or - // volumeName where the volume is currently attached. - GetVolumeAttach( - ctx context.Context, - args *api.GetVolumeAttachArgs) ([]*api.VolumeAttachment, error) - - // CreateSnapshot is a synch/async operation that returns snapshots that - // have been performed based on supplying a snapshotName, source volumeID, - // and optional description. - CreateSnapshot( - ctx context.Context, - args *api.CreateSnapshotArgs) ([]*api.Snapshot, error) - - // GetSnapshot returns a list of snapshots for a volume based on volumeID, - // snapshotID, or snapshotName. - GetSnapshot( - ctx context.Context, - args *api.GetSnapshotArgs) ([]*api.Snapshot, error) - - // RemoveSnapshot will remove a snapshot based on the snapshotID. - RemoveSnapshot( - ctx context.Context, - args *api.RemoveSnapshotArgs) error - - // CreateVolume is sync/async and will create an return a new/existing - // Volume based on volumeID/snapshotID with a name of volumeName and a size - // in GB. Optionally based on the storage driver, a volumeType, IOPS, and - // availabilityZone could be defined. - CreateVolume( - ctx context.Context, - args *api.CreateVolumeArgs) (*api.Volume, error) - - // RemoveVolume will remove a volume based on volumeID. - RemoveVolume( - ctx context.Context, - args *api.RemoveVolumeArgs) error - - // AttachVolume returns a list of VolumeAttachments is sync/async that will - // attach a volume to an instance based on volumeID and ctx. - AttachVolume( - ctx context.Context, - args *api.AttachVolumeArgs) ([]*api.VolumeAttachment, error) - - // DetachVolume is sync/async that will detach the volumeID from the local - // instance or the ctx. - DetachVolume( - ctx context.Context, - args *api.DetachVolumeArgs) error - - // CopySnapshot is a sync/async and returns a snapshot that will copy a - // snapshot based on volumeID/snapshotID/snapshotName and create a new - // snapshot of desinationSnapshotName in the destinationRegion location. - CopySnapshot( - ctx context.Context, - args *api.CopySnapshotArgs) (*api.Snapshot, error) - - // GetClientTool gets the client tool provided by the driver. This tool is - // executed on the client-side of the connection in order to discover - // information only available to the client, such as the client's instance - // ID or a local device map. - // - // The client tool is returned as a byte array that's either a binary file - // or a unicode-encoded, plain-text script file. Use the file extension - // of the client tool's file name to determine the file type. - GetClientTool( - ctx context.Context, - args *api.GetClientToolArgs) (*api.ClientTool, error) -} diff --git a/driver/driver_args.go b/driver/driver_args.go deleted file mode 100644 index ea1d768f..00000000 --- a/driver/driver_args.go +++ /dev/null @@ -1,102 +0,0 @@ -// +build OMIT - -package driver - -// GetVolumeMappingArgs are the arguments expected by the GetVolumeMapping -// function. -type GetVolumeMappingArgs struct { - Extensions map[string]interface{} `json:"extensions"` -} - -// GetInstanceArgs are the arguments expected by the GetInstance function. -type GetInstanceArgs struct { - Extensions map[string]interface{} `json:"extensions"` -} - -// GetVolumeArgs are the arguments expected by the GetVolume function. -type GetVolumeArgs struct { - VolumeID string `json:"volumeID"` - VolumeName string `json:"volumeName"` - Extensions map[string]interface{} `json:"extensions"` -} - -// GetVolumeAttachArgs are the arguments expected by the GetVolumeAttach -// function. -type GetVolumeAttachArgs struct { - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// CreateSnapshotArgs are the arguments expected by the CreateSnapshot function. -type CreateSnapshotArgs struct { - Description string `json:"description"` - SnapshotName string `json:"snapshotName"` - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// GetSnapshotArgs are the arguments expected by the GetSnapshot function. -type GetSnapshotArgs struct { - SnapshotID string `json:"snapshotID"` - SnapshotName string `json:"snapshotName"` - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// RemoveSnapshotArgs are the arguments expected by the RemoveSnapshot function. -type RemoveSnapshotArgs struct { - SnapshotID string `json:"snapshotID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// CreateVolumeArgs are the arguments expected by the CreateVolume function. -type CreateVolumeArgs struct { - VolumeID string `json:"volumeID"` - AvailabilityZone string `json:"availabilityZone"` - VolumeName string `json:"volumeName"` - Size int64 `json:"size"` - SnapshotID string `json:"snapshotID"` - IOPS int64 `json:"iops"` - VolumeType string `json:"volumeType"` - Extensions map[string]interface{} `json:"extensions"` -} - -// RemoveVolumeArgs are the arguments expected by the RemoveVolume function. -type RemoveVolumeArgs struct { - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// AttachVolumeArgs are the arguments expected by the AttachVolume function. -type AttachVolumeArgs struct { - NextDeviceName string `json:"nextDeviceName"` - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// DetachVolumeArgs are the arguments expected by the DetachVolume function. -type DetachVolumeArgs struct { - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// CopySnapshotArgs are the arguments expected by the CopySnapshot function. -type CopySnapshotArgs struct { - DestinationRegion string `json:"destinationRegion"` - DestinationSnapshotName string `json:"destinationSnapshotName"` - SnapshotID string `json:"snapshotID"` - SnapshotName string `json:"snapshotName"` - VolumeID string `json:"volumeID"` - Extensions map[string]interface{} `json:"extensions"` -} - -// GetClientToolNameArgs are the arguments expected by the GetClientToolName -// function. -type GetClientToolNameArgs struct { - Extensions map[string]interface{} `json:"extensions"` -} - -// GetClientToolArgs are the arguments expected by the GetClientTool function. -type GetClientToolArgs struct { - Extensions map[string]interface{} `json:"extensions"` -} diff --git a/drivers/drivers.go b/drivers/drivers.go new file mode 100644 index 00000000..05ce16d1 --- /dev/null +++ b/drivers/drivers.go @@ -0,0 +1,6 @@ +package drivers + +import ( + // import to load + _ "github.com/emccode/libstorage/drivers/storage" +) diff --git a/drivers/storage/mock/mock.go b/drivers/storage/mock/mock.go new file mode 100644 index 00000000..6c5d797c --- /dev/null +++ b/drivers/storage/mock/mock.go @@ -0,0 +1,178 @@ +// +build mock + +package mock + +import ( + "fmt" + + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/drivers" +) + +type driver struct { + config gofig.Config + name string + instanceID *types.InstanceID + nextDeviceInfo *types.NextDeviceInfo + volumes []*types.Volume + snapshots []*types.Snapshot + storageType types.StorageType +} + +const ( + // Driver1Name is the name of the first mock driver. + Driver1Name = "mockDriver1" + + // Driver2Name is the name of the second mock driver. + Driver2Name = "mockDriver2" + + // Driver3Name is the name of the third mock driver. + Driver3Name = "mockDriver3" +) + +func init() { + registry.RegisterStorageDriver(Driver1Name, newMockDriver1) + registry.RegisterStorageDriver(Driver2Name, newMockDriver2) + registry.RegisterStorageDriver(Driver3Name, newMockDriver3) +} + +func newMockDriver1() drivers.StorageDriver { + return newMockDriver(Driver1Name) +} + +func newMockDriver2() drivers.StorageDriver { + return newMockDriver(Driver2Name) +} + +func newMockDriver3() drivers.StorageDriver { + return newMockDriver(Driver3Name) +} + +func newMockDriver(name string) drivers.StorageDriver { + d := &driver{ + name: name, + storageType: types.Block, + instanceID: getInstanceID(name), + } + + d.nextDeviceInfo = &types.NextDeviceInfo{ + Prefix: "xvd", + Pattern: `\w`, + Ignore: getDeviceIgnore(name), + } + + d.volumes = []*types.Volume{ + &types.Volume{ + Name: d.pwn("Volume 0"), + ID: d.pwn("vol-000"), + AvailabilityZone: d.pwn("zone-000"), + Type: "gold", + Size: 10240, + }, + &types.Volume{ + Name: d.pwn("Volume 1"), + ID: d.pwn("vol-001"), + AvailabilityZone: d.pwn("zone-001"), + Type: "gold", + Size: 40960, + }, + &types.Volume{ + Name: d.pwn("Volume 2"), + ID: d.pwn("vol-002"), + AvailabilityZone: d.pwn("zone-002"), + Type: "gold", + Size: 163840, + }, + } + + d.snapshots = []*types.Snapshot{ + &types.Snapshot{ + Name: d.pwn("Snapshot 0"), + ID: d.pwn("snap-000"), + VolumeID: d.pwn("vol-000"), + }, + &types.Snapshot{ + Name: d.pwn("Snapshot 1"), + ID: d.pwn("snap-001"), + VolumeID: d.pwn("vol-001"), + }, + &types.Snapshot{ + Name: d.pwn("Snapshot 2"), + ID: d.pwn("snap-002"), + VolumeID: d.pwn("vol-002"), + }, + } + + return d +} + +func (d *driver) Init(config gofig.Config) error { + d.config = config + return nil +} + +func (d *driver) Name() string { + return d.name +} + +func (d *driver) Type() types.StorageType { + return d.storageType +} + +func pwn(name, v string) string { + return fmt.Sprintf("%s-%s", name, v) +} + +func (d *driver) pwn(v string) string { + return fmt.Sprintf("%s-%s", d.name, v) +} + +func getInstanceID(name string) *types.InstanceID { + return &types.InstanceID{ + ID: pwn(name, "InstanceID"), + Metadata: instanceIDMetadata(), + } +} + +func instanceIDMetadata() map[string]interface{} { + return map[string]interface{}{ + "min": 0, + "max": 10, + "rad": "cool", + "totally": "tubular", + } +} + +func getDeviceIgnore(driver string) bool { + if driver == Driver2Name { + return true + } + return false +} + +func getDeviceName(driver string) string { + var deviceName string + switch driver { + case Driver1Name: + deviceName = "/dev/xvdb" + case Driver2Name: + deviceName = "/dev/xvda" + case Driver3Name: + deviceName = "/dev/xvdc" + } + return deviceName +} + +func getNextDeviceName(driver string) string { + var deviceName string + switch driver { + case Driver1Name: + deviceName = "/dev/xvdc" + case Driver3Name: + deviceName = "/dev/xvdb" + } + return deviceName +} diff --git a/drivers/storage/mock/mock_driver.go b/drivers/storage/mock/mock_driver.go new file mode 100644 index 00000000..ea0c1437 --- /dev/null +++ b/drivers/storage/mock/mock_driver.go @@ -0,0 +1,290 @@ +// +build mock +// +build driver +// +build !executor + +package mock + +import ( + "fmt" + "strings" + + log "github.com/Sirupsen/logrus" + "github.com/akutz/goof" + + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" + "github.com/emccode/libstorage/api/utils" +) + +func (d *driver) NextDeviceInfo() *types.NextDeviceInfo { + return d.nextDeviceInfo +} + +func (d *driver) InstanceInspect( + ctx context.Context, + opts types.Store) (*types.Instance, error) { + return &types.Instance{InstanceID: d.instanceID}, nil +} + +func (d *driver) Volumes( + ctx context.Context, + opts *drivers.VolumesOpts) ([]*types.Volume, error) { + return d.volumes, nil +} + +func (d *driver) VolumeInspect( + ctx context.Context, + volumeID string, + opts *drivers.VolumeInspectOpts) (*types.Volume, error) { + + for _, v := range d.volumes { + if strings.ToLower(v.ID) == strings.ToLower(volumeID) { + return v, nil + } + } + return nil, utils.NewNotFoundError(volumeID) +} + +func (d *driver) VolumeCreate( + ctx context.Context, + name string, + opts *drivers.VolumeCreateOpts) (*types.Volume, error) { + + lenVols := len(d.volumes) + + volume := &types.Volume{ + Name: name, + ID: d.pwn(fmt.Sprintf("vol-%03d", lenVols+1)), + Fields: map[string]string{}, + } + + if opts.AvailabilityZone != nil { + volume.AvailabilityZone = *opts.AvailabilityZone + } + if opts.Type != nil { + volume.Type = *opts.Type + } + if opts.Size != nil { + volume.Size = *opts.Size + } + if opts.IOPS != nil { + volume.IOPS = *opts.IOPS + } + + if opts.Opts.IsSet("owner") { + volume.Fields["owner"] = opts.Opts.GetString("owner") + } + if opts.Opts.IsSet("priority") { + volume.Fields["priority"] = opts.Opts.GetString("priority") + } + + d.volumes = append(d.volumes, volume) + + return volume, nil +} + +func (d *driver) VolumeCreateFromSnapshot( + ctx context.Context, + snapshotID, volumeName string, + opts types.Store) (*types.Volume, error) { + + return nil, nil +} + +func (d *driver) VolumeCopy( + ctx context.Context, + volumeID, volumeName string, + opts types.Store) (*types.Volume, error) { + + ctx.Log().WithFields(log.Fields{ + "volumeID": volumeID, + "volumeName": volumeName, + }).Debug("mockDriver.VolumeCopy") + + lenVols := len(d.volumes) + + var ogvol *types.Volume + for _, v := range d.volumes { + if strings.ToLower(v.ID) == strings.ToLower(volumeID) { + ogvol = v + break + } + } + + volume := &types.Volume{ + Name: volumeName, + ID: d.pwn(fmt.Sprintf("vol-%03d", lenVols+1)), + AvailabilityZone: ogvol.AvailabilityZone, + Type: ogvol.Type, + Size: ogvol.Size, + Fields: map[string]string{}, + } + + for k, v := range ogvol.Fields { + volume.Fields[k] = v + } + + d.volumes = append(d.volumes, volume) + + return volume, nil + +} + +func (d *driver) VolumeSnapshot( + ctx context.Context, + volumeID, snapshotName string, + opts types.Store) (*types.Snapshot, error) { + + ctx.Log().WithFields(log.Fields{ + "volumeID": volumeID, + "snapshotName": snapshotName, + }).Debug("mockDriver.VolumeSnapshot") + + lenSnaps := len(d.snapshots) + + snapshot := &types.Snapshot{ + Name: snapshotName, + ID: d.pwn(fmt.Sprintf("snap-%03d", lenSnaps+1)), + VolumeID: volumeID, + Fields: map[string]string{}, + } + + d.snapshots = append(d.snapshots, snapshot) + + return snapshot, nil +} + +func (d *driver) VolumeRemove( + ctx context.Context, + volumeID string, + opts types.Store) error { + + ctx.Log().WithFields(log.Fields{ + "volumeID": volumeID, + }).Debug("mockDriver.VolumeRemove") + + var xToRemove int + var volume *types.Volume + for x, v := range d.volumes { + if strings.ToLower(v.ID) == strings.ToLower(volumeID) { + volume = v + xToRemove = x + break + } + } + + if volume == nil { + return utils.NewNotFoundError(volumeID) + } + + d.volumes = append(d.volumes[:xToRemove], d.volumes[xToRemove+1:]...) + + return nil +} + +func (d *driver) VolumeAttach( + ctx context.Context, + volumeID string, + opts *drivers.VolumeAttachByIDOpts) (*types.Volume, error) { + + return nil, nil +} + +func (d *driver) VolumeDetach( + ctx context.Context, + volumeID string, + opts types.Store) error { + + if strings.ToLower(ctx.Value("serviceID").(string)) == "testservice2" && + strings.ToLower(volumeID) == "mockdriver2-vol-001" { + return goof.New("volume detach error") + } + + return nil +} + +func (d *driver) Snapshots( + ctx context.Context, + opts types.Store) ([]*types.Snapshot, error) { + + return d.snapshots, nil +} + +func (d *driver) SnapshotInspect( + ctx context.Context, + snapshotID string, + opts types.Store) (*types.Snapshot, error) { + + for _, v := range d.snapshots { + if strings.ToLower(v.ID) == strings.ToLower(snapshotID) { + return v, nil + } + } + return nil, goof.New("invalid snapshot id") +} + +func (d *driver) SnapshotCopy( + ctx context.Context, + snapshotID, snapshotName, destinationID string, + opts types.Store) (*types.Snapshot, error) { + + ctx.Log().WithFields(log.Fields{ + "snapshotID": snapshotID, + "snapshotName": snapshotName, + "destinationID": destinationID, + }).Debug("mockDriver.SnapshotCopy") + + lenSnaps := len(d.snapshots) + + var ogsnap *types.Snapshot + for _, s := range d.snapshots { + if strings.ToLower(s.ID) == strings.ToLower(snapshotID) { + ogsnap = s + break + } + } + + snapshot := &types.Snapshot{ + Name: snapshotName, + ID: d.pwn(fmt.Sprintf("snap-%03d", lenSnaps+1)), + VolumeID: ogsnap.VolumeID, + Fields: map[string]string{}, + } + + for k, s := range ogsnap.Fields { + snapshot.Fields[k] = s + } + + d.snapshots = append(d.snapshots, snapshot) + + return snapshot, nil +} + +func (d *driver) SnapshotRemove( + ctx context.Context, + snapshotID string, + opts types.Store) error { + + ctx.Log().WithFields(log.Fields{ + "snapshotID": snapshotID, + }).Debug("mockDriver.SnapshotRemove") + + var xToRemove int + var snapshot *types.Snapshot + for x, s := range d.snapshots { + if strings.ToLower(s.ID) == strings.ToLower(snapshotID) { + snapshot = s + xToRemove = x + break + } + } + + if snapshot == nil { + return utils.NewNotFoundError(snapshotID) + } + + d.snapshots = append(d.snapshots[:xToRemove], d.snapshots[xToRemove+1:]...) + + return nil +} diff --git a/drivers/storage/mock/mock_executor.go b/drivers/storage/mock/mock_executor.go new file mode 100644 index 00000000..35b83627 --- /dev/null +++ b/drivers/storage/mock/mock_executor.go @@ -0,0 +1,34 @@ +// +build mock +// +build driver executor + +package mock + +import ( + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" +) + +var ( + nextDeviceVals = []string{"/dev/mock0", "/dev/mock1", "/dev/mock2"} +) + +// InstanceID returns the local system's InstanceID. +func (d *driver) InstanceID( + ctx context.Context, + opts types.Store) (*types.InstanceID, error) { + return d.instanceID, nil +} + +// NextDevice returns the next available device. +func (d *driver) NextDevice( + ctx context.Context, + opts types.Store) (string, error) { + return "", nil +} + +// LocalDevices returns a map of the system's local devices. +func (d *driver) LocalDevices( + ctx context.Context, + opts types.Store) (map[string]string, error) { + return nil, nil +} diff --git a/drivers/storage/mock/mock_nodriver.go b/drivers/storage/mock/mock_nodriver.go new file mode 100644 index 00000000..3f8c7609 --- /dev/null +++ b/drivers/storage/mock/mock_nodriver.go @@ -0,0 +1,105 @@ +// +build mock +// +build !driver + +package mock + +import ( + "github.com/emccode/libstorage/api/types" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" +) + +func (d *driver) InstanceInspect( + ctx context.Context, + opts types.Store) (*types.Instance, error) { + return nil, nil +} + +func (d *driver) Volumes( + ctx context.Context, + opts *drivers.VolumesOpts) ([]*types.Volume, error) { + return nil, nil +} + +func (d *driver) VolumeInspect( + ctx context.Context, + volumeID string, + opts *drivers.VolumeInspectOpts) (*types.Volume, error) { + return nil, nil +} + +func (d *driver) VolumeCreate( + ctx context.Context, + name string, + opts *drivers.VolumeCreateOpts) (*types.Volume, error) { + return nil, nil +} + +func (d *driver) VolumeCreateFromSnapshot( + ctx context.Context, + snapshotID, volumeName string, + opts types.Store) (*types.Volume, error) { + return nil, nil +} + +func (d *driver) VolumeCopy( + ctx context.Context, + volumeID, volumeName string, + opts types.Store) (*types.Volume, error) { + return nil, nil +} + +func (d *driver) VolumeSnapshot( + ctx context.Context, + volumeID, snapshotName string, + opts types.Store) (*types.Snapshot, error) { + return nil, nil +} + +func (d *driver) VolumeRemove( + ctx context.Context, + volumeID string, + opts types.Store) error { + return nil +} + +func (d *driver) VolumeAttach( + ctx context.Context, + volumeID string, + opts *drivers.VolumeAttachByIDOpts) (*types.Volume, error) { + return nil, nil +} + +func (d *driver) VolumeDetach( + ctx context.Context, + volumeID string, + opts types.Store) error { + return nil +} + +func (d *driver) Snapshots( + ctx context.Context, + opts types.Store) ([]*types.Snapshot, error) { + return nil, nil +} + +func (d *driver) SnapshotInspect( + ctx context.Context, + snapshotID string, + opts types.Store) (*types.Snapshot, error) { + return nil, nil +} + +func (d *driver) SnapshotCopy( + ctx context.Context, + snapshotID, snapshotName, destinationID string, + opts types.Store) (*types.Snapshot, error) { + return nil, nil +} + +func (d *driver) SnapshotRemove( + ctx context.Context, + snapshotID string, + opts types.Store) error { + return nil +} diff --git a/drivers/storage/mock/mock_noop.go b/drivers/storage/mock/mock_noop.go new file mode 100644 index 00000000..6db1fa17 --- /dev/null +++ b/drivers/storage/mock/mock_noop.go @@ -0,0 +1,3 @@ +// +build !mock + +package mock diff --git a/drivers/storage/storage.go b/drivers/storage/storage.go new file mode 100644 index 00000000..7307fd54 --- /dev/null +++ b/drivers/storage/storage.go @@ -0,0 +1,6 @@ +package storage + +import ( + // import to load + _ "github.com/emccode/libstorage/drivers/storage/mock" +) diff --git a/glide.yaml b/glide.yaml index 531c418a..6832124d 100644 --- a/glide.yaml +++ b/glide.yaml @@ -1,14 +1,15 @@ package: github.com/emccode/libstorage import: - - package: github.com/akutz/gofig - ref: master - repo: https://github.com/akutz/gofig + - package: github.com/Sirupsen/logrus + ref: feature/logrus-aware-types + repo: https://github.com/akutz/logrus vcs: git + - package: github.com/akutz/gofig - package: github.com/akutz/gotil - ref: master - repo: https://github.com/akutz/gotil - vcs: git - package: github.com/akutz/goof - ref: master - repo: https://github.com/akutz/goof + - package: github.com/akutz/golf + - package: github.com/blang/semver + ref: v3.0.1 vcs: git + - package: github.com/cesanta/validate-json + - package: github.com/stretchr/testify diff --git a/libstorage.apib b/libstorage.apib new file mode 100644 index 00000000..6014b39b --- /dev/null +++ b/libstorage.apib @@ -0,0 +1,2359 @@ +FORMAT: 1A + +# libStorage +`libStorage` provides a vendor agnostic storage orchestration model, API, and +reference client and server implementations. + +## Overview +`libStorage` enables storage consumption by leveraging methods commonly +available, locally and/or externally, to an operating system (OS). + +### The Past +The `libStorage` project and its architecture represents a culmination of +experience gained from the project authors' building of +[several](https://www.emc.com/cloud-virtualization/virtual-storage-integrator.htm) +different +[storage](https://www.emc.com/storage/storage-analytics.htm) +orchestration [tools](https://github.com/emccode/rexray). While created using +different languages and targeting disparate storage platforms, all the tools +were architecturally aligned and embedded functionality directly inside the +tools and affected storage platforms. + +This shared design goal enabled tools that natively consumed storage, sans +external dependencies. + +### The Present +Today `libStorage` focuses on adding value to container runtimes and storage +orchestration tools such as `Docker` and `Mesos`, however the `libStorage` +framework is available abstractly for more general usage across: + +* Operating systems +* Storage platforms +* Hardware platforms +* Virtualization platforms + +The client side implementation, focused on operating system activities, +has a minimal set of dependencies in order to avoid a large, runtime footprint. + +## Storage Orchestration Tools +Today there are many storage orchestration and abstraction tools relevant to +to container runtimes. These tools often must be installed locally and run +alongside the container runtime. + +![Storage Orchestration Tool Architecture Today](https://raw.githubusercontent.com/emccode/libstorage/master/.docs/.themes/yeti/img/architecture-today.png "Storage Orchestration Tool Architecture Today") + +*The solid green lines represent active communication paths. The dotted black +lines represent passive paths. The orange volume represents a operating system +device and volume path available to the container runtime.* + +## libStorage Architectures +`libStorage` is capable of adapting to nearly any environment. + +### libStorage Embedded Architecture +Embedding `libStorage` client and server components enable container +runtimes to communicate directly with storage platforms, the ideal +architecture. This design requires minimal operational dependencies and is +still able to provide volume management for container runtimes. + +![libStorage Embedded Architecture](https://raw.githubusercontent.com/emccode/libstorage/master/.docs/.themes/yeti/img/architecture-embeddedlibstorage.png "libStorage Embedded Architecture") + +### libStorage Centralized Architecture +In a centralized architecture, `libStorage` is hosted as a service, acting as a +go-between for container runtimes and backend storage platforms. + +The `libStorage` endpoint is advertised by a tool like [REX-Ray](https://github.com/emccode/rexray), run from anywhere, and is +responsible for all control plane operations to the storage platform along with +maintaining escalated credentials for these platforms. All client based +processes within the operating system are still embedded in the container +runtime. + +![libStorage Centralized Architecture](https://raw.githubusercontent.com/emccode/libstorage/master/.docs/.themes/yeti/img/architecture-centralized.png "libStorage Centralized Architecture") + +### libStorage Decentralized Architecture +Similar to the centralized architecture, this implementation design involves +running a separate `libStorage` process alongside each container runtime, across +one or several hosts. + +![libStorage De-Centralized Architecture](https://raw.githubusercontent.com/emccode/libstorage/master/.docs/.themes/yeti/img/architecture-decentralized.png "libStorage De-Centralized Architecture") + +## libStorage Design +The design of `libStorage` is broken apart into distinct but collaborative parts. + +### API +Central to `libStorage` is the `HTTP`/`JSON` API. It defines the control plane +calls that occur between the client and server. While the `libStorage` package +includes reference implementations of the client and server written using Go, +both the client and server could be written using any language as long as both +adhere to the published `libStorage` API. + +### Client +The `libStorage` client is responsible for discovering a host's instance ID +and the next, available device name. The client's reference implementation is +written using Go and is compatible with C++. + +The design goal of the client is to be lightweight, portable, and avoid +obsolescence by minimizing dependencies and focusing on deferring as much of +the logic as possible to the server. + +### Server +The `libStorage` server implements the `libStorage` API and is responsible for +coordinating requests between clients and backend orchestration packages. The +server's reference implementation is also written using Go. + +### Model +The `libStorage` [model](http://libstorage.rtfd.org/en/latest/user-guide/model/) +defines several data structures that are easily represented using Go structs or +a portable format such as JSON. + +## Documentation +The `libStorage` documentation is available at +[libstorage.rtfd.org](http://libstorage.rtfd.org). + +# Group Global + +# Default [/] +The root resource. + +## Get [GET] +Gets a list of the API's top-level resources. + ++ Response 200 (application/json) + + + Body + + [ + "/drivers", + "/services", + "/volumes", + "/snapshots" + ] + +# Volumes Collection [/volumes] +A collection of Volume objects for all configured services. + +## Get [GET /volumes] +Gets a list of Volume resources for all configured services. + ++ Response 200 (application/json) + + + Body + + { + "ec2-00": { + "vol-000": { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + }, + "vol-001": { + "id": "vol-001", + "name": "Volume-001", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + }, + "ec2-01": { + "vol-000": { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceVolumeMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Get with Attachments [GET /volumes?{attachments}] +Gets a list of Volume resources and their attached resources for all configured +services. + ++ Parameters + + + attachments (required) + + A flag that indicates that volume attachments should be retrieved. + If this flag is provided then the `LibStorage-InstanceID` header must + be set -- a comma-delimited list of service name/instance ID, + colon-delimited pairs. + ++ Request + + + Headers + + libStorage-InstanceID: ec2-00:eyJpZCI6ImlpZC0wMDAiLCJtZXRhZGF0YSI6eyJtYXgiOjEwLCJtaW4iOjAsInJhZCI6ImNvb2wiLCJ0b3RhbGx5IjoidHVidWxhciJ9fQ==,ec2-01:eyJpZCI6ImlpZC0wMDEiLCJtZXRhZGF0YSI6eyJtYXgiOjIwLCJtaW4iOjUsInJhZCI6ImhvdCJ9fQ== + ++ Response 200 (application/json) + + + Body + + { + "ec2-00": { + "vol-000": { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + }, + "vol-001": { + "id": "vol-001", + "name": "Volume-001", + "size": 10240, + "attachments": [ + { + "instanceID": { + "id": "iid-001" + }, + "volumeID": "vol-001", + "deviceName": "/dev/xvd01" + } + ], + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + }, + "ec2-01": { + "vol-000": { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceVolumeMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Detach All [POST /volumes?{detach}] +Detaches all volumes for all services. + ++ Parameters + + + detach (required) + + A flag indicating that the detach operation should be issued for + all volumes across all configured services. + ++ Request (application/json) + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volumeDetachAllRequest" } + ++ Response 200 (application/json) +A list of the detached volumes. + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceVolumeMap" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Snapshots Collection [/snapshots] +A collection of Snapshot objects for all configured services. + +## Get [GET] +Gets a list of Snapshot resources for all configured services. + ++ Response 200 (application/json) + + + Body + + { + "ec2-00": { + "snap-000": { + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + }, + "snap-001": { + "id": "snap-001", + "name": "Snapshot-001", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455846531, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + } + }, + "ec2-01": { + "snap-000": { + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceSnapshotMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Group Services + +# Services Collection [/services] + +## Get [GET] +Gets information about all of the configured services. + ++ Response 200 (application/json) + + + Body + + { + "ec2-00": { + "name": "ec2-00", + "driver": { + "name": "ec2", + "type": "nas", + "executors": [ + { + "name": "ec2-linux-executor.sh", + "md5checksum": "d2794c0df5b907fdace235a619d80314" + } + ] + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceInfoMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Service Inspector [/services/{service}] + ++ Parameters + + + service: `ec2-00` (string, required) + +## Inspect [GET] +Gets information about the service with the specified name. + ++ Response 200 (application/json) + + + Attributes (ServiceInfo) + + + Body + + { + "name": "ec2-00", + "driver": { + "name": "ec2", + "type": "nas", + "executors": [ + { + "name": "ec2-linux-executor.sh", + "md5checksum": "d2794c0df5b907fdace235a619d80314" + } + ] + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceInfo" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Group Drivers + +# Drivers Collection [/drivers] + +## Get [GET] +Gets information about all of the registered drivers. + ++ Response 200 (application/json) + + + Body + + { + "ec2": { + "name": "ec2", + "type": "nas", + "executors": [ + { + "name": "ec2-linux-executor.sh", + "md5checksum": "d2794c0df5b907fdace235a619d80314" + } + ] + }, + "scaleio": { + "name": "scaleio", + "type": "block", + "executors": [ + { + "name": "scaleio-linux-executor.sh", + "md5checksum": "6ae87ffe3787a7c9338afee160097c73" + } + ] + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/driverInfoMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Driver Inspector [/drivers/{driver}] + ++ Parameters + + + driver: `ec2` (string, required) + +## Inspect [GET] +Gets information about the driver with the specified name. + ++ Response 200 (application/json) + + + Attributes (DriverInfo) + + + Body + + { + "name": "ec2", + "type": "nas", + "executors": [ + { + "name": "ec2-linux-executor.sh", + "md5checksum": "d2794c0df5b907fdace235a619d80314" + } + ] + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/driverInfo" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Group Executors +A collection of resources and actions related to libStorage's client-side +executors. + +# Download Executor [/drivers/{driver}/executors/{name}] +Please read the following notes regarding downloading an Executor. + +## Performance +Due to the possible response size, the libStorage API does not support batch +retrieval of executors. While it is of course possible to invoke several calls +in parallel to this API from a client, it is not recommended as it may place +an undue burden on the server and result in overall, degraded performance. + +## Response Type +The response type differs based on the value of the `tool` parameter's file +extension. If the extension is `.sh` or `.ps1` then the response type will be +`text/plain`. Otherwise the tool will be treated as a binary file and the +response type will be `application/octet-stream`. + ++ Parameters + + + driver: `ec2` (string, required) + + The driver name + + + name: `libstor-ec2-tool` (string, required) + + The name of the executor to download. + +## Download [GET] + ++ Response 200 (application/octet-stream) + ++ Response 200 (text/plain) + + + Body + + #!/bin/sh + case "$1" in + "GetInstanceID") + echo '%s' + ;; + "GetNextAvailableDeviceName") + echo $BLOCK_DEVICES_JSON + ;; + esac + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Group Volumes +A collection of resources and actions related to libStorage Volumes. + +# Volumes Collection [/volumes/{service}] + ++ Parameters + + + service: `ec2-00` (string, required) + + The service name + +## Get [GET] +Gets a list of Volume objects for a single Service resource. + ++ Response 200 (application/json) + + + Body + + { + "vol-000": { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + }, + "vol-001": { + "id": "vol-001", + "name": "Volume-001", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volumeMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Get with Attachments [GET /volumes?{attachments}] +Gets a list of Volume resources and their attached resources for a single +service. + ++ Parameters + + + attachments (required) + + A flag that indicates that volume attachments should be retrieved. + If this flag is provided then the `LibStorage-InstanceID` header must + be set -- a comma-delimited list of service name/instance ID, + colon-delimited pairs. + ++ Request + + + Headers + + libStorage-InstanceID: ec2-00:eyJpZCI6ImlpZC0wMDAiLCJtZXRhZGF0YSI6eyJtYXgiOjEwLCJtaW4iOjAsInJhZCI6ImNvb2wiLCJ0b3RhbGx5IjoidHVidWxhciJ9fQ== + ++ Response 200 (application/json) + + + Body + + { + "vol-000": { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + }, + "vol-001": { + "id": "vol-001", + "name": "Volume-001", + "size": 10240, + "attachments": [ + { + "instanceID": { + "id": "iid-001" + }, + "volumeID": "vol-001", + "deviceName": "/dev/xvd01" + } + ], + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceVolumeMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Create [POST] +Create a new volume. + ++ Request (application/json) + + + Attributes + + + name (string, required) - The volume name + + availabilityZone (string, optional) - The zone for which the volume is available + + iops (number, optional) - The volume IOPs + + size (number, optional) - The volume size (GB) + + type (string, optional) - The volume type + + opts (object) - Optional request data + + + Body + + { + "name": "Volume-001", + "size": 10240, + "opts": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volumeCreateRequest" } + ++ Response 200 (application/json) +The newly created volume object. + + + Attributes (Volume) + + + Headers + + Location: /volumes/ec2-00/vol-001 + + + Body + + { + "id": "vol-001", + "name": "Volume-001", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volume" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Detach All [POST /volumes/{service}?{detach}] +Detaches all volumes for all services. + ++ Parameters + + + service: `ec2-00` (string, required) + + The service name + + + detach (required) + + A flag indicating that the detach operation should be issued for + all volumes for the specified service. + ++ Request (application/json) + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volumeDetachAllRequest" } + ++ Response 200 (application/json) +A list of the detached volumes. + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/serviceVolumeMap" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Volume Inspector [/volumes/{service}/{volumeID}] +A single Volume object. A central part of the libStorage +API, a Volume resource represents a storage volume. + ++ Parameters + + + service: `ec2-00` (string, required) + + The name of the service to which the Volume belongs + + + volumeID: `vol-000` (string, required) + + The volume's unique ID + +### Get [GET] +Retrieves a single volume. + +If the `libStorage-InstanceID` header is provided the volume's attachments are +also retrieved. Unlike when retrieving Volume collections, there's no need to +specify the `attachments` query parameter to request their retrieval. + ++ Request + + + Headers + + libStorage-InstanceID: ec2-00:eyJpZCI6ImlpZC0wMDAiLCJtZXRhZGF0YSI6eyJtYXgiOjEwLCJtaW4iOjAsInJhZCI6ImNvb2wiLCJ0b3RhbGx5IjoidHVidWxhciJ9fQ== + + ++ Response 200 (application/json) + + + Attributes (Volume) + + + Body + + { + "id": "vol-000", + "name": "Volume-000", + "size": 10240, + "attachments": [ + { + "instanceID": { + "id": "iid-000" + }, + "volumeID": "vol-000", + "deviceName": "/dev/xvd00" + } + ], + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volume" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +### Copy [POST /volumes/{service}/{volumeID}?{copy}] +Copies the volume. + ++ Parameters + + + service: `ec2-00` (string, required) + + The name of the service to which the Volume belongs + + + volumeID: `vol-000` (string, required) + + The volume's unique ID + + + copy (required) + + The operation flag indicating the copy operation + ++ Request (application/json) + + + Body + + { + "name": "Copy of Volume-000" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volumeCopyRequest" } + ++ Response 200 (application/json) + + + Attributes (Volume) + + + Body + + { + "id": "vol-002", + "name": "Copy of Volume-000", + "size": 10240, + "fields": { + "priority": 2, + "owner": "sakutz@gmail.com" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volume" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +### Snaphot [POST /volumes/{service}/{volumeID}?{snapshot}] +Takes a snapshot of the volume. + ++ Parameters + + + service: `ec2-00` (string, required) + + The name of the service to which the Volume belongs + + + volumeID: `vol-000` (string, required) + + The volume's unique ID + + + snapshot (required) + + The operation flag indicating the snapshot operation + ++ Request (application/json) + + + Attributes + + + snapshotName (string, required) - The name of the snapshot + + opts (object) - Optional request data + + + Body + + { + "snapshotName": "Snapshot-000" + } + ++ Response 200 (application/json) + + + Attributes (Snapshot) + + + Body + + { + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/snapshot" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +### Attach [POST /volumes/{service}/{volumeID}?{attach}] +Attaches the volume to an instance. + ++ Parameters + + + service: `ec2-00` (string, required) + + The name of the service to which the Volume belongs + + + volumeID: `vol-000` (string, required) + + The volume's unique ID + + + attach (required) + + The operation flag indicating the attach operation + ++ Request (application/json) + + + Attributes + + + nextDeviceName (string, optional) - The next device name + + opts (object) - Optional request data + + + Headers + + libStorage-InstanceID: ec2-00:eyJpZCI6ImlpZC0wMDAiLCJtZXRhZGF0YSI6eyJtYXgiOjEwLCJtaW4iOjAsInJhZCI6ImNvb2wiLCJ0b3RhbGx5IjoidHVidWxhciJ9fQ== + + + Body + + { + "nextDeviceName": "/dev/xvd02" + } + ++ Response 205 +Reset the view of the modified resource + + + Schema + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + + +### Detach [POST /volumes/{service}/{volumeID}?{detach}] +Detaches the volume. + ++ Parameters + + + service: `ec2-00` (string, required) + + The name of the service to which the Volume belongs + + + volumeID: `vol-000` (string, required) + + The volume's unique ID + + + detach (required) + + The operation flag indicating the detach operation + ++ Request (application/json) + + + Attributes + + + opts (object) - Optional request data + ++ Response 205 +Reset the view of the modified resource + + + Schema + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Remove [DELETE] +Removes the volume. + ++ Request (application/json) + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/volumeRemoveRequest" } + ++ Response 205 +Reset the view of the modified resource + + + Schema + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Group Snapshots +A collection of resources and actions related to libStorage Snapshots. + +# Snapshot Collection [/snapshots/{service}] +A collection of Snapshot resources that belong to Volumes for a specifc service. + ++ Parameters + + + service: `ec2-00` (string, required) + + The service name + +## Get [GET] +Gets a list of Snapshot objects for a single Service resource. + ++ Response 200 (application/json) + + + Body + + { + "snap-000": { + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + }, + "snap-001": { + "id": "snap-001", + "name": "Snapshot-001", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455846531, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/snapshotMap" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Create [POST] +Please note, that the Snapshot Collection resource is not used for the creation +of snapshots. Instead please see the documentation for the Volume Inspector for +how to create snapshots from existing volumes. + +Invoking this method will always result in an error. + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Snapshot Inspector [/snapshots/{service}/{snapshotID}] +A single Snapshot object. A central part of the libStorage API, a Snapshot +resource represents a snapshot of a storage volume. + ++ Parameters + + + service: `ec2-00` (string, required) + + The service name + + + snapshotID: `snap-000` (string, required) + + The snapshot's unique ID + +## Get [GET] +Retrieves a single snapshot. + ++ Response 200 (application/json) + + + Body + + { + "id": "snap-000", + "name": "Snapshot-000", + "description": "A snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/snapshot" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +### Copy [POST /snapshots/{service}/{snapshotID}?{copy}] +Copies the snapshot. + ++ Parameters + + + service: `ec2-00` (string, required) + + The name of the service to which the Volume belongs + + + snapshotID: `snap-000` (string, required) + + The snapshot's unique ID + + + copy (required) + + The operation flag indicating the copy operation + ++ Request (application/json) + + + Body + + { + "snapshotName": "Copy of Snapshot-000", + "destinationID": "ec2-01" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/snapshotCopyRequest" } + ++ Response 200 (application/json) + + + Attributes (Snapshot) + + + Body + + { + "id": "snap-002", + "name": "Copy of Snapshot-000", + "description": "A copy of the snapshot of Volume-000 (vol-000)", + "startTime": 1455826676, + "volumeID": "vol-000", + "volumeSize": 10240, + "fields": { + "sparse": true, + "region": "US" + } + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/snapshot" } + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +## Remove [DELETE] +Removes the snapshot. + ++ Request (application/json) + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/snapshotRemoveRequest" } + ++ Response 205 +Reset the view of the modified resource + + + Schema + ++ Response 400 (application/json) +Invalid request + + + Body + + { + "type": "invalidRequest", + "httpStatus": 400, + "message": "An invalid request was made" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/invalidRequestError" } + ++ Response 401 (application/json) +Unauthorized request + + + Body + + { + "type": "unauthorizedRequest", + "httpStatus": 401, + "message": "The requestor is unauthorized to access this resource" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/unauthorizedRequestError" } + ++ Response 404 (application/json) +The specified resource was not found + + + Body + + { + "type": "resourceNotFound", + "httpStatus": 404, + "message": "The requested resource was not found" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/resourceNotFoundError" } + ++ Response 500 (application/json) +Internal server error + + + Body + + { + "type": "internalServerError", + "httpStatus": 500, + "message": "An internal server error occurred" + } + + + Schema + + { "$ref": "https://raw.githubusercontent.com/akutz/libstorage/feature/dadd/libstorage.json#/definitions/internalServerError" } + +# Data Structures + +## InstanceID (object) +An InstanceID identifies a host to a remote storage platform. + +### Properties ++ id (string, required) - The instance ID. ++ data (object, optional) - Extra information about the instance ID. + +## ServiceInfo (object) +The ServiceInfo object contains informationa about a configured service. + +### Properties ++ name (string, required) - The service name ++ driver (DriverInfo, required) - The name of the driver used by the service + +## DriverInfo (object) +The DriverInfo object contains information about a registered driver. + +### Properties ++ name (string, required) ++ type (object, required) ++ nextDevice (NextDeviceInfo) ++ executors (array[ExecutorInfo]) + +## NextDeviceInfo (object) +NextDeviceInfo assists the libStorage client in determining the +next available device name by providing the driver's device prefix and +optional pattern. + +For example, the Amazon Web Services (AWS) device prefix is "xvd" and its +pattern is "[a-z]". These two values would be used to determine on an EC2 +instance where "/dev/xvda" and "/dev/xvdb" are in use that the next +available device name is "/dev/xvdc". + +If the Ignore field is set to true then the client logic does not invoke the +GetNextAvailableDeviceName function prior to submitting an AttachVolume +request to the server. + +### Properties ++ ignore (boolean) ++ prefix (string) ++ pattern (string) + +## ExecutorInfo +ExecutorInfo contains information about a client-side executor, such as +its name and MD5 checksum. + +### Properties ++ name (string, optional) - The file name of the executor. + + The name include's the file name extension as well. + ++ md5Checksum (string, optional) - MD5Checksum is the MD5 checksum of the executor. + + This can be used to determine if a local copy of the executor needs to be + updated. + +## Volume (object, fixed) +A single Volume object. A central part of the libStorage +API, a Volume resource represents a backend storage +volume. + +### Properties ++ id (string, required) - The volume ID. ++ name (string, required) - The volume name. ++ type (string) - The volume type. ++ attachments (array, optional) - The volume's attachments. + + (VolumeAttachment) ++ availabilityZone (string) - The zone for which the volume is available. ++ iops (number) - The volume IOPs. ++ networkName (string) - The name of the network on which the volume resides. ++ size (number, required) - The volume size (GB). ++ status (string) - The volume status. ++ fields (object) - Fields are additional properties that can be defined for this type. + +## VolumeAttachment (object, fixed) +A single VolumeAttachment object contains information about a resource attached +to a storage volume. + +### Properties ++ deviceName (string, required) - The name of the device on which the volume to which the object is attached is mounted. ++ instanceID (InstanceID, required) - The ID of the instance on which the volume to which the attachment belongs is mounted. ++ status (string) - The status of the attachment. ++ volumeID (string, required) - The ID of the volume to which the attachment belongs. ++ fields (object) - Fields are additional properties that can be defined for this type. + +## Snapshot (object, fixed) +A single Snapshot object. Created by invoking the snapshot operation on a +Volume, the Snapshot resource can also be copied and used as the basis for +a new Volume resource. + +### Properties ++ id (string, required) - The snapshot ID. ++ name (string, required) - The snapshot name. ++ description (string, required) - The snapshot description. ++ startTime (number, required) - The time (epoch) at which the request to create the snapshot was submitted. ++ status (string) - The volume status. ++ volumeID (string, required) - The ID of the volume to which the snapshot is linked. ++ volumeSize (number, required) - The size (GB) of the volume to which the snapshot is linked. ++ fields (object) - Fields are additional properties that can be defined for this type. diff --git a/libstorage.go b/libstorage.go index eba25c12..592f4377 100644 --- a/libstorage.go +++ b/libstorage.go @@ -34,31 +34,49 @@ has a minimal set of dependencies in order to avoid a large, runtime footprint. package libstorage import ( + "io" + "github.com/akutz/gofig" - "golang.org/x/net/context" - "github.com/emccode/libstorage/client" - "github.com/emccode/libstorage/driver" - "github.com/emccode/libstorage/service" + "github.com/emccode/libstorage/api/client" + "github.com/emccode/libstorage/api/registry" + "github.com/emccode/libstorage/api/server" + "github.com/emccode/libstorage/api/types/context" + "github.com/emccode/libstorage/api/types/drivers" ) func init() { registerGofigDefaults() } -// RegisterDriver registers a new Driver with the libStorage service. -func RegisterDriver(driverName string, ctor driver.NewDriver) { - service.RegisterDriver(driverName, ctor) +// RegisterStorageDriver registers a new StorageDriver with the libStorage +// service. +func RegisterStorageDriver(name string, ctor drivers.NewStorageDriver) { + registry.RegisterStorageDriver(name, ctor) } -// Serve starts the reference implementation of a server hosting an -// HTTP/JSON service that implements the libStorage API endpoint. -// -// If the config parameter is nil a default instance is created. The -// libStorage service is served at the address specified by the configuration -// property libstorage.host. -func Serve(config gofig.Config) error { - return service.Serve(config) +// RegisterOSDriver registers a new StorageDriver with the libStorage +// service. +func RegisterOSDriver(name string, ctor drivers.NewOSDriver) { + registry.RegisterOSDriver(name, ctor) +} + +// RegisterIntegrationDriver registers a new IntegrationDriver with the +// libStorage service. +func RegisterIntegrationDriver(name string, ctor drivers.NewIntegrationDriver) { + registry.RegisterIntegrationDriver(name, ctor) +} + +/* +Serve starts the reference implementation of a server hosting an +HTTP/JSON service that implements the libStorage API endpoint. + +If the config parameter is nil a default instance is created. The +libStorage service is served at the address specified by the configuration +property libstorage.host. +*/ +func Serve(config gofig.Config) (io.Closer, <-chan error) { + return server.Serve(config) } // Dial opens a connection to a remote libStorage serice and returns the client diff --git a/libstorage.json b/libstorage.json new file mode 100644 index 00000000..f56dc581 --- /dev/null +++ b/libstorage.json @@ -0,0 +1,446 @@ +{ + "id": "https://github.com/emccode/libstorage", + "$schema": "http://json-schema.org/draft-04/schema#", + "definitions": { + "volume": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The volume ID." + }, + "name": { + "type": "string", + "description": "The volume name." + }, + "type": { + "type": "string", + "description": "The volume type." + }, + "attachments": { + "type": "array", + "description": "The volume's attachments.", + "items": { "$ref": "#/definitions/volumeAttachment" } + }, + "availabilityZone": { + "type": "string", + "description": "The zone for which the volume is available." + }, + "iops": { + "type": "number", + "description": "The volume IOPs." + }, + "networkName": { + "type": "string", + "description": "The name of the network on which the volume resides." + }, + "size": { + "type": "number", + "description": "The volume size (GB)." + }, + "status": { + "type": "string", + "description": "The volume status." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "id", "name", "size" ], + "additionalProperties": false + }, + + + "volumeAttachment": { + "type": "object", + "properties": { + "instanceID": { "$ref": "#/definitions/instanceID" }, + "deviceName": { + "type": "string", + "description": "The name of the device on to which the volume is mounted." + }, + "status": { + "type": "string", + "description": "The status of the attachment." + }, + "volumeID": { + "type": "string", + "description": "The ID of the volume to which the attachment belongs." + }, + "mountPoint": { + "type": "string", + "description": "The file system path to which the volume is mounted." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "instanceID", "deviceName", "volumeID" ], + "additionalProperties": false + }, + + + "instanceID": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The instance ID." + }, + "metadata": { + "type": "object", + "description": "Extra information about the instance ID." + } + }, + "required": [ "id" ], + "additionalProperties": false + }, + + + "instance": { + "type": "object", + "properties": { + "instanceID": { "$ref": "#/definitions/instanceID" }, + "name": { + "type": "string", + "description": "The name of the instance." + }, + "providerName": { + "type": "string", + "description": "The name of the provider that owns the object." + }, + "region": { + "type": "string", + "description": "The region from which the object originates." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "id" ], + "additionalProperties": false + }, + + + "snapshot": { + "type": "object", + "properties": { + "id": { + "type": "string", + "description": "The snapshot's ID." + }, + "name": { + "type": "string", + "description": "The name of the snapshot." + }, + "description": { + "type": "string", + "description": "A description of the snapshot." + }, + "startTime": { + "type": "number", + "description": "The time (epoch) at which the request to create the snapshot was submitted." + }, + "status": { + "type": "string", + "description": "The status of the snapshot." + }, + "volumeID": { + "type": "string", + "description": "The ID of the volume to which the snapshot belongs." + }, + "volumeSize": { + "type": "number", + "description": "The size of the volume to which the snapshot belongs." + }, + "fields": { "$ref": "#/definitions/fields" } + }, + "required": [ "id" ], + "additionalProperties": false + }, + + + "serviceInfo": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name is the service's name." + }, + "driver": { "$ref": "#/definitions/driverInfo" } + }, + "required": [ "name", "driver" ], + "additionalProperties": false + }, + + + "driverInfo": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server." + }, + "type": { + "type": "string", + "description": "Type is the type of storage the driver provides: block, nas, object." + }, + "nextDevice": { "$ref": "#/definitions/nextDeviceInfo" }, + "executors": { + "type": "array", + "description": "Executors is information about the driver's client-side executors.", + "items": { "$ref": "#/definitions/executorInfo" } + } + }, + "required": [ "name", "type" ], + "additionalProperties": false + }, + + + "executorInfo": { + "type": "object", + "properties": { + "md5checksum": { + "type": "string", + "description": "MD5Checksum is the MD5 checksum of the executor. This can be used to determine if a local copy of the executor needs to be updated." + }, + "name": { + "type": "string", + "description": "Name is the file name of the executor, including the file extension." + } + }, + "required": [ "md5checksum", "name" ], + "additionalProperties": false + }, + + + "nextDeviceInfo": { + "type": "object", + "properties": { + "ignore": { + "type": "boolean", + "description": "Ignore is a flag that indicates whether the client logic should invoke the GetNextAvailableDeviceName function prior to submitting an AttachVolume request to the server." + }, + "prefix": { + "type": "string", + "description": "Prefix is the first part of a device path's value after the \"/dev/\" portion. For example, the prefix in \"/dev/xvda\" is \"xvd\"." + }, + "pattern": { + "type": "string", + "description": "Pattern is the regex to match the part of a device path after the prefix." + } + }, + "additionalProperties": false + }, + + + "fields": { + "type": "object", + "description": "Fields are additional properties that can be defined for this type.", + "patternProperties": { + ".+": { "type": "string" } + }, + "additionalProperties": true + }, + + + "volumeMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/volume" } + }, + "additionalProperties": false + }, + + + "snapshotMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/snapshot" } + }, + "additionalProperties": false + }, + + + "serviceVolumeMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/volumeMap" } + }, + "additionalProperties": false + }, + + + "serviceSnapshotMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/snapshotMap" } + }, + "additionalProperties": false + }, + + + "serviceInfoMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/serviceInfo" } + }, + "additionalProperties": false + }, + + + "driverInfoMap": { + "type": "object", + "patternProperties": { + "^[a-zA-Z].+$": { "$ref": "#/definitions/driverInfo" } + }, + "additionalProperties": false + }, + + + "opts": { + "type": "object", + "description": "Opts are additional properties that can be defined for POST requests.", + "patternProperties": { + ".+": { + "anyOf": [ + { "type": "array" }, + { "type": "boolean" }, + { "type": "integer" }, + { "type": "number" }, + { "type": "null" }, + { "type": "string" }, + { "$ref": "#/definitions/opts" } + ] + } + }, + "additionalProperties": true + }, + + + "volumeCreateRequest": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "availabilityZone": { + "type": "string" + }, + "iops": { + "type": "number" + }, + "size": { + "type": "number" + }, + "type": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "name" ], + "additionalProperties": false + }, + + + "volumeCreateFromSnapshotRequest": { + "type": "object", + "properties": { + "volumeName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "volumeName" ], + "additionalProperties": false + }, + + + "volumeCopyRequest": { + "type": "object", + "properties": { + "volumeName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "volumeName" ], + "additionalProperties": false + }, + + + "volumeSnapshotRequest": { + "type": "object", + "properties": { + "snapshotName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "snapshotName" ], + "additionalProperties": false + }, + + + "volumeAttachRequest": { + "type": "object", + "properties": { + "nextDeviceName": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "additionalProperties": false + }, + + + "volumeDetachRequest": { + "type": "object", + "properties": { + "opts": { "$ref" : "#/definitions/opts" } + }, + "additionalProperties": false + }, + + + "snapshotCopyRequest": { + "type": "object", + "properties": { + "snapshotName": { + "type": "string" + }, + "destinationID": { + "type": "string" + }, + "opts": { "$ref" : "#/definitions/opts" } + }, + "required": [ "snapshotName", "destinationID" ], + "additionalProperties": false + }, + + + "snapshotRemoveRequest": { + "type": "object", + "properties": { + "opts": { "$ref" : "#/definitions/opts" } + }, + "additionalProperties": false + }, + + + "error": { + "type": "object", + "properties": { + "message": { + "type": "string", + "pattern": "^.{10,}|.*[Ee]rror$" + }, + "status": { + "type": "number", + "minimum": 400, + "maximum": 599 + }, + "error": { + "type": "object", + "additionalProperties": true + } + }, + "required": [ "message", "status" ], + "additionalProperties": false + } + } +} diff --git a/libstorage_mock_test.go b/libstorage_mock_test.go new file mode 100644 index 00000000..a42eb3a7 --- /dev/null +++ b/libstorage_mock_test.go @@ -0,0 +1,109 @@ +// +build mock + +package libstorage + +import ( + "bytes" + "fmt" + "testing" + + "github.com/akutz/gofig" + + "github.com/emccode/libstorage/drivers/storage/mock" +) + +func getConfig(host string, tls bool, t *testing.T) gofig.Config { + if host == "" { + host = "tcp://127.0.0.1:7979" + } + config := gofig.New() + + var clientTLS, serverTLS string + if tls { + clientTLS = fmt.Sprintf( + libStorageConfigClientTLS, + clientCrt, clientKey, trustedCerts) + serverTLS = fmt.Sprintf( + libStorageConfigServerTLS, + serverCrt, serverKey, trustedCerts) + } + configYaml := fmt.Sprintf( + libStorageConfigBase, + host, + clientToolDir, localDevicesFile, + clientTLS, serverTLS, + testServer1Name, mock.Driver1Name, + testServer2Name, mock.Driver2Name, + testServer3Name, mock.Driver3Name) + + configYamlBuf := []byte(configYaml) + if err := config.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { + panic(err) + } + return config +} + +const ( + /* + libStorageConfigBase is the base config for tests + + 01 - the host address to server and which the client uses + 02 - the executors directory + 03 - the local devices file + 04 - the client TLS section. use an empty string if TLS is disabled + 05 - the server TLS section. use an empty string if TLS is disabled + 06 - the first service name + 07 - the first service's driver type + 08 - the second service name + 09 - the second service's driver type + 10 - the third service name + 11 - the third service's driver type + */ + libStorageConfigBase = ` +libstorage: + host: %[1]s + driver: invalidDriverName + executorsDir: %[2]s + profiles: + enabled: true + groups: + - local=127.0.0.1 + client: + localdevicesfile: %[3]s%[4]s + server: + endpoints: + localhost: + address: %[1]s%[5]s + services: + %[6]s: + libstorage: + driver: %[7]s + profiles: + enabled: true + groups: + - remote=127.0.0.1 + %[8]s: + libstorage: + driver: %[9]s + %[10]s: + libstorage: + driver: %[11]s +` + + libStorageConfigClientTLS = ` + tls: + serverName: libstorage-server + certFile: %s + keyFile: %s + trustedCertsFile: %s +` + + libStorageConfigServerTLS = ` + tls: + serverName: libstorage-server + certFile: %s + keyFile: %s + trustedCertsFile: %s + clientCertRequired: true +` +) diff --git a/libstorage_nomock_test.go b/libstorage_nomock_test.go new file mode 100644 index 00000000..426d3b38 --- /dev/null +++ b/libstorage_nomock_test.go @@ -0,0 +1,13 @@ +// +build !mock + +package libstorage + +import ( + "testing" + + "github.com/akutz/gofig" +) + +func getConfig(host string, tls bool, t *testing.T) gofig.Config { + return gofig.New() +} diff --git a/libstorage_run_test.go b/libstorage_run_test.go new file mode 100644 index 00000000..8fd5426d --- /dev/null +++ b/libstorage_run_test.go @@ -0,0 +1,21 @@ +// +build run + +package libstorage + +import ( + "os" + "strconv" + "testing" +) + +func TestRun(t *testing.T) { + + host := os.Getenv("LIBSTORAGE_TESTRUN_HOST") + tls, _ := strconv.ParseBool(os.Getenv("LIBSTORAGE_TESTRUN_TLS")) + + _, _, errs := getServer(host, tls, t) + err := <-errs + if err != nil { + t.Error(err) + } +} diff --git a/libstorage_test.go b/libstorage_test.go index cb381639..ad3a23b9 100644 --- a/libstorage_test.go +++ b/libstorage_test.go @@ -1,489 +1,180 @@ package libstorage import ( - "bytes" "encoding/json" "fmt" + "io" "io/ioutil" - "net/http" "os" + "os/signal" + "strconv" + "syscall" "testing" log "github.com/Sirupsen/logrus" "github.com/akutz/gofig" "github.com/akutz/gotil" - "github.com/stretchr/testify/assert" - "github.com/emccode/libstorage/api" - "github.com/emccode/libstorage/client" - "github.com/emccode/libstorage/context" - "github.com/emccode/libstorage/driver" + "github.com/emccode/libstorage/api/client" ) -const ( - localDevicesFile = "/tmp/libstorage/partitions" - clientToolDir = "/tmp/libstorage/bin" - - testServer1Name = "testService1" - testServer2Name = "testService2" - testServer3Name = "testService3" - - mockDriver1Name = "mockDriver1" - mockDriver2Name = "mockDriver2" - mockDriver3Name = "mockDriver3" -) - -var ( - nextDeviceVals = []string{"/dev/mock0", "/dev/mock1", "/dev/mock2"} -) - -func newMockDriver1(config gofig.Config) driver.Driver { - md := &mockDriver{name: mockDriver1Name} - var d driver.Driver = md - return d -} - -func newMockDriver2(config gofig.Config) driver.Driver { - md := &mockDriver{name: mockDriver2Name} - var d driver.Driver = md - return d -} - -func newMockDriver3(config gofig.Config) driver.Driver { - md := &mockDriver{name: mockDriver3Name} - var d driver.Driver = md - return d -} - -func getConfig(host, service string, t *testing.T) gofig.Config { - if host == "" { - host = "tcp://127.0.0.1:0" - } - if service == "" { - service = testServer2Name - } - config := gofig.New() - configYaml := fmt.Sprintf(` -libstorage: - host: %s - service: %s - driver: invalidDriverName - profiles: - enabled: true - groups: - - local=127.0.0.1 - client: - tooldir: %s - localdevicesfile: %s - server: - services: - %s: - libstorage: - driver: %s - profiles: - enabled: true - groups: - - remote=127.0.0.1 - %s: - libstorage: - driver: %s - %s: - libstorage: - driver: %s -`, - host, service, - clientToolDir, localDevicesFile, - testServer1Name, mockDriver1Name, - testServer2Name, mockDriver2Name, - testServer3Name, mockDriver3Name) - - configYamlBuf := []byte(configYaml) - if err := config.ReadConfig(bytes.NewReader(configYamlBuf)); err != nil { - panic(err) - } - return config -} +func TestMain(m *testing.M) { -func mustGetClient( - config gofig.Config, - t *testing.T) (context.Context, client.Client) { - if config == nil { - config = getConfig("", "", t) - } - if err := Serve(config); err != nil { - t.Fatalf("error serving libStorage service %v", err) - } - ctx := context.Background() - c, err := Dial(ctx, config) - if err != nil { - t.Fatalf("error dialing libStorage service at '%s' %v", - config.Get("libstorage.host"), err) - } - return ctx, c -} + // make sure all servers get closed even if the test is abrubptly aborted + trapAbort() -func TestMain(m *testing.M) { os.MkdirAll(clientToolDir, 0755) ioutil.WriteFile(localDevicesFile, localDevicesFileBuf, 0644) - if os.Getenv("LIBSTORAGE_DEBUG") != "" { + if debug, _ := strconv.ParseBool(os.Getenv("LIBSTORAGE_DEBUG")); debug { log.SetLevel(log.DebugLevel) - os.Setenv("LIBSTORAGE_SERVICE_HTTP_LOGGING_ENABLED", "true") - os.Setenv("LIBSTORAGE_SERVICE_HTTP_LOGGING_LOGREQUEST", "true") - os.Setenv("LIBSTORAGE_SERVICE_HTTP_LOGGING_LOGRESPONSE", "true") + os.Setenv("LIBSTORAGE_SERVER_HTTP_LOGGING_ENABLED", "true") + os.Setenv("LIBSTORAGE_SERVER_HTTP_LOGGING_LOGREQUEST", "true") + os.Setenv("LIBSTORAGE_SERVER_HTTP_LOGGING_LOGRESPONSE", "true") os.Setenv("LIBSTORAGE_CLIENT_HTTP_LOGGING_ENABLED", "true") os.Setenv("LIBSTORAGE_CLIENT_HTTP_LOGGING_LOGREQUEST", "true") os.Setenv("LIBSTORAGE_CLIENT_HTTP_LOGGING_LOGRESPONSE", "true") } - RegisterDriver(mockDriver1Name, newMockDriver1) - RegisterDriver(mockDriver2Name, newMockDriver2) - RegisterDriver(mockDriver3Name, newMockDriver3) exitCode := m.Run() + if !closeAllServers() && exitCode == 0 { + exitCode = 1 + } os.RemoveAll(clientToolDir) os.RemoveAll(localDevicesFile) os.Exit(exitCode) } -func TestGetServiceConfig(t *testing.T) { - config := getConfig("", "", t) - if err := Serve(config); err != nil { - t.Fatalf("error serving libStorage service %v", err) - } - - host := config.GetString("libstorage.host") - _, laddr, err := gotil.ParseAddress(host) - if err != nil { - t.Fatal(err) - } - - url := fmt.Sprintf("http://%s/libStorage/config", laddr) - res, err := http.Get(url) - if err != nil { - t.Fatal(err) - } - b, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Fatal(err) - } - t.Log(string(b)) +func trapAbort() { + // make sure all servers get closed even if the test is abrubptly aborted + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, + syscall.SIGKILL, + syscall.SIGHUP, + syscall.SIGINT, + syscall.SIGTERM, + syscall.SIGQUIT) + go func() { + <-sigc + fmt.Println("received abort signal") + closeAllServers() + fmt.Println("all servers closed") + os.Exit(1) + }() } -func TestGetServiceInfo(t *testing.T) { - testGetServiceInfo(testServer1Name, mockDriver1Name, t) - testGetServiceInfo(testServer2Name, mockDriver2Name, t) - testGetServiceInfo(testServer3Name, mockDriver3Name, t) -} +const ( + localDevicesFile = "/tmp/libstorage/partitions" + clientToolDir = "/tmp/libstorage/bin" -func testGetServiceInfo(service, driver string, t *testing.T) { - config := getConfig("", service, t) - ctx, c := mustGetClient(config, t) - args := &api.GetServiceInfoArgs{} - info, err := c.GetServiceInfo(ctx, args) - if err != nil { - t.Fatal(err) - } - assert.Equal(t, service, info.Name) - assert.Equal(t, driver, info.Driver) - assert.Equal(t, 3, len(info.RegisteredDrivers)) - assert.True(t, gotil.StringInSlice(mockDriver1Name, info.RegisteredDrivers)) - assert.True(t, gotil.StringInSlice(mockDriver2Name, info.RegisteredDrivers)) - assert.True(t, gotil.StringInSlice(mockDriver3Name, info.RegisteredDrivers)) -} + testServer1Name = "testService1" + testServer2Name = "testService2" + testServer3Name = "testService3" +) -func TestGetVolumeMappingServerAndDriver1(t *testing.T) { - testGetVolumeMapping(testServer1Name, mockDriver1Name, t) -} +var ( + tlsPath = fmt.Sprintf( + "%s/src/github.com/emccode/libstorage/.tls", os.Getenv("GOPATH")) + serverCrt = fmt.Sprintf("%s/libstorage-server.crt", tlsPath) + serverKey = fmt.Sprintf("%s/libstorage-server.key", tlsPath) + clientCrt = fmt.Sprintf("%s/libstorage-client.crt", tlsPath) + clientKey = fmt.Sprintf("%s/libstorage-client.key", tlsPath) + trustedCerts = fmt.Sprintf("%s/libstorage-ca.crt", tlsPath) +) -func TestGetVolumeMappingServerAndDriver2(t *testing.T) { - testGetVolumeMapping(testServer2Name, mockDriver2Name, t) -} +var localDevicesFileBuf = []byte(` +major minor #blocks name + 11 0 4050944 sr0 + 8 0 67108864 sda + 8 1 512000 sda1 + 8 2 66595840 sda2 + 253 0 4079616 dm-0 + 253 1 42004480 dm-1 + 253 2 20508672 dm-2 + 1024 1 20508672 xvda + 7 0 104857600 loop0 + 7 1 2097152 loop1 + 253 3 104857600 dm-3 +`) -func TestGetVolumeMappingServerAndDriver3(t *testing.T) { - testGetVolumeMapping(testServer3Name, mockDriver3Name, t) -} +var servers []io.Closer -func testGetVolumeMapping(service, driver string, t *testing.T) { - config := getConfig("", service, t) - ctx, c := mustGetClient(config, t) - args := &api.GetVolumeMappingArgs{} - bds, err := c.GetVolumeMapping(ctx, args) - if err != nil { - t.Fatal(err) +func closeAllServers() bool { + noErrors := true + for _, server := range servers { + if err := server.Close(); err != nil { + noErrors = false + fmt.Printf("error closing server: %v\n", err) + } } - assert.Equal(t, 1, len(bds)) - bd := bds[0] - assert.Equal(t, getDeviceName(driver), bd.DeviceName) - assert.Equal(t, driver+"Provider", bd.ProviderName) - assert.Equal(t, driver+"Region", bd.Region) - assertInstanceID(t, driver, bd.InstanceID) -} - -func TestGetNextAvailableDeviceNameServerAndDriver1(t *testing.T) { - testGetNextAvailableDeviceName(testServer1Name, mockDriver1Name, t) -} - -func TestGetNextAvailableDeviceNameServerAndDriver2(t *testing.T) { - testGetNextAvailableDeviceName(testServer2Name, mockDriver2Name, t) + return noErrors } -func TestGetNextAvailableDeviceNameServerAndDriver3(t *testing.T) { - testGetNextAvailableDeviceName(testServer3Name, mockDriver3Name, t) -} +func getServer( + host string, tls bool, t *testing.T) (gofig.Config, io.Closer, <-chan error) { -func testGetNextAvailableDeviceName(service, driver string, t *testing.T) { - config := getConfig("", service, t) - ctx, c := mustGetClient(config, t) - name, err := c.GetNextAvailableDeviceName( - ctx, &api.GetNextAvailableDeviceNameArgs{}) - if err != nil { - t.Fatal(err) + if host == "" { + host = fmt.Sprintf("tcp://localhost:%d", gotil.RandomTCPPort()) } - assert.Equal(t, name, getNextDeviceName(driver)) -} - -func TestGetClientTool(t *testing.T) { - ctx, c := mustGetClient(nil, t) - args := &api.GetClientToolArgs{} - _, err := c.GetClientTool(ctx, args) - if err != nil { - t.Fatal(err) + config := getConfig(host, tls, t) + server, errs := Serve(config) + if server != nil { + servers = append(servers, server) } + return config, server, errs } -func assertInstanceID(t *testing.T, driver string, iid *api.InstanceID) { - assert.NotNil(t, iid) - assert.Equal(t, driver+"InstanceID", iid.ID) - md := iid.Metadata - assert.NotNil(t, md) - tmd, ok := md.(map[string]interface{}) - assert.True(t, ok, "metadata is map") - emd := mockDriverInstanceIDMetadata() - assert.EqualValues(t, emd["min"], tmd["min"]) - assert.EqualValues(t, emd["max"], tmd["max"]) - assert.EqualValues(t, emd["rad"], tmd["rad"]) - assert.EqualValues(t, emd["totally"], tmd["totally"]) -} - -type mockDriver struct { - config gofig.Config - name string -} +func getClient(host string, tls bool, t *testing.T) client.Client { -func (m *mockDriver) pwn(v string) string { - return fmt.Sprintf("%s%s", m.name, v) -} - -func mockDriverInstanceIDMetadata() map[string]interface{} { - return map[string]interface{}{ - "min": 0, - "max": 10, - "rad": "cool", - "totally": "tubular", - } -} + config, _, _ := getServer(host, tls, t) -func (m *mockDriver) iid() *api.InstanceID { - return &api.InstanceID{ - ID: m.pwn("InstanceID"), - Metadata: mockDriverInstanceIDMetadata(), + c, err := Dial(nil, config) + if err != nil { + t.Fatalf("error dialing libStorage service at '%s' %v", + config.Get("libstorage.host"), err) } + return c } -func (m *mockDriver) Init() error { - return nil -} - -func (m *mockDriver) Name() string { - return m.name -} - -func (m *mockDriver) GetNextAvailableDeviceName( - ctx context.Context, - args *api.GetNextAvailableDeviceNameArgs) ( - *api.NextAvailableDeviceName, error) { - return &api.NextAvailableDeviceName{ - Prefix: "xvd", - Pattern: `\w`, - Ignore: getDeviceIgnore(m.name), - }, nil -} - -func (m *mockDriver) GetVolumeMapping( - ctx context.Context, - args *api.GetVolumeMappingArgs) ([]*api.BlockDevice, error) { - return []*api.BlockDevice{&api.BlockDevice{ - DeviceName: getDeviceName(m.name), - ProviderName: m.pwn("Provider"), - InstanceID: m.iid(), - Region: m.pwn("Region"), - }}, nil -} - -func (m *mockDriver) GetInstance( - ctx context.Context, - args *api.GetInstanceArgs) (*api.Instance, error) { - return &api.Instance{ - Name: m.pwn("Name"), - InstanceID: m.iid(), - ProviderName: m.pwn("Provider"), - Region: m.pwn("Region"), - }, nil -} - -func (m *mockDriver) GetVolume( - ctx context.Context, - args *api.GetVolumeArgs) ([]*api.Volume, error) { - return []*api.Volume{&api.Volume{ - Name: m.pwn(args.Optional.VolumeName), - VolumeID: m.pwn("VolumeID"), - AvailabilityZone: m.pwn("AvailabilityZone"), - }}, nil -} - -func (m *mockDriver) GetVolumeAttach( - ctx context.Context, - args *api.GetVolumeAttachArgs) ([]*api.VolumeAttachment, error) { - return []*api.VolumeAttachment{&api.VolumeAttachment{ - VolumeID: m.pwn("VolumeID"), - }}, nil -} - -func (m *mockDriver) CreateSnapshot( - ctx context.Context, - args *api.CreateSnapshotArgs) ([]*api.Snapshot, error) { - return []*api.Snapshot{&api.Snapshot{ - VolumeID: m.pwn("VolumeID"), - }}, nil -} - -func (m *mockDriver) GetSnapshot( - ctx context.Context, - args *api.GetSnapshotArgs) ([]*api.Snapshot, error) { - return []*api.Snapshot{&api.Snapshot{ - VolumeID: m.pwn("VolumeID"), - }}, nil -} - -func (m *mockDriver) RemoveSnapshot( - ctx context.Context, - args *api.RemoveSnapshotArgs) error { - return nil -} - -func (m *mockDriver) CreateVolume( - ctx context.Context, - args *api.CreateVolumeArgs) (*api.Volume, error) { - return &api.Volume{ - Name: m.pwn(args.Optional.VolumeName), - VolumeID: m.pwn("VolumeID"), - AvailabilityZone: m.pwn("AvailabilityZone"), - }, nil -} +type testFunc func(client client.Client, t *testing.T) -func (m *mockDriver) RemoveVolume( - ctx context.Context, - args *api.RemoveVolumeArgs) error { - return nil +func testTCP(tf testFunc, t *testing.T) { + client := getClient("", false, t) + tf(client, t) } -func (m *mockDriver) AttachVolume( - ctx context.Context, - args *api.AttachVolumeArgs) ([]*api.VolumeAttachment, error) { - return []*api.VolumeAttachment{&api.VolumeAttachment{ - VolumeID: m.pwn("VolumeID"), - }}, nil +func testTCPTLS(tf testFunc, t *testing.T) { + client := getClient("", true, t) + tf(client, t) } -func (m *mockDriver) DetachVolume( - ctx context.Context, - args *api.DetachVolumeArgs) error { - return nil +func testSock(tf testFunc, t *testing.T) { + sock := fmt.Sprintf("unix://%s", getTempSockFile()) + client := getClient(sock, false, t) + tf(client, t) } -func (m *mockDriver) CopySnapshot( - ctx context.Context, - args *api.CopySnapshotArgs) (*api.Snapshot, error) { - return &api.Snapshot{ - VolumeID: m.pwn("VolumeID"), - }, nil +func testSockTLS(tf testFunc, t *testing.T) { + sock := fmt.Sprintf("unix://%s", getTempSockFile()) + client := getClient(sock, true, t) + tf(client, t) } -func (m *mockDriver) GetClientTool( - ctx context.Context, - args *api.GetClientToolArgs) (*api.ClientTool, error) { - - clientTool := &api.ClientTool{ - Name: m.pwn("-clientTool.sh"), - } - - if args.Optional.OmitBinary { - return clientTool, nil - } - - jsonBuf, err := json.Marshal(m.iid()) +func getTempSockFile() string { + f, err := ioutil.TempFile("", "") if err != nil { - return nil, err - } - script := fmt.Sprintf(`#!/bin/sh - case "$1" in - "GetInstanceID") - echo '%s' - ;; - "GetNextAvailableDeviceName") - echo $BLOCK_DEVICES_JSON - ;; - esac -`, string(jsonBuf)) - - clientTool.Data = []byte(script) - return clientTool, nil -} - -func getDeviceName(driver string) string { - var deviceName string - switch driver { - case mockDriver1Name: - deviceName = "/dev/xvdb" - case mockDriver2Name: - deviceName = "/dev/xvda" - case mockDriver3Name: - deviceName = "/dev/xvdc" - } - return deviceName -} - -func getNextDeviceName(driver string) string { - var deviceName string - switch driver { - case mockDriver1Name: - deviceName = "/dev/xvdc" - case mockDriver3Name: - deviceName = "/dev/xvdb" + panic(err) } - return deviceName + name := f.Name() + os.RemoveAll(name) + return fmt.Sprintf("%s.sock", name) } -func getDeviceIgnore(driver string) bool { - if driver == mockDriver2Name { - return true +func testLogAsJSON(i interface{}, t *testing.T) { + buf, err := json.MarshalIndent(i, "", " ") + if err != nil { + panic(err) } - return false + t.Logf("%s\n", string(buf)) } - -var localDevicesFileBuf = []byte(` -major minor #blocks name - - 11 0 4050944 sr0 - 8 0 67108864 sda - 8 1 512000 sda1 - 8 2 66595840 sda2 - 253 0 4079616 dm-0 - 253 1 42004480 dm-1 - 253 2 20508672 dm-2 - 1024 1 20508672 xvda - 7 0 104857600 loop0 - 7 1 2097152 loop1 - 253 3 104857600 dm-3 -`) diff --git a/libstorage_tests_test.go b/libstorage_tests_test.go new file mode 100644 index 00000000..e74ec298 --- /dev/null +++ b/libstorage_tests_test.go @@ -0,0 +1,41 @@ +// +build !run + +package libstorage + +import ( + "testing" + + "github.com/emccode/libstorage/api/client" +) + +func TestRoot(t *testing.T) { + + tf := func(client client.Client, t *testing.T) { + reply, err := client.Root() + if err != nil { + t.Fatal(err) + } + testLogAsJSON(reply, t) + } + + testTCP(tf, t) + testTCPTLS(tf, t) + testSock(tf, t) + testSockTLS(tf, t) +} + +func TestVolumes(t *testing.T) { + + tf := func(client client.Client, t *testing.T) { + reply, err := client.Volumes() + if err != nil { + t.Fatal(err) + } + testLogAsJSON(reply, t) + } + + testTCP(tf, t) + testTCPTLS(tf, t) + testSock(tf, t) + testSockTLS(tf, t) +} diff --git a/service/server/server.go b/service/server/server.go deleted file mode 100644 index 4cca4d8f..00000000 --- a/service/server/server.go +++ /dev/null @@ -1,230 +0,0 @@ -package server - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "io" - golog "log" - "net" - "net/http" - "time" - - log "github.com/Sirupsen/logrus" - "github.com/akutz/gofig" - "github.com/akutz/goof" - "github.com/akutz/gotil" - gcontext "github.com/gorilla/context" - "github.com/gorilla/mux" - "github.com/gorilla/rpc" - gjson "github.com/gorilla/rpc/json" - - "github.com/emccode/libstorage/api" - "github.com/emccode/libstorage/service/server/handlers" -) - -var ( - noInstanceIDMethods = []string{ - "libStorage.GetServiceInfo", - "libStorage.GetClientTool", - } -) - -// ServiceInfo is information used to serve a service. -type ServiceInfo struct { - Name string `json:"name"` - Service api.ServiceEndpoint `json:"-"` - Config gofig.Config `json:"config"` - server *rpc.Server -} - -// Serve serves one or more services via HTTP/JSON. -func Serve( - serviceInfo map[string]*ServiceInfo, - config gofig.Config) (err error) { - - if err = initServers(serviceInfo, config); err != nil { - return - } - - var l net.Listener - var host, laddr string - if host, laddr, l, err = getNetInfo(config); err != nil { - return - } - log.WithField("host", host).Debug("ready to listen") - - doLogs := config.GetBool("libstorage.server.http.logging.enabled") - var stdOut, stdErr io.WriteCloser - if doLogs { - stdOut = handlers.GetLogIO( - "libstorage.server.http.logging.out", config) - stdErr = handlers.GetLogIO( - "libstorage.server.http.logging.err", config) - } - - r := mux.NewRouter() - var configHandler, serversHandler http.Handler - - configHandler = http.HandlerFunc( - func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(http.StatusOK) - w.Header().Add("Content-Type", "application/json") - - configJSON, _ := json.MarshalIndent(config, "", " ") - w.Write(configJSON) - }) - - serversHandler = http.HandlerFunc( - func(w http.ResponseWriter, req *http.Request) { - vars := mux.Vars(req) - name := vars["name"] - serviceInfo[name].server.ServeHTTP(w, req) - }) - - if doLogs { - configHandler = handlers.NewLoggingHandler( - stdOut, stdErr, configHandler, config) - - serversHandler = handlers.NewLoggingHandler( - stdOut, stdErr, serversHandler, config) - } - - r.Handle("/libStorage/config", - gcontext.ClearHandler(configHandler)) - - r.Handle("/libStorage/services/{name}", - gcontext.ClearHandler(serversHandler)) - - hs := &http.Server{ - Addr: laddr, - Handler: r, - ReadTimeout: getReadTimeout(config) * time.Second, - WriteTimeout: getWriteTimeout(config) * time.Second, - MaxHeaderBytes: 1 << 20, - } - - if doLogs { - hs.ErrorLog = golog.New(stdErr, "", 0) - } - - go func() { - defer func() { - if stdOut != nil { - stdOut.Close() - } - if stdErr != nil { - stdErr.Close() - } - - r := recover() - switch tr := r.(type) { - case error: - log.Panic( - "unhandled exception when serving libStorage service", tr) - } - }() - - if err := hs.Serve(l); err != nil { - log.Panic("error serving libStorage service", err) - } - }() - - updatedHost := fmt.Sprintf("%s://%s", - l.Addr().Network(), - l.Addr().String()) - - if updatedHost != host { - host = updatedHost - lsmap := config.Get("libstorage").(map[string]interface{}) - lsmap["host"] = host - } - log.WithField("host", host).Debug("listening") - - return nil -} - -func initServers( - serviceInfo map[string]*ServiceInfo, - config gofig.Config) error { - - for _, si := range serviceInfo { - s := rpc.NewServer() - s.RegisterBeforeFunc(func(i *rpc.RequestInfo) { - if !gotil.StringInSlice(i.Method, noInstanceIDMethods) { - initInstanceID(i.Request) - } - }) - - codec := gjson.NewCodec() - s.RegisterCodec(codec, "application/json") - s.RegisterCodec(codec, "application/json;charset=UTF-8") - - if err := s.RegisterService(si.Service, "libStorage"); err != nil { - return err - } - - si.server = s - } - - return nil -} - -func getNetInfo(config gofig.Config) ( - host, laddr string, - l net.Listener, err error) { - - host = config.GetString("libstorage.host") - if host == "" { - host = "tcp://127.0.0.1:0" - } - - var netProto string - if netProto, laddr, err = gotil.ParseAddress(host); err != nil { - return - } - - if l, err = net.Listen(netProto, laddr); err != nil { - return - } - - return -} - -func initInstanceID(req *http.Request) { - iidb64 := req.Header.Get("libStorage-InstanceID") - if iidb64 == "" { - panic(goof.New("instanceID required")) - } - - iidJSON, err := base64.URLEncoding.DecodeString(iidb64) - if err != nil { - panic(err) - } - - var iid *api.InstanceID - err = json.Unmarshal(iidJSON, &iid) - if err != nil { - panic(err) - } - - log.WithField("instanceID", iid).Debug("request's instance ID") - - gcontext.Set(req, "instanceID", iid) -} - -func getReadTimeout(config gofig.Config) time.Duration { - t := config.GetInt("libstorage.server.readtimeout") - if t == 0 { - return 60 - } - return time.Duration(t) -} - -func getWriteTimeout(config gofig.Config) time.Duration { - t := config.GetInt("libstorage.server.writetimeout") - if t == 0 { - return 60 - } - return time.Duration(t) -} diff --git a/service/service.go b/service/service.go deleted file mode 100644 index 93526b5f..00000000 --- a/service/service.go +++ /dev/null @@ -1,398 +0,0 @@ -package service - -import ( - "crypto/md5" - "fmt" - "net/http" - "strings" - "sync" - - log "github.com/Sirupsen/logrus" - "github.com/akutz/gofig" - "github.com/akutz/goof" - "github.com/akutz/gotil" - gcontext "github.com/gorilla/context" - - "github.com/emccode/libstorage/api" - "github.com/emccode/libstorage/context" - "github.com/emccode/libstorage/driver" - "github.com/emccode/libstorage/service/server" -) - -var ( - errNoStorageDetected = goof.New("no storage detected") - errDriverBlockDeviceDiscovery = goof.New("no block devices discovered") - - driverCtors = map[string]driver.NewDriver{} - driverCtorsLock = &sync.RWMutex{} -) - -type service struct { - name string - config gofig.Config - constructedDrivers map[string]driver.Driver - driver driver.Driver - driversInitialized bool - initializingDriver *sync.Mutex -} - -// RegisterDriver is used by drivers to indicate their availability to -// libstorage. -func RegisterDriver(driverName string, ctor driver.NewDriver) { - driverCtorsLock.Lock() - defer driverCtorsLock.Unlock() - driverCtors[strings.ToLower(driverName)] = ctor -} - -// Serve starts the reference implementation of a server hosting an -// HTTP/JSON service that implements the libStorage API endpoint. -// -// If the config parameter is nil a default instance is created. The -// libStorage service is served at the address specified by the configuration -// property libstorage.host. -// -// If libstorage.host specifies a tcp network and the provided port is 0, -// aka "select any avaialble port", then after this function exists -// successfully the libstorage.host property is updated to reflect the port -// being used. For example, if the original value is tcp://127.0.0.1:0 and -// the operating system (OS) selects port 3756, then the libstorage.host -// property is updated to tcp://127.0.0.1:3756. -func Serve(config gofig.Config) error { - - si := map[string]*server.ServiceInfo{} - - services := config.Get("libstorage.server.services") - servicesMap := services.(map[string]interface{}) - log.WithField("count", len(servicesMap)).Debug("got services map") - - for name := range servicesMap { - log.WithField("name", name).Debug("processing service config") - - scope := fmt.Sprintf("libstorage.server.services.%s", name) - log.WithField("scope", scope).Debug("getting scoped config for service") - sc := config.Scope(scope) - - service, err := newService(name, sc) - if err != nil { - return err - } - - si[name] = &server.ServiceInfo{ - Name: name, - Service: service, - Config: sc, - } - } - - if err := server.Serve(si, config); err != nil { - return err - } - - return nil -} - -func (s *service) GetServiceInfo( - req *http.Request, - args *api.GetServiceInfoArgs, - reply *api.GetServiceInfoReply) error { - reply.Name = s.name - reply.Driver = s.driver.Name() - reply.RegisteredDrivers = s.getRegisteredDriverNames() - return nil -} - -func (s *service) GetNextAvailableDeviceName( - req *http.Request, - args *api.GetNextAvailableDeviceNameArgs, - reply *api.GetNextAvailableDeviceNameReply) (err error) { - reply.Next, err = - s.driver.GetNextAvailableDeviceName(s.ctx(req), args) - return nil -} - -func (s *service) GetVolumeMapping( - req *http.Request, - args *api.GetVolumeMappingArgs, - reply *api.GetVolumeMappingReply) error { - - bds, err := s.driver.GetVolumeMapping(s.ctx(req), args) - if err != nil { - return errDriverBlockDeviceDiscovery - } - - for _, bd := range bds { - reply.BlockDevices = append(reply.BlockDevices, bd) - } - - if len(reply.BlockDevices) == 0 { - return errNoStorageDetected - } - - log.WithField( - "len(blockDevices)", len(reply.BlockDevices)).Debug( - "libStorage.Server.GetVolumeMapping") - - return nil -} - -func (s *service) GetInstance( - req *http.Request, - args *api.GetInstanceArgs, - reply *api.GetInstanceReply) (err error) { - reply.Instance, err = s.driver.GetInstance(s.ctx(req), args) - return -} - -func (s *service) GetVolume( - req *http.Request, - args *api.GetVolumeArgs, - reply *api.GetVolumeReply) (err error) { - - ctx := s.ctx(req) - unprofile := profile(ctx, &args.Optional.VolumeName) - defer unprofile() - - if reply.Volumes, err = s.driver.GetVolume(ctx, args); err != nil { - return - } - if reply.Volumes != nil && - !s.config.GetBool("libstorage.profiles.client") { - for _, v := range reply.Volumes { - np := strings.Split(v.Name, "-") - if len(np) == 1 { - v.Name = np[0] - } else { - v.Name = np[1] - } - } - } - log.WithField( - "len(volumes)", len(reply.Volumes)).Debug( - "libStorage.Server.GetVolume") - return -} - -func (s *service) GetVolumeAttach( - req *http.Request, - args *api.GetVolumeAttachArgs, - reply *api.GetVolumeAttachReply) (err error) { - reply.Attachments, err = s.driver.GetVolumeAttach(s.ctx(req), args) - return -} - -func (s *service) CreateSnapshot( - req *http.Request, - args *api.CreateSnapshotArgs, - reply *api.CreateSnapshotReply) (err error) { - reply.Snapshots, err = s.driver.CreateSnapshot(s.ctx(req), args) - return -} - -func (s *service) GetSnapshot( - req *http.Request, - args *api.GetSnapshotArgs, - reply *api.GetSnapshotReply) (err error) { - reply.Snapshots, err = s.driver.GetSnapshot(s.ctx(req), args) - return -} - -func (s *service) RemoveSnapshot( - req *http.Request, - args *api.RemoveSnapshotArgs, - reply *api.RemoveSnapshotReply) (err error) { - err = s.driver.RemoveSnapshot(s.ctx(req), args) - return -} - -func (s *service) CreateVolume( - req *http.Request, - args *api.CreateVolumeArgs, - reply *api.CreateVolumeReply) (err error) { - - ctx := s.ctx(req) - unprofile := profile(ctx, &args.Optional.VolumeName) - defer unprofile() - if reply.Volume, err = s.driver.CreateVolume(ctx, args); err != nil { - return - } - if s.config.GetBool("libstorage.profiles.client") { - return - } - - np := strings.Split(reply.Volume.Name, "-") - if len(np) == 1 { - reply.Volume.Name = np[0] - } else { - reply.Volume.Name = np[1] - } - return -} - -func (s *service) RemoveVolume( - req *http.Request, - args *api.RemoveVolumeArgs, - reply *api.RemoveVolumeReply) (err error) { - err = s.driver.RemoveVolume(s.ctx(req), args) - return -} - -func (s *service) AttachVolume( - req *http.Request, - args *api.AttachVolumeArgs, - reply *api.AttachVolumeReply) (err error) { - reply.Attachments, err = s.driver.AttachVolume(s.ctx(req), args) - return -} - -func (s *service) DetachVolume( - req *http.Request, - args *api.DetachVolumeArgs, - reply *api.DetachVolumeReply) (err error) { - err = s.driver.DetachVolume(s.ctx(req), args) - return -} - -func (s *service) CopySnapshot( - req *http.Request, - args *api.CopySnapshotArgs, - reply *api.CopySnapshotReply) (err error) { - reply.Snapshot, err = s.driver.CopySnapshot(s.ctx(req), args) - return -} - -func (s *service) GetClientTool( - req *http.Request, - args *api.GetClientToolArgs, - reply *api.GetClientToolReply) (err error) { - reply.ClientTool, err = s.driver.GetClientTool(s.ctx(req), args) - - // calculate the checksum of the tool if the tool is set and no checksum - // exists for it - if len(reply.ClientTool.Data) > 0 && reply.ClientTool.MD5Checksum == "" { - reply.ClientTool.MD5Checksum = fmt.Sprintf("%x", - md5.Sum(reply.ClientTool.Data)) - } - return -} - -func newService(name string, config gofig.Config) (*service, error) { - s := &service{ - name: name, - config: config, - constructedDrivers: map[string]driver.Driver{}, - initializingDriver: &sync.Mutex{}, - } - - driverCtorsLock.RLock() - defer driverCtorsLock.RUnlock() - for name, ctor := range driverCtors { - d := ctor(config) - s.constructedDrivers[name] = d - log.WithField("driverName", name).Debug("constructed driver") - } - - if err := s.initDriver(); err != nil { - return nil, err - } - - return s, nil -} - -func getGroupForIP(ip, groupMap string) string { - gmp := strings.Split(groupMap, "=") - group := gmp[0] - ipMap := strings.Split(gmp[1], ",") - if gotil.StringInSlice(ip, ipMap) { - return group - } - return "" -} - -func profile(ctx context.Context, val *string) func() { - originalVal := *val - p := ctx.Value("profile") - switch tp := p.(type) { - case string: - *val = fmt.Sprintf("%s-%s", tp, originalVal) - } - log.WithFields(log.Fields{ - "old": originalVal, - "new": *val, - }).Debug("prefixed value w/ profile") - return func() { *val = originalVal } -} - -func hash(text string) string { - return fmt.Sprintf("%x", md5.Sum([]byte(text))) -} - -func (s *service) ctx(req *http.Request) context.Context { - - ctx := context.Background() - iid := gcontext.Get(req, "instanceID") - switch tiid := iid.(type) { - case (*api.InstanceID): - ctx = context.WithInstanceID(ctx, tiid) - } - - if !s.config.GetBool("libstorage.profiles.enabled") { - log.Debug("not using server-side profiles") - return ctx - } - - ra := req.RemoteAddr - rap := strings.Split(ra, ":") - ip := rap[0] - - g := "" - grps := s.config.GetStringSlice("libstorage.profiles.groups") - if grps != nil { - for _, gmap := range grps { - if g = getGroupForIP(ip, gmap); g != "" { - break - } - } - } - - var profile string - if profile = g; profile == "" { - profile = hash(ip) - } - log.WithField("profile", profile).Debug("using server-side profile") - return context.WithValue(ctx, "profile", profile) -} - -// initDriver initializes the drivers for the libStorage instance. -func (s *service) initDriver() error { - - driverName := s.config.GetString("libstorage.driver") - log.WithField("driverName", driverName).Debug("libstorage driver") - - var ok bool - s.driver, ok = s.constructedDrivers[strings.ToLower(driverName)] - if !ok { - return errNoStorageDetected - } - - if err := s.driver.Init(); err != nil { - return goof.WithFields(goof.Fields{ - "name": s.driver.Name(), - "error": err, - }, "error initializing libstorage driver") - } - - log.WithFields(log.Fields{ - "registeredDriverNames": s.getRegisteredDriverNames(), - "driverName": s.driver.Name(), - }).Debug("libStorage.Server.InitDrivers") - - return nil -} - -func (s *service) getRegisteredDriverNames() []string { - driverNames := []string{} - for dn := range s.constructedDrivers { - driverNames = append(driverNames, dn) - } - return driverNames -}