Merge pull request #143 from v0ctor/feature/improve-generation

Improve code generation

.env.example

@@ -0,0 +1,2 @@
+# Docker
+#USER_ID=1000

.github/workflows/main.yml

@@ -19,7 +19,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: License check
-        run: ./scripts/licensecheck.sh
+        run: ./scripts/license-check.sh
 
       - name: Go installation
         uses: actions/setup-go@v2

.gitignore

@@ -1,3 +1,3 @@
 .DS_Store
+.env
 .gobincache/
-

Dockerfile

@@ -0,0 +1,42 @@
+ARG GO_VERSION=1.19.5
+ARG ALPINE_VERSION=3.17
+ARG GOIMPORTS_VERSION=0.5.0
+ARG DELVE_VERSION=1.20.1
+ARG SWAGGER_VERSION=0.30.3
+
+
+## Base image
+FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS base
+
+WORKDIR /go/src/app
+
+ENV CGO_ENABLED=0
+
+
+## Development image
+FROM base AS development
+
+ARG GOIMPORTS_VERSION
+ARG DELVE_VERSION
+ARG SWAGGER_VERSION
+
+RUN apk add \
+        bash \
+        curl \
+        git \
+        jq \
+        python3 \
+        zsh \
+ && go install golang.org/x/tools/cmd/goimports@v${GOIMPORTS_VERSION} \
+ && go install github.com/go-delve/delve/cmd/dlv@v${DELVE_VERSION} \
+ && go install github.com/go-swagger/go-swagger/cmd/swagger@v${SWAGGER_VERSION}
+
+ARG USER_ID=1000
+ENV USER_NAME=default
+
+ENV PROMPT="%B%F{cyan}%n%f@%m:%F{yellow}%~%f %F{%(?.green.red[%?] )}>%f %b"
+
+RUN adduser -D -u $USER_ID ${USER_NAME} \
+ && chown -R ${USER_NAME}: /go/pkg
+
+USER ${USER_NAME}

Makefile

@@ -1,12 +1,40 @@
-deps:
-	go mod download
-	go install github.com/go-swagger/go-swagger/cmd/swagger@latest
+# Helpers
+IS_DARWIN := $(filter Darwin,$(shell uname -s))
 
-generate: deps
-	swagger generate client --target=./netbox --copyright-file=copyright_header.txt
+define set_env
+	sed $(if $(IS_DARWIN),-i "",-i) -e "s/^#*\($(1)=\).*/$(if $(2),,#)\1$(2)/" .env
+endef
 
-clean:
-	rm -rf netbox/client netbox/models
+EXEC := docker compose exec app
 
-integration:
-	go test ./... -tags=integration
+# Environment recipes
+.PHONY: default
+default: init up
+
+.PHONY: init
+init:
+	test -f .env || cp .env.example .env
+	$(call set_env,USER_ID,$(shell id -u))
+
+.PHONY: up
+up:
+	DOCKER_BUILDKIT=1 docker compose up -d --build
+
+.PHONY: down
+down:
+	docker compose down
+
+.PHONY: shell
+shell:
+	$(EXEC) zsh
+
+# Project recipes
+.PHONY: build
+build:
+	$(EXEC) ./scripts/set-versions.sh $(NETBOX_VERSION) $(NETBOX_DOCKER_VERSION)
+	./scripts/fetch-spec.sh $$(cat api/netbox_version) $$(cat api/netbox_docker_version)
+	$(EXEC) ./scripts/generate-code.sh
+
+.PHONY: test
+test:
+	$(EXEC) go test ./... -tags=integration

README.md

@@ -1,14 +1,12 @@
-go-netbox 
-=========
+# go-netbox
 
 [![GoDoc](http://godoc.org/github.com/netbox-community/go-netbox?status.svg)](http://godoc.org/github.com/netbox-community/go-netbox) [![Build Status](https://github.com/netbox-community/go-netbox/workflows/main/badge.svg?branch=master)](https://github.com/netbox-community/go-netbox/actions) [![Report Card](https://goreportcard.com/badge/github.com/netbox-community/go-netbox)](https://goreportcard.com/report/github.com/netbox-community/go-netbox)
 
 Package `netbox` provides an API 3.2 client for [netbox-community's Netbox](https://github.com/netbox-community/netbox) IPAM and DCIM service.
 
 This package assumes you are using Netbox 3.2, as the Netbox 1.0 API no longer exists. If you need support for previous Netbox versions, you can still import the corresponding release of the library. For example, run `go get github.com/netbox-community/go-netbox@netbox_v2.11` if you need compatibility with Netbox 2.11.
 
-Using the client
-================
+## Using the client
 
 The `github.com/netbox-community/go-netbox/netbox` package has some convenience functions for creating clients with the most common
 configurations you are likely to need while connecting to NetBox. `NewNetboxAt` allows you to specify a hostname
@@ -68,16 +66,14 @@ func main() {
 }
 ```
 
-Go Module support
-================
+## Go Module support
 
 Go 1.16+
 
 `go get github.com/netbox-community/go-netbox`
 
 
-More complex client configuration
-=================================
+## More complex client configuration
 
 The client is generated using [go-swagger](https://github.com/go-swagger/go-swagger). This means the generated client
 makes use of [github.com/go-openapi/runtime/client](https://godoc.org/github.com/go-openapi/runtime/client). If you need
@@ -88,14 +84,53 @@ The [godocs for the go-openapi/runtime/client module](https://godoc.org/github.c
 the client options in detail, including different authentication and debugging options. One thing I want to flag because
 it is so useful: setting the `DEBUG` environment variable will dump all requests to standard out.
 
-Regenerating the client
-=======================
+## Development
 
-To regenerate the client with a new or different swagger schema, first clean the existing client, then replace
-swagger.json and finally re-generate:
+The project comes with a containerized development environment that can be used from any platform. It is only required to have [Git](https://git-scm.com) and [Docker Desktop](https://www.docker.com/products/docker-desktop/) (or, separately, [Docker](https://docs.docker.com/engine/install) and [Docker Compose](https://docs.docker.com/compose/install/)) installed on the machine.
+
+To start the development environment, run the following command.
+
+```bash
+make
+```
+
+Then, to attach a shell in the container, run the command below.
+
+```bash
+make shell
 ```
-make clean
-./scripts/swagger_modifier.py new_swagger_file.json
-mv swagger_transformed.json swagger.json
-make generate
+
+Finally, to stop the development environment, run the following command.
+
+```bash
+make down
+```
+
+### Considerations
+
+The library is almost entirely generated from the Netbox [OpenAPI](https://www.openapis.org/) specification. Therefore, files under directories `netbox/client` and `netbox/models` should not be directly modified, as they will be overwritten in the next regeneration (see next section).
+
+To fix issues in generated code, there are two options:
+
+- Change the OpenAPI spec with pre-generation hooks (see [`scripts/pre-generation`](scripts/pre-generation)).
+- If the above is not possible, change the generated code with post-generation hooks (see [`scripts/post-generation`](scripts/post-generation)).
+
+### Regenerate the library
+
+To update the OpenAPI specification to the latest Netbox version and regenerate the library, run the following command.
+
+```bash
+make build
+```
+
+If regeneration of the library is needed for a specific Netbox version other than the latest one, pass the corresponding argument.
+
+```bash
+make build NETBOX_VERSION=3.0.0
+```
+
+In order to obtain the OpenAPI specification, the version of _[netbox-docker](https://github.com/netbox-community/netbox-docker)_ corresponding to the given Netbox version is used. However, it is also possible to provide a specific version of _netbox-docker_.
+
+```bash
+make build NETBOX_VERSION=3.0.0 NETBOX_DOCKER_VERSION=1.3.1
 ```

api/netbox_docker_version

@@ -0,0 +1 @@
+2.3.0

api/netbox_version

@@ -0,0 +1 @@
+3.3.10

docker-compose.yml

@@ -0,0 +1,16 @@
+services:
+  app:
+    build:
+      context: .
+      target: development
+      args:
+        USER_ID: ${USER_ID:-1000}
+    volumes:
+      - .:/go/src/app
+      - cache:/go/pkg
+    env_file: .env
+    hostname: app
+    tty: true
+
+volumes:
+  cache:

scripts/fetch-spec.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+NETBOX_VERSION="$1"
+NETBOX_DOCKER_VERSION="$2"
+
+REPO_DIR='/tmp/netbox-docker'
+
+rm -rf "${REPO_DIR}"
+
+git clone https://github.com/netbox-community/netbox-docker.git \
+  --config advice.detachedHead=false \
+  --branch ${NETBOX_DOCKER_VERSION} \
+  --depth=1 \
+  --quiet \
+  "${REPO_DIR}"
+
+mv "${REPO_DIR}/docker-compose.override.yml.example" "${REPO_DIR}/docker-compose.override.yml"
+
+export VERSION="v${NETBOX_VERSION}"
+docker compose --project-directory="${REPO_DIR}" up --detach --quiet-pull
+
+curl --silent http://127.0.0.1:8000/api/docs/?format=openapi > api/openapi.json
+
+docker compose --project-directory="${REPO_DIR}" down

scripts/generate-code.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+export SRC_DIR='./netbox'
+
+# Prepare environment
+rm -rf netbox/client netbox/models
+go mod download
+
+# Run pre-generation hooks
+for SCRIPT in scripts/pre-generation/*; do
+  "${SCRIPT}"
+done
+
+# Generate code
+swagger generate client -f api/openapi.json --target="${SRC_DIR}" --copyright-file=assets/copyright_header.txt
+go mod tidy
+
+# Run post-generation hooks
+for SCRIPT in scripts/post-generation/*; do
+  "${SCRIPT}"
+done
+
+# Format generated code and fix imports
+goimports -w "${SRC_DIR}"

scripts/post-generation/fix-integers.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Maximum and minimum values in spec that correspond to 
+# `math.MaxInt64` and `math.MinInt64`, respectively, are not rendered
+# properly by "go-swagger", thus causing the following errors.
+#
+# cannot use -9.223372036854776e+18 (untyped float constant -9.22337e+18) as int64 value in argument to validate.MinimumInt (truncated)
+# cannot use 9.223372036854776e+18 (untyped float constant 9.22337e+18) as int64 value in argument to validate.MaximumInt (truncated)
+#
+# See: https://github.com/go-swagger/go-swagger/issues/2755
+
+set -euo pipefail
+
+find "${SRC_DIR}" -type f -name '*.go' -exec \
+  sed -i \
+    -e 's/-9.223372036854776e+18/math.MinInt64/' \
+    -e 's/9.223372036854776e+18/math.MaxInt64/' \
+    {} \;

scripts/swagger_modifier.py → scripts/pre-generation/fix-schema.py

@@ -44,19 +44,19 @@ def cli_arguments():
     :return: argparse.parser object
     """
     parser = argparse.ArgumentParser(
-        description=('Modifications on swagger.json.'))
+        description=('Modifications on openapi.json.'))
     parser.add_argument('-v',
                         '--verbose',
                         action='count',
                         default=0,
                         help='Increase output verbosity.')
     parser.add_argument('-o',
                         '--output',
-                        default='swagger_transformed.json',
+                        default='api/openapi.json',
                         help='Output file')
     parser.add_argument('input',
                         nargs='?',
-                        default='swagger.json',
+                        default='api/openapi.json',
                         help='Original input json file')
 
     return parser.parse_args()
@@ -124,17 +124,6 @@ def complete_data(input_file, log):
                             def_properties[def_property].update(
                                 modify_properties[prop])
 
-            for p in def_properties:
-                # The maximum value (9223372036854775807) set here lead to an error
-                # cannot use 9.223372036854776e+18 (untyped float constant 9.22337e+18) as int64 value in argument to validate.MaximumInt (truncated) # noqa E501
-                # There's issue opened on this https://github.com/go-swagger/go-swagger/issues/2755 # noqa E501
-                if def_properties[p].get('maximum', 0) == 9223372036854775807:
-                    log.info('Changing maximum value for {}'.format(p))
-                    def_properties[p]['maximum'] = 2147483647
-                # The minimum value also has same issue.
-                if def_properties[p].get('minimum', 0) == -9223372036854775808:
-                    log.info('Changing minimum value for {}'.format(p))
-                    def_properties[p]['minimum'] = -2147483648
     return (data)
 
 
@@ -160,7 +149,7 @@ def write_results(data, output_file, log):
 def main():
     args = cli_arguments()
     # Get logger
-    log = init_logger('swagger', args.verbose, logfile='/tmp/swagger.log')
+    log = init_logger('openapi', args.verbose, logfile='/tmp/openapi.log')
 
     modified_data = complete_data(args.input, log)
     write_results(modified_data, args.output, log)

scripts/set-versions.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+NETBOX_VERSION='latest'
+NETBOX_DOCKER_VERSION='auto'
+
+set -euo pipefail
+
+# Parse Netbox version
+if [ "$#" -gt 0 ]; then
+  NETBOX_VERSION="$1"
+fi
+
+if [ "${NETBOX_VERSION}" == 'latest' ]; then
+  NETBOX_VERSION=$(curl --silent https://api.github.com/repos/netbox-community/netbox/releases/latest |
+    jq '.tag_name' |
+    sed -E 's/"v([^"]+)"/\1/')
+fi
+
+# Parse Netbox Docker version
+if [ "$#" -gt 1 ]; then
+  NETBOX_DOCKER_VERSION="$2"
+fi
+
+if [ "${NETBOX_DOCKER_VERSION}" == 'auto' ]; then
+  NEXT='https://registry.hub.docker.com/v2/repositories/netboxcommunity/netbox/tags/?page=1&page_size=1000'
+  NETBOX_DOCKER_VERSION=''
+
+  while [ "${NEXT}" != "null" ] && [ "${NETBOX_DOCKER_VERSION}" == "" ]; do
+    RESPONSE=$(curl --silent "${NEXT}")
+    NEXT=$(echo -E "${RESPONSE}" | jq '.next')
+
+    NETBOX_DOCKER_VERSION=$(echo -E "${RESPONSE}" |
+      jq -r ".results[] | select(.name | startswith(\"v${NETBOX_VERSION}-\")) | .name" |
+      cut -d "-" -f 2)
+  done
+
+  unset NEXT
+  unset RESPONSE
+fi
+
+# Print variables
+echo ${NETBOX_VERSION} > api/netbox_version
+echo ${NETBOX_DOCKER_VERSION} > api/netbox_docker_version