-
Notifications
You must be signed in to change notification settings - Fork 52
/
tasks.py
281 lines (216 loc) · 8.23 KB
/
tasks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
"""Tasks for use with Invoke."""
import os
import sys
from invoke import task
try:
import toml
except ImportError:
sys.exit("Please make sure to `pip install toml` or enable the Poetry shell and run `poetry install`.")
def is_truthy(arg):
"""Convert "truthy" strings into Booleans.
Args:
arg (str): Truthy string (True values are y, yes, t, true, on and 1; false values are n, no,
f, false, off and 0. Raises ValueError if val is anything else.
Examples:
>>> is_truthy('yes')
True
"""
if isinstance(arg, bool):
return arg
val = str(arg).lower()
if val in ("y", "yes", "t", "true", "on", "1"):
return True
if val in ("n", "no", "f", "false", "off", "0"):
return False
raise ValueError(f"Invalid truthy value: `{arg}`")
PYPROJECT_CONFIG = toml.load("pyproject.toml")
TOOL_CONFIG = PYPROJECT_CONFIG["tool"]["poetry"]
# Can be set to a separate Python version to be used for launching or building image
PYTHON_VER = os.getenv("PYTHON_VER", "3.13")
# Name of the docker image/image
IMAGE_NAME = os.getenv("IMAGE_NAME", TOOL_CONFIG["name"])
# Tag for the image
IMAGE_VER = os.getenv("IMAGE_VER", f"{TOOL_CONFIG['version']}-py{PYTHON_VER}")
# Gather current working directory for Docker commands
PWD = os.getcwd()
# Local or Docker execution provide "local" to run locally without docker execution
INVOKE_LOCAL = is_truthy(os.getenv("INVOKE_LOCAL", False)) # pylint: disable=W1508
# Get project name from the toml file
PROJECT_NAME = PYPROJECT_CONFIG["tool"]["poetry"]["name"]
# Get current project version from the toml file
PROJECT_VERSION = PYPROJECT_CONFIG["tool"]["poetry"]["version"]
def run_cmd(context, exec_cmd, local=INVOKE_LOCAL, port=None):
"""Wrapper to run the invoke task commands.
Args:
context ([invoke.task]): Invoke task object.
exec_cmd ([str]): Command to run.
local (bool): Define as `True` to execute locally
Returns:
result (obj): Contains Invoke result from running task.
"""
if is_truthy(local):
print(f"LOCAL - Running command {exec_cmd}")
result = context.run(exec_cmd, pty=True)
else:
print(f"DOCKER - Running command: {exec_cmd} container: {IMAGE_NAME}:{IMAGE_VER}")
if port:
result = context.run(
f"docker run -it -p {port} -v {PWD}:/local {IMAGE_NAME}:{IMAGE_VER} sh -c '{exec_cmd}'", pty=True
)
else:
result = context.run(
f"docker run -it -v {PWD}:/local {IMAGE_NAME}:{IMAGE_VER} sh -c '{exec_cmd}'", pty=True
)
return result
@task
def build(context, nocache=False, forcerm=False, hide=False): # pylint: disable=too-many-arguments
"""Build a Docker image.
Args:
context (obj): Used to run specific commands
nocache (bool): Do not use cache when building the image
forcerm (bool): Always remove intermediate containers
hide (bool): Hide output of Docker image build
"""
print(f"Building image {IMAGE_NAME}:{IMAGE_VER}")
command = f"docker build --tag {IMAGE_NAME}:{IMAGE_VER} --build-arg PYTHON_VER={PYTHON_VER} -f Dockerfile ."
if nocache:
command += " --no-cache"
if forcerm:
command += " --force-rm"
result = context.run(command, hide=hide)
if result.exited != 0:
print(f"Failed to build image {IMAGE_NAME}:{IMAGE_VER}\nError: {result.stderr}")
@task
def clean(context):
"""Remove the project specific image.
Args:
context (obj): Used to run specific commands
"""
print(f"Attempting to forcefully remove image {IMAGE_NAME}:{IMAGE_VER}")
context.run(f"docker rmi {IMAGE_NAME}:{IMAGE_VER} --force")
print(f"Successfully removed image {IMAGE_NAME}:{IMAGE_VER}")
@task
def rebuild(context):
"""Clean the Docker image and then rebuild without using cache.
Args:
context (obj): Used to run specific commands
"""
clean(context)
build(context)
@task
def coverage(context, local=INVOKE_LOCAL):
"""Run the coverage report against pytest.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "coverage run --source=netutils -m pytest"
run_cmd(context, exec_cmd, local)
run_cmd(context, "coverage report", local)
run_cmd(context, "coverage html", local)
@task
def pytest(context, local=INVOKE_LOCAL):
"""Run pytest for the specified name and Python version.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "pytest -vv --doctest-modules netutils/ && coverage run --source=netutils -m pytest && coverage report"
run_cmd(context, exec_cmd, local)
@task
def black(context, local=INVOKE_LOCAL):
"""Run black to check that Python files adherence to black standards.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "black --check --diff ."
run_cmd(context, exec_cmd, local)
@task
def flake8(context, local=INVOKE_LOCAL):
"""Run flake8 for the specified name and Python version.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "flake8 ."
run_cmd(context, exec_cmd, local)
@task
def pylint(context, local=INVOKE_LOCAL):
"""Run pylint for the specified name and Python version.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = 'find . -name "*.py" | grep -vE "(tests/unit/mock|netutils/data_files)" | xargs pylint'
run_cmd(context, exec_cmd, local)
@task
def yamllint(context, local=INVOKE_LOCAL):
"""Run yamllint to validate formatting adheres to NTC defined YAML standards.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "yamllint ."
run_cmd(context, exec_cmd, local)
@task
def pydocstyle(context, local=INVOKE_LOCAL):
"""Run pydocstyle to validate docstring formatting adheres to NTC defined standards.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "pydocstyle ."
run_cmd(context, exec_cmd, local)
@task
def bandit(context, local=INVOKE_LOCAL):
"""Run bandit to validate basic static code security analysis.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "bandit --recursive ./ --configfile .bandit.yml"
run_cmd(context, exec_cmd, local)
@task
def mypy(context, local=INVOKE_LOCAL):
"""Run mypy to validate typing-hints.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
exec_cmd = "mypy ./netutils"
run_cmd(context, exec_cmd, local)
@task
def cli(context):
"""Enter the image to perform troubleshooting or dev work.
Args:
context (obj): Used to run specific commands
"""
dev = f"docker run -it -v {PWD}:/local {IMAGE_NAME}:{IMAGE_VER} /bin/bash"
context.run(f"{dev}", pty=True)
@task
def tests(context, local=INVOKE_LOCAL):
"""Run all tests for the specified name and Python version.
Args:
context (obj): Used to run specific commands
local (bool): Define as `True` to execute locally
"""
black(context, local)
flake8(context, local)
pylint(context, local)
yamllint(context, local)
pydocstyle(context, local)
bandit(context, local)
mypy(context, local)
pytest(context, local)
print("All tests have passed!")
@task
def clean_container(context):
"""Remove stopped containers that source for image `netutils:`."""
exec_cmd = r"""docker container rm $(docker container ls -a | grep -E "^\S+\s+netutils:" | awk 'NR>1 {print $1}')"""
run_cmd(context, exec_cmd, local=True)
@task
def docs(context, local=INVOKE_LOCAL):
"""Build and serve docs locally for development."""
exec_cmd = "mkdocs serve -v --dev-addr=0.0.0.0:8001"
run_cmd(context, exec_cmd, local, port="8001:8001")