Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Radically redesign the build plan form #646

Merged
merged 3 commits into from
Nov 26, 2024
Merged

feat: Radically redesign the build plan form #646

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0e42fd0
chore: rename things in package.py
ahlinc Nov 18, 2024
fc9cf82
fix: radically redesign the build plan form
ahlinc Nov 26, 2024
63a4d7d
fix: tests
ahlinc Nov 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
280 changes: 150 additions & 130 deletions package.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -679,9 +679,10 @@ def plan(self, source_path, query):

source_paths = []
build_plan = []
build_step = []

def step(*x):
build_plan.append(x)
build_step.append(x)

def hash(path):
source_paths.append(path)
Expand Down Expand Up @@ -754,30 +755,30 @@ def commands_step(path, commands):

if path:
path = os.path.normpath(path)
step("set:workdir", path)

batch = []
for c in commands:
if isinstance(c, str):
if c.startswith(":zip"):
if path:
hash(path)
if batch:
step("sh", path, "\n".join(batch))
step("sh", "\n".join(batch))
batch.clear()
c = shlex.split(c)
if len(c) == 3:
n = len(c)
if n == 3:
_, _path, prefix = c
prefix = prefix.strip()
_path = os.path.normpath(os.path.join(path, _path))
_path = os.path.normpath(_path)
step("zip:embedded", _path, prefix)
elif len(c) == 2:
elif n == 2:
_, _path = c
prefix = None
_path = os.path.normpath(_path)
step("zip:embedded", _path, prefix)
elif len(c) == 1:
prefix = None
_path = None
step("zip:embedded", _path, prefix)
step("zip:embedded", _path)
elif n == 1:
step("zip:embedded")
else:
raise ValueError(
":zip invalid call signature, use: "
Expand All @@ -786,11 +787,9 @@ def commands_step(path, commands):
else:
batch.append(c)
if batch:
step("sh", path, "\n".join(batch))
step("sh", "\n".join(batch))
batch.clear()

step("reset:workdir")

for claim in claims:
if isinstance(claim, str):
path = claim
Expand Down Expand Up @@ -877,12 +876,13 @@ def commands_step(path, commands):
hash(path_from_pattern)
else:
hash(path)

if patterns:
step("clear:filter")
else:
raise ValueError("Unsupported source_path item: {}".format(claim))

if build_step:
build_plan.append(build_step)
build_step = []

self._source_paths = source_paths
return build_plan

Expand All @@ -895,125 +895,145 @@ def execute(self, build_plan, zip_stream, query):
sh_work_dir = None
pf = None

for action in build_plan:
cmd = action[0]
if cmd.startswith("zip"):
ts = 0 if cmd == "zip:embedded" else None
source_path, prefix = action[1:]
if not sh_work_dir:
sh_work_dir = tf_work_dir
log.debug("WORKDIR: %s", sh_work_dir)
if source_path:
if not os.path.isabs(source_path):
source_path = os.path.normpath(
os.path.join(sh_work_dir, source_path)
)
else:
source_path = sh_work_dir
if os.path.isdir(source_path):
if pf:
self._zip_write_with_filter(
zs, pf, source_path, prefix, timestamp=ts
)
for step in build_plan:
# init step
sh_work_dir = tf_work_dir
if pf:
pf.reset()
pf = None

log.debug("STEPDIR: %s", sh_work_dir)

# execute step actions
for action in step:
cmd = action[0]
if cmd.startswith("zip"):
ts = 0 if cmd == "zip:embedded" else None

source_path, prefix = None, None
n = len(action)
if n == 2:
source_path = action[1]
elif n == 3:
source_path, prefix = action[1:]

if source_path:
if not os.path.isabs(source_path):
source_path = os.path.normpath(
os.path.join(sh_work_dir, source_path)
)
else:
zs.write_dirs(source_path, prefix=prefix, timestamp=ts)
else:
zs.write_file(source_path, prefix=prefix, timestamp=ts)
elif cmd == "pip":
runtime, pip_requirements, prefix, tmp_dir = action[1:]
with install_pip_requirements(query, pip_requirements, tmp_dir) as rd:
if rd:
source_path = sh_work_dir
if os.path.isdir(source_path):
if pf:
self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0)
else:
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "poetry":
(
runtime,
path,
poetry_export_extra_args,
prefix,
) = action[1:]
log.info("poetry_export_extra_args: %s", poetry_export_extra_args)
with install_poetry_dependencies(
query, path, poetry_export_extra_args
) as rd:
if rd:
if pf:
self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0)
else:
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "npm":
runtime, npm_requirements, prefix, tmp_dir = action[1:]
with install_npm_requirements(query, npm_requirements, tmp_dir) as rd:
if rd:
if pf:
self._zip_write_with_filter(zs, pf, rd, prefix, timestamp=0)
self._zip_write_with_filter(
zs, pf, source_path, prefix, timestamp=ts
)
else:
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "sh":
with tempfile.NamedTemporaryFile(mode="w+t", delete=True) as temp_file:
path, script = action[1:]

if not path:
path = tf_work_dir
if not os.path.isabs(path):
path = os.path.normpath(os.path.join(tf_work_dir, path))

if log.isEnabledFor(DEBUG2):
log.debug("exec shell script ...")
for line in script.splitlines():
sh_log.debug(line)

script = "\n".join(
(
script,
# NOTE: Execute `pwd` to determine the subprocess shell's
# working directory after having executed all other commands.
"retcode=$?",
f"pwd >{temp_file.name}",
"exit $retcode",
zs.write_dirs(source_path, prefix=prefix, timestamp=ts)
else:
zs.write_file(source_path, prefix=prefix, timestamp=ts)
elif cmd == "pip":
runtime, pip_requirements, prefix, tmp_dir = action[1:]
with install_pip_requirements(
query, pip_requirements, tmp_dir
) as rd:
if rd:
if pf:
self._zip_write_with_filter(
zs, pf, rd, prefix, timestamp=0
)
else:
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "poetry":
(
runtime,
path,
poetry_export_extra_args,
prefix,
) = action[1:]
log.info("poetry_export_extra_args: %s", poetry_export_extra_args)
with install_poetry_dependencies(
query, path, poetry_export_extra_args
) as rd:
if rd:
if pf:
self._zip_write_with_filter(
zs, pf, rd, prefix, timestamp=0
)
else:
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "npm":
runtime, npm_requirements, prefix, tmp_dir = action[1:]
with install_npm_requirements(
query, npm_requirements, tmp_dir
) as rd:
if rd:
if pf:
self._zip_write_with_filter(
zs, pf, rd, prefix, timestamp=0
)
else:
# XXX: timestamp=0 - what actually do with it?
zs.write_dirs(rd, prefix=prefix, timestamp=0)
elif cmd == "sh":
with tempfile.NamedTemporaryFile(
mode="w+t", delete=True
) as temp_file:
script = action[1]

if log.isEnabledFor(DEBUG2):
log.debug("exec shell script ...")
for line in script.splitlines():
sh_log.debug(line)

script = "\n".join(
(
script,
# NOTE: Execute `pwd` to determine the subprocess shell's
# working directory after having executed all other commands.
"retcode=$?",
f"pwd >{temp_file.name}",
"exit $retcode",
)
)
)

p = subprocess.Popen(
script,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=path,
)
p = subprocess.Popen(
script,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=sh_work_dir,
)

call_stdout, call_stderr = p.communicate()
exit_code = p.returncode
log.debug("exit_code: %s", exit_code)
if exit_code != 0:
raise RuntimeError(
"Script did not run successfully, exit code {}: {} - {}".format(
exit_code,
call_stdout.decode("utf-8").strip(),
call_stderr.decode("utf-8").strip(),
call_stdout, call_stderr = p.communicate()
exit_code = p.returncode
log.debug("exit_code: %s", exit_code)
if exit_code != 0:
raise RuntimeError(
"Script did not run successfully, exit code {}: {} - {}".format(
exit_code,
call_stdout.decode("utf-8").strip(),
call_stderr.decode("utf-8").strip(),
)
)
)

temp_file.seek(0)
# NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop
sh_work_dir = temp_file.read().strip()
temp_file.seek(0)
# NOTE: This var `sh_work_dir` is consumed in cmd == "zip" loop
sh_work_dir = temp_file.read().strip()
log.debug("WORKDIR: %s", sh_work_dir)

elif cmd == "set:workdir":
path = action[1]
sh_work_dir = os.path.normpath(os.path.join(tf_work_dir, path))
log.debug("WORKDIR: %s", sh_work_dir)

elif cmd == "reset:workdir":
sh_work_dir = tf_work_dir
log.debug("WORKDIR: %s", sh_work_dir)
elif cmd == "set:filter":
patterns = action[1]
pf = ZipContentFilter(args=self._args)
pf.compile(patterns)
elif cmd == "clear:filter":
pf.reset()
pf = None
elif cmd == "set:filter":
patterns = action[1]
pf = ZipContentFilter(args=self._args)
pf.compile(patterns)

@staticmethod
def _zip_write_with_filter(
Expand Down Expand Up @@ -1616,11 +1636,11 @@ def prepare_command(args):
content_hash = content_hash.hexdigest()

# Generate a unique filename based on the hash.
filename = os.path.join(artifacts_dir, "{}.zip".format(content_hash))
zip_filename = os.path.join(artifacts_dir, "{}.zip".format(content_hash))

# Compute timestamp trigger
was_missing = False
filename_path = os.path.join(os.getcwd(), filename)
filename_path = os.path.join(os.getcwd(), zip_filename)
if recreate_missing_package:
if os.path.exists(filename_path):
st = os.stat(filename_path)
Expand All @@ -1633,7 +1653,7 @@ def prepare_command(args):

# Replace variables in the build command with calculated values.
build_data = {
"filename": filename,
"filename": zip_filename,
"runtime": runtime,
"artifacts_dir": artifacts_dir,
"build_plan": build_plan,
Expand All @@ -1653,7 +1673,7 @@ def prepare_command(args):
# Output the result to Terraform.
json.dump(
{
"filename": filename,
"filename": zip_filename,
"build_plan": build_plan,
"build_plan_filename": build_plan_filename,
"timestamp": str(timestamp),
Expand Down
2 changes: 1 addition & 1 deletion tests/test_package_toml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def test_build_manager_failing_command():
bpm = BuildPlanManager(args=Mock())
with raises(Exception):
bpm.execute(
build_plan=[["sh", "/tmp", "NOTACOMMAND"]],
build_plan=[[["sh", "/tmp", "NOTACOMMAND"]]],
zip_stream=None,
query=None,
)
Expand Down
Loading