Skip to content

Commit

Permalink
took functions to format CMD and ENV for passing to daemon
Browse files Browse the repository at this point in the history
  • Loading branch information
@TimoKramer
TimoKramer committed Feb 25, 2020
1 parent 1217e31 commit 9d05e0d
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 68 deletions.
95 changes: 32 additions & 63 deletions src/bob/execution/internals.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,24 +100,24 @@
(if (contains? result :message)
(u/log-and-fail "Could not build image:" (:message result))
(:Id result))))
([image step evars] (let [resource (:needs_resource step)
working-dir (when resource (str "/root/" resource))
cmd (:cmd step)
_ (log/debugf "Creating new container with:
image: %s
cmd: %s
evars: %s
working-dir: %s"
image
cmd
evars
working-dir)
evars-nilled (if (empty? evars) nil evars)
result (docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image image
:Cmd cmd
:Env evars-nilled
:WorkingDir working-dir}}})]
([image step evars] (let [resource (:needs_resource step)
working-dir (when resource (str "/root/" resource))
cmd (u/sh-tokenize! (:cmd step))
formatted-evars (u/format-env-vars evars)
_ (log/debugf "Creating new container with:
image: %s
cmd: %s
evars: %s
working-dir: %s"
image
cmd
formatted-evars
working-dir)
result (docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image image
:Cmd cmd
:Env formatted-evars
:WorkingDir working-dir}}})]
(if (contains? result :message)
(u/log-and-fail "Could not build image:" (:message result))
(:Id result)))))
Expand All @@ -137,11 +137,7 @@
"Get stdout and stderr logs from a container."
[id reaction-fn]
(let [log-client (docker/client {:category :containers
:conn (docker/connect {:uri "unix:///var/run/docker.sock"
:connect-timeout 10
:read-timeout 2000
:write-timeout 2000
:call-timeout 30000})})
:conn (docker/connect {:uri "unix:///var/run/docker.sock"})})
log-stream (docker/invoke log-client {:op :ContainerLogs
:params
{:id id
Expand Down Expand Up @@ -199,8 +195,7 @@
result (docker/invoke states/containers
{:op :ContainerDelete
:params {:id id
:force force-flag}})
_ (log/debugf (str result))]
:force force-flag}})]
(if-let [message (get result :message)]
(u/log-and-fail "Could not delete container:" message)
result)))
Expand All @@ -212,22 +207,18 @@
(docker/doc states/images :ImageCreate)
(docker/invoke states/images {:op :ImageCreate :params {:fromImage "clojure" :tag "latest"}})
(docker/invoke states/images {:op :ImageDelete :params {:name "clojure:latest"}})
(def myid (:Id (docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image "oracle/graalvm-ce:19.3.0"
:Cmd "sh -c 'touch test.txt && echo $PATH >> test.txt'"
:Env ""
:WorkingDir nil}}})))
(docker/invoke states/containers {:op :ContainerStart :params {:id myid}})
(:StatusCode (docker/invoke states/containers {:op :ContainerWait :params {:id myid}}))

(docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image "oracle/graalvm-ce:19.3.0"
:Cmd "gu install native-image"
:Env {}
:WorkingDir nil}}})

(docker/invoke states/containers {:op :ContainerCreate
:params {:body {:Image "bobcd/bob:latest"
:Cmd ["ls"]
:Env ["FOO=bar"]
:WorkingDir ""}}})
(build "dbobcd/bob:latest"
(build "bobcd/bob:latest"
{:needs_resource "source"
:cmd ["ls"]}
["FOO=bar"])
:cmd "ls"}
{:FOO "bar"})

(defn validate-name [name]
(let [split-name (cs/split name #":")]
Expand Down Expand Up @@ -261,32 +252,10 @@
(docker/invoke states/images {:op :ImageCreate
:params {:fromImage "alpine"
:tag "3.11.3"}})
(status-of "lucid_pasteur")
(:ExitCode (:State (docker/invoke states/containers {:op :ContainerInspect :params {:id "82edce1bfdea"}})))

(future
(with-open [rdr (clojure.java.io/reader (char-array "foobar\n"))]
(loop [r (java.io.BufferedReader. rdr)]
(when-let [line (.readLine r)]
((fn [l] (prn l)) line)
(recur r)))))
((fn [l] l) "foobar")

(docker/invoke states/containers {:op :ContainerStart :params {:id "kind_burnell"}})
(run "kind_burnell" "run-id")

(log/infof "Creating new container with:
image: %s
entry point: %s
env vars: %s
working dir: %s"
"busybox:musl"
"echo Hello"
{:k1 "v1"
:k2 "v2"}
"/root")

(docker/invoke states/containers {:op :ContainerWait :params {:id "civicrm"}})
(docker/invoke states/containers {:op :ContainerStart :params {:id "interesting_swirles"}})
(:StatusCode (docker/invoke states/containers {:op :ContainerWait :params {:id "interesting_swirles"}}))

(docker/invoke states/containers {:op :ContainerDelete :params {:id "16a4b4e860b4" :force true}})
(delete-container "3c12417d5a7e" true)
Expand Down
60 changes: 59 additions & 1 deletion src/bob/util.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,5 +55,63 @@
(do (log/errorf errormessage)
(f/fail errormessage))))

;; TODO: Optimize as mentioned in:
;; https://www.reddit.com/r/Clojure/comments/8zurv4/critical_code_review_and_feedback/
(defn sh-tokenize!
"Tokenizes a shell command given as a string into the command and its args.
Either returns a list of tokens or throws an IllegalStateException.
Sample input: sh -c 'while sleep 1; do echo \\\"${RANDOM}\\\"; done'
Output: [sh, -c, while sleep 1; do echo \"${RANDOM}\"; done]"
[^String command]
(let [[escaped?
current-arg
args
state] (loop [cmd command
escaped? false
state :no-token
current-arg ""
args []]
(if (or (nil? cmd)
(zero? (count cmd)))
[escaped? current-arg args state]
(let [char ^Character (first cmd)]
(if escaped?
(recur (rest cmd) false state (str current-arg char) args)
(case state
:single-quote (if (= char \')
(recur (rest cmd) escaped? :normal current-arg args)
(recur (rest cmd) escaped? state (str current-arg char) args))
:double-quote (case char
\" (recur (rest cmd) escaped? :normal current-arg args)
\\ (let [next (second cmd)]
(if (or (= next \")
(= next \\))
(recur (drop 2 cmd) escaped? state (str current-arg next) args)
(recur (drop 2 cmd) escaped? state (str current-arg char next) args)))
(recur (rest cmd) escaped? state (str current-arg char) args))
(:no-token :normal) (case char
\\ (recur (rest cmd) true :normal current-arg args)
\' (recur (rest cmd) escaped? :single-quote current-arg args)
\" (recur (rest cmd) escaped? :double-quote current-arg args)
(if-not (Character/isWhitespace char)
(recur (rest cmd) escaped? :normal (str current-arg char) args)
(if (= state :normal)
(recur (rest cmd) escaped? :no-token "" (conj args current-arg))
(recur (rest cmd) escaped? state current-arg args))))
(throw (IllegalStateException.
(format "Invalid shell command: %s, unexpected token %s found." command state))))))))]
(if escaped?
(conj args (str current-arg \\))
(if (not= state :no-token)
(conj args current-arg)
args))))

(defn format-env-vars
[env-vars]
(map #(format "%s=%s" (name (first %)) (last %))
env-vars))

(comment
(log-and-fail "foo" :bar))
(log-and-fail "foo" :bar)

(db/logs-of states/db {:run-id "1"}))
8 changes: 4 additions & 4 deletions test/bob/execution/internals_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@
(with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})}
#(is (= "83996a1d" (build "foo:bar"
{:needs_resource "source"
:cmd ["ls"]}
["FOO=bar"])))))
:cmd "ls"}
{:FOO "bar"})))))

(testing "successful build single param"
(with-redefs-fn {#'docker/invoke (constantly {:Id "83996a1d", :Warnings []})}
Expand All @@ -94,8 +94,8 @@
(with-redefs-fn {#'docker/invoke (constantly {:message "failed"})}
#(is (f/failed? (build "foo:bar"
{:needs_resource "source"
:cmd ["ls"]}
["FOO=bar"]))))))
:cmd "ls"}
{:FOO "bar"}))))))

(deftest container-status
(testing "successful status fetch"
Expand Down
14 changes: 14 additions & 0 deletions test/bob/util_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,17 @@
(gen/generate gen/string-alphanumeric)
(gen/generate gen/string-alphanumeric)
(gen/generate gen/string-alphanumeric))))))

(deftest shell-arg-tokenize-test
(testing "tokenizing a Shell command"
(is (= (sh-tokenize! "sh -c \"while sleep 1; do echo ${RANDOM}; done\"")
["sh" "-c" "while sleep 1; do echo ${RANDOM}; done"])))
(testing "tokenizing a Shell command with escaped double quotes"
(is (= (sh-tokenize! "sort -t \"\t\" -k2 test > test-sorted")
["sort" "-t" "\t" "-k2" "test" ">" "test-sorted"]))))

(deftest format-env-vars-test
(testing "format a map of env vars for Docker"
(is (= (format-env-vars {:var1 "val1"
:var2 "val2"})
["var1=val1" "var2=val2"]))))

0 comments on commit 9d05e0d

Please sign in to comment.