Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Respect CLJ_JVM_OPTS env var options when downloading clojure-tools #73

Merged
merged 2 commits into from
Oct 25, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
@@ -61,15 +61,23 @@ jobs:
- v1-dependencies-{{ checksum "deps.edn" }}
# fallback to using the latest cache if no exact match is found
- v1-dependencies-
- run:
name: Install babashka
command: |
bash <(curl -s https://raw.githubusercontent.com/borkdude/babashka/master/install) --dir ~
sudo mv ~/bb /usr/local/bin/bb
# - run:
# name: Run babashka tests
# name: Install babashka
# command: |
# bb babashka-test
# bash <(curl -s https://raw.githubusercontent.com/borkdude/babashka/master/install) --dir ~
# sudo mv ~/bb /usr/local/bin/bb

- run:
name: Install babashka REVIEW
command: |
curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
chmod +x ./install
sudo ./install --version 1.0.165-SNAPSHOT

- run:
name: Run babashka tests
command: |
bb babashka-test
- save_cache:
paths:
- ~/.m2
25 changes: 22 additions & 3 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
@@ -30,10 +30,29 @@ jobs:
- name: Install Clojure
uses: DeLaGuardo/[email protected]
with:
bb: '0.10.163'
# bb: '0.10.163'
cli: '1.10.3.1013'
lein: '2.9.10'

- name: Install Babashka REVIEW
if: "!startsWith (matrix.os, 'win')"
run: |
curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
chmod +x ./install
./install --version 1.0.165-SNAPSHOT

- name: Install Babashka REVIEW (win)
if: "startsWith (matrix.os, 'win')"
shell: cmd
run: |
set LOCAL_BIN_DIR=%USERPROFILE%\.local\bin

mkdir %LOCAL_BIN_DIR%

curl.exe -fsSL https://github.com/babashka/babashka-dev-builds/releases/download/v1.0.165-SNAPSHOT/babashka-1.0.165-SNAPSHOT-windows-amd64.zip -o bb-snap.zip || exit /b
unzip bb-snap.zip -d %LOCAL_BIN_DIR% || exit /b
echo %LOCAL_BIN_DIR%>> %GITHUB_PATH%

- name: Cache clojure dependencies
uses: actions/cache@v3
with:
@@ -46,8 +65,8 @@ jobs:
- name: Run JVM tests
run: bb jvm-test

# - name: Run babashka tests
# run: bb babashka-test
- name: Run babashka tests
run: bb babashka-test

- name: Create ubejar
if: "startsWith (matrix.os, env.UBERJAR_OS) && env.UBERJAR_JDK == matrix.jdk"
104 changes: 59 additions & 45 deletions src/borkdude/deps.clj
Original file line number Diff line number Diff line change
@@ -309,35 +309,34 @@ For more info, see:
[dest-dir]
(let [dest-file (.getCanonicalPath (io/file dest-dir "ClojureToolsDownloader.java"))]
(spit dest-file
(str "/** Auto-generated by " *file* ". **/"
"package borkdude.deps;"
"import java.io.FileOutputStream;"
"import java.io.IOException;"
"import java.net.URLConnection;"
"import java.net.HttpURLConnection;"
"import java.net.URL;"
"import java.nio.channels.Channels;"
"import java.nio.channels.FileChannel;"
"import java.nio.channels.ReadableByteChannel;"
"public class ClojureToolsDownloader {"
" public static void main (String[] args) {"
" try {"
" URL url = new URL(args[0]);"
;; " System.err.println (\":0 \" +args [0]+ \" :1 \"+args [1]);"
" URLConnection conn = url.openConnection();"
" if (conn instanceof HttpURLConnection)"
" {((HttpURLConnection) conn).setInstanceFollowRedirects(true);}"
" ReadableByteChannel readableByteChannel = Channels.newChannel(conn.getInputStream());"
" FileOutputStream fileOutputStream = new FileOutputStream(args[1]);"
" FileChannel fileChannel = fileOutputStream.getChannel();"
" fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);"
" System.exit(0);"
" } catch (IOException e) {"
" e.printStackTrace();"
" System.exit(1); }}}"))
(str "
/** Auto-generated by " *file* ". **/
package borkdude.deps;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URLConnection;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.channels.Channels;import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
public class ClojureToolsDownloader {
public static void main (String[] args) {
try {
URL url = new URL(args[0]);
// System.err.println (\":0 \" +args [0]+ \" :1 \"+args [1]);
URLConnection conn = url.openConnection();
if (conn instanceof HttpURLConnection)
{((HttpURLConnection) conn).setInstanceFollowRedirects(true);}
ReadableByteChannel readableByteChannel = Channels.newChannel(conn.getInputStream());
FileOutputStream fileOutputStream = new FileOutputStream(args[1]);
FileChannel fileChannel = fileOutputStream.getChannel();
fileOutputStream.getChannel().transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
System.exit(0);
} catch (IOException e) {
e.printStackTrace();
System.exit(1); }}}"))
dest-file))


(defn- clojure-tools-download-java
"Downloads URL file to DEST-ZIP-FILE by invoking `java` with JVM-OPTS
on a `.java` program file, and returns true on success. Requires
@@ -363,32 +362,30 @@ For more info, see:
and extracts in-place the clojure tools jar file and other important
files.

The download is attempted by the following two methods in order:

1. via java, invoking a java subprocess with
JAVA-ARGS-WITH-CLJ-JVM-OPTS options, or

2. directly from this process.
The download is attempted directly from this process, unless
JAVA-ARGS-WITH-CLJ-JVM-OPTS is set, in which case a java subprocess
is created to download the archive passing in its value as command
line options.

It calls `*exit-fn*` if it cannot download the archive, with
instructions how to manually download it."
[out-dir java-args-with-clj-jvm-opts]
[out-dir java-args-with-clj-jvm-opts & {:keys [debug?] :as _opts}]
(let [{:keys [ct-error-exit-code ct-url-str ct-zip-name]} @clojure-tools-info*
dir (io/file out-dir)
zip-file (io/file out-dir ct-zip-name)]
(when-not (.exists zip-file)
(warn "Downloading" ct-url-str "to" (str zip-file))
(.mkdirs dir)
(or (do (warn "\nAttempting download using " ::java-subprocess-method...
"(respects CLJ_JVM_OPTS, requires Java11+)")
(clojure-tools-download-java ct-url-str (str zip-file) java-args-with-clj-jvm-opts))
(do (warn "\nAttempting download using " ::direct-method...)
(or (when java-args-with-clj-jvm-opts
(when debug? (warn "Attempting download using java subprocess... (requires Java11+"))
(clojure-tools-download-java ct-url-str (str zip-file) java-args-with-clj-jvm-opts))
(do (when debug? (warn "Attempting direct download..."))
(clojure-tools-download-direct ct-url-str zip-file))
(*exit-fn* ct-error-exit-code (str "\nError: Cannot download Clojure tools."
(*exit-fn* ct-error-exit-code (str "Error: Cannot download Clojure tools."
" Please download manually from " ct-url-str
" to " (str (io/file dir ct-zip-name)) ".\n"))
" to " (str (io/file dir ct-zip-name))))
{:url ct-url-str :dest-dir (str dir)}))
(warn "\nUnziping" (str zip-file) "...")
(warn "Unziping" (str zip-file) "...")
(unzip zip-file (.getPath dir))
(.delete zip-file))
(warn "Successfully installed clojure tools!"))
@@ -558,10 +555,26 @@ For more info, see:
f))

(defn -main
"See `help-text`."
"See `help-text`.

In addition

- the values of the `CLJ_JVM_OPTS` and `JAVA_OPTIONS` environment
variables are passed to the java subprocess as command line options
when downloading dependencies and running any other commands
respectively.

- if the clojure tools jar cannot be located and the clojure tools
archive is not found, an attempt is made to download the archive
from the official site and extract its contents locally. The archive
is downloaded from this process directly, unless the `CLJ_JVM_OPTS`
env variable is set and a succesful attempt is made to download the
archive by invoking a java subprocess passing the env variable value
as command line options."
[& command-line-args]
(let [opts (parse-args command-line-args)
{:keys [ct-base-dir ct-jar-name]} @clojure-tools-info*
debug? (*getenv-fn* "DEPS_CLJ_DEBUG")
java-cmd (get-java-cmd)
env-tools-dir (or
;; legacy name
@@ -588,9 +601,10 @@ For more info, see:
(when (.exists tools-jar) (.getPath tools-jar))
(binding [*out* *err*]
(warn "Clojure tools not yet in expected location:" (str tools-jar))
(clojure-tools-jar-download libexec-dir (vec (concat clj-jvm-opts
proxy-settings
["-Xms256m"])))
(let [java-clj-jvm-opts (when clj-jvm-opts (vec (concat clj-jvm-opts
proxy-settings
["-Xms256m"])))]
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the -Xms256m argument for?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I picked it up from the clj-main-cmd a little bit further down the code, I assumed it was for some unknown to me use, shall I remove it?

(clojure-tools-jar-download libexec-dir java-clj-jvm-opts :debug? debug?))
tools-jar))
mode (:mode opts)
exec? (= :exec mode)
34 changes: 9 additions & 25 deletions test/borkdude/deps_test.clj
Original file line number Diff line number Diff line change
@@ -239,10 +239,10 @@
file)))

(deftest clojure-tools-download
;; Test clojure tools download methods:
;; Test clojure tools download methods
;;
;; - via java subprocess, respecting CLJ_JVM_OPTS (requires java11+).
;; - direct download, if the above fails.
;; - via java subprocess, when CLJ_JVM_OPTS (requires java11+).
;; - direct download, when the above is not ran or fails.
;; - manual download, simulating a user following manual instructions.
(let [java-version (java-major-version-get)
{:keys [ct-error-exit-code ct-jar-name ct-url-str ct-zip-name]} @@#'deps/clojure-tools-info*]
@@ -261,25 +261,12 @@
(is (fs/exists? dest-zip-file)))))))

(when (>= java-version 11)
(testing "java downloader called from -main"
(testing "java downloader called from -main when CLJ_JVM_OPTS is set"
(fs/with-temp-dir
[temp-dir {}]
(let [dest-jar-file (fs/file temp-dir ct-jar-name)]
(with-redefs [deps/clojure-tools-download-direct
(fn [& _] (throw (Exception. "Direct hould not be called.")))]
(binding [deps/*getenv-fn* #(or (get {"DEPS_CLJ_TOOLS_DIR" (str temp-dir)
"CLJ_JVM_OPTS" nil} %)
(System/getenv %))]

(deps-main-throw "--version")
(is (fs/exists? dest-jar-file)))))))

(testing "java downloader called from -main with CLJ_JVM_OPTS set"
(fs/with-temp-dir
[temp-dir {}]
(let [dest-jar-file (fs/file temp-dir ct-jar-name)]
(with-redefs [deps/clojure-tools-download-direct
(fn [& _] (throw (Exception. "Direct hould not be called.")))]
(fn [& _] (throw (Exception. "Direct should not be called.")))]
(let [xx-pclf "-XX:+PrintCommandLineFlags"
xx-gc-threads "-XX:ConcGCThreads=1"
sh-args (get-shell-command-args
@@ -299,15 +286,12 @@
(is (= true (deps/clojure-tools-download-direct url-str dest-zip-file)))
(is (fs/exists? dest-zip-file)))))

(testing "direct downloader called from -main"
(testing "direct downloader called from -main (CLJ_JVM_OPTS not set)"
(fs/with-temp-dir
[temp-dir {}]
(let [dest-jar-file (fs/file temp-dir ct-jar-name)]
(with-redefs [deps/clojure-tools-download-java
;; indicate failure so that direct downloader is
;; called.
(fn [_url _dest-zip-file _jvm-opts]
false)]
(fn [& _] (throw (Exception. "Java subprocess should not be called.")))]
(binding [deps/*getenv-fn* #(or (get {"DEPS_CLJ_TOOLS_DIR" (str temp-dir)
"CLJ_JVM_OPTS" nil} %)
(System/getenv %))]
@@ -323,9 +307,9 @@
dest-jar-file (fs/file temp-dir ct-jar-name)]
(fs/copy tools-zip-file dest-zip-file) ;; user copies downloaded file
(with-redefs [deps/clojure-tools-download-java
(fn [& _] (throw (Exception. "Java hould not be called.")))
(fn [& _] (throw (Exception. "Java should not be called.")))
deps/clojure-tools-download-direct
(fn [& _] (throw (Exception. "Direct hould not be called.")))]
(fn [& _] (throw (Exception. "Direct should not be called.")))]
(binding [deps/*getenv-fn* #(or (get {"DEPS_CLJ_TOOLS_DIR" (str temp-dir)} %)
(System/getenv %))]