RFC: Simpelify packaging by splitting out collection of git version info

DISTRIBUTING.md

@@ -16,6 +16,23 @@ GPL licensed, as various dependent libraries such as `FFTW`, `Rmath`,
 `SuiteSparse`, and `git` are GPL licensed. We do hope to have a
 non-GPL distribution of Julia in the future.
 
+Versioning and Git
+------------------
+The Makefile uses both the `VERSION` file and commit hashes and tags from the
+git repository to generate the `base/version_git.jl` with information we use to
+fill the splash screen and the `versioninfo()` output. If you for some reason
+don't want to have the git repository availible when building you should
+pregenerate the `base/version_git.jl` file with:
+
+    make -C base version_git.jl.phony
+
+For some dependencies we use patched versions that will be fetched via
+`git submodules`. Those might be prefetched by:
+
+    git submodule update --init
+
+See example for how this is done for building nigtlies at [the debian build servers](https://github.com/staticfloat/julia-nightly-packaging/blob/master/build_ubuntu.sh)
+
 When compiling a tagged release in the git repository, we don't display the
 branch/commit hash info in the splash screen. You can use this line to show
 a release description of up to 45 characters. To set this line you have
@@ -36,16 +53,6 @@ for Debian and Ubuntu-based systems. Although we have not yet experimented
 with it, [Alien](https://wiki.debian.org/Alien) could be used to
 generate Julia packages for various Linux distributions.
 
-Julia looks for git versioning information when building.  If it does
-not find the git executable or the `.git/` directory is unreadable,
-the build process will continue, however all versioning information
-will be unavailable.  This is the case, for instance, on Canonical's
-build servers where the [Ubuntu
-nightlies](https://launchpad.net/~staticfloat/+archive/julianightlies)
-are built.  Therefore, a workaround must be enacted, where the git
-versioning information [is encoded into the
-source](https://github.com/staticfloat/julia-nightly-packaging/blob/master/build_ubuntu.sh#L76-78) before upload for building, and the source is modified to [not attempt to look for it](https://github.com/staticfloat/julia-nightly-packaging/blob/master/nogit-workaround.patch) when building.
-
 By default, Julia loads `$PREFIX/etc/julia/juliarc.jl` as an
 installation-wide initialization file. This file can be used by
 distribution managers to provide paths to various binaries such as a

Make.inc

@@ -49,8 +49,19 @@ endif
 # disable automatic Makefile rules
 .SUFFIXES:
 
+# find out if git repository is availible
+ifeq ($(shell [ -e $(JULIAHOME)/.git ] && echo true || echo "Warning: git information unavailable; versioning information limited" >&2), true)
+NO_GIT = 0
+else
+NO_GIT = 1
+endif
+
 JULIA_VERSION = $(shell cat $(JULIAHOME)/VERSION)
+ifneq ($(NO_GIT), 1)
 JULIA_COMMIT = $(shell git rev-parse --short=10 HEAD)
+else
+JULIA_COMMIT = $JULIA_VERSION
+endif
 
 # LLVM Options
 LLVMROOT = $(BUILD)

Makefile

@@ -42,8 +42,12 @@ julia-release-symlink:
 	@ln -sf $(BUILD)/bin/julia-$(DEFAULT_REPL) julia
 
 julia-debug julia-release:
+ifneq ($(NO_GIT), 1)
 	@-git submodule init --quiet
 	@-git submodule update
+else
+	$(warn "Submodules could not be updated because git is unavailible")
+endif
 	@$(MAKE) $(QUIET_MAKE) -C deps
 	@$(MAKE) $(QUIET_MAKE) -C src lib$@
 	@$(MAKE) $(QUIET_MAKE) -C base

base/.gitignore

@@ -4,3 +4,4 @@
 /fenv_constants.jl
 /file_constants.jl
 /uv_constants.jl
+/version_git.jl

base/Makefile

@@ -3,38 +3,9 @@ include ../Make.inc
 
 PCRE_CONST = 0x[0-9a-fA-F]+|[-+]?\s*[0-9]+
 
-# These are all the values needed for the BUILD_INFO struct in build_h.jl
-version_string = $(shell cat ../VERSION)
-commit = $(shell git rev-parse HEAD 2>/dev/null)
-commit_short = $(shell git rev-parse --short HEAD 2>/dev/null)
-git_branch = $(shell git branch 2>/dev/null | sed -n '/\* /s///p')
-
-last_tag = $(shell git describe --tags --abbrev=0 2>/dev/null)
-tagged_commit = $(shell [ $$(git describe --tags --exact-match 2>/dev/null) ] && echo true || echo false)
-
-origin = $(shell [ -d ../.git/refs/remotes/origin ] && echo "origin/")
-
-build_number = $(shell git rev-list --count HEAD ^"$(last_tag)" 2>/dev/null || echo 0)
-fork_master_distance = $(shell git rev-list --count HEAD ^"$(origin)master" 2>/dev/null || echo 0)
-fork_master_timestamp = $(shell git show -s $$(git merge-base HEAD $(origin)master 2>/dev/null) --format=format:"%ct" 2>/dev/null || echo 0)
-
-git_time = $(shell git log -1 --pretty=format:%ct 2>/dev/null)
-ifneq ($(git_time), )
-	ifneq (,$(filter $(OS), Darwin FreeBSD))
-		date_string = "$(shell /bin/date -jr $(git_time) -u '+%Y-%m-%d %H:%M %Z')"
-	else
-		date_string = "$(shell /bin/date --date=@$(git_time) -u '+%Y-%m-%d %H:%M %Z')"
-	endif
-else
-	date_string = ""
-endif
-
-dirty = $(shell [ -z "$(shell git status --porcelain 2>/dev/null)" ] && echo "" || echo "*" )
-
 TAGGED_RELEASE_BANNER = ""
 
-
-all: pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl
+all: pcre_h.jl errno_h.jl build_h.jl.phony fenv_constants.jl file_constants.jl uv_constants.jl version_git.jl.phony
 
 pcre_h.jl:
 	@$(call PRINT_PERL, $(CPP) -dM $(shell $(PCRE_CONFIG) --prefix)/include/pcre.h | perl -nle '/^\s*#define\s+PCRE_(\w*)\s*\(?($(PCRE_CONST))\)?\s*$$/ and print "const $$1 = uint32($$2)"' | sort > $@)
@@ -69,33 +40,8 @@ else
 	@echo "const USE_BLAS64 = false" >> $@
 endif
 	@echo "const SYSCONFDIR = \"$(SYSCONFDIR)\"" >> $@
-
-	@echo "immutable BuildInfo" >> $@
-	@echo "    version_string::String" >> $@
-	@echo "    commit::String" >> $@
-	@echo "    commit_short::String" >> $@
-	@echo "    branch::String" >> $@
-	@echo "    build_number::Int" >> $@
-	@echo "    date_string::String" >> $@
-	@echo "    tagged_commit::Bool" >> $@
-	@echo "    fork_master_distance::Int" >> $@
-	@echo "    fork_master_timestamp::Float64" >> $@
-	@echo "    TAGGED_RELEASE_BANNER::String" >> $@
-	@echo "end" >> $@
-
-
-	@echo "const BUILD_INFO = BuildInfo( \
-			'\"$(version_string)\"', \
-			'\"$(commit)\"', \
-			'\"$(commit_short)$(dirty)\"', \
-			'\"$(git_branch)\"', \
-			$(build_number), \
-			'\"$(date_string)\"', \
-			$(tagged_commit), \
-			$(fork_master_distance), \
-			$(fork_master_timestamp)., \
-			'\"$(TAGGED_RELEASE_BANNER)\"' \
-			)" | xargs >> $@
+	@echo "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@
+	@echo "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@
 
 	@# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl,
 	@# ensuring we rebuild the system image as infrequently as possible
@@ -106,7 +52,30 @@ endif
 		rm -f $@; \
 	fi
 
-.PHONY: build_h.jl.phony
+version_git.jl.phony:
+ifneq ($(NO_GIT), 1)
+	@sh version_git.sh > $@
+	@# This to avoid touching git_version.jl when it is not modified,
+	@# so that the system image does not need to be rebuilt.
+	@if ! cmp -s $@ version_git.jl; then \
+	    $(call PRINT_PERL,) \
+	    mv $@ version_git.jl; \
+	else \
+	    rm -f $@; \
+	fi
+else
+ifeq ($(shell [ -f version_git.jl ] && echo "true"), true)
+	@# Give warning if boilerplate git is used
+	@if grep -q "Default output if git is not available" version_git.jl; then \
+	    echo "WARNING: Using boilerplate git version info" >&2; \
+	fi
+else
+	$(warning "WARNING: Using boilerplate git version info")
+	@sh version_git.sh NO_GIT > version_git.jl
+endif
+endif
+
+.PHONY: build_h.jl.phony version_git.jl.phony
 
 
 
@@ -115,6 +84,9 @@ clean:
 	rm -f pcre_h.jl
 	rm -f errno_h.jl
 	rm -f build_h.jl
+	rm -f build_h.jl.phony
 	rm -f fenv_constants.jl
 	rm -f uv_constants.jl
 	rm -f file_constants.jl
+	rm -f version_git.jl
+	rm -f version_git.jl.phony

base/pkg/generate.jl

@@ -60,7 +60,7 @@ function package(
                 years:    $years
                 user:     $user
 
-            Julia Version $VERSION [$(Base.BUILD_INFO.commit[1:10])]
+            Julia Version $VERSION [$(Base.GIT_VERSION_INFO.short_commit)]
             """
 
             if isnew

base/sysimg.jl

@@ -22,6 +22,7 @@ end
 include("base.jl")
 include("reflection.jl")
 include("build_h.jl")
+include("version_git.jl")
 include("c.jl")
 
 # core operations & types

base/util.jl

@@ -345,8 +345,8 @@ end
 
 function versioninfo(io::IO=STDOUT, verbose::Bool=false)
     println(io,             "Julia Version $VERSION")
-    if !isempty(BUILD_INFO.commit_short)
-      println(io,             "Commit $(BUILD_INFO.commit_short) ($(BUILD_INFO.date_string))")
+    if !isempty(GIT_VERSION_INFO.commit_short)
+      println(io,             "Commit $(GIT_VERSION_INFO.commit_short) ($(GIT_VERSION_INFO.date_string))")
     end
     println(io,             "Platform Info:")
     println(io,             "  System: ", Sys.OS_NAME, " (", Sys.MACHINE, ")")

base/version.jl

@@ -182,27 +182,27 @@ end
 
 begin
 # Include build number if we've got at least some distance from a tag (e.g. a release)
-build_number = BUILD_INFO.build_number != 0 ? "+$(BUILD_INFO.build_number)" : ""
 try
-    global const VERSION = convert(VersionNumber, "$(BUILD_INFO.version_string)$(build_number)")
+    build_number = GIT_VERSION_INFO.build_number != 0 ? "+$(GIT_VERSION_INFO.build_number)" : ""
+    global const VERSION = convert(VersionNumber, "$(VERSION_STRING)$(build_number)")
 catch e
     println("while creating Base.VERSION, ignoring error $e")
     global const VERSION = VersionNumber(0)
 end
 
-if BUILD_INFO.tagged_commit
-    const commit_string = BUILD_INFO.TAGGED_RELEASE_BANNER
-elseif BUILD_INFO.commit == ""
+if GIT_VERSION_INFO.tagged_commit
+    const commit_string = TAGGED_RELEASE_BANNER
+elseif GIT_VERSION_INFO.commit == ""
     const commit_string = ""
 else
-    local days = int(floor((ccall(:clock_now, Float64, ()) - BUILD_INFO.fork_master_timestamp) / (60 * 60 * 24)))
-    if BUILD_INFO.fork_master_distance == 0
-        const commit_string = "Commit $(BUILD_INFO.commit_short) ($(days) days old master)"
+    local days = int(floor((ccall(:clock_now, Float64, ()) - GIT_VERSION_INFO.fork_master_timestamp) / (60 * 60 * 24)))
+    if GIT_VERSION_INFO.fork_master_distance == 0
+        const commit_string = "Commit $(GIT_VERSION_INFO.commit_short) ($(days) days old master)"
     else
-        const commit_string = "$(BUILD_INFO.branch)/$(BUILD_INFO.commit_short) (fork: $(BUILD_INFO.fork_master_distance) commits, $(days) days)"
+        const commit_string = "$(GIT_VERSION_INFO.branch)/$(GIT_VERSION_INFO.commit_short) (fork: $(GIT_VERSION_INFO.fork_master_distance) commits, $(days) days)"
     end
 end
-commit_date = BUILD_INFO.date_string != "" ? " ($(BUILD_INFO.date_string))": ""
+commit_date = GIT_VERSION_INFO.date_string != "" ? " ($(GIT_VERSION_INFO.date_string))": ""
 
 const banner_plain =
 """

base/version_git.sh

@@ -0,0 +1,61 @@
+# This file collects git info and create a julia file with the GIT_VERSION_INFO struct
+
+echo "# This file was autogenerated in base/version_git.sh"
+echo "immutable GitVersionInfo"
+echo "    commit::String"
+echo "    commit_short::String"
+echo "    branch::String"
+echo "    build_number::Int"
+echo "    date_string::String"
+echo "    tagged_commit::Bool"
+echo "    fork_master_distance::Int"
+echo "    fork_master_timestamp::Float64"
+echo "end"
+echo ""
+
+# If the script didn't ask not to use git info
+if [  "$#" = "1"  -a "$1" = "NO_GIT" ]; then
+    # this comment is used in base/Makefile to distinguish boilerplate
+    echo "# Default output if git is not available."
+    echo "const GIT_VERSION_INFO = GitVersionInfo(\"\" ,\"\" ,\"\" ,0 ,\"\" ,true ,0 ,0.)"
+    exit 0
+fi
+# Collect temporary variables
+origin=$(git config -l 2>/dev/null | grep 'remote\.\w*\.url.*JuliaLang/julia.git' | sed -n 's/remote\.\([a-zA-Z]*\)\..*/\1\//p')
+last_tag=$(git describe --tags --abbrev=0)
+git_time=$(git log -1 --pretty=format:%ct)
+
+#collect the contents
+commit=$(git rev-parse HEAD)
+commit_short=$(git rev-parse --short HEAD)
+if [ -n "$(git status --porcelain)" ]; then
+    # append dirty mark '*' if the repository has uncommited changes
+    commit_short="$commit_short"*
+fi
+branch=$(git branch | sed -n '/\* /s///p')
+build_number=$(git rev-list --count HEAD ^$last_tag)
+
+date_string=$git_time
+if [ "$(uname)" = "Darwin" ] || [ "$(uname)" = "FreeBSD" ]; then
+    date_string="$(/bin/date -jr $git_time -u '+%Y-%m-%d %H:%M %Z')"
+else
+    date_string="$(/bin/date --date="@$git_time" -u '+%Y-%m-%d %H:%M %Z')"
+fi
+if [ $(git describe --tags --exact-match 2> /dev/null) ]; then
+    tagged_commit="true"
+else
+    tagged_commit="false"
+fi
+fork_master_distance=$(git rev-list --count HEAD ^"$(echo $origin)master")
+fork_master_timestamp=$(git show -s $(git merge-base HEAD $(echo $origin)master) --format=format:"%ct")
+
+echo "const GIT_VERSION_INFO = GitVersionInfo("
+echo "    \"$commit\","
+echo "    \"$commit_short\","
+echo "    \"$branch\","
+echo "    $build_number,"
+echo "    \"$date_string\","
+echo "    $tagged_commit,"
+echo "    $fork_master_distance,"
+echo "    $fork_master_timestamp."
+echo ")"