build: backport config for new CI infrastructure

PR-URL: #3642 Reviewed-By: Johan Bergström <[email protected]> Reviewed-By: Alexis Campailla <[email protected]>
nodejs · Nov 24, 2015 · 6fb0b92 · 6fb0b92
1 parent caa16b4
commit 6fb0b92
Show file tree

Hide file tree

Showing 8 changed files with 280 additions and 100 deletions.
diff --git a/Makefile b/Makefile
@@ -7,7 +7,14 @@ DESTDIR ?=
 SIGN ?=
 PREFIX ?= /usr/local
 FLAKY_TESTS ?= run
+STAGINGSERVER ?= node-www
 
+OSTYPE := $(shell uname -s | tr '[A-Z]' '[a-z]')
+
+# Flags for packaging.
+BUILD_DOWNLOAD_FLAGS ?= --download=all
+BUILD_INTL_FLAGS ?= --with-intl=small-icu
+BUILD_RELEASE_FLAGS ?= $(BUILD_DOWNLOAD_FLAGS) $(BUILD_INTL_FLAGS)
 NODE ?= ./node
 
 # Default to verbose builds.
@@ -251,9 +258,45 @@ run-ci:
 
 RAWVER=$(shell $(PYTHON) tools/getnodeversion.py)
 VERSION=v$(RAWVER)
+
+# For nightly builds, you must set DISTTYPE to "nightly", "next-nightly" or
+# "custom". For the nightly and next-nightly case, you need to set DATESTRING
+# and COMMIT in order to properly name the build.
+# For the rc case you need to set CUSTOMTAG to an appropriate CUSTOMTAG number
+
+ifndef DISTTYPE
+DISTTYPE=release
+endif
+ifeq ($(DISTTYPE),release)
+FULLVERSION=$(VERSION)
+else # ifeq ($(DISTTYPE),release)
+ifeq ($(DISTTYPE),custom)
+ifndef CUSTOMTAG
+$(error CUSTOMTAG is not set for DISTTYPE=custom)
+endif # ifndef CUSTOMTAG
+TAG=$(CUSTOMTAG)
+else # ifeq ($(DISTTYPE),custom)
+ifndef DATESTRING
+$(error DATESTRING is not set for nightly)
+endif # ifndef DATESTRING
+ifndef COMMIT
+$(error COMMIT is not set for nightly)
+endif # ifndef COMMIT
+ifneq ($(DISTTYPE),nightly)
+ifneq ($(DISTTYPE),next-nightly)
+$(error DISTTYPE is not release, custom, nightly or next-nightly)
+endif # ifneq ($(DISTTYPE),next-nightly)
+endif # ifneq ($(DISTTYPE),nightly)
+TAG=$(DISTTYPE)$(DATESTRING)$(COMMIT)
+endif # ifeq ($(DISTTYPE),custom)
+FULLVERSION=$(VERSION)-$(TAG)
+endif # ifeq ($(DISTTYPE),release)
+
+DISTTYPEDIR ?= $(DISTTYPE)
+RELEASE=$(shell sed -ne 's/\#define NODE_VERSION_IS_RELEASE \([01]\)/\1/p' src/node_version.h)
 NODE_DOC_VERSION=$(VERSION)
-RELEASE=$(shell $(PYTHON) tools/getnodeisrelease.py)
-PLATFORM=$(shell uname | tr '[:upper:]' '[:lower:]')
+NPMVERSION=v$(shell cat deps/npm/package.json | grep '"version"' | sed 's/^[^:]*: "\([^"]*\)",.*/\1/')
+
 ifeq ($(findstring x86_64,$(shell uname -m)),x86_64)
 DESTCPU ?= x64
 else
@@ -268,61 +311,62 @@ else
 ARCH=x86
 endif
 endif
-TARNAME=node-$(VERSION)
-ifdef NIGHTLY
-TAG = nightly-$(NIGHTLY)
-TARNAME=node-$(VERSION)-$(TAG)
-endif
-TARBALL=$(TARNAME).tar.gz
-BINARYNAME=$(TARNAME)-$(PLATFORM)-$(ARCH)
-BINARYTAR=$(BINARYNAME).tar.gz
-PKG=out/$(TARNAME).pkg
-PACKAGEMAKER ?= /Developer/Applications/Utilities/PackageMaker.app/Contents/MacOS/PackageMaker
 
-PKGSRC=nodejs-$(DESTCPU)-$(RAWVER).tgz
-ifdef NIGHTLY
-PKGSRC=nodejs-$(DESTCPU)-$(RAWVER)-$(TAG).tgz
+# enforce "x86" over "ia32" as the generally accepted way of referring to 32-bit intel
+ifeq ($(ARCH),ia32)
+override ARCH=x86
+endif
+ifeq ($(DESTCPU),ia32)
+override DESTCPU=x86
 endif
 
-dist: doc $(TARBALL) $(PKG)
-
+TARNAME=node-$(FULLVERSION)
+TARBALL=$(TARNAME).tar
+BINARYNAME=$(TARNAME)-$(OSTYPE)-$(ARCH)
+BINARYTAR=$(BINARYNAME).tar
+PKG=$(TARNAME).pkg
+PACKAGEMAKER ?= /Developer/Applications/Utilities/PackageMaker.app/Contents/MacOS/PackageMaker
 PKGDIR=out/dist-osx
 
 release-only:
 	@if [ "$(shell git status --porcelain | egrep -v '^\?\? ')" = "" ]; then \
 		exit 0 ; \
 	else \
-	  echo "" >&2 ; \
+		echo "" >&2 ; \
 		echo "The git repository is not clean." >&2 ; \
 		echo "Please commit changes before building release tarball." >&2 ; \
 		echo "" >&2 ; \
 		git status --porcelain | egrep -v '^\?\?' >&2 ; \
 		echo "" >&2 ; \
 		exit 1 ; \
 	fi
-	@if [ "$(NIGHTLY)" != "" -o "$(RELEASE)" = "1" ]; then \
+	@if [ "$(DISTTYPE)" != "release" -o "$(RELEASE)" = "1" ]; then \
 		exit 0; \
 	else \
-	  echo "" >&2 ; \
+		echo "" >&2 ; \
 		echo "#NODE_VERSION_IS_RELEASE is set to $(RELEASE)." >&2 ; \
-	  echo "Did you remember to update src/node_version.cc?" >&2 ; \
-	  echo "" >&2 ; \
+		echo "Did you remember to update src/node_version.h?" >&2 ; \
+		echo "" >&2 ; \
 		exit 1 ; \
 	fi
 
-pkg: $(PKG)
-
 $(PKG): release-only
 	rm -rf $(PKGDIR)
 	rm -rf out/deps out/Release
-	$(PYTHON) ./configure --download=all --with-intl=small-icu \
-		--without-snapshot --dest-cpu=ia32 --tag=$(TAG)
+	$(PYTHON) ./configure \
+		--dest-cpu=ia32 \
+		--tag=$(TAG) \
+		--without-snapshot \
+		$(CONFIG_FLAGS) $(BUILD_RELEASE_FLAGS)
 	$(MAKE) install V=$(V) DESTDIR=$(PKGDIR)/32
 	rm -rf out/deps out/Release
-	$(PYTHON) ./configure --download=all --with-intl=small-icu \
-		--without-snapshot --dest-cpu=x64 --tag=$(TAG)
+	$(PYTHON) ./configure \
+		--dest-cpu=x64 \
+		--tag=$(TAG) \
+		--without-snapshot \
+		$(CONFIG_FLAGS) $(BUILD_RELEASE_FLAGS)
 	$(MAKE) install V=$(V) DESTDIR=$(PKGDIR)
-	SIGN="$(APP_SIGN)" PKGDIR="$(PKGDIR)" bash tools/osx-codesign.sh
+	SIGN="$(CODESIGN_CERT)" PKGDIR="$(PKGDIR)" bash tools/osx-codesign.sh
 	lipo $(PKGDIR)/32/usr/local/bin/node \
 		$(PKGDIR)/usr/local/bin/node \
 		-output $(PKGDIR)/usr/local/bin/node-universal \
@@ -333,7 +377,15 @@ $(PKG): release-only
 		--id "org.nodejs.Node" \
 		--doc tools/osx-pkg.pmdoc \
 		--out $(PKG)
-	SIGN="$(INT_SIGN)" PKG="$(PKG)" bash tools/osx-productsign.sh
+	SIGN="$(PRODUCTSIGN_CERT)" PKG="$(PKG)" bash tools/osx-productsign.sh
+
+pkg: $(PKG)
+
+pkg-upload: pkg
+	ssh $(STAGINGSERVER) "mkdir -p nodejs/$(DISTTYPEDIR)/$(FULLVERSION)"
+	chmod 664 node-$(FULLVERSION).pkg
+	scp -p node-$(FULLVERSION).pkg $(STAGINGSERVER):nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION).pkg
+	ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION).pkg.done"
 
 $(TARBALL): release-only node doc
 	git archive --format=tar --prefix=$(TARNAME)/ HEAD | tar xf -
@@ -349,6 +401,39 @@ $(TARBALL): release-only node doc
 
 tar: $(TARBALL)
 
+tar-upload: tar
+	ssh $(STAGINGSERVER) "mkdir -p nodejs/$(DISTTYPEDIR)/$(FULLVERSION)"
+	chmod 664 node-$(FULLVERSION).tar.gz
+	scp -p node-$(FULLVERSION).tar.gz $(STAGINGSERVER):nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION).tar.gz
+	ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION).tar.gz.done"
+
+doc-upload: tar
+	ssh $(STAGINGSERVER) "mkdir -p nodejs/$(DISTTYPEDIR)/$(FULLVERSION)"
+	chmod -R ug=rw-x+X,o=r+X out/doc/
+	scp -pr out/doc/ $(STAGINGSERVER):nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/docs/
+	ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/docs.done"
+
+$(TARBALL)-headers: config.gypi release-only
+	$(PYTHON) ./configure \
+		--prefix=/ \
+		--dest-cpu=$(DESTCPU) \
+		--tag=$(TAG) \
+		$(CONFIG_FLAGS) $(BUILD_RELEASE_FLAGS)
+	HEADERS_ONLY=1 $(PYTHON) tools/install.py install '$(TARNAME)' '/'
+	find $(TARNAME)/ -type l | xargs rm # annoying on windows
+	tar -cf $(TARNAME)-headers.tar $(TARNAME)
+	rm -rf $(TARNAME)
+	gzip -c -f -9 $(TARNAME)-headers.tar > $(TARNAME)-headers.tar.gz
+	rm $(TARNAME)-headers.tar
+
+tar-headers: $(TARBALL)-headers
+
+tar-headers-upload: tar-headers
+	ssh $(STAGINGSERVER) "mkdir -p nodejs/$(DISTTYPEDIR)/$(FULLVERSION)"
+	chmod 664 $(TARNAME)-headers.tar.gz
+	scp -p $(TARNAME)-headers.tar.gz $(STAGINGSERVER):nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/$(TARNAME)-headers.tar.gz
+	ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/$(TARNAME)-headers.tar.gz.done"
+
 $(BINARYTAR): release-only
 	rm -rf $(BINARYNAME)
 	rm -rf out/deps out/Release
@@ -364,6 +449,35 @@ $(BINARYTAR): release-only
 
 binary: $(BINARYTAR)
 
+binary-upload-arch: binary
+	ssh $(STAGINGSERVER) "mkdir -p nodejs/$(DISTTYPEDIR)/$(FULLVERSION)"
+	chmod 664 node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.gz
+	scp -p node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.gz $(STAGINGSERVER):nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.gz
+	ssh $(STAGINGSERVER) "touch nodejs/$(DISTTYPEDIR)/$(FULLVERSION)/node-$(FULLVERSION)-$(OSTYPE)-$(ARCH).tar.gz.done"
+
+ifeq ($(OSTYPE),darwin)
+binary-upload:
+	$(MAKE) binary-upload-arch \
+		DESTCPU=ia32 \
+		ARCH=x86 \
+		DISTTYPE=$(DISTTYPE) \
+		DATESTRING=$(DATESTRING) \
+		COMMIT=$(COMMIT) \
+		CUSTOMTAG=$(CUSTOMTAG) \
+		CONFIG_FLAGS=$(CONFIG_FLAGS)
+	$(MAKE) binary-upload-arch \
+		DESTCPU=x64 \
+		ARCH=x64 \
+		DISTTYPE=$(DISTTYPE) \
+		DATESTRING=$(DATESTRING) \
+		COMMIT=$(COMMIT) \
+		CUSTOMTAG=$(CUSTOMTAG) \
+		CONFIG_FLAGS=$(CONFIG_FLAGS)
+else
+binary-upload: binary-upload-arch
+endif
+
+
 $(PKGSRC): release-only
 	rm -rf dist out
 	$(PYTHON) configure --prefix=/ --without-snapshot --download=all \
@@ -378,11 +492,6 @@ $(PKGSRC): release-only
 
 pkgsrc: $(PKGSRC)
 
-dist-upload: $(TARBALL) $(PKG)
-	ssh [email protected] mkdir -p web/nodejs.org/dist/$(VERSION)
-	scp $(TARBALL) [email protected]:~/web/nodejs.org/dist/$(VERSION)/$(TARBALL)
-	scp $(PKG) [email protected]:~/web/nodejs.org/dist/$(VERSION)/$(TARNAME).pkg
-
 wrkclean:
 	$(MAKE) -C tools/wrk/ clean
 	rm tools/wrk/wrk

diff --git a/configure b/configure
@@ -267,6 +267,11 @@ parser.add_option('--with-icu-source',
     dest='with_icu_source',
     help='Intl mode: optional local path to icu/ dir, or path/URL of icu source archive.')
 
+parser.add_option('--download-path',
+    dest='download_path',
+    default=os.path.join(root_dir, 'deps'),
+    help='Download directory [default: %default]')
+
 parser.add_option('--with-perfctr',
     action='store_true',
     dest='with_perfctr',
@@ -517,6 +522,10 @@ def configure_node(o):
 
   host_arch = host_arch_win() if os.name == 'nt' else host_arch_cc()
   target_arch = options.dest_cpu or host_arch
+  # ia32 is preferred by the build tools (GYP) over x86 even if we prefer the latter
+  # the Makefile resets this to x86 afterward
+  if target_arch == 'x86':
+    target_arch = 'ia32'
   o['variables']['host_arch'] = host_arch
   o['variables']['target_arch'] = target_arch
 
@@ -762,11 +771,15 @@ def configure_intl(o):
   ]
   def icu_download(path):
     # download ICU, if needed
+    if not os.access(options.download_path, os.W_OK):
+      print 'Error: cannot write to desired download path. ' \
+            'Either create it or verify permissions.'
+      sys.exit(1)
     for icu in icus:
       url = icu['url']
       md5 = icu['md5']
       local = url.split('/')[-1]
-      targetfile = os.path.join(root_dir, 'deps', local)
+      targetfile = os.path.join(options.download_path, local)
       if not os.path.isfile(targetfile):
         if nodedownload.candownload(auto_downloads, "icu"):
           nodedownload.retrievefile(url, targetfile)

diff --git a/src/node_version.h b/src/node_version.h
@@ -28,25 +28,31 @@
 
 #define NODE_VERSION_IS_RELEASE 0
 
-#ifndef NODE_TAG
-# define NODE_TAG ""
-#endif
-
 #ifndef NODE_STRINGIFY
 #define NODE_STRINGIFY(n) NODE_STRINGIFY_HELPER(n)
 #define NODE_STRINGIFY_HELPER(n) #n
 #endif
 
-#if NODE_VERSION_IS_RELEASE
-# define NODE_VERSION_STRING  NODE_STRINGIFY(NODE_MAJOR_VERSION) "." \
-                              NODE_STRINGIFY(NODE_MINOR_VERSION) "." \
-                              NODE_STRINGIFY(NODE_PATCH_VERSION)     \
-                              NODE_TAG
+#ifndef NODE_TAG
+# if NODE_VERSION_IS_RELEASE
+#  define NODE_TAG ""
+# else
+#  define NODE_TAG "-pre"
+# endif
 #else
+// NODE_TAG is passed without quotes when rc.exe is run from msbuild
+# define NODE_EXE_VERSION NODE_STRINGIFY(NODE_MAJOR_VERSION) "." \
+                          NODE_STRINGIFY(NODE_MINOR_VERSION) "." \
+                          NODE_STRINGIFY(NODE_PATCH_VERSION)     \
+                          NODE_STRINGIFY(NODE_TAG)
+#endif
+
 # define NODE_VERSION_STRING  NODE_STRINGIFY(NODE_MAJOR_VERSION) "." \
                               NODE_STRINGIFY(NODE_MINOR_VERSION) "." \
                               NODE_STRINGIFY(NODE_PATCH_VERSION)     \
-                              NODE_TAG "-pre"
+                              NODE_TAG
+#ifndef NODE_EXE_VERSION
+# define NODE_EXE_VERSION NODE_VERSION_STRING
 #endif
 
 #define NODE_VERSION "v" NODE_VERSION_STRING

diff --git a/tools/msvs/msi/nodemsi.wixproj b/tools/msvs/msi/nodemsi.wixproj
@@ -6,33 +6,33 @@
     <ProductVersion>3.5</ProductVersion>
     <ProjectGuid>{1d808ff0-b5a9-4be9-859d-b334b6f48be2}</ProjectGuid>
     <SchemaVersion>2.0</SchemaVersion>
-    <OutputName>node-v$(NodeVersion)-$(Platform)</OutputName>
+    <OutputName>node-v$(FullVersion)-$(Platform)</OutputName>
     <OutputType>Package</OutputType>
     <EnableProjectHarvesting>True</EnableProjectHarvesting>
     <WixTargetsPath Condition=" '$(WixTargetsPath)' == '' AND '$(MSBuildExtensionsPath32)' != '' ">$(MSBuildExtensionsPath32)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
     <WixTargetsPath Condition=" '$(WixTargetsPath)' == '' ">$(MSBuildExtensionsPath)\Microsoft\WiX\v3.x\Wix.targets</WixTargetsPath>
     <NodeVersion Condition=" '$(NodeVersion)' == '' ">0.0.0.0</NodeVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x86' ">
-    <OutputPath>..\..\..\$(Configuration)\</OutputPath>
+    <OutputPath>..\..\..\</OutputPath>
     <IntermediateOutputPath>obj\$(Configuration)\</IntermediateOutputPath>
-    <DefineConstants>Debug;ProductVersion=$(NodeVersion);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFilesFolder</DefineConstants>
+    <DefineConstants>Debug;ProductVersion=$(NodeVersion);FullVersion=$(FullVersion);DistTypeDir=$(DistTypeDir);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFilesFolder</DefineConstants>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x86' ">
-    <OutputPath>..\..\..\$(Configuration)\</OutputPath>
+    <OutputPath>..\..\..\</OutputPath>
     <IntermediateOutputPath>obj\$(Configuration)\</IntermediateOutputPath>
-    <DefineConstants>Debug;ProductVersion=$(NodeVersion);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFilesFolder</DefineConstants>
+    <DefineConstants>Debug;ProductVersion=$(NodeVersion);FullVersion=$(FullVersion);DistTypeDir=$(DistTypeDir);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFilesFolder</DefineConstants>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|x64' ">
-    <OutputPath>..\..\..\$(Configuration)\</OutputPath>
+    <OutputPath>..\..\..\</OutputPath>
     <IntermediateOutputPath>obj\$(Configuration)\</IntermediateOutputPath>
-    <DefineConstants>Debug;ProductVersion=$(NodeVersion);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFiles64Folder</DefineConstants>
+    <DefineConstants>Debug;ProductVersion=$(NodeVersion);FullVersion=$(FullVersion);DistTypeDir=$(DistTypeDir);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFiles64Folder</DefineConstants>
     <Cultures>en-US</Cultures>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|x64' ">
-    <OutputPath>..\..\..\$(Configuration)\</OutputPath>
+    <OutputPath>..\..\..\</OutputPath>
     <IntermediateOutputPath>obj\$(Configuration)\</IntermediateOutputPath>
-    <DefineConstants>Debug;ProductVersion=$(NodeVersion);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFiles64Folder</DefineConstants>
+    <DefineConstants>Debug;ProductVersion=$(NodeVersion);FullVersion=$(FullVersion);DistTypeDir=$(DistTypeDir);NoETW=$(NoETW);NoPerfCtr=$(NoPerfCtr);NpmSourceDir=..\..\..\deps\npm\;ProgramFilesFolderId=ProgramFiles64Folder</DefineConstants>
   </PropertyGroup>
   <PropertyGroup>
     <EnableProjectHarvesting>True</EnableProjectHarvesting>