[DEVEL] Tensorflow build recipe cleanup

eigen-toolfile.spec

@@ -14,6 +14,9 @@ cat << \EOF_TOOLFILE >%i/etc/scram.d/eigen.xml
     <environment name="INCLUDE"      default="$EIGEN_BASE/include/eigen3"/>
   </client>
   <flags CPPDEFINES="EIGEN_DONT_PARALLELIZE"/>
+  <flags CPPDEFINES="EIGEN_MPL2_ONLY"/>
+  <flags CPPDEFINES="EIGEN_MAX_ALIGN_BYTES=64"/>
+  <flags CPPDEFINES="EIGEN_HAS_TYPE_TRAITS=0"/>
 </tool>
 EOF_TOOLFILE
 

pip/py2-protobuf.file

@@ -1 +1 @@
-Requires: py2-six py2-packaging py2-setuptools py2-pyparsing py2-appdirs
+Requires: py2-six py2-packaging py2-setuptools py2-pyparsing py2-appdirs protobuf

pip/py2-tensorboard.file

@@ -0,0 +1,15 @@
+Requires: py2-Werkzeug py2-absl-py py2-numpy py2-futures py2-protobuf py2-wheel py2-Markdown py2-grpcio py2-six
+%define pip_name %(echo %n | cut -f2-10 -d-)
+%define PipDownloadSourceType none
+%define PipDownloadOptions --no-deps%%20--no-binary%%3D:%{PipDownloadSourceType}:
+%define source0 none
+
+#Python3 Sources which are default in build-with-pip
+Source3: pip://%{pip_name}/%{realversion}?pip_options=%{PipDownloadOptions}&pip=pip3&output=/py3-source.tar.gz
+%define PipPreBuildPy3 tar xfz %{_sourcedir}/py3-source.tar.gz ; export PIPFILE=`cat files.list`
+
+#Python2 Sources
+Source2: pip://%{pip_name}/%{realversion}?pip_options=%{PipDownloadOptions}&pip=pip2&output=/py2-source.tar.gz
+%define PipPreBuildPy2 tar xfz %{_sourcedir}/py2-source.tar.gz ; export PIPFILE=`cat files.list`
+
+%define PipPostBuild perl -p -i -e "s|^#!.*python(.*)|#!/usr/bin/env python$1|" %{i}/bin/*

pip/py2-tensorflow-estimator.file

@@ -0,0 +1 @@
+%define PipDownloadSourceType none

pip/py2-tensorflow.file

@@ -0,0 +1,13 @@
+BuildRequires: tensorflow-sources tensorflow-python3-sources
+## INCLUDE tensorflow-deps
+
+%define source0  none
+%define PipPostBuild perl -p -i -e "s|^#!.*python(.*)|#!/usr/bin/env python$1|" %{i}/bin/*
+
+#Python3 Sources which are default in build-with-pip
+%define py3_source tensorflow-%{realversion}-cp36-cp36m-linux_%{_arch}.whl
+%define PipPreBuildPy3 cp ${TENSORFLOW_PYTHON3_SOURCES_ROOT}/%{py3_source} %{py3_source} ; export PIPFILE=%{py3_source}
+
+#Python2 Sources
+%define py2_source tensorflow-%{realversion}-cp27-cp27mu-linux_%{_arch}.whl
+%define PipPreBuildPy2 cp ${TENSORFLOW_SOURCES_ROOT}/%{py2_source} %{py2_source} ; export PIPFILE=%{py2_source}

pip/requirements.txt

@@ -2,6 +2,7 @@ absl-py==0.7.1
 appdirs==1.4.3
 argparse==1.4.0
 asn1crypto==0.24.0
+astor==0.6.2
 astroid==1.6.6
 atomicwrites==1.3.0
 attrs==19.1.0
@@ -51,6 +52,7 @@ gast==0.2.2
 gitdb2==2.0.5
 GitPython==2.1.11
 google-common==0.0.1
+google-pasta==0.1.7
 grpcio==1.19.0
 h5py-cache==1.0
 h5py==2.9.0
@@ -175,6 +177,10 @@ stevedore==1.30.1
 subprocess32==3.5.3
 sympy==1.4
 tables==3.4.4
+tensorboard==1.13.1
+tensorflow==1.13.1
+tensorflow-estimator==1.14.0
+termcolor==1.1.0
 terminado==0.8.2
 testpath==0.4.2
 theanets==0.7.3

protobuf.spec

@@ -28,7 +28,7 @@ BuildRequires: autotools
 ./configure --prefix %{i} \
     --disable-static \
     --disable-dependency-tracking \
-    CXXFLAGS="-I${ZLIB_ROOT}/include" \
+    CXXFLAGS="-std=c++1z -I${ZLIB_ROOT}/include" \
     CFLAGS="-I${ZLIB_ROOT}/include" \
     LDFLAGS="-L${ZLIB_ROOT}/lib"
 make %{makeprocesses}

tensorflow-deps.file

@@ -0,0 +1,5 @@
+# Dependency of tensorflow soruces and python package
+Requires: eigen java-env libjpeg-turbo nasm protobuf py2-absl-py py2-appdirs py2-astor py2-backports-weakref py2-enum34
+Requires: py2-funcsigs py2-future py2-gast py2-google-pasta py2-keras-applications py2-keras-preprocessing py2-mock
+Requires: py2-numpy py2-packaging py2-pbr py2-protobuf py2-pyparsing py2-setuptools py2-six py2-tensorboard
+Requires: py2-tensorflow-estimator py2-termcolor py2-Werkzeug py2-wheel py2-wrapt python swig

tensorflow-python3-sources.spec

@@ -1,109 +1,8 @@
 ### RPM external tensorflow-python3-sources 1.13.1
-#Source: https://github.com/tensorflow/tensorflow/archive/v%{realversion}.tar.gz
-# NOTE: whenever the version of tensorflow changes, update it also in tensorflow-c tensorflow-cc and py2-tensorflow
-%define isslc6amd64 %(case %{cmsplatf} in (slc6_amd64_*) echo 1 ;; (*) echo 0 ;; esac)
-%define tag a977edf5ae572444fc45f39038e73098b4830801
-%define branch cms/v%{realversion}
-%define github_user cms-externals
-Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag}&export=tensorflow-%{realversion}&output=/tensorflow-%{realversion}-%{tag}.tgz
-
-BuildRequires: bazel
-Requires: gcc protobuf java-env  libjpeg-turbo eigen  python3 nasm py2-numpy py2-enum34 py2-mock py2-wheel py2-keras-applications py2-keras-preprocessing py2-setuptools
-
-%prep
-
-%setup -q -n tensorflow-%{realversion}
-
-%build
-
-export PYTHON_BIN_PATH=`which python3`
-# introduce a hack to link calls to python -> python3
-mkdir mybin
-ln -s ${PYTHON_BIN_PATH} mybin/python
-export PATH=`pwd`/mybin:${PATH}
-# end of hack, remove when fixed
-export TF_NEED_JEMALLOC=0
-export TF_NEED_HDFS=0
-export CC_OPT_FLAGS=-march=core2
-export CXX_OPT_FLAGS=-std=c++1z
-export TF_NEED_GCP=0
-export TF_ENABLE_XLA=0
-export TF_NEED_OPENCL=0
-export TF_NEED_CUDA=0
-export TF_NEED_VERBS=0
-export TF_NEED_MKL=0
-export TF_NEED_MPI=0
-export USE_DEFAULT_PYTHON_LIB_PATH=1
-export TF_NEED_S3=0
-export TF_NEED_GDR=0
-export TF_NEED_OPENCL_SYCL=0
-export TF_SET_ANDROID_WORKSPACE=false
-export TF_NEED_KAFKA=false
-export TF_NEED_AWS=0
-export TF_DOWNLOAD_CLANG=0
-export TF_NEED_IGNITE=false
-export TF_NEED_ROCM=false
-
-#and source locations
-export EIGEN_SOURCE=${EIGEN_SOURCE} # we are using tf own eigen now
-export PROTOBUF_SOURCE=${PROTOBUF_SOURCE}
-#export ZLIB_SOURCE=${ZLIB_SOURCE}
-export LIBJPEG_TURBO_SOURCE=${LIBJPEG_TURBO_SOURCE}
-#export LIBJPEG_TURBO_SOURCE="https://github.com/libjpeg-turbo/libjpeg-turbo/archive/2.0.1.tar.gz"
-
-#${LIBJPEG_TURBO_SOURCE}
-
-export EIGEN_STRIP_PREFIX=${EIGEN_STRIP_PREFIX} # we are using tf own eigen now
-export PROTOBUF_STRIP_PREFIX=${PROTOBUF_STRIP_PREFIX}
-#export ZLIB_STRIP_PREFIX= ${ZLIB_STRIP_PREFIX}
-export LIBJPEG_TURBO_STRIP_PREFIX=${LIBJPEG_TURBO_STRIP_PREFIX}
-
-#temp directory
-rm -rf ../build
-
-./configure
-
-BAZEL_OPTS="--output_user_root ../build build -s --verbose_failures -c opt --config monolithic --cxxopt=${CXX_OPT_FLAGS}"
-BAZEL_EXTRA_OPTS="--action_env PYTHONPATH=${PYTHON3PATH} --distinct_host_configuration=false"
-
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/tools/pip_package:build_pip_package
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow:libtensorflow_cc.so
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/tools/lib_package:libtensorflow
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/python/tools:tools_pip
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/tools/graph_transforms:transform_graph
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/compiler/tf2xla:tf2xla
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/compiler/tf2xla:cpu_function_runtime
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/compiler/tf2xla:xla_compiled_cpu_function
-%ifnarch ppc64le
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow/compiler/aot:tfcompile
-%endif
-bazel $BAZEL_OPTS $BAZEL_EXTRA_OPTS //tensorflow:install_headers
-
-bazel shutdown
-
-#Copying out what was built by bazel
-incdir="$PWD/tensorflow_cc/include"
-libdir="$PWD/tensorflow_cc/lib"
-bindir="$PWD/tensorflow_cc/bin"
-
-# Make directory and clean it
-rm -rf $incdir $libdir $bindir
-mkdir -p $incdir $libdir $bindir
-
-cp -rp $PWD/bazel-out/*-opt/genfiles/tensorflow/include/* $incdir/
-cp  $PWD/bazel-bin/tensorflow/libtensorflow_cc.so $libdir
-cp  $PWD/bazel-bin/tensorflow/compiler/tf2xla/libcpu_function_runtime.so $libdir
-cp  $PWD/bazel-bin/tensorflow/compiler/tf2xla/libxla_compiled_cpu_function.so $libdir
-%ifnarch ppc64le
-cp  $PWD/bazel-bin/tensorflow/compiler/aot/tfcompile $bindir
-%endif
-
-#Download depencies used by tensorflow and copy to include dir
-
-%install
-
-bazel-bin/tensorflow/tools/pip_package/build_pip_package %{i}
-
-cp $PWD/bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz %{i}
-cd tensorflow_cc
-tar cfz %{i}/libtensorflow_cc.tar.gz .
+%define python_cmd python3
+%define python_env PYTHON3PATH
+%define build_type opt
+%define pythonOnly yes
+#Just to make sure that only one tensorflow-source package built at a time
+BuildRequires: tensorflow-sources
+## INCLUDE tensorflow-sources

tensorflow-sources.file

@@ -0,0 +1,128 @@
+############################################
+# IMPORTANT
+# - Whenever the version of tensorflow changes,
+#   update it also in tensorflow-c tensorflow-cc
+#   and py2-tensorflow version too
+# - Need to check protobuf and eigen versions
+# - Check eigen flags in third_party/eigen.BUILD
+#   and update eigen-toolfile.spec accordingly
+############################################
+%define isslc6amd64 %(case %{cmsplatf} in (slc6_amd64_*) echo 1 ;; (*) echo 0 ;; esac)
+%define tag         185f77b7ad50087a2ae2d68459b95b66244002da
+%define branch      cms/v6612da89
+%define github_user cms-externals
+Source: git+https://github.com/%{github_user}/tensorflow.git?obj=%{branch}/%{tag}&export=tensorflow-%{realversion}&output=/tensorflow-%{realversion}-%{tag}.tgz
+
+%if "%{?build_type:set}" != "set"
+%define build_type opt
+%endif
+%if "%{?pythonOnly:set}" != "set"
+%define pythonOnly no
+%endif
+
+BuildRequires: bazel
+## INCLUDE tensorflow-deps
+
+%prep
+
+%setup -q -n tensorflow-%{realversion}
+
+%build
+
+export PYTHON_BIN_PATH=`which %{python_cmd}`
+export TF_NEED_JEMALLOC=0
+export TF_NEED_HDFS=0
+export CC_OPT_FLAGS=-march=core2
+export CXX_OPT_FLAGS=-std=c++1z
+export TF_NEED_GCP=0
+export TF_ENABLE_XLA=0
+export TF_NEED_OPENCL=0
+export TF_NEED_CUDA=0
+export TF_NEED_VERBS=0
+export TF_NEED_MKL=0
+export TF_NEED_MPI=0
+export USE_DEFAULT_PYTHON_LIB_PATH=1
+export TF_NEED_S3=0
+export TF_NEED_GDR=0
+export TF_NEED_OPENCL_SYCL=0
+export TF_SET_ANDROID_WORKSPACE=false
+export TF_NEED_KAFKA=false
+export TF_NEED_AWS=0
+export TF_DOWNLOAD_CLANG=0
+export TF_NEED_IGNITE=false
+export TF_NEED_ROCM=false
+export TEST_TMPDIR=%{_builddir}/build
+
+#temp directory
+rm -rf ../build
+
+./configure
+
+BAZEL_OPTS=" --output_user_root %{_builddir}/build build --jobs %{compiling_processes} -s --verbose_failures"
+BAZEL_OPTS="${BAZEL_OPTS} -c %{build_type} --config=noaws --config=nogcp --config=nohdfs --config=noignite --config=nokafka"
+case %{build_type} in
+  dbg) BAZEL_OPTS="${BAZEL_OPTS} --strip=never --compilation_mode=%{build_type}" ;;
+esac
+BAZEL_OPTS="${BAZEL_OPTS} --cxxopt=${CXX_OPT_FLAGS} --linkopt=-L${PROTOBUF_ROOT}/lib"
+BAZEL_OPTS="${BAZEL_OPTS} --action_env=PATH --action_env=TMPDIR --action_env=LD_LIBRARY_PATH --action_env=%{python_env} --distinct_host_configuration=false"
+BAZEL_OPTS="${BAZEL_OPTS} --test_env=PATH --test_env=LD_LIBRARY_PATH"
+BAZEL_OPTS="${BAZEL_OPTS} --action_env TF_SYSTEM_LIBS=protobuf_archive"
+BAZEL_OPTS="${BAZEL_OPTS} --action_env PROTOBUF_ROOT=${PROTOBUF_ROOT} --cxxopt=-Ibazel-out/k8-%{build_type}/genfiles/external/protobuf_archive"
+
+#Fix Eigen headers list
+echo "HEADERS = [" > protobuf.BUILD
+find ${PROTOBUF_ROOT}/include/google/protobuf -name '*.h' -type f | sed 's|.*/include/google/protobuf/|  "google/protobuf/|;s|$|",|' >> protobuf.BUILD
+echo "]" >> protobuf.BUILD
+
+sed -e 's|^\(\s*\)\(HEADERS\s*=\)|\1OLD_\2|;s|$(INCLUDEDIR)/|$$PROTOBUF_ROOT/include/|' third_party/systemlibs/protobuf.BUILD >> protobuf.BUILD
+mv protobuf.BUILD third_party/systemlibs/protobuf.BUILD
+
+%if "%{pythonOnly}" == "no"
+bazel $BAZEL_OPTS //tensorflow:libtensorflow_cc.so
+bazel $BAZEL_OPTS //tensorflow/tools/lib_package:libtensorflow
+bazel $BAZEL_OPTS //tensorflow/tools/graph_transforms:transform_graph
+bazel $BAZEL_OPTS //tensorflow/compiler/tf2xla:tf2xla
+bazel $BAZEL_OPTS //tensorflow/compiler/tf2xla:xla_compiled_cpu_function
+bazel $BAZEL_OPTS //tensorflow/compiler/tf2xla:cpu_function_runtime
+%ifnarch ppc64le
+bazel $BAZEL_OPTS //tensorflow/compiler/aot:tfcompile
+%endif
+bazel $BAZEL_OPTS //tensorflow:install_headers
+%endif
+bazel $BAZEL_OPTS //tensorflow/python/tools:tools_pip
+bazel $BAZEL_OPTS //tensorflow/tools/pip_package:build_pip_package
+
+bazel shutdown
+
+%if "%{pythonOnly}" == "no"
+#Copying out what was built by bazel
+incdir="$PWD/tensorflow_cc/include"
+libdir="$PWD/tensorflow_cc/lib"
+bindir="$PWD/tensorflow_cc/bin"
+
+# Make directory and clean it
+rm -rf $incdir $libdir $bindir
+mkdir -p $incdir $libdir $bindir
+
+#Install libs/bins
+cp -fP $PWD/bazel-bin/tensorflow/lib*.so $libdir/
+cp -fP $PWD/bazel-bin/tensorflow/compiler/tf2xla/lib*.so $libdir/
+%ifnarch ppc64le
+cp  $PWD/bazel-bin/tensorflow/compiler/aot/tfcompile $bindir
+%endif
+#install Headers
+rsync -a $PWD/bazel-genfiles/tensorflow/include/ $incdir/
+if [ ! -d $incdir/tensorflow/compiler ] ; then
+  cp -rf $PWD/tensorflow/compiler $incdir/tensorflow/compiler
+fi
+%endif
+
+%install
+
+bazel-bin/tensorflow/tools/pip_package/build_pip_package %{i}
+
+%if "%{pythonOnly}" == "no"
+cp $PWD/bazel-bin/tensorflow/tools/lib_package/libtensorflow.tar.gz %{i}
+cd tensorflow_cc
+tar cfz %{i}/libtensorflow_cc.tar.gz .
+%endif