From 8a2ceda67659f52a83212ca15aff7b057e63d50c Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 10:48:37 +0800 Subject: [PATCH 01/22] Add externel google's dependencies --- cmake/external/gflags.cmake | 39 ++++++++++++++++++++++++++ cmake/external/glog.cmake | 40 +++++++++++++++++++++++++++ cmake/external/gtest.cmake | 48 ++++++++++++++++++++++++++++++++ cmake/external/protobuf.cmake | 52 +++++++++++++++++++++++++++++++++++ 4 files changed, 179 insertions(+) create mode 100644 cmake/external/gflags.cmake create mode 100644 cmake/external/glog.cmake create mode 100644 cmake/external/gtest.cmake create mode 100644 cmake/external/protobuf.cmake diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake new file mode 100644 index 0000000000000..128d50cec8553 --- /dev/null +++ b/cmake/external/gflags.cmake @@ -0,0 +1,39 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(GFLAGS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/gflags) +SET(GFLAGS_INSTALL_DIR ${PROJECT_BINARY_DIR}/gflags) + +ExternalProject_Add( + gflags + GIT_REPOSITORY "https://github.com/gflags/gflags.git" + PREFIX ${GFLAGS_SOURCES_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} + CMAKE_ARGS -DBUILD_TESTING=OFF + LOG_DOWNLOAD =ON + UPDATE_COMMAND "" +) + +SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE) +INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) + +IF(WIN32) + set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/gflags.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) +ELSE(WIN32) + set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.a" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) +ENDIF(WIN32) + +LIST(APPEND external_project_dependencies gflags) diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake new file mode 100644 index 0000000000000..8a4b9d5996f59 --- /dev/null +++ b/cmake/external/glog.cmake @@ -0,0 +1,40 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(GLOG_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/glog) +SET(GLOG_INSTALL_DIR ${PROJECT_BINARY_DIR}/glog) + +ExternalProject_Add( + glog + GIT_REPOSITORY "https://github.com/google/glog.git" + PREFIX ${GLOG_SOURCES_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} + CMAKE_ARGS -DWITH_GFLAGS=OFF + CMAKE_ARGS -DBUILD_TESTING=OFF + LOG_DOWNLOAD =ON + UPDATE_COMMAND "" +) + +SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE) +INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR}) + +IF(WIN32) + SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.lib" CACHE FILEPATH "glog library." FORCE) +ELSE(WIN32) + SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.a" CACHE FILEPATH "glog library." FORCE) +ENDIF(WIN32) + +LIST(APPEND external_project_dependencies glog) diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake new file mode 100644 index 0000000000000..533104422acae --- /dev/null +++ b/cmake/external/gtest.cmake @@ -0,0 +1,48 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(GTEST_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/gtest) +SET(GTEST_INSTALL_DIR ${PROJECT_BINARY_DIR}/gtest) + +ExternalProject_Add( + gtest + GIT_REPOSITORY "https://github.com/google/googletest.git" + GIT_TAG "release-1.8.0" + PREFIX ${GTEST_SOURCES_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} + CMAKE_ARGS -DBUILD_GMOCK=ON + CMAKE_ARGS -Dgtest_disable_pthreads=ON + CMAKE_ARGS -Dgtest_force_shared_crt=ON + LOG_DOWNLOAD =ON + UPDATE_COMMAND "" +) + +SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) +INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIR}) + +IF(WIN32) + set(GTEST_LIBRARIES + "${GTEST_INSTALL_DIR}/lib/gtest.lib" + "${GTEST_INSTALL_DIR}/lib/gtest_main.lib" CACHE FILEPATH "gtest libraries." FORCE) +ELSE(WIN32) + set(GTEST_LIBRARIES + "${GTEST_INSTALL_DIR}/lib/libgtest.a" + "${GTEST_INSTALL_DIR}/lib/libgtest_main.a" CACHE FILEPATH "gtest libraries." FORCE) +ENDIF(WIN32) + +ENABLE_TESTING() + +LIST(APPEND external_project_dependencies gtest) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake new file mode 100644 index 0000000000000..8acc6325b9e55 --- /dev/null +++ b/cmake/external/protobuf.cmake @@ -0,0 +1,52 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(PROTOBUF_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf) +SET(PROTOBUF_INSTALL_DIR ${PROJECT_BINARY_DIR}/protobuf) + +ExternalProject_Add( + protobuf + PREFIX ${PROTOBUF_SOURCES_DIR} + DEPENDS zlib + GIT_REPOSITORY "https://github.com/google/protobuf.git" + GIT_TAG "v3.0.0" + CONFIGURE_COMMAND + ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/protobuf/cmake + -Dprotobuf_BUILD_TESTS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} + UPDATE_COMMAND "" +) + +SET(PROTOBUF_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" CACHE PATH "protobuf include directory." FORCE) +INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) + +IF(WIN32) + SET(PROTOBUF_LIBRARIES + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.lib" + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.lib" + "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protobuf libraries." FORCE) + SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc.exe" CACHE FILEPATH "protobuf executable." FORCE) +ELSE(WIN32) + SET(PROTOBUF_LIBRARIES + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.a" + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.a" + "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.a" CACHE FILEPATH "protobuf libraries." FORCE) + SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc" CACHE FILEPATH "protobuf executable." FORCE) +ENDIF(WIN32) + +LIST(APPEND external_project_dependencies protobuf) From 280994fafac08ac77e22b796aced5535fdc3a009 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 11:25:28 +0800 Subject: [PATCH 02/22] Add other extern dependencies --- cmake/external/numpy.cmake | 57 ++++++++++++++++++++++++ cmake/external/openblas.cmake | 44 +++++++++++++++++++ cmake/external/python.cmake | 83 +++++++++++++++++++++++++++++++++++ cmake/external/swig.cmake | 75 +++++++++++++++++++++++++++++++ cmake/external/warpctc.cmake | 39 ++++++++++++++++ cmake/external/zlib.cmake | 44 +++++++++++++++++++ 6 files changed, 342 insertions(+) create mode 100644 cmake/external/numpy.cmake create mode 100644 cmake/external/openblas.cmake create mode 100644 cmake/external/python.cmake create mode 100644 cmake/external/swig.cmake create mode 100644 cmake/external/warpctc.cmake create mode 100644 cmake/external/zlib.cmake diff --git a/cmake/external/numpy.cmake b/cmake/external/numpy.cmake new file mode 100644 index 0000000000000..de3e6492cdfca --- /dev/null +++ b/cmake/external/numpy.cmake @@ -0,0 +1,57 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(NUMPY_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/numpy) +SET(NUMPY_INSTALL_DIR ${PROJECT_BINARY_DIR}/numpy) +set(NUMPY_VERSION "v1.11.3") + +# setuptools +ExternalProject_Add(setuptools + PREFIX ${PYTHON_SOURCES_DIR}/setuptools + URL http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz + URL_MD5 7df2a529a074f613b509fb44feefe74e + BUILD_IN_SOURCE 1 + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python zlib +) + +ExternalProject_Add(cython + PREFIX ${PYTHON_SOURCES_DIR}/cython + GIT_REPOSITORY https://github.com/cython/cython.git + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + UPDATE_COMMAND "" + PATCH_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python +) + +ExternalProject_Add(numpy + GIT_REPOSITORY https://github.com/numpy/numpy.git + GIT_TAG ${NUMPY_VERSION} + CONFIGURE_COMMAND "" + UPDATE_COMMAND "" + PREFIX ${NUMPY_SOURCES_DIR} + BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py build + INSTALL_COMMAND ${PYTHON_EXECUTABLE} setup.py install + BUILD_IN_SOURCE 1 + DEPENDS python setuptools cython +) diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake new file mode 100644 index 0000000000000..d1220036a0f0e --- /dev/null +++ b/cmake/external/openblas.cmake @@ -0,0 +1,44 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# INCLUDE(cblas) + +INCLUDE(ExternalProject) + +SET(CBLAS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/openblas) +SET(CBLAS_INSTALL_DIR ${PROJECT_BINARY_DIR}/openblas) + +ExternalProject_Add( + openblas + GIT_REPOSITORY "https://github.com/xianyi/OpenBLAS.git" + GIT_TAG v0.2.19 + PREFIX ${CBLAS_SOURCES_DIR} + INSTALL_DIR ${CBLAS_INSTALL_DIR} + BUILD_IN_SOURCE 1 + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND cd ${CBLAS_SOURCES_DIR}/src/openblas && make -j4 + INSTALL_COMMAND cd ${CBLAS_SOURCES_DIR}/src/openblas && make install PREFIX= +) + +SET(CBLAS_INCLUDE_DIR "${CBLAS_INSTALL_DIR}/include" CACHE PATH "openblas include directory." FORCE) +INCLUDE_DIRECTORIES(${CBLAS_INCLUDE_DIR}) + +IF(WIN32) + set(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/openblas.lib" CACHE FILEPATH "openblas library." FORCE) +ELSE(WIN32) + set(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/libopenblas.a" CACHE FILEPATH "openblas library" FORCE) +ENDIF(WIN32) + +LIST(APPEND external_project_dependencies openblas) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake new file mode 100644 index 0000000000000..b459913314025 --- /dev/null +++ b/cmake/external/python.cmake @@ -0,0 +1,83 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python) +SET(PYTHON_INSTALL_DIR ${PROJECT_BINARY_DIR}/python) + +if(MSVC) + list(APPEND EXTERNAL_PROJECT_OPTIONAL_ARGS + PATCH_COMMAND ${CMAKE_COMMAND} + -DPYTHON_SRC_DIR:PATH=${_python_SOURCE_DIR} + -P ${CMAKE_CURRENT_LIST_DIR}/PythonPatch.cmake + ) +endif() + +if(APPLE) + list(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS + -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON + ) +endif() + +set(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS) + +# Force Python build to "Release". +if(CMAKE_CONFIGURATION_TYPES) + set(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR}) + set(CMAKE_CFG_INTDIR "Release") +else() + list(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=Release) +endif() + +ExternalProject_Add(python + GIT_REPOSITORY "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git" + GIT_TAG "ed5f9bcee540e47f82fa17f8360b820591aa6d66" + PREFIX ${PYTHON_SOURCES_DIR} + UPDATE_COMMAND "" + CMAKE_CACHE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR} + -DBUILD_SHARED:BOOL=OFF + -DBUILD_STATIC:BOOL=ON + -DUSE_SYSTEM_LIBRARIES:BOOL=OFF + -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT} + -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR} + -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES} + -DDOWNLOAD_SOURCES:BOOL=ON + -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF + ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS} + ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS} + DEPENDS zlib +) + +set(_python_DIR ${PYTHON_INSTALL_DIR}) + +if(UNIX) + set(_python_IMPORT_SUFFIX so) + if(APPLE) + set(_python_IMPORT_SUFFIX dylib) + endif() + set(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE) + set(PYTHON_LIBRARY "${PYTHON_INSTALL_DIR}/lib/libpython2.7.${_python_IMPORT_SUFFIX}" CACHE FILEPATH "Python library" FORCE) + set(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE) + set(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE) +elseif(WIN32) + set(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE) + set(PYTHON_LIBRARY "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE) + set(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE) + set(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE) +else() + message(FATAL_ERROR "Unknown system !") +endif() diff --git a/cmake/external/swig.cmake b/cmake/external/swig.cmake new file mode 100644 index 0000000000000..9dc112b98ecc0 --- /dev/null +++ b/cmake/external/swig.cmake @@ -0,0 +1,75 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Look for system swig +# FIND_PACKAGE(SWIG) + +#IF(NOT ${SWIG_FOUND}) + # build swig as an external project + INCLUDE(ExternalProject) + SET(SWIG_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/swig) + SET(SWIG_INSTALL_DIR ${PROJECT_BINARY_DIR}/swig) + SET(SWIG_TARGET_VERSION "3.0.2") + SET(SWIG_DOWNLOAD_SRC_MD5 "62f9b0d010cef36a13a010dc530d0d41") + SET(SWIG_DOWNLOAD_WIN_MD5 "3f18de4fc09ab9abb0d3be37c11fbc8f") + + IF(WIN32) + # swig.exe available as pre-built binary on Windows: + ExternalProject_Add(swig + URL http://prdownloads.sourceforge.net/swig/swigwin-${SWIG_TARGET_VERSION}.zip + URL_MD5 ${SWIG_DOWNLOAD_WIN_MD5} + SOURCE_DIR ${SWIG_SOURCES_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + ) + SET(SWIG_DIR ${SWIG_SOURCES_DIR} CACHE FILEPATH "SWIG Directory" FORCE) + SET(SWIG_EXECUTABLE ${SWIG_SOURCES_DIR}/swig.exe CACHE FILEPATH "SWIG Executable" FORCE) + + ELSE(WIN32) + # From PCRE configure + ExternalProject_Add(pcre + GIT_REPOSITORY https://github.com/svn2github/pcre.git + PREFIX ${SWIG_SOURCES_DIR}/pcre + UPDATE_COMMAND "" + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SWIG_INSTALL_DIR}/pcre + ) + + # swig uses bison find it by cmake and pass it down + FIND_PACKAGE(BISON) + + # From SWIG configure + ExternalProject_Add(swig + URL https://github.com/swig/swig/archive/rel-3.0.10.tar.gz + PREFIX ${SWIG_SOURCES_DIR} + UPDATE_COMMAND "" + CONFIGURE_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && ./autogen.sh + CONFIGURE_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && + env "PCRE_LIBS=${SWIG_INSTALL_DIR}/pcre/lib/libpcre.a \ + ${SWIG_INSTALL_DIR}/pcre/lib/libpcrecpp.a \ + ${SWIG_INSTALL_DIR}/pcre/lib/libpcreposix.a" + ./configure + --prefix=${SWIG_INSTALL_DIR} + --with-pcre-prefix=${SWIG_INSTALL_DIR}/pcre + --with-python=${PYTHON_EXECUTABLE} + BUILD_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make + INSTALL_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make install + DEPENDS pcre python + ) + + set(SWIG_DIR ${SWIG_INSTALL_DIR}/share/swig/${SWIG_TARGET_VERSION} CACHE FILEPATH "SWIG Directory" FORCE) + set(SWIG_EXECUTABLE ${SWIG_INSTALL_DIR}/bin/swig CACHE FILEPATH "SWIG Executable" FORCE) + ENDIF(WIN32) +#ENDIF() + diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake new file mode 100644 index 0000000000000..57864aca69cf5 --- /dev/null +++ b/cmake/external/warpctc.cmake @@ -0,0 +1,39 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(WARPCTC_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/warpctc) +SET(WARPCTC_INSTALL_DIR ${PROJECT_BINARY_DIR}/warpctc) + +ExternalProject_Add( + warpctc + GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git" + PREFIX ${WARPCTC_SOURCES_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} + CMAKE_ARGS -DWITH_GPU=${CUDA_FOUND} +) + +SET(WARPCTC_INCLUDE_DIR "${WARP_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) +INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) + +IF(WIN32) + set(WARPCTC_LIBRARIES + "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) +ELSE(WIN32) + set(WARPCTC_LIBRARIES + "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.so" CACHE FILEPATH "Warp-ctc Library" FORCE) +ENDIF(WIN32) + +LIST(APPEND external_project_dependencies warpctc) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake new file mode 100644 index 0000000000000..ec44467aa70a6 --- /dev/null +++ b/cmake/external/zlib.cmake @@ -0,0 +1,44 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +INCLUDE(ExternalProject) + +SET(ZLIB_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/zlib) +SET(ZLIB_INSTALL_DIR ${PROJECT_BINARY_DIR}/zlib) + +ExternalProject_Add( + zlib + GIT_REPOSITORY "https://github.com/madler/zlib.git" + GIT_TAG "v1.2.8" + PREFIX ${ZLIB_SOURCES_DIR} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${ZLIB_INSTALL_DIR} + CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON + CMAKE_ARGS -DCMAKE_MACOSX_RPATH=ON + LOG_DOWNLOAD =ON + UPDATE_COMMAND "" +) + +SET(ZLIB_ROOT ${ZLIB_INSTALL_DIR} CACHE PATH "zlib root directory." FORCE) + +SET(ZLIB_INCLUDE_DIR "${ZLIB_INSTALL_DIR}/include" CACHE PATH "zlib include directory." FORCE) +INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) + +IF(WIN32) + SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/zlibstatic.lib" CACHE FILEPATH "zlib library." FORCE) +ELSE(WIN32) + set(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) +ENDIF(WIN32) + +LIST(APPEND external_project_dependencies zlib) From 6cd4b6e041c09a4ba12a5bfd236da2891364dcb4 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 17:11:26 +0800 Subject: [PATCH 03/22] Update external libs --- cmake/external/numpy.cmake | 104 ++++++++++++++++----------- cmake/external/openblas.cmake | 2 - cmake/external/python.cmake | 131 ++++++++++++++++++---------------- cmake/external/swig.cmake | 24 +++++-- cmake/external/warpctc.cmake | 11 ++- 5 files changed, 159 insertions(+), 113 deletions(-) diff --git a/cmake/external/numpy.cmake b/cmake/external/numpy.cmake index de3e6492cdfca..607ff317899aa 100644 --- a/cmake/external/numpy.cmake +++ b/cmake/external/numpy.cmake @@ -12,46 +12,64 @@ # See the License for the specific language governing permissions and # limitations under the License. -INCLUDE(ExternalProject) - -SET(NUMPY_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/numpy) -SET(NUMPY_INSTALL_DIR ${PROJECT_BINARY_DIR}/numpy) -set(NUMPY_VERSION "v1.11.3") - -# setuptools -ExternalProject_Add(setuptools - PREFIX ${PYTHON_SOURCES_DIR}/setuptools - URL http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz - URL_MD5 7df2a529a074f613b509fb44feefe74e - BUILD_IN_SOURCE 1 - UPDATE_COMMAND "" - PATCH_COMMAND "" - CONFIGURE_COMMAND "" - INSTALL_COMMAND "" - BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install - DEPENDS python zlib -) - -ExternalProject_Add(cython - PREFIX ${PYTHON_SOURCES_DIR}/cython - GIT_REPOSITORY https://github.com/cython/cython.git - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - UPDATE_COMMAND "" - PATCH_COMMAND "" - INSTALL_COMMAND "" - BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install - DEPENDS python -) - -ExternalProject_Add(numpy - GIT_REPOSITORY https://github.com/numpy/numpy.git - GIT_TAG ${NUMPY_VERSION} - CONFIGURE_COMMAND "" - UPDATE_COMMAND "" - PREFIX ${NUMPY_SOURCES_DIR} - BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py build - INSTALL_COMMAND ${PYTHON_EXECUTABLE} setup.py install - BUILD_IN_SOURCE 1 - DEPENDS python setuptools cython -) +FIND_PACKAGE(NumPy) + +IF(NOT ${NUMPY_FOUND}) + + INCLUDE(ExternalProject) + + SET(NUMPY_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/numpy) + SET(NUMPY_INSTALL_DIR ${PROJECT_BINARY_DIR}/numpy) + set(NUMPY_VERSION "v1.11.3") + + ExternalProject_Add(setuptools + PREFIX ${PYTHON_SOURCES_DIR}/setuptools + URL http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz + URL_MD5 7df2a529a074f613b509fb44feefe74e + BUILD_IN_SOURCE 1 + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python zlib + ) + + ExternalProject_Add(cython + PREFIX ${PYTHON_SOURCES_DIR}/cython + GIT_REPOSITORY https://github.com/cython/cython.git + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + UPDATE_COMMAND "" + PATCH_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python + ) + + ExternalProject_Add(numpy + GIT_REPOSITORY https://github.com/numpy/numpy.git + GIT_TAG ${NUMPY_VERSION} + CONFIGURE_COMMAND "" + UPDATE_COMMAND "" + PREFIX ${NUMPY_SOURCES_DIR} + BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py build + INSTALL_COMMAND ${PYTHON_EXECUTABLE} setup.py install + BUILD_IN_SOURCE 1 + DEPENDS python setuptools cython + ) + + # find numpy include directory + FILE(WRITE ${PROJECT_BINARY_DIR}/FindNumpyPath.py + "try: import numpy; print(numpy.get_include())\nexcept:pass\n") + + EXEC_PROGRAM("${PYTHON_EXECUTABLE}" ${PROJECT_BINARY_DIR} + ARGS "FindNumpyPath.py" + OUTPUT_VARIABLE NUMPY_PATH) + + FIND_PATH(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h + HINTS "${NUMPY_PATH}" "${PYTHON_INCLUDE_PATH}") + + INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) + +ENDIF() diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index d1220036a0f0e..2683153b49769 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# INCLUDE(cblas) - INCLUDE(ExternalProject) SET(CBLAS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/openblas) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index b459913314025..2354f555db89d 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -12,72 +12,81 @@ # See the License for the specific language governing permissions and # limitations under the License. -INCLUDE(ExternalProject) +FIND_PACKAGE(PythonLibs 2.7) +FIND_PACKAGE(PythonInterp 2.7) -SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python) -SET(PYTHON_INSTALL_DIR ${PROJECT_BINARY_DIR}/python) +IF((NOT ${PYTHONINTERP_FOUND}) OR (NOT ${PYTHONLIBS_FOUND})) -if(MSVC) - list(APPEND EXTERNAL_PROJECT_OPTIONAL_ARGS - PATCH_COMMAND ${CMAKE_COMMAND} - -DPYTHON_SRC_DIR:PATH=${_python_SOURCE_DIR} - -P ${CMAKE_CURRENT_LIST_DIR}/PythonPatch.cmake - ) -endif() + INCLUDE(ExternalProject) -if(APPLE) - list(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS - -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON - ) -endif() + SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python) + SET(PYTHON_INSTALL_DIR ${PROJECT_BINARY_DIR}/python) -set(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS) + IF(MSVC) + LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_ARGS + PATCH_COMMAND ${CMAKE_COMMAND} + -DPYTHON_SRC_DIR:PATH=${_python_SOURCE_DIR} + -P ${CMAKE_CURRENT_LIST_DIR}/PythonPatch.cmake + ) + ENDIF() -# Force Python build to "Release". -if(CMAKE_CONFIGURATION_TYPES) - set(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR}) - set(CMAKE_CFG_INTDIR "Release") -else() - list(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release) -endif() + IF(APPLE) + LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS + -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON + ) + ENDIF() -ExternalProject_Add(python - GIT_REPOSITORY "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git" - GIT_TAG "ed5f9bcee540e47f82fa17f8360b820591aa6d66" - PREFIX ${PYTHON_SOURCES_DIR} - UPDATE_COMMAND "" - CMAKE_CACHE_ARGS - -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR} - -DBUILD_SHARED:BOOL=OFF - -DBUILD_STATIC:BOOL=ON - -DUSE_SYSTEM_LIBRARIES:BOOL=OFF - -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT} - -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR} - -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES} - -DDOWNLOAD_SOURCES:BOOL=ON - -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF - ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS} - ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS} - DEPENDS zlib -) + SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS) -set(_python_DIR ${PYTHON_INSTALL_DIR}) + # Force Python build to "Release". + IF(CMAKE_CONFIGURATION_TYPES) + SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR}) + SET(CMAKE_CFG_INTDIR "Release") + ELSE() + LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS + -DCMAKE_BUILD_TYPE:STRING=Release) + ENDIF() -if(UNIX) - set(_python_IMPORT_SUFFIX so) - if(APPLE) - set(_python_IMPORT_SUFFIX dylib) - endif() - set(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE) - set(PYTHON_LIBRARY "${PYTHON_INSTALL_DIR}/lib/libpython2.7.${_python_IMPORT_SUFFIX}" CACHE FILEPATH "Python library" FORCE) - set(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE) - set(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE) -elseif(WIN32) - set(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE) - set(PYTHON_LIBRARY "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE) - set(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE) - set(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE) -else() - message(FATAL_ERROR "Unknown system !") -endif() + ExternalProject_Add(python + GIT_REPOSITORY "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git" + GIT_TAG "ed5f9bcee540e47f82fa17f8360b820591aa6d66" + PREFIX ${PYTHON_SOURCES_DIR} + UPDATE_COMMAND "" + CMAKE_CACHE_ARGS + -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR} + -DBUILD_SHARED:BOOL=OFF + -DBUILD_STATIC:BOOL=ON + -DUSE_SYSTEM_LIBRARIES:BOOL=OFF + -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT} + -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR} + -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES} + -DDOWNLOAD_SOURCES:BOOL=ON + -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF + ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS} + ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS} + DEPENDS zlib + ) + + SET(_python_DIR ${PYTHON_INSTALL_DIR}) + + IF(UNIX) + SET(_python_IMPORT_SUFFIX a) + IF(APPLE) + SET(_python_IMPORT_SUFFIX lib) + ENDIF() + SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE) + SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.${_python_IMPORT_SUFFIX}" CACHE FILEPATH "Python library" FORCE) + SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE) + SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE) + ELSEIF(WIN32) + SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE) + SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE) + SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE) + SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE) + ELSE() + MESSAGE(FATAL_ERROR "Unknown system !") + ENDIF() + +INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) + +ENDIF() diff --git a/cmake/external/swig.cmake b/cmake/external/swig.cmake index 9dc112b98ecc0..1ec61660bc069 100644 --- a/cmake/external/swig.cmake +++ b/cmake/external/swig.cmake @@ -13,9 +13,9 @@ # limitations under the License. # Look for system swig -# FIND_PACKAGE(SWIG) +FIND_PACKAGE(SWIG) -#IF(NOT ${SWIG_FOUND}) +IF(NOT ${SWIG_FOUND}) # build swig as an external project INCLUDE(ExternalProject) SET(SWIG_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/swig) @@ -62,14 +62,28 @@ ./configure --prefix=${SWIG_INSTALL_DIR} --with-pcre-prefix=${SWIG_INSTALL_DIR}/pcre - --with-python=${PYTHON_EXECUTABLE} BUILD_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make INSTALL_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make install - DEPENDS pcre python + DEPENDS pcre ) set(SWIG_DIR ${SWIG_INSTALL_DIR}/share/swig/${SWIG_TARGET_VERSION} CACHE FILEPATH "SWIG Directory" FORCE) set(SWIG_EXECUTABLE ${SWIG_INSTALL_DIR}/bin/swig CACHE FILEPATH "SWIG Executable" FORCE) ENDIF(WIN32) -#ENDIF() +ENDIF() +FUNCTION(generate_python_api target_name) + ADD_CUSTOM_COMMAND(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py + ${PROJ_ROOT}/paddle/Paddle_wrap.cxx + ${PROJ_ROOT}/paddle/Paddle_wrap.h + COMMAND ${SWIG_EXECUTABLE} -python -c++ -outcurrentdir -I../ api/Paddle.swig + && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py + DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig + ${PROJ_ROOT}/paddle/api/PaddleAPI.h + WORKING_DIRECTORY ${PROJ_ROOT}/paddle + COMMENT "Generate Python API from swig") + ADD_CUSTOM_TARGET(${target_name} ALL DEPENDS + ${PROJ_ROOT}/paddle/Paddle_wrap.cxx + ${PROJ_ROOT}/paddle/Paddle_wrap.h + ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py) +ENDFUNCTION(generate_python_api) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 57864aca69cf5..4fdd47acdb75e 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -17,22 +17,29 @@ INCLUDE(ExternalProject) SET(WARPCTC_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/warpctc) SET(WARPCTC_INSTALL_DIR ${PROJECT_BINARY_DIR}/warpctc) +IF(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + SET(USE_OMP ON) +ELSE() + SET(USE_OMP OFF) +ENDIF() + ExternalProject_Add( warpctc GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git" PREFIX ${WARPCTC_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} CMAKE_ARGS -DWITH_GPU=${CUDA_FOUND} + CMAKE_ARGS -DWITH_OMP=${USE_OMP} ) SET(WARPCTC_INCLUDE_DIR "${WARP_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) IF(WIN32) - set(WARPCTC_LIBRARIES + SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) ELSE(WIN32) - set(WARPCTC_LIBRARIES + SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.so" CACHE FILEPATH "Warp-ctc Library" FORCE) ENDIF(WIN32) From 662f174b856353105b5291be8f3a855ed54f5f6d Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 17:12:39 +0800 Subject: [PATCH 04/22] Refine cmake file names --- .gitignore | 1 + cmake/{FindPythonModule.cmake => python_module.cmake} | 0 cmake/{FindAVX.cmake => simd.cmake} | 0 3 files changed, 1 insertion(+) rename cmake/{FindPythonModule.cmake => python_module.cmake} (100%) rename cmake/{FindAVX.cmake => simd.cmake} (100%) diff --git a/.gitignore b/.gitignore index 1c9730a5ad57c..0a15b996e26e0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ build/ .pydevproject Makefile .test_env/ +third_party/ *~ bazel-* diff --git a/cmake/FindPythonModule.cmake b/cmake/python_module.cmake similarity index 100% rename from cmake/FindPythonModule.cmake rename to cmake/python_module.cmake diff --git a/cmake/FindAVX.cmake b/cmake/simd.cmake similarity index 100% rename from cmake/FindAVX.cmake rename to cmake/simd.cmake From 62b55cc6ab9c21c9a09b3ad0c9fd94e5e8a3bfd6 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 17:13:21 +0800 Subject: [PATCH 05/22] Remove paddle internals --- python/CMakeLists.txt | 5 ----- python/setup.py.in | 5 ----- 2 files changed, 10 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index dce0b90952436..6b80e4d58ee17 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -10,11 +10,6 @@ set(PY_FILES paddle/__init__.py ${HELPERS_PY_FILES} ${UTILS_PY_FILES}) -set(PADDLE_INTERNAL_PACKAGE "") -if (PADDLE_WITH_INTERNAL) - set(PADDLE_INTERNAL_PACKAGE "paddle.internals") -endif() - configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py) diff --git a/python/setup.py.in b/python/setup.py.in index d2fb95f27ff2f..b66a42e87c787 100644 --- a/python/setup.py.in +++ b/python/setup.py.in @@ -1,16 +1,11 @@ from setuptools import setup -INTERNAL_PACKAGE='${PADDLE_INTERNAL_PACKAGE}' - packages=['paddle', 'paddle.proto', 'paddle.trainer', 'paddle.trainer_config_helpers', 'paddle.utils'] -if len(INTERNAL_PACKAGE) != 0: - packages.append(INTERNAL_PACKAGE) - setup(name='paddle', version='${PADDLE_VERSION}', description='Parallel Distributed Deep Learning', From a02ec8c9323558d035fc2ca78d5076decca52c6f Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 17:13:39 +0800 Subject: [PATCH 06/22] Refine CMakeLists --- CMakeLists.txt | 180 ++++++++++--------------------------- cmake/check_packages.cmake | 31 +------ cmake/definitions.cmake | 62 +++++++++++++ cmake/rdma.cmake | 132 ++++++++++++++------------- cmake/swig.cmake | 15 ---- cmake/util.cmake | 34 +------ cmake/version.cmake | 1 + 7 files changed, 184 insertions(+), 271 deletions(-) create mode 100644 cmake/definitions.cmake delete mode 100644 cmake/swig.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 65fbbb481c432..7db4c9f1d1f2c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,163 +4,75 @@ project(paddle CXX C) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") set(PROJ_ROOT ${CMAKE_SOURCE_DIR}) -include(package) -find_package(SWIG 2.0) -find_package(CUDA QUIET) -find_package(Protobuf REQUIRED) - -# Check protobuf library version. -execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --version - OUTPUT_VARIABLE PROTOBUF_VERSION) -string(REPLACE "libprotoc " "" PROTOBUF_VERSION ${PROTOBUF_VERSION}) -set(PROTOBUF_3 OFF) -if (${PROTOBUF_VERSION} VERSION_GREATER "3.0.0" OR ${PROTOBUF_VERSION} VERSION_EQUAL "3.0.0") - set(PROTOBUF_3 ON) -endif() - -find_package(PythonLibs 2.7 REQUIRED) -find_package(PythonInterp 2.7 REQUIRED) -find_package(ZLIB REQUIRED) -find_package(NumPy REQUIRED) -find_package(Threads REQUIRED) -find_package(AVX QUIET) -find_package(Glog REQUIRED) -find_package(Gflags REQUIRED) -find_package(GTest) find_package(Sphinx) find_package(Doxygen) -include(cblas) -find_program(M4_EXECUTABLE m4) -###################### Configurations ########################### +find_package(CUDA QUIET) +find_package(Git REQUIRED) +find_package(Threads REQUIRED) + +include(simd) + +###################### Configurations ############################ option(WITH_DSO "Compile PaddlePaddle with dynamic linked libraries" ON) option(WITH_GPU "Compile PaddlePaddle with gpu" ${CUDA_FOUND}) option(WITH_DOUBLE "Compile PaddlePaddle with double precision, otherwise use single precision" OFF) option(WITH_AVX "Compile PaddlePaddle with avx intrinsics" ${AVX_FOUND}) option(WITH_PYTHON "Compile PaddlePaddle with python interpreter" ON) -option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ${PYTHONINTERP_FOUND}) +option(WITH_STYLE_CHECK "Style Check for PaddlePaddle" ON) option(WITH_RDMA "Compile PaddlePaddle with rdma support" OFF) option(WITH_TIMER "Compile PaddlePaddle use timer" OFF) option(WITH_PROFILER "Compile PaddlePaddle use gpu profiler" OFF) -option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ${GTEST_FOUND}) +option(WITH_TESTING "Compile and run unittest for PaddlePaddle" ON) option(WITH_DOC "Compile PaddlePaddle with documentation" OFF) -option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ${SWIG_FOUND}) +option(WITH_SWIG_PY "Compile PaddlePaddle with py PaddlePaddle prediction api" ON) option(ON_TRAVIS "Running test on travis-ci or not." OFF) option(ON_COVERALLS "Generating code coverage data on coveralls or not." OFF) option(COVERALLS_UPLOAD "Uploading the generated coveralls json." ON) +include(external/zlib) # download, build, install zlib +include(external/gflags) # download, build, install gflags +include(external/glog) # download, build, install glog +include(external/gtest) # download, build, install gtest +include(external/protobuf) # download, build, install protobuf +include(external/openblas) # download, build, install openblas +include(external/python) # download, build, install python +include(external/numpy) # download, build, install numpy +include(external/swig) # download, build, install swig +include(external/warpctc) # download, build, install warpctc + +include(package) # set paddle packages +include(cpplint) # set paddle c++ style +include(ccache) # set ccache for compilation +include(util) # set unittest and link libs +include(rdma) # set rdma libraries +include(flags) # set paddle compile flags +include(cudnn) # set cudnn libraries +include(version) # set PADDLE_VERSION +include(coveralls) # set code coverage +include(python_module) # set python module + +include(check_packages) # check configuration +include(definitions) # add paddle definitions -include(cpplint) -include(ccache) -if(WITH_RDMA) - include(rdma) -endif() -include(util) -include(flags) -include(cudnn) -include(FindPythonModule) -include(check_packages) -include(swig) -include(coveralls) - -# Set PaddlePaddle version to Git tag name or Git commit ID. -find_package(Git REQUIRED) -# version.cmake will get the current PADDLE_VERSION -include(version) -add_definitions(-DPADDLE_VERSION=${PADDLE_VERSION}) - -if(NOT WITH_GPU) - add_definitions(-DPADDLE_ONLY_CPU) - add_definitions(-DHPPL_STUB_FUNC) - - list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) -else() - if(${CUDA_VERSION_MAJOR} VERSION_LESS 7) - message(FATAL_ERROR "Paddle need CUDA >= 7.0 to compile") - endif() - - if(NOT CUDNN_FOUND) - message(FATAL_ERROR "Paddle need cudnn to compile") - endif() - - if(WITH_AVX) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}") - else(WITH_AVX) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}") - endif(WITH_AVX) - - # Include cuda and cudnn - include_directories(${CUDNN_INCLUDE_DIR}) - include_directories(${CUDA_TOOLKIT_INCLUDE}) -endif(NOT WITH_GPU) - -if(WITH_DSO) - add_definitions(-DPADDLE_USE_DSO) -endif(WITH_DSO) - -if(WITH_DOUBLE) - add_definitions(-DPADDLE_TYPE_DOUBLE) - set(ACCURACY double) -else(WITH_DOUBLE) - set(ACCURACY float) -endif(WITH_DOUBLE) - -if(NOT WITH_TIMER) - add_definitions(-DPADDLE_DISABLE_TIMER) -endif(NOT WITH_TIMER) - -if(NOT WITH_PROFILER) - add_definitions(-DPADDLE_DISABLE_PROFILER) -endif(NOT WITH_PROFILER) - -if(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}") -else(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}") -endif(WITH_AVX) - -if(WITH_PYTHON) - include_directories(${PYTHON_INCLUDE_DIR}) - include_directories(${PYTHON_NUMPY_INCLUDE_DIR}) -else(WITH_PYTHON) - add_definitions(-DPADDLE_NO_PYTHON) -endif(WITH_PYTHON) - -if(WITH_RDMA) - include_directories("${RDMA_INC_DIR}") -else(WITH_RDMA) - add_definitions(-DPADDLE_DISABLE_RDMA) -endif(WITH_RDMA) - -# glog -include_directories(${LIBGLOG_INCLUDE_DIR}) - -#gflags -add_definitions(-DGFLAGS_NS=${GFLAGS_NAMESPACE}) -include_directories(${GFLAGS_INCLUDE_DIRS}) - -if(WITH_TESTING) - enable_testing() - include_directories(${GTEST_INCLUDE_DIRS}) -endif() - -include_directories("${CBLAS_INC_DIR}") include_directories("${PROJ_ROOT}") include_directories("${PROJ_ROOT}/paddle/cuda/include") -include_directories(${PROTOBUF_INCLUDE_DIRS}) include_directories("${CMAKE_CURRENT_BINARY_DIR}/proto") -if(EXISTS "${PROJ_ROOT}/paddle/internals/CMakeLists.txt") - set(PADDLE_WITH_INTERNAL ON) - include(paddle/internals/CMakeLists.txt) -else() - set(PADDLE_WITH_INTERNAL OFF) - set(INTERNAL_PROTO_PATH "") -endif() + +set(EXTERNAL_LIBS + # have not include gtest here. + ${GFLAGS_LIBRARIES} + ${GLOG_LIBRARIES} + ${CBLAS_LIBRARIES} + ${PROTOBUF_LIBRARIES} + ${WARPCTC_LIBRARIES} + ${ZLIB_LIBRARIES} +) + add_subdirectory(proto) add_subdirectory(paddle) add_subdirectory(python) + if(WITH_DOC) add_subdirectory(doc) endif() diff --git a/cmake/check_packages.cmake b/cmake/check_packages.cmake index afb84c6ff52af..8f0ed26256650 100644 --- a/cmake/check_packages.cmake +++ b/cmake/check_packages.cmake @@ -2,38 +2,13 @@ if(WITH_GPU) find_package(CUDA REQUIRED) # CUDA is required when use gpu -endif() - -if(WITH_PYTHON) - find_package(PythonLibs 2.6 REQUIRED) - find_package(PythonInterp REQUIRED) - find_package(NumPy REQUIRED) -endif() - -if(WITH_STYLE_CHECK) - find_package(PythonInterp REQUIRED) -endif() - -find_package(Glog REQUIRED) - -find_package(Gflags REQUIRED) - -if(WITH_TESTING) - find_package(GTest REQUIRED) -endif() +endif(WITH_GPU) if(WITH_DOC) find_package(Sphinx REQUIRED) find_python_module(recommonmark REQUIRED) -endif() +endif(WITH_DOC) if(WITH_SWIG_PY) - if(NOT SWIG_FOUND) - message(FATAL_ERROR "SWIG is not found. Please install swig or disable WITH_SWIG_PY") - endif() find_python_module(wheel REQUIRED) # package wheel -endif() - -if(NOT M4_EXECUTABLE) - message(FATAL_ERROR "Paddle need m4 to generate proto file.") -endif() +endif(WITH_SWIG_PY) diff --git a/cmake/definitions.cmake b/cmake/definitions.cmake new file mode 100644 index 0000000000000..99a52ad764b90 --- /dev/null +++ b/cmake/definitions.cmake @@ -0,0 +1,62 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(WITH_DSO) + add_definitions(-DPADDLE_USE_DSO) +endif(WITH_DSO) + +if(WITH_DOUBLE) + add_definitions(-DPADDLE_TYPE_DOUBLE) +endif(WITH_DOUBLE) + +if(NOT WITH_TIMER) + add_definitions(-DPADDLE_DISABLE_TIMER) +endif(NOT WITH_TIMER) + +if(NOT WITH_PROFILER) + add_definitions(-DPADDLE_DISABLE_PROFILER) +endif(NOT WITH_PROFILER) + +if(NOT WITH_GPU) + add_definitions(-DPADDLE_ONLY_CPU) + add_definitions(-DHPPL_STUB_FUNC) + + list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) +else() + if(${CUDA_VERSION_MAJOR} VERSION_LESS 7) + message(FATAL_ERROR "Paddle need CUDA >= 7.0 to compile") + endif() + + if(NOT CUDNN_FOUND) + message(FATAL_ERROR "Paddle need cudnn to compile") + endif() + + if(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}") + else(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}") + endif(WITH_AVX) + + # Include cuda and cudnn + include_directories(${CUDNN_INCLUDE_DIR}) + include_directories(${CUDA_TOOLKIT_INCLUDE}) +endif(NOT WITH_GPU) + +if(WITH_AVX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}") +else(WITH_AVX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}") +endif(WITH_AVX) diff --git a/cmake/rdma.cmake b/cmake/rdma.cmake index e9a4da79aa92a..9ff1a77cac74f 100644 --- a/cmake/rdma.cmake +++ b/cmake/rdma.cmake @@ -5,72 +5,76 @@ # svn co https://svn.baidu.com/sys/ip/trunk/rdma/thirdparty rdma/ # we use static output in svn repositories to avoid implict bugs from not standard runtime env. -set(RDMA_ROOT $ENV{RDMA_ROOT} CACHE PATH "Folder contains RDMA sock library and thirdparty library") +if(WITH_RDMA) + set(RDMA_ROOT $ENV{RDMA_ROOT} CACHE PATH "Folder contains RDMA sock library and thirdparty library") -function(generate_rdma_links) - #redirect to current DIR to isolate the pollution from system runtime environment - #it can benifits unified control for different gcc environment. - #e.g, by default gcc48 did not refer /usr/lib64 which could contain low version - #runtime libraries that will crash process while loading it. That redirect trick - #can fix it. - execute_process( - COMMAND mkdir -p librdma - COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so.1 - COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so - COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so.1 - COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - ) -endfunction(generate_rdma_links) - - -#check and set headers -find_path(RDMA_INC_SXISOCK sxi_sock.h PATHS ${RDMA_ROOT}/sockrdmav1/output/include) -find_path(RDMA_INC_XIO libxio.h PATHS ${RDMA_ROOT}/thirdparty/output/accelio) -find_path(RDMA_INC_EVENT event2 PATHS ${RDMA_ROOT}/thirdparty/output/libevent) -find_path(RDMA_INC_NUMA numa.h PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) - -#check and set libs -find_library(RDMA_LIB_SXISOCK NAMES sxisock PATHS ${RDMA_ROOT}/sockrdmav1/output) -find_library(RDMA_LIB_XIO NAMES xio PATHS ${RDMA_ROOT}/thirdparty/output/accelio) -find_library(RDMA_LIB_EVENT NAMES event PATHS ${RDMA_ROOT}/thirdparty/output/libevent) -find_library(RDMA_LIB_EVENT_CORE NAMES event_core PATHS ${RDMA_ROOT}/thirdparty/output/libevent) -find_library(RDMA_LIB_EVENT_EXTRA NAMES event_extra PATHS ${RDMA_ROOT}/thirdparty/output/libevent) -find_library(RDMA_LIB_EVENT_PTHREADS NAMES event_pthreads PATHS ${RDMA_ROOT}/thirdparty/output/libevent) -find_library(RDMA_LIB_NUMA NAMES numa PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) - -if( - RDMA_INC_SXISOCK AND - RDMA_INC_XIO AND - RDMA_INC_EVENT AND - RDMA_INC_NUMA AND - RDMA_LIB_SXISOCK AND - RDMA_LIB_XIO AND - RDMA_LIB_EVENT AND - RDMA_LIB_EVENT_CORE AND - RDMA_LIB_EVENT_EXTRA AND - RDMA_LIB_EVENT_PTHREADS AND - RDMA_LIB_NUMA + function(generate_rdma_links) + #redirect to current DIR to isolate the pollution from system runtime environment + #it can benifits unified control for different gcc environment. + #e.g, by default gcc48 did not refer /usr/lib64 which could contain low version + #runtime libraries that will crash process while loading it. That redirect trick + #can fix it. + execute_process( + COMMAND mkdir -p librdma + COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so.1 + COMMAND ln -s -f /usr/lib64/libibverbs.so.1.0.0 librdma/libibverbs.so + COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so.1 + COMMAND ln -s -f /usr/lib64/librdmacm.so.1.0.0 librdma/librdmacm.so + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} ) + endfunction(generate_rdma_links) - set(RDMA_INC_DIR - ${RDMA_INC_SXISOCK} - ${RDMA_INC_XIO} - ${RDMA_INC_EVENT} - ${RDMA_INC_NUMA}) - set(RDMA_LIBS - ${RDMA_LIB_SXISOCK} - ${RDMA_LIB_XIO} - ${RDMA_LIB_EVENT} - ${RDMA_LIB_EVENT_CORE} - ${RDMA_LIB_EVENT_EXTRA} - ${RDMA_LIB_EVENT_PTHREADS} - ${RDMA_LIB_NUMA} - ) - set(RDMA_LD_FLAGS "-L./librdma -libverbs -lrdmacm -Xlinker -rpath ./librdma") - return() -endif() + #check and set headers + find_path(RDMA_INC_SXISOCK sxi_sock.h PATHS ${RDMA_ROOT}/sockrdmav1/output/include) + find_path(RDMA_INC_XIO libxio.h PATHS ${RDMA_ROOT}/thirdparty/output/accelio) + find_path(RDMA_INC_EVENT event2 PATHS ${RDMA_ROOT}/thirdparty/output/libevent) + find_path(RDMA_INC_NUMA numa.h PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) + + #check and set libs + find_library(RDMA_LIB_SXISOCK NAMES sxisock PATHS ${RDMA_ROOT}/sockrdmav1/output) + find_library(RDMA_LIB_XIO NAMES xio PATHS ${RDMA_ROOT}/thirdparty/output/accelio) + find_library(RDMA_LIB_EVENT NAMES event PATHS ${RDMA_ROOT}/thirdparty/output/libevent) + find_library(RDMA_LIB_EVENT_CORE NAMES event_core PATHS ${RDMA_ROOT}/thirdparty/output/libevent) + find_library(RDMA_LIB_EVENT_EXTRA NAMES event_extra PATHS ${RDMA_ROOT}/thirdparty/output/libevent) + find_library(RDMA_LIB_EVENT_PTHREADS NAMES event_pthreads PATHS ${RDMA_ROOT}/thirdparty/output/libevent) + find_library(RDMA_LIB_NUMA NAMES numa PATHS ${RDMA_ROOT}/thirdparty/output/libnuma) -#if this module is not called, RDMA_INC_DIR RDMA_LIBS will be null, so top module always refer this variable + if( + RDMA_INC_SXISOCK AND + RDMA_INC_XIO AND + RDMA_INC_EVENT AND + RDMA_INC_NUMA AND + RDMA_LIB_SXISOCK AND + RDMA_LIB_XIO AND + RDMA_LIB_EVENT AND + RDMA_LIB_EVENT_CORE AND + RDMA_LIB_EVENT_EXTRA AND + RDMA_LIB_EVENT_PTHREADS AND + RDMA_LIB_NUMA + ) -message(FATAL_ERROR, "RDMA libraries are not found, try to set RDMA_ROOT or check all related libraries.") + set(RDMA_INC_DIR + ${RDMA_INC_SXISOCK} + ${RDMA_INC_XIO} + ${RDMA_INC_EVENT} + ${RDMA_INC_NUMA}) + set(RDMA_LIBS + ${RDMA_LIB_SXISOCK} + ${RDMA_LIB_XIO} + ${RDMA_LIB_EVENT} + ${RDMA_LIB_EVENT_CORE} + ${RDMA_LIB_EVENT_EXTRA} + ${RDMA_LIB_EVENT_PTHREADS} + ${RDMA_LIB_NUMA} + ) + set(RDMA_LD_FLAGS "-L./librdma -libverbs -lrdmacm -Xlinker -rpath ./librdma") + include_directories("${RDMA_INC_DIR}") + else() + #if this module is not called, RDMA_INC_DIR RDMA_LIBS will be null, so top module always refer this variable + message(FATAL_ERROR, "RDMA libraries are not found, try to set RDMA_ROOT or check all related libraries.") + endif() +else(WITH_RDMA) + set(RDMA_LIBS "") + set(RDMA_LD_FLAGS "") + add_definitions(-DPADDLE_DISABLE_RDMA) +endif(WITH_RDMA) diff --git a/cmake/swig.cmake b/cmake/swig.cmake deleted file mode 100644 index 97e87aa947791..0000000000000 --- a/cmake/swig.cmake +++ /dev/null @@ -1,15 +0,0 @@ -function(generate_python_api target_name) - add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py - ${PROJ_ROOT}/paddle/Paddle_wrap.cxx - ${PROJ_ROOT}/paddle/Paddle_wrap.h - COMMAND swig -python -c++ -outcurrentdir -I../ api/Paddle.swig - && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py - DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig - ${PROJ_ROOT}/paddle/api/PaddleAPI.h - WORKING_DIRECTORY ${PROJ_ROOT}/paddle - COMMENT "Generate Python API from swig") - add_custom_target(${target_name} ALL DEPENDS - ${PROJ_ROOT}/paddle/Paddle_wrap.cxx - ${PROJ_ROOT}/paddle/Paddle_wrap.h - ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py) -endfunction(generate_python_api) diff --git a/cmake/util.cmake b/cmake/util.cmake index 8a71b23c62d9f..1b1e630661e63 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -81,18 +81,6 @@ function(link_paddle_exe TARGET_NAME) set(METRIC_LIBS "") endif() - if(PADDLE_WITH_INTERNAL) - set(INTERAL_LIBS paddle_internal_gserver paddle_internal_parameter) - target_circle_link_libraries(${TARGET_NAME} - ARCHIVE_START - paddle_internal_gserver - paddle_internal_owlqn - ARCHIVE_END - paddle_internal_parameter) - else() - set(INTERAL_LIBS "") - endif() - target_circle_link_libraries(${TARGET_NAME} ARCHIVE_START paddle_gserver @@ -109,20 +97,11 @@ function(link_paddle_exe TARGET_NAME) paddle_cuda paddle_test_main ${METRIC_LIBS} - ${PROTOBUF_LIBRARY} - ${LIBGLOG_LIBRARY} - ${GFLAGS_LIBRARIES} + ${EXTERNAL_LIBS} ${CMAKE_THREAD_LIBS_INIT} - ${CBLAS_LIBS} - ${ZLIB_LIBRARIES} - ${INTERAL_LIBS} - ${CMAKE_DL_LIBS}) - - if(WITH_RDMA) - target_link_libraries(${TARGET_NAME} - ${RDMA_LD_FLAGS} - ${RDMA_LIBS}) - endif() + ${CMAKE_DL_LIBS} + ${RDMA_LD_FLAGS} + ${RDMA_LIBS}) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} @@ -142,11 +121,6 @@ function(link_paddle_exe TARGET_NAME) target_link_libraries(${TARGET_NAME} rt) endif() endif() - - if(NOT WITH_DSO) - target_link_libraries(${TARGET_NAME} - ${WARPCTC_LIBRARY}) - endif() endfunction() # link_paddle_test diff --git a/cmake/version.cmake b/cmake/version.cmake index a0518e07e88a1..ac1583a24c828 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -21,4 +21,5 @@ while ("${PADDLE_VERSION}" STREQUAL "") endif() endwhile() +add_definitions(-DPADDLE_VERSION=${PADDLE_VERSION}) message(STATUS "Paddle version is ${PADDLE_VERSION}") From 46cadaeaa98668ba868f7f9eef406a59b0176f46 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 23:23:25 +0800 Subject: [PATCH 07/22] Update external dependencies cmake --- cmake/external/numpy.cmake | 5 +++-- cmake/external/protobuf.cmake | 2 +- cmake/external/python.cmake | 8 ++++---- cmake/external/warpctc.cmake | 7 ++++--- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cmake/external/numpy.cmake b/cmake/external/numpy.cmake index 607ff317899aa..9d686ecaac981 100644 --- a/cmake/external/numpy.cmake +++ b/cmake/external/numpy.cmake @@ -70,6 +70,7 @@ IF(NOT ${NUMPY_FOUND}) FIND_PATH(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h HINTS "${NUMPY_PATH}" "${PYTHON_INCLUDE_PATH}") - INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) - ENDIF() + +INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) + diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 8acc6325b9e55..0138f082d942c 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -22,7 +22,7 @@ ExternalProject_Add( PREFIX ${PROTOBUF_SOURCES_DIR} DEPENDS zlib GIT_REPOSITORY "https://github.com/google/protobuf.git" - GIT_TAG "v3.0.0" +# GIT_TAG "v3.1.0" CONFIGURE_COMMAND ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/protobuf/cmake -Dprotobuf_BUILD_TESTS=OFF diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 2354f555db89d..7b66cb44e4f9f 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -19,8 +19,8 @@ IF((NOT ${PYTHONINTERP_FOUND}) OR (NOT ${PYTHONLIBS_FOUND})) INCLUDE(ExternalProject) - SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python) - SET(PYTHON_INSTALL_DIR ${PROJECT_BINARY_DIR}/python) + SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/Python) + SET(PYTHON_INSTALL_DIR ${PROJECT_BINARY_DIR}/Python) IF(MSVC) LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_ARGS @@ -87,6 +87,6 @@ IF((NOT ${PYTHONINTERP_FOUND}) OR (NOT ${PYTHONLIBS_FOUND})) MESSAGE(FATAL_ERROR "Unknown system !") ENDIF() -INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) - ENDIF() + +INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 4fdd47acdb75e..d59e5e9c3af48 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -17,7 +17,7 @@ INCLUDE(ExternalProject) SET(WARPCTC_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/warpctc) SET(WARPCTC_INSTALL_DIR ${PROJECT_BINARY_DIR}/warpctc) -IF(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") +IF(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" ) SET(USE_OMP ON) ELSE() SET(USE_OMP OFF) @@ -28,8 +28,9 @@ ExternalProject_Add( GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git" PREFIX ${WARPCTC_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} - CMAKE_ARGS -DWITH_GPU=${CUDA_FOUND} + CMAKE_ARGS -DWITH_GPU=${WITH_GPU} CMAKE_ARGS -DWITH_OMP=${USE_OMP} + UPDATE_COMMAND "" ) SET(WARPCTC_INCLUDE_DIR "${WARP_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) @@ -40,7 +41,7 @@ IF(WIN32) "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) ELSE(WIN32) SET(WARPCTC_LIBRARIES - "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.so" CACHE FILEPATH "Warp-ctc Library" FORCE) + "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.dylib" CACHE FILEPATH "Warp-ctc Library" FORCE) ENDIF(WIN32) LIST(APPEND external_project_dependencies warpctc) From de3c15277f2b08e88c3b7f84156064fef0831b90 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 23:25:32 +0800 Subject: [PATCH 08/22] Fix glog check type unmatch --- paddle/function/cross_map_normal_op.cpp | 16 ++++++++-------- paddle/gserver/layers/ConvProjection.cpp | 3 ++- paddle/gserver/tests/LayerGradUtil.cpp | 2 +- paddle/gserver/tests/test_BatchNorm.cpp | 4 ++-- paddle/gserver/tests/test_PyDataProvider2.cpp | 2 +- paddle/math/Matrix.cpp | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/paddle/function/cross_map_normal_op.cpp b/paddle/function/cross_map_normal_op.cpp index a9c7693830542..74094bc4fc805 100644 --- a/paddle/function/cross_map_normal_op.cpp +++ b/paddle/function/cross_map_normal_op.cpp @@ -128,11 +128,11 @@ class CrossMapNormalFunc : public FunctionBase { void calc(const Arguments& inputs, const Arguments& outputs, const Arguments& inouts) override { - CHECK_EQ(1, inputs.size()); - CHECK_EQ(2, outputs.size()); - CHECK_EQ(0, inouts.size()); + CHECK_EQ(1, static_cast(inputs.size())); + CHECK_EQ(2, static_cast(outputs.size())); + CHECK_EQ(0, static_cast(inouts.size())); - CHECK_EQ(inputs[0].dims_.size(), 4); + CHECK_EQ(static_cast(inputs[0].dims_.size()), 4); for (size_t i = 0; i < inputs[0].dims_.size(); i++) { CHECK_EQ(inputs[0].dims_[i], outputs[0].dims_[i]); CHECK_EQ(inputs[0].dims_[i], outputs[1].dims_[i]); @@ -180,11 +180,11 @@ class CrossMapNormalGradFunc : public FunctionBase { void calc(const Arguments& inputs, const Arguments& outputs, const Arguments& inouts) override { - CHECK_EQ(4, inputs.size()); - CHECK_EQ(1, outputs.size()); - CHECK_EQ(0, inouts.size()); + CHECK_EQ(4, static_cast(inputs.size())); + CHECK_EQ(1, static_cast(outputs.size())); + CHECK_EQ(0, static_cast(inouts.size())); - CHECK_EQ(inputs[0].dims_.size(), 4); + CHECK_EQ(static_cast(inputs[0].dims_.size()), 4); for (size_t i = 0; i < inputs[0].dims_.size(); i++) { CHECK_EQ(inputs[0].dims_[i], inputs[1].dims_[i]); CHECK_EQ(inputs[0].dims_[i], inputs[2].dims_[i]); diff --git a/paddle/gserver/layers/ConvProjection.cpp b/paddle/gserver/layers/ConvProjection.cpp index e1c4b91ace215..0281170bc5985 100644 --- a/paddle/gserver/layers/ConvProjection.cpp +++ b/paddle/gserver/layers/ConvProjection.cpp @@ -130,7 +130,8 @@ void ConvProjection::reshapeTensorDesc(int batchSize) { void ConvProjection::reshape(int batchSize) { size_t width = calOutputSize(); CHECK_EQ(width, out_->value->getWidth()); - CHECK_EQ(channels_ * imageH_ * imageW_, in_->value->getWidth()) + CHECK_EQ(static_cast(channels_ * imageH_ * imageW_), + in_->value->getWidth()) << "Wrong input size for convolution" << " channels=" << channels_ << " imageH=" << imageH_ << " imageW=" << imageW_ << " inputSize=" << in_->value->getWidth(); diff --git a/paddle/gserver/tests/LayerGradUtil.cpp b/paddle/gserver/tests/LayerGradUtil.cpp index 57c176810fddf..ae016e74eaa84 100644 --- a/paddle/gserver/tests/LayerGradUtil.cpp +++ b/paddle/gserver/tests/LayerGradUtil.cpp @@ -310,7 +310,7 @@ void initDataLayer(TestConfig testConf, testConf.inputDefs[i].labelSeqStartPositions; if (labelSeqStartPositions.size() != 0) { CHECK(!sequenceStartPositions); - CHECK_GE(labelSeqStartPositions.size(), 2); + CHECK_GE(static_cast(labelSeqStartPositions.size()), 2); sequenceStartPositions = ICpuGpuVector::create(labelSeqStartPositions.size(), useGpu); diff --git a/paddle/gserver/tests/test_BatchNorm.cpp b/paddle/gserver/tests/test_BatchNorm.cpp index 7f5fcb670b70a..e000c6994407c 100644 --- a/paddle/gserver/tests/test_BatchNorm.cpp +++ b/paddle/gserver/tests/test_BatchNorm.cpp @@ -114,8 +114,8 @@ TEST(Layer, batchNorm) { bnLayer->forward(PASS_GC); convLayer->forward(PASS_GC); - CHECK_EQ(convLayer->getOutputValue()->getHeight(), 100); - CHECK_EQ(convLayer->getOutputValue()->getWidth(), 576); + CHECK_EQ(static_cast(convLayer->getOutputValue()->getHeight()), 100); + CHECK_EQ(static_cast(convLayer->getOutputValue()->getWidth()), 576); } int main(int argc, char** argv) { diff --git a/paddle/gserver/tests/test_PyDataProvider2.cpp b/paddle/gserver/tests/test_PyDataProvider2.cpp index 5f8bc5ecd0f77..7e193eb31a03e 100644 --- a/paddle/gserver/tests/test_PyDataProvider2.cpp +++ b/paddle/gserver/tests/test_PyDataProvider2.cpp @@ -293,7 +293,7 @@ TEST(PyDataProvider2, can_over_batch_size) { while (true) { int64_t realBatchSize = provider->getNextBatchInternal(batchSize, &batch); if (realBatchSize) { - CHECK_LE(realBatchSize, batchSize); + CHECK_LE(static_cast(realBatchSize), batchSize); } else { break; } diff --git a/paddle/math/Matrix.cpp b/paddle/math/Matrix.cpp index 50d2e3eb67102..b281d5eb02f6d 100644 --- a/paddle/math/Matrix.cpp +++ b/paddle/math/Matrix.cpp @@ -2268,7 +2268,7 @@ void CpuMatrix::contextProjectionBackward(Matrix* inputGrad, int64_t inputDim = inputGrad ? inputGrad->getWidth() : weightGrad ? weightGrad->getWidth() : 0; - CHECK_EQ(getWidth(), inputDim * contextLength); + CHECK_EQ(getWidth(), static_cast(inputDim * contextLength)); const int* starts = sequence.getData(); size_t numSequences = sequence.getSize() - 1; From 0b956711d994d101f7e68aa59b586d2f21645195 Mon Sep 17 00:00:00 2001 From: liaogang Date: Tue, 27 Dec 2016 23:25:50 +0800 Subject: [PATCH 09/22] Add external_project_dependencies for targets --- cmake/util.cmake | 1 + paddle/cuda/CMakeLists.txt | 2 ++ paddle/cuda/include/hl_warpctc_wrap.h | 2 +- paddle/function/CMakeLists.txt | 2 ++ proto/CMakeLists.txt | 8 ++++---- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cmake/util.cmake b/cmake/util.cmake index 1b1e630661e63..b8d20266f4bcf 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -121,6 +121,7 @@ function(link_paddle_exe TARGET_NAME) target_link_libraries(${TARGET_NAME} rt) endif() endif() + add_dependencies(${TARGET_NAME} ${external_project_dependencies}) endfunction() # link_paddle_test diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index aa1ff4a771c4a..57fb89608f4bc 100755 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -88,6 +88,8 @@ else() ${CUDA_CXX_SOURCES}) endif() +add_dependencies(paddle_cuda ${external_project_dependencies}) + add_style_check_target(paddle_cuda ${CUDA_SOURCES} ${CUDA_HEADERS} diff --git a/paddle/cuda/include/hl_warpctc_wrap.h b/paddle/cuda/include/hl_warpctc_wrap.h index 79bf6c3db7f87..7885ae570148c 100644 --- a/paddle/cuda/include/hl_warpctc_wrap.h +++ b/paddle/cuda/include/hl_warpctc_wrap.h @@ -15,8 +15,8 @@ limitations under the License. */ #ifndef HL_WARPCTC_WRAP_H_ #define HL_WARPCTC_WRAP_H_ +#include "ctc.h" #include "hl_base.h" -#include "warp-ctc/include/ctc.h" typedef ctcStatus_t hl_warpctc_status_t; typedef ctcOptions hl_warpctc_options_t; diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index 0697842bbef62..1de887b7dda61 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -10,8 +10,10 @@ if(WITH_GPU) endif() add_library(paddle_function STATIC ${cpp_files} ${cu_objs}) +add_dependencies(paddle_function ${external_project_dependencies}) add_library(paddle_test_main STATIC TestMain.cpp) +add_dependencies(paddle_test_main ${external_project_dependencies}) if(WITH_GPU) # TODO: diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index 2c40070eca44d..c4e170b10fb78 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -20,8 +20,8 @@ foreach(filename ${proto_filenames}) add_custom_command(OUTPUT ${CUR_PROTO_GEN} COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --cpp_out ${CMAKE_CURRENT_BINARY_DIR} - --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} - DEPENDS ${filename}) + --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} + DEPENDS ${filename} ${external_project_dependencies}) set(CUR_PROTO_GEN_PY ${PROJ_ROOT}/paddle/python/paddle/proto/${base_filename}_pb2.py) @@ -30,8 +30,8 @@ foreach(filename ${proto_filenames}) ${PROTO_GEN_PY}) add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY} COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto - --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} - DEPENDS ${filename}) + --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} + DEPENDS ${filename} ${external_project_dependencies}) endforeach() include_directories(${CMAKE_CURRENT_BINARY_DIR}/proto) From 338dc3ee923e0757420b6466f91dc60d19850b2a Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Dec 2016 00:20:15 +0800 Subject: [PATCH 10/22] Fix external denpendencies build order --- cmake/external/numpy.cmake | 2 ++ cmake/external/protobuf.cmake | 12 +++++++++--- cmake/external/python.cmake | 3 +++ cmake/external/swig.cmake | 7 ++++++- cmake/external/warpctc.cmake | 8 +++++++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cmake/external/numpy.cmake b/cmake/external/numpy.cmake index 9d686ecaac981..d01cff9722b84 100644 --- a/cmake/external/numpy.cmake +++ b/cmake/external/numpy.cmake @@ -59,6 +59,8 @@ IF(NOT ${NUMPY_FOUND}) DEPENDS python setuptools cython ) + LIST(APPEND external_project_dependencies numpy) + # find numpy include directory FILE(WRITE ${PROJECT_BINARY_DIR}/FindNumpyPath.py "try: import numpy; print(numpy.get_include())\nexcept:pass\n") diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index 0138f082d942c..efd07eb807a17 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -42,10 +42,16 @@ IF(WIN32) "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protobuf libraries." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc.exe" CACHE FILEPATH "protobuf executable." FORCE) ELSE(WIN32) + FIND_PATH(PROTOBUF_LIBS_DIR libprotoc.a + PATHS + ${PROTOBUF_INSTALL_DIR}/lib64 + ${PROTOBUF_INSTALL_DIR}/lib + NO_DEFAULT_PATH + ) SET(PROTOBUF_LIBRARIES - "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.a" - "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.a" - "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.a" CACHE FILEPATH "protobuf libraries." FORCE) + "${PROTOBUF_LIBS_DIR}/libprotobuf-lite.a" + "${PROTOBUF_LIBS_DIR}/libprotobuf.a" + "${PROTOBUF_LIBS_DIR}/libprotoc.a" CACHE FILEPATH "protobuf libraries." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc" CACHE FILEPATH "protobuf executable." FORCE) ENDIF(WIN32) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 7b66cb44e4f9f..d6cdf535fee4e 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -87,6 +87,9 @@ IF((NOT ${PYTHONINTERP_FOUND}) OR (NOT ${PYTHONLIBS_FOUND})) MESSAGE(FATAL_ERROR "Unknown system !") ENDIF() +LIST(APPEND external_project_dependencies python) + ENDIF() INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) + diff --git a/cmake/external/swig.cmake b/cmake/external/swig.cmake index 1ec61660bc069..2da826d375003 100644 --- a/cmake/external/swig.cmake +++ b/cmake/external/swig.cmake @@ -70,6 +70,9 @@ IF(NOT ${SWIG_FOUND}) set(SWIG_DIR ${SWIG_INSTALL_DIR}/share/swig/${SWIG_TARGET_VERSION} CACHE FILEPATH "SWIG Directory" FORCE) set(SWIG_EXECUTABLE ${SWIG_INSTALL_DIR}/bin/swig CACHE FILEPATH "SWIG Executable" FORCE) ENDIF(WIN32) + + LIST(APPEND external_project_dependencies swig) + ENDIF() FUNCTION(generate_python_api target_name) @@ -80,10 +83,12 @@ FUNCTION(generate_python_api target_name) && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig ${PROJ_ROOT}/paddle/api/PaddleAPI.h + ${external_project_dependencies} WORKING_DIRECTORY ${PROJ_ROOT}/paddle COMMENT "Generate Python API from swig") ADD_CUSTOM_TARGET(${target_name} ALL DEPENDS ${PROJ_ROOT}/paddle/Paddle_wrap.cxx ${PROJ_ROOT}/paddle/Paddle_wrap.h - ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py) + ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py + ${external_project_dependencies}) ENDFUNCTION(generate_python_api) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index d59e5e9c3af48..2e678aadcf91b 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -40,8 +40,14 @@ IF(WIN32) SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) ELSE(WIN32) + IF(APPLE) + SET(_warpctc_SHARED_SUFFIX dylib) + ELSE(APPLE) + SET(_warpctc_SHARED_SUFFIX so) + ENDIF(APPLE) + SET(WARPCTC_LIBRARIES - "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.dylib" CACHE FILEPATH "Warp-ctc Library" FORCE) + "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.${_warpctc_SHARED_SUFFIX}" CACHE FILEPATH "Warp-ctc Library" FORCE) ENDIF(WIN32) LIST(APPEND external_project_dependencies warpctc) From 2ffb6dbb171b426ef85db88df4f5ac2493abb8d8 Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Dec 2016 09:58:54 +0800 Subject: [PATCH 11/22] Fix warpctc header directory --- cmake/external/warpctc.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 2e678aadcf91b..b3dea19cebd27 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -33,7 +33,7 @@ ExternalProject_Add( UPDATE_COMMAND "" ) -SET(WARPCTC_INCLUDE_DIR "${WARP_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) +SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) IF(WIN32) From aee0857838ee41f46237f0b6561242d523116a0c Mon Sep 17 00:00:00 2001 From: liaogang Date: Wed, 28 Dec 2016 13:49:01 +0800 Subject: [PATCH 12/22] Clean Travis ci and fix bug --- .gitmodules | 3 --- .pre-commit-config.yaml | 4 ++-- .travis.yml | 6 ----- CMakeLists.txt | 1 - cmake/external/gflags.cmake | 1 + cmake/external/glog.cmake | 1 + cmake/external/gtest.cmake | 1 + cmake/external/protobuf.cmake | 23 +++++++++++-------- cmake/external/warpctc.cmake | 1 + paddle/api/paddle_api_config.py.in | 10 ++++---- paddle/api/paddle_ld_flags.py | 6 ++--- paddle/cuda/CMakeLists.txt | 2 +- paddle/gserver/tests/CMakeLists.txt | 2 +- paddle/scripts/travis/before_install.linux.sh | 18 --------------- paddle/scripts/travis/before_install.osx.sh | 8 +------ paddle/scripts/travis/build_submodules.sh | 20 ---------------- 16 files changed, 30 insertions(+), 77 deletions(-) delete mode 100644 .gitmodules delete mode 100755 paddle/scripts/travis/before_install.linux.sh delete mode 100755 paddle/scripts/travis/build_submodules.sh diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index f635e65784af4..0000000000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "warp-ctc"] - path = warp-ctc - url = https://github.com/baidu-research/warp-ctc.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b9902a863d864..a6e45028ebc3f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ sha: c25201a00e6b0514370501050cf2a8538ac12270 hooks: - id: remove-crlf - files: (?!.*warp-ctc)^.*$ + files: (?!.*third_party)^.*$ - repo: https://github.com/reyoung/mirrors-yapf.git sha: v0.13.2 hooks: @@ -15,7 +15,7 @@ - id: check-merge-conflict - id: check-symlinks - id: detect-private-key - files: (?!.*warp-ctc)^.*$ + files: (?!.*third_party)^.*$ - id: end-of-file-fixer - repo: https://github.com/PaddlePaddle/clang-format-pre-commit-hook.git sha: 28c0ea8a67a3e2dbbf4822ef44e85b63a0080a29 diff --git a/.travis.yml b/.travis.yml index 047ca6ffe79bd..b49d4638d710b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,16 +24,11 @@ addons: - wget - git - build-essential - - libatlas-base-dev - python - python-pip - python2.7-dev - - m4 - python-numpy - python-wheel - - libgoogle-glog-dev - - libgflags-dev - - libgtest-dev - curl - lcov - graphviz @@ -53,7 +48,6 @@ before_install: fi fi fi - - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then sudo paddle/scripts/travis/before_install.linux.sh; fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi - pip install wheel protobuf sphinx recommonmark virtualenv numpy sphinx_rtd_theme pre-commit requests==2.9.2 LinkChecker diff --git a/CMakeLists.txt b/CMakeLists.txt index 7db4c9f1d1f2c..784876f0899c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,7 +65,6 @@ set(EXTERNAL_LIBS ${GLOG_LIBRARIES} ${CBLAS_LIBRARIES} ${PROTOBUF_LIBRARIES} - ${WARPCTC_LIBRARIES} ${ZLIB_LIBRARIES} ) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 128d50cec8553..55f9a4c3e6b28 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -22,6 +22,7 @@ ExternalProject_Add( GIT_REPOSITORY "https://github.com/gflags/gflags.git" PREFIX ${GFLAGS_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DBUILD_TESTING=OFF LOG_DOWNLOAD =ON UPDATE_COMMAND "" diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 8a4b9d5996f59..473071a72a61b 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -22,6 +22,7 @@ ExternalProject_Add( GIT_REPOSITORY "https://github.com/google/glog.git" PREFIX ${GLOG_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DWITH_GFLAGS=OFF CMAKE_ARGS -DBUILD_TESTING=OFF LOG_DOWNLOAD =ON diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index 533104422acae..a6ed9e9b9f82a 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -23,6 +23,7 @@ ExternalProject_Add( GIT_TAG "release-1.8.0" PREFIX ${GTEST_SOURCES_DIR} CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DBUILD_GMOCK=ON CMAKE_ARGS -Dgtest_disable_pthreads=ON CMAKE_ARGS -Dgtest_force_shared_crt=ON diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index efd07eb807a17..f42e42ef681b0 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -36,22 +36,25 @@ SET(PROTOBUF_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" CACHE PATH "protobuf INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) IF(WIN32) - SET(PROTOBUF_LIBRARIES - "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.lib" - "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.lib" - "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protobuf libraries." FORCE) + SET(PROTOBUF_LITE_LIBRARY + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite.lib" CACHE FILEPATH "protobuf lite library." FORCE) + SET(PROTOBUF_LIBRARY + "${PROTOBUF_INSTALL_DIR}/lib/libprotobuf.lib" CACHE FILEPATH "protobuf library." FORCE) + SET(PROTOBUF_PROTOC_LIBRARY + "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protoc library." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc.exe" CACHE FILEPATH "protobuf executable." FORCE) ELSE(WIN32) FIND_PATH(PROTOBUF_LIBS_DIR libprotoc.a - PATHS - ${PROTOBUF_INSTALL_DIR}/lib64 ${PROTOBUF_INSTALL_DIR}/lib + ${PROTOBUF_INSTALL_DIR}/lib64 NO_DEFAULT_PATH ) - SET(PROTOBUF_LIBRARIES - "${PROTOBUF_LIBS_DIR}/libprotobuf-lite.a" - "${PROTOBUF_LIBS_DIR}/libprotobuf.a" - "${PROTOBUF_LIBS_DIR}/libprotoc.a" CACHE FILEPATH "protobuf libraries." FORCE) + SET(PROTOBUF_LITE_LIBRARY + "${PROTOBUF_LIBS_DIR}/libprotobuf-lite.a" CACHE FILEPATH "protobuf lite library." FORCE) + SET(PROTOBUF_LIBRARY + "${PROTOBUF_LIBS_DIR}/libprotobuf.a" CACHE FILEPATH "protobuf library." FORCE) + SET(PROTOBUF_PROTOC_LIBRARY + "${PROTOBUF_LIBS_DIR}/libprotoc.a" CACHE FILEPATH "protoc library." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc" CACHE FILEPATH "protobuf executable." FORCE) ENDIF(WIN32) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index b3dea19cebd27..6a88c87df6496 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -36,6 +36,7 @@ ExternalProject_Add( SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) +SET(WARPCTC_LIB_DIR "${WARPCTC_INSTALL_DIR}/lib" CACHE PATH "Warp-ctc Library Directory" FORCE) IF(WIN32) SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) diff --git a/paddle/api/paddle_api_config.py.in b/paddle/api/paddle_api_config.py.in index 23542b952b769..e11ee920362ae 100644 --- a/paddle/api/paddle_api_config.py.in +++ b/paddle/api/paddle_api_config.py.in @@ -1,17 +1,17 @@ PADDLE_BUILD_DIR="@CMAKE_CURRENT_BINARY_DIR@/../" WITH_GPU="@WITH_GPU@" -PROTOBUF_LIB="@PROTOBUF_LIBRARY@" -ZLIB_LIB="@ZLIB_LIBRARIES@" +PROTOBUF_LIBRARY="@PROTOBUF_LIBRARY@" +ZLIB_LIBRARIES="@ZLIB_LIBRARIES@" CMAKE_THREAD_LIB="@CMAKE_THREAD_LIBS_INIT@" CMAKE_DL_LIBS="@CMAKE_DL_LIBS@" WITH_PYTHON="@WITH_PYTHON@" PYTHON_LIBRARIES="@PYTHON_LIBRARIES@" -LIBGLOG_LIBRARY="@LIBGLOG_LIBRARY@" +GLOG_LIBRARIES="@GLOG_LIBRARIES@" GFLAGS_LIBRARIES="@GFLAGS_LIBRARIES@" GFLAGS_LOCATION="@GFLAGS_LOCATION@" -CBLAS_LIBRARIES="@CBLAS_LIBS@" +CBLAS_LIBRARIES="@CBLAS_LIBRARIES@" -CUDA_LIBRARIES="@CUDA_LIBRARIES@" +CUDA_LIBRARIES="@CUDA_cudart_shared_LIBRARY@" WITH_COVERALLS="@ON_COVERALLS@" diff --git a/paddle/api/paddle_ld_flags.py b/paddle/api/paddle_ld_flags.py index b4d27b1cc728f..ad5dce209bf8e 100644 --- a/paddle/api/paddle_ld_flags.py +++ b/paddle/api/paddle_ld_flags.py @@ -40,14 +40,14 @@ def __init__(self): self.paddle_build_dir = PADDLE_BUILD_DIR self.paddle_build_dir = os.path.abspath(self.paddle_build_dir) self.with_gpu = PaddleLDFlag.cmake_bool(WITH_GPU) - self.protolib = PROTOBUF_LIB - self.zlib = ZLIB_LIB + self.protolib = PROTOBUF_LIBRARY + self.zlib = ZLIB_LIBRARIES self.thread = CMAKE_THREAD_LIB self.dl_libs = CMAKE_DL_LIBS self.with_python = PaddleLDFlag.cmake_bool(WITH_PYTHON) self.python_libs = PYTHON_LIBRARIES - self.glog_libs = LIBGLOG_LIBRARY + self.glog_libs = GLOG_LIBRARIES self.with_coverage = PaddleLDFlag.cmake_bool(WITH_COVERALLS) self.gflags_libs = GFLAGS_LIBRARIES diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index 57fb89608f4bc..0a058978540a4 100755 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -88,7 +88,7 @@ else() ${CUDA_CXX_SOURCES}) endif() -add_dependencies(paddle_cuda ${external_project_dependencies}) +add_dependencies(paddle_cuda warpctc) add_style_check_target(paddle_cuda ${CUDA_SOURCES} diff --git a/paddle/gserver/tests/CMakeLists.txt b/paddle/gserver/tests/CMakeLists.txt index c26a2a7f06bc1..4190892db1fbf 100644 --- a/paddle/gserver/tests/CMakeLists.txt +++ b/paddle/gserver/tests/CMakeLists.txt @@ -92,7 +92,7 @@ if(NOT WITH_DOUBLE) TestUtil.cpp) add_test(NAME test_WarpCTCLayer - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_WarpCTCLayer --warpctc_dir=${PROJ_ROOT}/warp-ctc/build + COMMAND ${CMAKE_CURRENT_BINARY_DIR}/test_WarpCTCLayer --warpctc_dir=${WARPCTC_LIB_DIR} WORKING_DIRECTORY ${PROJ_ROOT}/paddle) endif() diff --git a/paddle/scripts/travis/before_install.linux.sh b/paddle/scripts/travis/before_install.linux.sh deleted file mode 100755 index 9620bff6bcf77..0000000000000 --- a/paddle/scripts/travis/before_install.linux.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e -pip install protobuf -cd /tmp -wget https://github.com/google/protobuf/archive/v3.0.2.tar.gz -O protobuf.tar.gz -tar xf protobuf.tar.gz -cd protobuf* -./autogen.sh -./configure --prefix=/usr/ -make -j 2 install -cd .. -rm -rf protobuf* - -pushd /usr/src/gtest -cmake . -make -sudo cp *.a /usr/lib -popd diff --git a/paddle/scripts/travis/before_install.osx.sh b/paddle/scripts/travis/before_install.osx.sh index bd88ed39132f1..89742d67f54c9 100755 --- a/paddle/scripts/travis/before_install.osx.sh +++ b/paddle/scripts/travis/before_install.osx.sh @@ -3,10 +3,4 @@ brew update brew tap homebrew/science brew install python sudo pip install --upgrade protobuf -brew install cmake python glog gflags openblas wget md5sha1sum protobuf - -wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz -O gtest.tar.gz -tar xf gtest.tar.gz -cd googletest-release-1.8.0/ -cmake . -make install +brew install cmake python wget md5sha1sum diff --git a/paddle/scripts/travis/build_submodules.sh b/paddle/scripts/travis/build_submodules.sh deleted file mode 100755 index d458bf92bf455..0000000000000 --- a/paddle/scripts/travis/build_submodules.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -set -e -WORK_DIR=$PWD -PROJ_ROOT=$(git rev-parse --show-cdup) -SUBMODULES=$(grep path ${PROJ_ROOT}.gitmodules | sed 's/^.*path = //') - -for module in $SUBMODULES -do - case $module in - "warp-ctc") - if [ -d ${PROJ_ROOT}warp-ctc/build ]; then - rm -rf ${PROJ_ROOT}warp-ctc/build - fi - mkdir ${PROJ_ROOT}warp-ctc/build - cd ${PROJ_ROOT}warp-ctc/build - cmake ..; make - ;; - esac -done -cd $WORK_DIR From 8acd1ac31a741558725bebe1c97fcce7e28bdef5 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 00:23:23 +0800 Subject: [PATCH 13/22] Add extern python interp --- paddle/api/CMakeLists.txt | 32 ++++++++++++++++++- paddle/api/test/CMakeLists.txt | 2 +- paddle/api/test/run_tests.sh | 10 +++--- paddle/scripts/travis/build_and_test.sh | 9 ++---- paddle/setup.py.in | 2 ++ paddle/trainer/tests/CMakeLists.txt | 5 +-- paddle/trainer/tests/test_Trainer.cpp | 5 --- paddle/utils/.gitignore | 1 + paddle/utils/CMakeLists.txt | 2 ++ .../{PythonUtil.cpp => PythonUtil.cpp.in} | 2 ++ proto/CMakeLists.txt | 4 +-- python/CMakeLists.txt | 6 +--- .../tests/CMakeLists.txt | 8 ++--- .../tests/configs/generate_protostr.sh | 8 ++--- .../tests/configs/run_tests.sh | 2 +- 15 files changed, 62 insertions(+), 36 deletions(-) rename paddle/utils/{PythonUtil.cpp => PythonUtil.cpp.in} (98%) diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index da6dad10cd807..dd617e32681e7 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -1,3 +1,21 @@ +FUNCTION(generate_python_api target_name) + ADD_CUSTOM_COMMAND(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py + ${PROJ_ROOT}/paddle/Paddle_wrap.cxx + ${PROJ_ROOT}/paddle/Paddle_wrap.h + COMMAND ${SWIG_EXECUTABLE} -python -c++ -outcurrentdir -I../ api/Paddle.swig + && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py + DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig + ${PROJ_ROOT}/paddle/api/PaddleAPI.h + ${external_project_dependencies} + WORKING_DIRECTORY ${PROJ_ROOT}/paddle + COMMENT "Generate Python API from swig") + ADD_CUSTOM_TARGET(${target_name} ALL DEPENDS + ${PROJ_ROOT}/paddle/Paddle_wrap.cxx + ${PROJ_ROOT}/paddle/Paddle_wrap.h + ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py + ${external_project_dependencies}) +ENDFUNCTION(generate_python_api) + set(API_SOURCES Arguments.cpp ConfigParser.cpp @@ -42,7 +60,7 @@ file(GLOB PY_PADDLE_PYTHON_FILES ${PROJ_ROOT}/paddle/py_paddle/*.py) # TODO(yuyang18) : make wheel name calculated by cmake add_custom_command(OUTPUT ${PROJ_ROOT}/paddle/dist/.timestamp - COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel + COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch dist/.timestamp COMMAND rm -rf py_paddle.egg-info build WORKING_DIRECTORY ${PROJ_ROOT}/paddle @@ -76,5 +94,17 @@ add_dependencies(python_api_wheel python_swig_sources paddle_cuda) if(WITH_TESTING) + SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip) + ExternalProject_Add(pip + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY https://github.com/pypa/pip.git + GIT_TAG 9.0.1 + PREFIX ${PIP_SOURCES_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + BUILD_IN_SOURCE 1 + DEPENDS python setuptools python_api_wheel + ) add_subdirectory(test) endif() diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt index 08a0fe96a004d..985df6f56ea52 100644 --- a/paddle/api/test/CMakeLists.txt +++ b/paddle/api/test/CMakeLists.txt @@ -1,2 +1,2 @@ add_test(NAME test_swig_api - COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh) + COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE} ${PYTHON_INSTALL_DIR}/bin/pip) diff --git a/paddle/api/test/run_tests.sh b/paddle/api/test/run_tests.sh index 2f12ba026430b..f00ec2c967d8c 100755 --- a/paddle/api/test/run_tests.sh +++ b/paddle/api/test/run_tests.sh @@ -20,11 +20,11 @@ popd > /dev/null cd $SCRIPTPATH -rm -rf .test_env -virtualenv .test_env -source .test_env/bin/activate +# rm -rf .test_env +# virtualenv .test_env +# source .test_env/bin/activate -pip --timeout 600 install ../../dist/*.whl +$1 -m pip install ../../dist/*.whl test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py" @@ -33,7 +33,7 @@ export PYTHONPATH=$PWD/../../../python/ for fn in $test_list do echo "test $fn" - python $fn + $1 $fn if [ $? -ne 0 ]; then exit 1 fi diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh index 9caeb21beb15e..fb21712188def 100755 --- a/paddle/scripts/travis/build_and_test.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -1,15 +1,13 @@ #!/bin/bash -./build_submodules.sh source ./common.sh -CMAKE_EXTRA="" + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - CMAKE_EXTRA="-DPYTHON_LIBRARY=/usr/local/Cellar/python/2.7.12_1/Frameworks/Python.framework/Versions/2.7/lib/python2.7/config/libpython2.7.dylib" + CMAKE_EXTRA="-DWITH_SWIG_PY=OFF" else CMAKE_EXTRA="-DWITH_SWIG_PY=ON" fi - -cmake .. -DCMAKE_BUILD_TYPE=Debug -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA} +cmake .. -DWITH_GPU=OFF -DWITH_DOC=OFF -DWITH_TESTING=ON -DON_TRAVIS=ON -DON_COVERALLS=ON ${CMAKE_EXTRA} NPROC=1 if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then @@ -22,6 +20,5 @@ elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC" fi - sudo make install sudo paddle version diff --git a/paddle/setup.py.in b/paddle/setup.py.in index 464ad632868bd..e3650bf1c0c46 100644 --- a/paddle/setup.py.in +++ b/paddle/setup.py.in @@ -14,7 +14,9 @@ # This file is used to build paddle python binding package. # It will be invoked by Makefile that generated by COMAKE + from setuptools import setup, Extension + import numpy as np import api.paddle_ld_flags import platform diff --git a/paddle/trainer/tests/CMakeLists.txt b/paddle/trainer/tests/CMakeLists.txt index 28c3d6f2631f9..22e07bd0e98a4 100644 --- a/paddle/trainer/tests/CMakeLists.txt +++ b/paddle/trainer/tests/CMakeLists.txt @@ -17,9 +17,10 @@ add_test(NAME test_Compare ################# test_Trainer ########################### add_unittest_without_exec(test_Trainer test_Trainer.cpp) -set(diy_dll_dir ${CMAKE_CURRENT_BINARY_DIR}/../../gserver/tests) add_test(NAME test_Trainer COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/paddle/trainer/tests/gen_proto_data.py && + ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ ${CMAKE_CURRENT_BINARY_DIR}/test_Trainer WORKING_DIRECTORY ${PROJ_ROOT}/paddle/) @@ -82,5 +83,5 @@ add_test(NAME test_PyDataProviderWrapper #################### test_config_parser ######################### add_test(NAME test_config_parser COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - python ${PROJ_ROOT}/paddle/trainer/tests/config_parser_test.py + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/paddle/trainer/tests/config_parser_test.py WORKING_DIRECTORY ${PROJ_ROOT}/paddle/) diff --git a/paddle/trainer/tests/test_Trainer.cpp b/paddle/trainer/tests/test_Trainer.cpp index 371282dd6bb9a..264bc46ebcd0a 100644 --- a/paddle/trainer/tests/test_Trainer.cpp +++ b/paddle/trainer/tests/test_Trainer.cpp @@ -96,11 +96,6 @@ TEST(checkGradient, multi) { TEST(checkGradient, hsigmoid) { checkGradientTest(configFile2, false, false); } TEST(checkGradient, chunk) { -#if defined(__APPLE__) || defined(__OSX__) - EXPECT_EQ(0, system("python trainer/tests/gen_proto_data.py")); -#else - EXPECT_EQ(0, system("python2 trainer/tests/gen_proto_data.py")); -#endif checkGradientTest(configFile3, false, false); #ifndef PADDLE_ONLY_CPU checkGradientTest(configFile3, true, true); diff --git a/paddle/utils/.gitignore b/paddle/utils/.gitignore index f2cfd7409412d..956b606a18cae 100644 --- a/paddle/utils/.gitignore +++ b/paddle/utils/.gitignore @@ -1 +1,2 @@ enable_virtualenv.c +PythonUtil.cpp diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index 45240b5002aa1..10d906ee16656 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -1,5 +1,7 @@ # The utilities for paddle +configure_file(PythonUtil.cpp.in ${PROJ_ROOT}/paddle/utils/PythonUtil.cpp) + file(GLOB UTIL_HEADERS . *.h) file(GLOB UTIL_SOURCES . *.cpp) create_resources(enable_virtualenv.py enable_virtualenv.c) diff --git a/paddle/utils/PythonUtil.cpp b/paddle/utils/PythonUtil.cpp.in similarity index 98% rename from paddle/utils/PythonUtil.cpp rename to paddle/utils/PythonUtil.cpp.in index 7faeff55c28b9..e0caaf4cd6cf4 100644 --- a/paddle/utils/PythonUtil.cpp +++ b/paddle/utils/PythonUtil.cpp.in @@ -195,6 +195,8 @@ extern const char enable_virtualenv_py[]; } void initPython(int argc, char** argv) { #ifndef PADDLE_NO_PYTHON + char PythonHome[] = "@PYTHON_INSTALL_DIR@"; // NOLINT + Py_SetPythonHome(PythonHome); Py_SetProgramName(argv[0]); Py_Initialize(); PySys_SetArgv(argc, argv); diff --git a/proto/CMakeLists.txt b/proto/CMakeLists.txt index c4e170b10fb78..e854b2b427e55 100644 --- a/proto/CMakeLists.txt +++ b/proto/CMakeLists.txt @@ -18,7 +18,7 @@ foreach(filename ${proto_filenames}) ${PROTO_GEN} ${CUR_PROTO_GEN}) add_custom_command(OUTPUT ${CUR_PROTO_GEN} - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} --cpp_out ${CMAKE_CURRENT_BINARY_DIR} --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} DEPENDS ${filename} ${external_project_dependencies}) @@ -29,7 +29,7 @@ foreach(filename ${proto_filenames}) ${CUR_PROTO_GEN_PY} ${PROTO_GEN_PY}) add_custom_command(OUTPUT ${CUR_PROTO_GEN_PY} - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto + COMMAND env ${py_env} ${PROTOBUF_PROTOC_EXECUTABLE} --python_out ${PROJ_ROOT}/python/paddle/proto --proto_path ${PROJ_ROOT}/proto ${PROJ_ROOT}/proto/${filename} DEPENDS ${filename} ${external_project_dependencies}) endforeach() diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 6b80e4d58ee17..0a3599a47aee8 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -14,17 +14,13 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in ${CMAKE_CURRENT_BINARY_DIR}/setup.py) add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp - COMMAND ${PYTHON_EXECUTABLE} setup.py bdist_wheel + COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp DEPENDS gen_proto_py ${PY_FILES}) add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp) -find_python_module(pip REQUIRED) -find_python_module(wheel REQUIRED) -find_python_module(google.protobuf REQUIRED) - add_subdirectory(paddle/trainer_config_helpers/tests) install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/dist/ diff --git a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt index d1a9843d32666..403aafabe9143 100644 --- a/python/paddle/trainer_config_helpers/tests/CMakeLists.txt +++ b/python/paddle/trainer_config_helpers/tests/CMakeLists.txt @@ -1,12 +1,12 @@ #################### test_config_parser ######################### add_test(NAME layers_test COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/layers_test.py WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) add_test(NAME test_reset_hook COMMAND ${PROJ_ROOT}/paddle/.set_python_path.sh -d ${PROJ_ROOT}/python/ - python ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/test_reset_hook.py + ${PYTHON_EXECUTABLE} ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/test_reset_hook.py WORKING_DIRECTORY ${PROJ_ROOT}/python/paddle) if (PROTOBUF_3) @@ -14,12 +14,12 @@ if (PROTOBUF_3) ProtobufEqualMain.cpp) add_test(NAME test_layerHelpers COMMAND - ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh + ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/protobuf_equal ) else() add_test(NAME test_layerHelpers COMMAND - ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh + ${PROJ_ROOT}/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh ${PYTHON_EXECUTABLE} ) endif() diff --git a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh index a54af94ce3db4..ee5961af75ebb 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/generate_protostr.sh @@ -10,13 +10,13 @@ protostr=$PWD/protostr for conf in ${configs[*]} do echo "Generating " $conf - python -m paddle.utils.dump_config $conf.py > $protostr/$conf.protostr.unittest - cat ${conf}.py |python test_config_parser_for_non_file_config.py > $protostr/$conf.protostr.non_file_config.unittest + $1 -m paddle.utils.dump_config $conf.py > $protostr/$conf.protostr.unittest + cat ${conf}.py |$1 test_config_parser_for_non_file_config.py > $protostr/$conf.protostr.non_file_config.unittest done for conf in ${whole_configs[*]} do echo "Generating " $conf - python -m paddle.utils.dump_config $conf.py "" --whole > $protostr/$conf.protostr.unittest - cat ${conf}.py |python test_config_parser_for_non_file_config.py --whole > $protostr/$conf.protostr.non_file_config.unittest + $1 -m paddle.utils.dump_config $conf.py "" --whole > $protostr/$conf.protostr.unittest + cat ${conf}.py |$1 test_config_parser_for_non_file_config.py --whole > $protostr/$conf.protostr.non_file_config.unittest done diff --git a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh index e984ee7062545..a37eb6439e6d2 100755 --- a/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh +++ b/python/paddle/trainer_config_helpers/tests/configs/run_tests.sh @@ -7,7 +7,7 @@ protostr=`dirname $0`/protostr files=`ls $protostr | grep -v "unittest"` -./generate_protostr.sh +./generate_protostr.sh $1 . ./file_list.sh From 9e7f2b8de830335cb2d645cf993c8dd82b6450cc Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 00:24:08 +0800 Subject: [PATCH 14/22] Add system configure --- cmake/configure.cmake | 64 +++++++++++++++++++++++++++++++++++++++++++ cmake/system.cmake | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 cmake/configure.cmake create mode 100644 cmake/system.cmake diff --git a/cmake/configure.cmake b/cmake/configure.cmake new file mode 100644 index 0000000000000..ae0ec01d94da4 --- /dev/null +++ b/cmake/configure.cmake @@ -0,0 +1,64 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(WITH_DSO) + add_definitions(-DPADDLE_USE_DSO) +endif(WITH_DSO) + +if(WITH_DOUBLE) + add_definitions(-DPADDLE_TYPE_DOUBLE) +endif(WITH_DOUBLE) + +if(NOT WITH_TIMER) + add_definitions(-DPADDLE_DISABLE_TIMER) +endif(NOT WITH_TIMER) + +if(NOT WITH_PROFILER) + add_definitions(-DPADDLE_DISABLE_PROFILER) +endif(NOT WITH_PROFILER) + +if(NOT WITH_GPU) + add_definitions(-DPADDLE_ONLY_CPU) + add_definitions(-DHPPL_STUB_FUNC) + + list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) +else() + FIND_PACKAGE(CUDA REQUIRED) + + if(${CUDA_VERSION_MAJOR} VERSION_LESS 7) + message(FATAL_ERROR "Paddle need CUDA >= 7.0 to compile") + endif() + + if(NOT CUDNN_FOUND) + message(FATAL_ERROR "Paddle need cudnn to compile") + endif() + + if(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}") + else(WITH_AVX) + set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}") + endif(WITH_AVX) + + # Include cuda and cudnn + include_directories(${CUDNN_INCLUDE_DIR}) + include_directories(${CUDA_TOOLKIT_INCLUDE}) +endif(NOT WITH_GPU) + +if(WITH_AVX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}") +else(WITH_AVX) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}") +endif(WITH_AVX) diff --git a/cmake/system.cmake b/cmake/system.cmake new file mode 100644 index 0000000000000..788db404ebfb6 --- /dev/null +++ b/cmake/system.cmake @@ -0,0 +1,53 @@ +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +IF(WIN32) + SET(HOST_SYSTEM "win32") +ELSE(WIN32) + IF(APPLE) + EXEC_PROGRAM (sw_vers ARGS -productVersion OUTPUT_VARIABLE MACOSX_VERSION) + STRING(REGEX MATCH "[0-9]+.[0-9]+" VERSION "${MACOSX_VERSION}") + SET(MACOS_VERSION ${VERSION}) + SET(HOST_SYSTEM "macosx") + ELSE(APPLE) + IF(EXISTS "/etc/issue") + FILE(READ "/etc/issue" LINUX_ISSUE) + IF(LINUX_ISSUE MATCHES "CentOS") + SET(HOST_SYSTEM "centos") + ELSEIF(LINUX_ISSUE MATCHES "Debian") + SET(HOST_SYSTEM "debian") + ELSEIF(LINUX_ISSUE MATCHES "Ubuntu") + SET(HOST_SYSTEM "ubuntu") + ENDIF() + ENDIF(EXISTS "/etc/issue") + ENDIF(APPLE) +ENDIF(WIN32) + +# query number of logical cores +CMAKE_HOST_SYSTEM_INFORMATION(RESULT CPU_CORES QUERY NUMBER_OF_LOGICAL_CORES) + +MARK_AS_ADVANCED(HOST_SYSTEM CPU_CORES) + +MESSAGE(STATUS "Found Paddle host system: ${HOST_SYSTEM}") +MESSAGE(STATUS "Found Paddle host system's CPU: ${CPU_CORES} cores") + +# external dependencies log output +SET(EXTERNAL_PROJECT_LOG_ARGS + LOG_DOWNLOAD 0 # Wrap download in script to log output + LOG_UPDATE 1 # Wrap update in script to log output + LOG_CONFIGURE 1 # Wrap configure in script to log output + LOG_BUILD 1 # Wrap build in script to log output + LOG_TEST 1 # Wrap test in script to log output + LOG_INSTALL 1 # Wrap install in script to log output +) From e2d0e09a8c92de7faf7d36bfcc0bdbeeda9184e0 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 00:25:04 +0800 Subject: [PATCH 15/22] Update external dependencies --- cmake/external/gflags.cmake | 23 ++-- cmake/external/glog.cmake | 24 ++-- cmake/external/gtest.cmake | 64 ++++----- cmake/external/numpy.cmake | 78 ----------- cmake/external/openblas.cmake | 50 +++---- cmake/external/protobuf.cmake | 51 +++---- cmake/external/python.cmake | 242 ++++++++++++++++++++++++++-------- cmake/external/swig.cmake | 124 +++++++---------- cmake/external/warpctc.cmake | 40 +++--- cmake/external/zlib.cmake | 27 ++-- 10 files changed, 379 insertions(+), 344 deletions(-) delete mode 100644 cmake/external/numpy.cmake diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 55f9a4c3e6b28..d38b7d1ba2a74 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -15,26 +15,25 @@ INCLUDE(ExternalProject) SET(GFLAGS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/gflags) -SET(GFLAGS_INSTALL_DIR ${PROJECT_BINARY_DIR}/gflags) +SET(GFLAGS_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/gflags) +SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE) +IF(WIN32) + set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/gflags.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) +ELSE(WIN32) + set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.a" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) +ENDIF(WIN32) + +INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) ExternalProject_Add( gflags + ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/gflags/gflags.git" PREFIX ${GFLAGS_SOURCES_DIR} + UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DBUILD_TESTING=OFF - LOG_DOWNLOAD =ON - UPDATE_COMMAND "" ) -SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE) -INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) - -IF(WIN32) - set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/gflags.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) -ELSE(WIN32) - set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/libgflags.a" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) -ENDIF(WIN32) - LIST(APPEND external_project_dependencies gflags) diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 473071a72a61b..bec69f3ddf093 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -15,27 +15,27 @@ INCLUDE(ExternalProject) SET(GLOG_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/glog) -SET(GLOG_INSTALL_DIR ${PROJECT_BINARY_DIR}/glog) +SET(GLOG_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/glog) +SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE) + +IF(WIN32) + SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.lib" CACHE FILEPATH "glog library." FORCE) +ELSE(WIN32) + SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.a" CACHE FILEPATH "glog library." FORCE) +ENDIF(WIN32) + +INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR}) ExternalProject_Add( glog + ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/google/glog.git" PREFIX ${GLOG_SOURCES_DIR} + UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DWITH_GFLAGS=OFF CMAKE_ARGS -DBUILD_TESTING=OFF - LOG_DOWNLOAD =ON - UPDATE_COMMAND "" ) -SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE) -INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR}) - -IF(WIN32) - SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.lib" CACHE FILEPATH "glog library." FORCE) -ELSE(WIN32) - SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/libglog.a" CACHE FILEPATH "glog library." FORCE) -ENDIF(WIN32) - LIST(APPEND external_project_dependencies glog) diff --git a/cmake/external/gtest.cmake b/cmake/external/gtest.cmake index a6ed9e9b9f82a..2fcb7893fa30e 100644 --- a/cmake/external/gtest.cmake +++ b/cmake/external/gtest.cmake @@ -12,38 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. -INCLUDE(ExternalProject) +IF(WITH_TESTING) + ENABLE_TESTING() + INCLUDE(ExternalProject) -SET(GTEST_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/gtest) -SET(GTEST_INSTALL_DIR ${PROJECT_BINARY_DIR}/gtest) + SET(GTEST_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/gtest) + SET(GTEST_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/gtest) + SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) -ExternalProject_Add( - gtest - GIT_REPOSITORY "https://github.com/google/googletest.git" - GIT_TAG "release-1.8.0" - PREFIX ${GTEST_SOURCES_DIR} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} - CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON - CMAKE_ARGS -DBUILD_GMOCK=ON - CMAKE_ARGS -Dgtest_disable_pthreads=ON - CMAKE_ARGS -Dgtest_force_shared_crt=ON - LOG_DOWNLOAD =ON - UPDATE_COMMAND "" -) + INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIR}) -SET(GTEST_INCLUDE_DIR "${GTEST_INSTALL_DIR}/include" CACHE PATH "gtest include directory." FORCE) -INCLUDE_DIRECTORIES(${GTEST_INCLUDE_DIR}) + IF(WIN32) + set(GTEST_LIBRARIES + "${GTEST_INSTALL_DIR}/lib/gtest.lib" CACHE FILEPATH "gtest libraries." FORCE) + set(GTEST_MAIN_LIBRARIES + "${GTEST_INSTALL_DIR}/lib/gtest_main.lib" CACHE FILEPATH "gtest main libraries." FORCE) + ELSE(WIN32) + set(GTEST_LIBRARIES + "${GTEST_INSTALL_DIR}/lib/libgtest.a" CACHE FILEPATH "gtest libraries." FORCE) + set(GTEST_MAIN_LIBRARIES + "${GTEST_INSTALL_DIR}/lib/libgtest_main.a" CACHE FILEPATH "gtest main libraries." FORCE) + ENDIF(WIN32) -IF(WIN32) - set(GTEST_LIBRARIES - "${GTEST_INSTALL_DIR}/lib/gtest.lib" - "${GTEST_INSTALL_DIR}/lib/gtest_main.lib" CACHE FILEPATH "gtest libraries." FORCE) -ELSE(WIN32) - set(GTEST_LIBRARIES - "${GTEST_INSTALL_DIR}/lib/libgtest.a" - "${GTEST_INSTALL_DIR}/lib/libgtest_main.a" CACHE FILEPATH "gtest libraries." FORCE) -ENDIF(WIN32) - -ENABLE_TESTING() - -LIST(APPEND external_project_dependencies gtest) + ExternalProject_Add( + gtest + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY "https://github.com/google/googletest.git" + GIT_TAG "release-1.8.0" + PREFIX ${GTEST_SOURCES_DIR} + UPDATE_COMMAND "" + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GTEST_INSTALL_DIR} + CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON + CMAKE_ARGS -DBUILD_GMOCK=ON + CMAKE_ARGS -Dgtest_disable_pthreads=ON + CMAKE_ARGS -Dgtest_force_shared_crt=ON + ) + LIST(APPEND external_project_dependencies gtest) +ENDIF(WITH_TESTING) diff --git a/cmake/external/numpy.cmake b/cmake/external/numpy.cmake deleted file mode 100644 index d01cff9722b84..0000000000000 --- a/cmake/external/numpy.cmake +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FIND_PACKAGE(NumPy) - -IF(NOT ${NUMPY_FOUND}) - - INCLUDE(ExternalProject) - - SET(NUMPY_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/numpy) - SET(NUMPY_INSTALL_DIR ${PROJECT_BINARY_DIR}/numpy) - set(NUMPY_VERSION "v1.11.3") - - ExternalProject_Add(setuptools - PREFIX ${PYTHON_SOURCES_DIR}/setuptools - URL http://pypi.python.org/packages/source/s/setuptools/setuptools-0.6c11.tar.gz - URL_MD5 7df2a529a074f613b509fb44feefe74e - BUILD_IN_SOURCE 1 - UPDATE_COMMAND "" - PATCH_COMMAND "" - CONFIGURE_COMMAND "" - INSTALL_COMMAND "" - BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install - DEPENDS python zlib - ) - - ExternalProject_Add(cython - PREFIX ${PYTHON_SOURCES_DIR}/cython - GIT_REPOSITORY https://github.com/cython/cython.git - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - UPDATE_COMMAND "" - PATCH_COMMAND "" - INSTALL_COMMAND "" - BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py install - DEPENDS python - ) - - ExternalProject_Add(numpy - GIT_REPOSITORY https://github.com/numpy/numpy.git - GIT_TAG ${NUMPY_VERSION} - CONFIGURE_COMMAND "" - UPDATE_COMMAND "" - PREFIX ${NUMPY_SOURCES_DIR} - BUILD_COMMAND ${PYTHON_EXECUTABLE} setup.py build - INSTALL_COMMAND ${PYTHON_EXECUTABLE} setup.py install - BUILD_IN_SOURCE 1 - DEPENDS python setuptools cython - ) - - LIST(APPEND external_project_dependencies numpy) - - # find numpy include directory - FILE(WRITE ${PROJECT_BINARY_DIR}/FindNumpyPath.py - "try: import numpy; print(numpy.get_include())\nexcept:pass\n") - - EXEC_PROGRAM("${PYTHON_EXECUTABLE}" ${PROJECT_BINARY_DIR} - ARGS "FindNumpyPath.py" - OUTPUT_VARIABLE NUMPY_PATH) - - FIND_PATH(PYTHON_NUMPY_INCLUDE_DIR numpy/arrayobject.h - HINTS "${NUMPY_PATH}" "${PYTHON_INCLUDE_PATH}") - -ENDIF() - -INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) - diff --git a/cmake/external/openblas.cmake b/cmake/external/openblas.cmake index 2683153b49769..677999cc9f5d3 100644 --- a/cmake/external/openblas.cmake +++ b/cmake/external/openblas.cmake @@ -12,31 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -INCLUDE(ExternalProject) +INCLUDE(cblas) -SET(CBLAS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/openblas) -SET(CBLAS_INSTALL_DIR ${PROJECT_BINARY_DIR}/openblas) +IF(NOT ${CBLAS_FOUND}) + INCLUDE(ExternalProject) -ExternalProject_Add( - openblas - GIT_REPOSITORY "https://github.com/xianyi/OpenBLAS.git" - GIT_TAG v0.2.19 - PREFIX ${CBLAS_SOURCES_DIR} - INSTALL_DIR ${CBLAS_INSTALL_DIR} - BUILD_IN_SOURCE 1 - UPDATE_COMMAND "" - CONFIGURE_COMMAND "" - BUILD_COMMAND cd ${CBLAS_SOURCES_DIR}/src/openblas && make -j4 - INSTALL_COMMAND cd ${CBLAS_SOURCES_DIR}/src/openblas && make install PREFIX= -) + SET(CBLAS_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/openblas) + SET(CBLAS_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/openblas) + SET(CBLAS_INC_DIR "${CBLAS_INSTALL_DIR}/include" CACHE PATH "openblas include directory." FORCE) -SET(CBLAS_INCLUDE_DIR "${CBLAS_INSTALL_DIR}/include" CACHE PATH "openblas include directory." FORCE) -INCLUDE_DIRECTORIES(${CBLAS_INCLUDE_DIR}) + IF(WIN32) + SET(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/openblas.lib" CACHE FILEPATH "openblas library." FORCE) + ELSE(WIN32) + SET(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/libopenblas.a" CACHE FILEPATH "openblas library" FORCE) + ENDIF(WIN32) -IF(WIN32) - set(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/openblas.lib" CACHE FILEPATH "openblas library." FORCE) -ELSE(WIN32) - set(CBLAS_LIBRARIES "${CBLAS_INSTALL_DIR}/lib/libopenblas.a" CACHE FILEPATH "openblas library" FORCE) -ENDIF(WIN32) + ExternalProject_Add( + openblas + ${EXTERNAL_PROJECT_LOG_ARGS} + URL "https://github.com/xianyi/OpenBLAS/archive/v0.2.19.tar.gz" + PREFIX ${CBLAS_SOURCES_DIR} + INSTALL_DIR ${CBLAS_INSTALL_DIR} + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + BUILD_COMMAND make CC=${CMAKE_C_COMPILER} FC=${CMAKE_Fortran_COMPILER} + INSTALL_COMMAND make install PREFIX= + UPDATE_COMMAND "" + ) -LIST(APPEND external_project_dependencies openblas) + LIST(APPEND external_project_dependencies openblas) +ENDIF() + +INCLUDE_DIRECTORIES(${CBLAS_INC_DIR}) diff --git a/cmake/external/protobuf.cmake b/cmake/external/protobuf.cmake index f42e42ef681b0..2f2769b4c628d 100644 --- a/cmake/external/protobuf.cmake +++ b/cmake/external/protobuf.cmake @@ -15,24 +15,9 @@ INCLUDE(ExternalProject) SET(PROTOBUF_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/protobuf) -SET(PROTOBUF_INSTALL_DIR ${PROJECT_BINARY_DIR}/protobuf) - -ExternalProject_Add( - protobuf - PREFIX ${PROTOBUF_SOURCES_DIR} - DEPENDS zlib - GIT_REPOSITORY "https://github.com/google/protobuf.git" -# GIT_TAG "v3.1.0" - CONFIGURE_COMMAND - ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/protobuf/cmake - -Dprotobuf_BUILD_TESTS=OFF - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} - UPDATE_COMMAND "" -) - +SET(PROTOBUF_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/protobuf) SET(PROTOBUF_INCLUDE_DIR "${PROTOBUF_INSTALL_DIR}/include" CACHE PATH "protobuf include directory." FORCE) + INCLUDE_DIRECTORIES(${PROTOBUF_INCLUDE_DIR}) IF(WIN32) @@ -44,18 +29,34 @@ IF(WIN32) "${PROTOBUF_INSTALL_DIR}/lib/libprotoc.lib" CACHE FILEPATH "protoc library." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc.exe" CACHE FILEPATH "protobuf executable." FORCE) ELSE(WIN32) - FIND_PATH(PROTOBUF_LIBS_DIR libprotoc.a - ${PROTOBUF_INSTALL_DIR}/lib - ${PROTOBUF_INSTALL_DIR}/lib64 - NO_DEFAULT_PATH - ) + IF(${HOST_SYSTEM} STREQUAL "centos") + SET(LIB "lib64") + ELSE() + SET(LIB "lib") + ENDIF() SET(PROTOBUF_LITE_LIBRARY - "${PROTOBUF_LIBS_DIR}/libprotobuf-lite.a" CACHE FILEPATH "protobuf lite library." FORCE) + "${PROTOBUF_INSTALL_DIR}/${LIB}/libprotobuf-lite.a" CACHE FILEPATH "protobuf lite library." FORCE) SET(PROTOBUF_LIBRARY - "${PROTOBUF_LIBS_DIR}/libprotobuf.a" CACHE FILEPATH "protobuf library." FORCE) + "${PROTOBUF_INSTALL_DIR}/${LIB}/libprotobuf.a" CACHE FILEPATH "protobuf library." FORCE) SET(PROTOBUF_PROTOC_LIBRARY - "${PROTOBUF_LIBS_DIR}/libprotoc.a" CACHE FILEPATH "protoc library." FORCE) + "${PROTOBUF_INSTALL_DIR}/${LIB}/libprotoc.a" CACHE FILEPATH "protoc library." FORCE) SET(PROTOBUF_PROTOC_EXECUTABLE "${PROTOBUF_INSTALL_DIR}/bin/protoc" CACHE FILEPATH "protobuf executable." FORCE) ENDIF(WIN32) +ExternalProject_Add( + protobuf + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${PROTOBUF_SOURCES_DIR} + UPDATE_COMMAND "" + DEPENDS zlib + GIT_REPOSITORY "https://github.com/google/protobuf.git" + GIT_TAG "9f75c5aa851cd877fb0d93ccc31b8567a6706546" + CONFIGURE_COMMAND + ${CMAKE_COMMAND} ${PROTOBUF_SOURCES_DIR}/src/protobuf/cmake + -Dprotobuf_BUILD_TESTS=OFF + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_INSTALL_PREFIX=${PROTOBUF_INSTALL_DIR} +) + LIST(APPEND external_project_dependencies protobuf) diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index d6cdf535fee4e..479ec301b72f3 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -12,84 +12,212 @@ # See the License for the specific language governing permissions and # limitations under the License. -FIND_PACKAGE(PythonLibs 2.7) -FIND_PACKAGE(PythonInterp 2.7) +INCLUDE(ExternalProject) -IF((NOT ${PYTHONINTERP_FOUND}) OR (NOT ${PYTHONLIBS_FOUND})) - INCLUDE(ExternalProject) +##################################### PYTHON ######################################## +SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/python) +SET(PYTHON_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/python) +SET(_python_DIR ${PYTHON_INSTALL_DIR}) - SET(PYTHON_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/Python) - SET(PYTHON_INSTALL_DIR ${PROJECT_BINARY_DIR}/Python) +IF(UNIX) + SET(PYTHON_FOUND ON) + SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE) + SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.a" CACHE FILEPATH "Python library" FORCE) + SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE) + SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE) +ELSEIF(WIN32) + SET(PYTHON_FOUND ON) + SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE) + SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE) + SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE) + SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE) +ELSE() + MESSAGE(FATAL_ERROR "Unknown system !") +ENDIF() + +SET(py_env + PATH=${PYTHON_INSTALL_DIR}/bin/:$ENV{PATH} + PYTHONHOME=${PYTHON_INSTALL_DIR} + PYTHONPATH=${PYTHON_INSTALL_DIR}/lib:${PYTHON_INSTALL_DIR}/lib/python2.7:${PY_SITE_PACKAGES_PATH}) - IF(MSVC) - LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_ARGS - PATCH_COMMAND ${CMAKE_COMMAND} - -DPYTHON_SRC_DIR:PATH=${_python_SOURCE_DIR} - -P ${CMAKE_CURRENT_LIST_DIR}/PythonPatch.cmake - ) - ENDIF() +INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) - IF(APPLE) +IF(APPLE) LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS - -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON - ) - ENDIF() + -DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON + ) +ENDIF() - SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS) +SET(EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS) - # Force Python build to "Release". - IF(CMAKE_CONFIGURATION_TYPES) +# Force Python build to "Release". +IF(CMAKE_CONFIGURATION_TYPES) SET(SAVED_CMAKE_CFG_INTDIR ${CMAKE_CFG_INTDIR}) SET(CMAKE_CFG_INTDIR "Release") - ELSE() +ELSE() LIST(APPEND EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=Release) - ENDIF() + -DCMAKE_BUILD_TYPE:STRING=Release + ) +ENDIF() - ExternalProject_Add(python +ExternalProject_Add(python + ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/python-cmake-buildsystem/python-cmake-buildsystem.git" - GIT_TAG "ed5f9bcee540e47f82fa17f8360b820591aa6d66" PREFIX ${PYTHON_SOURCES_DIR} UPDATE_COMMAND "" + CMAKE_ARGS -DPYTHON_VERSION=2.7.12 + CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} CMAKE_CACHE_ARGS - -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR} - -DBUILD_SHARED:BOOL=OFF - -DBUILD_STATIC:BOOL=ON - -DUSE_SYSTEM_LIBRARIES:BOOL=OFF - -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT} - -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR} - -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES} - -DDOWNLOAD_SOURCES:BOOL=ON - -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF - ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS} - ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS} + -DCMAKE_INSTALL_PREFIX:PATH=${PYTHON_INSTALL_DIR} + -DBUILD_LIBPYTHON_SHARED:BOOL=OFF + -DUSE_SYSTEM_LIBRARIES:BOOL=OFF + -DZLIB_ROOT:FILEPATH=${ZLIB_ROOT} + -DZLIB_INCLUDE_DIR:PATH=${ZLIB_INCLUDE_DIR} + -DZLIB_LIBRARY:FILEPATH=${ZLIB_LIBRARIES} + -DDOWNLOAD_SOURCES:BOOL=ON + -DINSTALL_WINDOWS_TRADITIONAL:BOOL=OFF + ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_CACHE_ARGS} + ${EXTERNAL_PROJECT_OPTIONAL_CMAKE_ARGS} DEPENDS zlib - ) +) +#################################################################################### + +##################################### SETUPTOOLS ################################### +SET(SETUPTOOLS_SOURCES_DIR ${PYTHON_SOURCES_DIR}/setuptools) +ExternalProject_Add(setuptools + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${SETUPTOOLS_SOURCES_DIR} + URL "https://pypi.python.org/packages/source/s/setuptools/setuptools-18.3.2.tar.gz" + BUILD_IN_SOURCE 1 + PATCH_COMMAND "" + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python zlib +) +##################################################################################### + +##################################### SIX ########################################### +SET(SIX_SOURCES_DIR ${PYTHON_SOURCES_DIR}/six) +ExternalProject_Add(six + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${SIX_SOURCES_DIR} + URL https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz + BUILD_IN_SOURCE 1 + PATCH_COMMAND "" + UPDATE_COMMAND "" + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python setuptools +) +##################################################################################### - SET(_python_DIR ${PYTHON_INSTALL_DIR}) +##################################### CYTHON ######################################## +SET(CYTHON_SOURCES_DIR ${PYTHON_SOURCES_DIR}/cython) +ExternalProject_Add(cython + ${EXTERNAL_PROJECT_LOG_ARGS} + PREFIX ${CYTHON_SOURCES_DIR} + URL https://github.com/cython/cython/archive/0.25.2.tar.gz + GIT_TAG 0.25.2 + BUILD_IN_SOURCE 1 + CONFIGURE_COMMAND "" + PATCH_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND "" + BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python +) +#################################################################################### - IF(UNIX) - SET(_python_IMPORT_SUFFIX a) +##################################### NUMPY ######################################## +SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy) +SET(NUMPY_TAG_VERSION "v1.11.3") +SET(NUMPY_VERSION "1.11.3") + +IF(WIN32) + SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg") +ELSE(WIN32) IF(APPLE) - SET(_python_IMPORT_SUFFIX lib) - ENDIF() - SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include/python2.7" CACHE PATH "Python include dir" FORCE) - SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/lib/libpython2.7.${_python_IMPORT_SUFFIX}" CACHE FILEPATH "Python library" FORCE) - SET(PYTHON_EXECUTABLE ${PYTHON_INSTALL_DIR}/bin/python CACHE FILEPATH "Python executable" FORCE) - SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/lib/python2.7/site-packages" CACHE PATH "Python site-packages path" FORCE) - ELSEIF(WIN32) - SET(PYTHON_INCLUDE_DIR "${PYTHON_INSTALL_DIR}/include" CACHE PATH "Python include dir" FORCE) - SET(PYTHON_LIBRARIES "${PYTHON_INSTALL_DIR}/libs/python27.lib" CACHE FILEPATH "Python library" FORCE) - SET(PYTHON_EXECUTABLE "${PYTHON_INSTALL_DIR}/bin/python.exe" CACHE FILEPATH "Python executable" FORCE) - SET(PY_SITE_PACKAGES_PATH "${PYTHON_INSTALL_DIR}/Lib/site-packages" CACHE PATH "Python site-packages path" FORCE) - ELSE() - MESSAGE(FATAL_ERROR "Unknown system !") - ENDIF() + SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}-x86_64.egg") + ELSE(APPLE) + SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux-x86_64.egg") + ENDIF(APPLE) +ENDIF(WIN32) -LIST(APPEND external_project_dependencies python) +SET(PYTHON_NUMPY_INCLUDE_DIR "${PY_SITE_PACKAGES_PATH}/${EGG_NAME}/numpy/core/include") +IF(${PYTHON_FOUND}) # local python + SET(PYTHON_NUMPY_INCLUDE_DIR + "${PY_SITE_PACKAGES_PATH}/${EGG_NAME}/numpy/core/include") +ELSE(${PYTHON_FOUND}) # global python + SET(PYTHON_NUMPY_INCLUDE_DIR "") + SET(PY_SITE_PACKAGES_DIR "") + FILE(WRITE ${PROJECT_BINARY_DIR}/FindNumpyPath.py + "try: import site; print(site.getsitepackages())\nexcept:pass\n") + EXEC_PROGRAM("env ${py_env} ${PYTHON_EXECUTABLE}" ${PROJECT_BINARY_DIR} + ARGS "FindNumpyPath.py" OUTPUT_VARIABLE NUMPY_PATH) -ENDIF() + STRING(REPLACE "[" "" NUMPY_PATH "${NUMPY_PATH}") + STRING(REPLACE "]" "" NUMPY_PATH "${NUMPY_PATH}") + STRING(REPLACE "'" "" NUMPY_PATH "${NUMPY_PATH}") + STRING(REPLACE ", " ";" SITE_DIRS "${NUMPY_PATH}") -INCLUDE_DIRECTORIES(${PYTHON_INCLUDE_DIR}) + FOREACH(SITE_DIR ${SITE_DIRS}) + IF(EXISTS ${SITE_DIR}) + LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR + "${SITE_DIR}/${EGG_NAME}/numpy/core/include") + SET(PY_SITE_PACKAGES_DIR "${SITE_DIR}") + ENDIF() + ENDFOREACH() +ENDIF(${PYTHON_FOUND}) + +INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) + +ExternalProject_Add(numpy + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY https://github.com/numpy/numpy.git + GIT_TAG ${NUMPY_TAG_VERSION} + CONFIGURE_COMMAND "" + UPDATE_COMMAND "" + PREFIX ${NUMPY_SOURCES_DIR} + BUILD_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py build + INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + BUILD_IN_SOURCE 1 + DEPENDS python setuptools cython +) +#################################################################################### + +##################################### WHEEL ######################################## +SET(WHEEL_SOURCES_DIR ${PYTHON_SOURCES_DIR}/wheel) +ExternalProject_Add(wheel + ${EXTERNAL_PROJECT_LOG_ARGS} + URL https://pypi.python.org/packages/source/w/wheel/wheel-0.29.0.tar.gz + PREFIX ${WHEEL_SOURCES_DIR} + CONFIGURE_COMMAND "" + UPDATE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + BUILD_IN_SOURCE 1 + DEPENDS python setuptools +) +#################################################################################### + +################################### PROTOBUF ####################################### +SET(PY_PROTOBUF_SOURCES_DIR ${PYTHON_SOURCES_DIR}/protobuf) +ExternalProject_Add(python-protobuf + ${EXTERNAL_PROJECT_LOG_ARGS} + URL https://pypi.python.org/packages/e0/b0/0a1b364fe8a7d177b4b7d4dca5b798500dc57a7273b93cca73931b305a6a/protobuf-3.1.0.post1.tar.gz + URL_MD5 38b5fb160c768d2f8444d0c6d637ff91 + PREFIX ${PY_PROTOBUF_SOURCES_DIR} + BUILD_IN_SOURCE 1 + PATCH_COMMAND "" + CONFIGURE_COMMAND "" + BUILD_COMMAND env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py build + INSTALL_COMMAND env PATH=${PROTOBUF_INSTALL_DIR}/bin:$ENV{PATH} ${py_env} ${PYTHON_EXECUTABLE} setup.py install + DEPENDS python setuptools six +) +LIST(APPEND external_project_dependencies python setuptools six cython numpy wheel python-protobuf) diff --git a/cmake/external/swig.cmake b/cmake/external/swig.cmake index 2da826d375003..5460b02c37ec3 100644 --- a/cmake/external/swig.cmake +++ b/cmake/external/swig.cmake @@ -12,83 +12,59 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Look for system swig -FIND_PACKAGE(SWIG) +# build swig as an external project +INCLUDE(ExternalProject) -IF(NOT ${SWIG_FOUND}) - # build swig as an external project - INCLUDE(ExternalProject) - SET(SWIG_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/swig) - SET(SWIG_INSTALL_DIR ${PROJECT_BINARY_DIR}/swig) - SET(SWIG_TARGET_VERSION "3.0.2") - SET(SWIG_DOWNLOAD_SRC_MD5 "62f9b0d010cef36a13a010dc530d0d41") - SET(SWIG_DOWNLOAD_WIN_MD5 "3f18de4fc09ab9abb0d3be37c11fbc8f") +SET(SWIG_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/swig) +SET(SWIG_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/swig) +SET(SWIG_TARGET_VERSION "3.0.2") +SET(SWIG_DOWNLOAD_SRC_MD5 "62f9b0d010cef36a13a010dc530d0d41") +SET(SWIG_DOWNLOAD_WIN_MD5 "3f18de4fc09ab9abb0d3be37c11fbc8f") - IF(WIN32) - # swig.exe available as pre-built binary on Windows: - ExternalProject_Add(swig - URL http://prdownloads.sourceforge.net/swig/swigwin-${SWIG_TARGET_VERSION}.zip - URL_MD5 ${SWIG_DOWNLOAD_WIN_MD5} - SOURCE_DIR ${SWIG_SOURCES_DIR} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - ) - SET(SWIG_DIR ${SWIG_SOURCES_DIR} CACHE FILEPATH "SWIG Directory" FORCE) - SET(SWIG_EXECUTABLE ${SWIG_SOURCES_DIR}/swig.exe CACHE FILEPATH "SWIG Executable" FORCE) +IF(WIN32) + # swig.exe available as pre-built binary on Windows: + ExternalProject_Add(swig + URL http://prdownloads.sourceforge.net/swig/swigwin-${SWIG_TARGET_VERSION}.zip + URL_MD5 ${SWIG_DOWNLOAD_WIN_MD5} + SOURCE_DIR ${SWIG_SOURCES_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + UPDATE_COMMAND "" + ) + SET(SWIG_DIR ${SWIG_SOURCES_DIR} CACHE FILEPATH "SWIG Directory" FORCE) + SET(SWIG_EXECUTABLE ${SWIG_SOURCES_DIR}/swig.exe CACHE FILEPATH "SWIG Executable" FORCE) +ELSE(WIN32) + # From PCRE configure + ExternalProject_Add(pcre + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY https://github.com/svn2github/pcre.git + PREFIX ${SWIG_SOURCES_DIR}/pcre + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SWIG_INSTALL_DIR}/pcre + ) - ELSE(WIN32) - # From PCRE configure - ExternalProject_Add(pcre - GIT_REPOSITORY https://github.com/svn2github/pcre.git - PREFIX ${SWIG_SOURCES_DIR}/pcre - UPDATE_COMMAND "" - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SWIG_INSTALL_DIR}/pcre - ) + # swig uses bison find it by cmake and pass it down + FIND_PACKAGE(BISON) - # swig uses bison find it by cmake and pass it down - FIND_PACKAGE(BISON) + # From SWIG configure + ExternalProject_Add(swig + GIT_REPOSITORY https://github.com/swig/swig.git + GIT_TAG rel-3.0.10 + PREFIX ${SWIG_SOURCES_DIR} + CONFIGURE_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && ./autogen.sh + CONFIGURE_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && + env "PCRE_LIBS=${SWIG_INSTALL_DIR}/pcre/lib/libpcre.a ${SWIG_INSTALL_DIR}/pcre/lib/libpcrecpp.a ${SWIG_INSTALL_DIR}/pcre/lib/libpcreposix.a" + ./configure + --prefix=${SWIG_INSTALL_DIR} + --with-pcre-prefix=${SWIG_INSTALL_DIR}/pcre + BUILD_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make + INSTALL_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make install + UPDATE_COMMAND "" + DEPENDS pcre + ) - # From SWIG configure - ExternalProject_Add(swig - URL https://github.com/swig/swig/archive/rel-3.0.10.tar.gz - PREFIX ${SWIG_SOURCES_DIR} - UPDATE_COMMAND "" - CONFIGURE_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && ./autogen.sh - CONFIGURE_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && - env "PCRE_LIBS=${SWIG_INSTALL_DIR}/pcre/lib/libpcre.a \ - ${SWIG_INSTALL_DIR}/pcre/lib/libpcrecpp.a \ - ${SWIG_INSTALL_DIR}/pcre/lib/libpcreposix.a" - ./configure - --prefix=${SWIG_INSTALL_DIR} - --with-pcre-prefix=${SWIG_INSTALL_DIR}/pcre - BUILD_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make - INSTALL_COMMAND cd ${SWIG_SOURCES_DIR}/src/swig && make install - DEPENDS pcre - ) + SET(SWIG_DIR ${SWIG_INSTALL_DIR}/share/swig/${SWIG_TARGET_VERSION}) + SET(SWIG_EXECUTABLE ${SWIG_INSTALL_DIR}/bin/swig) +ENDIF(WIN32) - set(SWIG_DIR ${SWIG_INSTALL_DIR}/share/swig/${SWIG_TARGET_VERSION} CACHE FILEPATH "SWIG Directory" FORCE) - set(SWIG_EXECUTABLE ${SWIG_INSTALL_DIR}/bin/swig CACHE FILEPATH "SWIG Executable" FORCE) - ENDIF(WIN32) - - LIST(APPEND external_project_dependencies swig) - -ENDIF() - -FUNCTION(generate_python_api target_name) - ADD_CUSTOM_COMMAND(OUTPUT ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py - ${PROJ_ROOT}/paddle/Paddle_wrap.cxx - ${PROJ_ROOT}/paddle/Paddle_wrap.h - COMMAND ${SWIG_EXECUTABLE} -python -c++ -outcurrentdir -I../ api/Paddle.swig - && mv ${PROJ_ROOT}/paddle/swig_paddle.py ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py - DEPENDS ${PROJ_ROOT}/paddle/api/Paddle.swig - ${PROJ_ROOT}/paddle/api/PaddleAPI.h - ${external_project_dependencies} - WORKING_DIRECTORY ${PROJ_ROOT}/paddle - COMMENT "Generate Python API from swig") - ADD_CUSTOM_TARGET(${target_name} ALL DEPENDS - ${PROJ_ROOT}/paddle/Paddle_wrap.cxx - ${PROJ_ROOT}/paddle/Paddle_wrap.h - ${PROJ_ROOT}/paddle/py_paddle/swig_paddle.py - ${external_project_dependencies}) -ENDFUNCTION(generate_python_api) +LIST(APPEND external_project_dependencies swig) diff --git a/cmake/external/warpctc.cmake b/cmake/external/warpctc.cmake index 6a88c87df6496..d90768b6f1576 100644 --- a/cmake/external/warpctc.cmake +++ b/cmake/external/warpctc.cmake @@ -15,28 +15,13 @@ INCLUDE(ExternalProject) SET(WARPCTC_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/warpctc) -SET(WARPCTC_INSTALL_DIR ${PROJECT_BINARY_DIR}/warpctc) - -IF(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" ) - SET(USE_OMP ON) -ELSE() - SET(USE_OMP OFF) -ENDIF() - -ExternalProject_Add( - warpctc - GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git" - PREFIX ${WARPCTC_SOURCES_DIR} - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} - CMAKE_ARGS -DWITH_GPU=${WITH_GPU} - CMAKE_ARGS -DWITH_OMP=${USE_OMP} - UPDATE_COMMAND "" -) - +SET(WARPCTC_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/warpctc) SET(WARPCTC_INCLUDE_DIR "${WARPCTC_INSTALL_DIR}/include" CACHE PATH "Warp-ctc Directory" FORCE) + INCLUDE_DIRECTORIES(${WARPCTC_INCLUDE_DIR}) SET(WARPCTC_LIB_DIR "${WARPCTC_INSTALL_DIR}/lib" CACHE PATH "Warp-ctc Library Directory" FORCE) + IF(WIN32) SET(WARPCTC_LIBRARIES "${WARPCTC_INSTALL_DIR}/lib/warpctc.dll" CACHE FILEPATH "Warp-ctc Library" FORCE) @@ -51,4 +36,23 @@ ELSE(WIN32) "${WARPCTC_INSTALL_DIR}/lib/libwarpctc.${_warpctc_SHARED_SUFFIX}" CACHE FILEPATH "Warp-ctc Library" FORCE) ENDIF(WIN32) +IF(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang" ) + SET(USE_OMP OFF) +ELSE() + SET(USE_OMP ON) +ENDIF() + +ExternalProject_Add( + warpctc + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY "https://github.com/gangliao/warp-ctc.git" + PREFIX ${WARPCTC_SOURCES_DIR} + UPDATE_COMMAND "" + CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + CMAKE_ARGS -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${WARPCTC_INSTALL_DIR} + CMAKE_ARGS -DWITH_GPU=${WITH_GPU} + CMAKE_ARGS -DWITH_OMP=${USE_OMP} +) + LIST(APPEND external_project_dependencies warpctc) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index ec44467aa70a6..916f6816aae99 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -15,30 +15,29 @@ INCLUDE(ExternalProject) SET(ZLIB_SOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/zlib) -SET(ZLIB_INSTALL_DIR ${PROJECT_BINARY_DIR}/zlib) +SET(ZLIB_INSTALL_DIR ${CMAKE_CURRENT_SOURCE_DIR}/third_party/install/zlib) +SET(ZLIB_ROOT ${ZLIB_INSTALL_DIR} CACHE FILEPATH "zlib root directory." FORCE) +SET(ZLIB_INCLUDE_DIR "${ZLIB_INSTALL_DIR}/include" CACHE PATH "zlib include directory." FORCE) + +IF(WIN32) + SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/zlibstatic.lib" CACHE FILEPATH "zlib library." FORCE) +ELSE(WIN32) + set(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) +ENDIF(WIN32) + +INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) ExternalProject_Add( zlib + ${EXTERNAL_PROJECT_LOG_ARGS} GIT_REPOSITORY "https://github.com/madler/zlib.git" GIT_TAG "v1.2.8" PREFIX ${ZLIB_SOURCES_DIR} + UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${ZLIB_INSTALL_DIR} CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF CMAKE_ARGS -DCMAKE_POSITION_INDEPENDENT_CODE=ON CMAKE_ARGS -DCMAKE_MACOSX_RPATH=ON - LOG_DOWNLOAD =ON - UPDATE_COMMAND "" ) -SET(ZLIB_ROOT ${ZLIB_INSTALL_DIR} CACHE PATH "zlib root directory." FORCE) - -SET(ZLIB_INCLUDE_DIR "${ZLIB_INSTALL_DIR}/include" CACHE PATH "zlib include directory." FORCE) -INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) - -IF(WIN32) - SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/zlibstatic.lib" CACHE FILEPATH "zlib library." FORCE) -ELSE(WIN32) - set(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) -ENDIF(WIN32) - LIST(APPEND external_project_dependencies zlib) From 642dc356aef25b13c08b5fc2d64c9bbe600f707d Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 00:25:46 +0800 Subject: [PATCH 16/22] clean travis ci --- .travis.yml | 9 ++------- paddle/scripts/travis/before_install.osx.sh | 4 +--- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index b49d4638d710b..61d31132b73be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,15 +24,10 @@ addons: - wget - git - build-essential - - python + - libatlas-base-dev - python-pip - - python2.7-dev - - python-numpy - - python-wheel - curl - - lcov - graphviz - - swig - clang-format-3.8 - automake - libtool @@ -50,7 +45,7 @@ before_install: fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi - - pip install wheel protobuf sphinx recommonmark virtualenv numpy sphinx_rtd_theme pre-commit requests==2.9.2 LinkChecker + - pip install protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker script: - paddle/scripts/travis/main.sh notifications: diff --git a/paddle/scripts/travis/before_install.osx.sh b/paddle/scripts/travis/before_install.osx.sh index 89742d67f54c9..fd113d313e314 100755 --- a/paddle/scripts/travis/before_install.osx.sh +++ b/paddle/scripts/travis/before_install.osx.sh @@ -1,6 +1,4 @@ #!/bin/bash brew update brew tap homebrew/science -brew install python -sudo pip install --upgrade protobuf -brew install cmake python wget md5sha1sum +brew install openblas md5sha1sum From 572d8254ea63a88b9c382ba2e91e53e90994e620 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 00:25:58 +0800 Subject: [PATCH 17/22] Clean cmake --- CMakeLists.txt | 26 +++++++++++----- cmake/cblas.cmake | 24 ++++++++------- cmake/check_packages.cmake | 14 --------- cmake/cpplint.cmake | 2 +- cmake/definitions.cmake | 62 -------------------------------------- cmake/util.cmake | 4 +-- paddle/cuda/CMakeLists.txt | 2 +- 7 files changed, 36 insertions(+), 98 deletions(-) delete mode 100644 cmake/check_packages.cmake delete mode 100644 cmake/definitions.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 784876f0899c6..9ed757bd1bfbd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,18 @@ -cmake_minimum_required(VERSION 2.8) +# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +cmake_minimum_required(VERSION 3.0) project(paddle CXX C) @@ -6,11 +20,11 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake") set(PROJ_ROOT ${CMAKE_SOURCE_DIR}) find_package(Sphinx) -find_package(Doxygen) find_package(CUDA QUIET) find_package(Git REQUIRED) find_package(Threads REQUIRED) +include(system) include(simd) ###################### Configurations ############################ @@ -35,9 +49,8 @@ include(external/gflags) # download, build, install gflags include(external/glog) # download, build, install glog include(external/gtest) # download, build, install gtest include(external/protobuf) # download, build, install protobuf -include(external/openblas) # download, build, install openblas include(external/python) # download, build, install python -include(external/numpy) # download, build, install numpy +include(external/openblas) # download, build, install openblas include(external/swig) # download, build, install swig include(external/warpctc) # download, build, install warpctc @@ -52,8 +65,7 @@ include(version) # set PADDLE_VERSION include(coveralls) # set code coverage include(python_module) # set python module -include(check_packages) # check configuration -include(definitions) # add paddle definitions +include(configure) # add paddle env configuration include_directories("${PROJ_ROOT}") include_directories("${PROJ_ROOT}/paddle/cuda/include") @@ -64,7 +76,7 @@ set(EXTERNAL_LIBS ${GFLAGS_LIBRARIES} ${GLOG_LIBRARIES} ${CBLAS_LIBRARIES} - ${PROTOBUF_LIBRARIES} + ${PROTOBUF_LIBRARY} ${ZLIB_LIBRARIES} ) diff --git a/cmake/cblas.cmake b/cmake/cblas.cmake index 685334c658506..4e1ae7dc81231 100644 --- a/cmake/cblas.cmake +++ b/cmake/cblas.cmake @@ -13,6 +13,7 @@ # system paths. # +set(CBLAS_FOUND OFF) ## Find MKL First. set(MKL_ROOT $ENV{MKL_ROOT} CACHE PATH "Folder contains MKL") @@ -35,11 +36,12 @@ find_library(MKL_INTEL_LP64 NAMES mkl_intel_lp64 PATHS if(MKL_INCLUDE_DIR AND MKL_CORE_LIB AND MKL_SEQUENTIAL_LIB AND MKL_INTEL_LP64) set(CBLAS_PROVIDER MKL) set(CBLAS_INC_DIR ${MKL_INCLUDE_DIR}) - set(CBLAS_LIBS ${MKL_INTEL_LP64} + set(CBLAS_LIBRARIES ${MKL_INTEL_LP64} ${MKL_SEQUENTIAL_LIB} ${MKL_CORE_LIB}) add_definitions(-DPADDLE_USE_MKL) - message(STATUS "Found MKL (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") + message(STATUS "Found MKL (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})") + set(CBLAS_FOUND ON) return() # return file. endif() @@ -68,9 +70,10 @@ find_library(ATLAS_LIB NAMES lapack_atlas liblapack_atlas.so.3 if(ATLAS_INC_DIR AND ATLAS_CBLAS_LIB AND ATLAS_LIB) set(CBLAS_PROVIDER ATLAS) set(CBLAS_INC_DIR ${ATLAS_INC_DIR} ${ATLAS_CLAPACK_INC_DIR}) - set(CBLAS_LIBS ${ATLAS_LIB} ${ATLAS_CBLAS_LIB}) + set(CBLAS_LIBRARIES ${ATLAS_LIB} ${ATLAS_CBLAS_LIB}) add_definitions(-DPADDLE_USE_ATLAS) - message(STATUS "Found Atlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") + message(STATUS "Found Atlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})") + set(CBLAS_FOUND ON) return() endif() @@ -98,8 +101,9 @@ find_library(OPENBLAS_LIB NAMES openblas if(OPENBLAS_INC_DIR AND OPENBLAS_LIB) set(CBLAS_PROVIDER OPENBLAS) set(CBLAS_INC_DIR ${OPENBLAS_INC_DIR}) - set(CBLAS_LIBS ${OPENBLAS_LIB}) - message(STATUS "Found OpenBlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") + set(CBLAS_LIBRARIES ${OPENBLAS_LIB}) + message(STATUS "Found OpenBlas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBRARIES})") + set(CBLAS_FOUND ON) return() endif() @@ -130,9 +134,7 @@ find_library(REFERENCE_CBLAS_LIBRARY NAMES cblas PATHS if (REFERENCE_CBLAS_INCLUDE_DIR AND REFERENCE_CBLAS_LIBRARY) set(CBLAS_PROVIDER REFERENCE) set(CBLAS_INC_DIR ${REFERENCE_CBLAS_INCLUDE_DIR}) - set(CBLAS_LIBS ${REFERENCE_CBLAS_LIBRARY}) - return() + set(CBLAS_LIBRARIES ${REFERENCE_CBLAS_LIBRARY}) + message(STATUS "Found reference-cblas (include: ${CBLAS_INC_DIR}, library: ${CBLAS_LIBS})") + set(CBLAS_FOUND ON) endif() - -message(FATAL_ERROR "CBlas must be set. Paddle support MKL, ATLAS, OpenBlas, reference-cblas." - " Try set MKL_ROOT, ATLAS_ROOT, OPENBLAS_ROOT or REFERENCE_CBLAS_ROOT.") diff --git a/cmake/check_packages.cmake b/cmake/check_packages.cmake deleted file mode 100644 index 8f0ed26256650..0000000000000 --- a/cmake/check_packages.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# Check package for each cmake option - -if(WITH_GPU) - find_package(CUDA REQUIRED) # CUDA is required when use gpu -endif(WITH_GPU) - -if(WITH_DOC) - find_package(Sphinx REQUIRED) - find_python_module(recommonmark REQUIRED) -endif(WITH_DOC) - -if(WITH_SWIG_PY) - find_python_module(wheel REQUIRED) # package wheel -endif(WITH_SWIG_PY) diff --git a/cmake/cpplint.cmake b/cmake/cpplint.cmake index 241af9a0835b2..38c636b30edc0 100644 --- a/cmake/cpplint.cmake +++ b/cmake/cpplint.cmake @@ -53,7 +53,7 @@ macro(add_style_check_target TARGET_NAME) if(LINT MATCHES ON) add_custom_command(TARGET ${TARGET_NAME} PRE_BUILD - COMMAND "${PYTHON_EXECUTABLE}" "${PROJ_ROOT}/paddle/scripts/cpplint.py" + COMMAND env ${py_env} "${PYTHON_EXECUTABLE}" "${PROJ_ROOT}/paddle/scripts/cpplint.py" "--filter=${STYLE_FILTER}" ${filename} WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}) endif() diff --git a/cmake/definitions.cmake b/cmake/definitions.cmake deleted file mode 100644 index 99a52ad764b90..0000000000000 --- a/cmake/definitions.cmake +++ /dev/null @@ -1,62 +0,0 @@ -# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -if(WITH_DSO) - add_definitions(-DPADDLE_USE_DSO) -endif(WITH_DSO) - -if(WITH_DOUBLE) - add_definitions(-DPADDLE_TYPE_DOUBLE) -endif(WITH_DOUBLE) - -if(NOT WITH_TIMER) - add_definitions(-DPADDLE_DISABLE_TIMER) -endif(NOT WITH_TIMER) - -if(NOT WITH_PROFILER) - add_definitions(-DPADDLE_DISABLE_PROFILER) -endif(NOT WITH_PROFILER) - -if(NOT WITH_GPU) - add_definitions(-DPADDLE_ONLY_CPU) - add_definitions(-DHPPL_STUB_FUNC) - - list(APPEND CMAKE_CXX_SOURCE_FILE_EXTENSIONS cu) -else() - if(${CUDA_VERSION_MAJOR} VERSION_LESS 7) - message(FATAL_ERROR "Paddle need CUDA >= 7.0 to compile") - endif() - - if(NOT CUDNN_FOUND) - message(FATAL_ERROR "Paddle need cudnn to compile") - endif() - - if(WITH_AVX) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${AVX_FLAG}") - else(WITH_AVX) - set(CUDA_NVCC_FLAGS ${CUDA_NVCC_FLAGS} "-Xcompiler ${SSE3_FLAG}") - endif(WITH_AVX) - - # Include cuda and cudnn - include_directories(${CUDNN_INCLUDE_DIR}) - include_directories(${CUDA_TOOLKIT_INCLUDE}) -endif(NOT WITH_GPU) - -if(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${AVX_FLAG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${AVX_FLAG}") -else(WITH_AVX) - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SSE3_FLAG}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SSE3_FLAG}") -endif(WITH_AVX) diff --git a/cmake/util.cmake b/cmake/util.cmake index b8d20266f4bcf..c9b48e5f8fc3b 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -24,7 +24,7 @@ function(target_circle_link_libraries TARGET_NAME) list(APPEND libsInArgn ${arg}) endif() endforeach() - if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" OR "${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang") list(APPEND LIBS "-undefined dynamic_lookup") endif() list(REVERSE libsInArgn) @@ -105,7 +105,7 @@ function(link_paddle_exe TARGET_NAME) if(WITH_PYTHON) target_link_libraries(${TARGET_NAME} - ${PYTHON_LIBRARIES}) + ${PYTHON_LIBRARIES} util) endif() if(WITH_GPU) diff --git a/paddle/cuda/CMakeLists.txt b/paddle/cuda/CMakeLists.txt index 0a058978540a4..57fb89608f4bc 100755 --- a/paddle/cuda/CMakeLists.txt +++ b/paddle/cuda/CMakeLists.txt @@ -88,7 +88,7 @@ else() ${CUDA_CXX_SOURCES}) endif() -add_dependencies(paddle_cuda warpctc) +add_dependencies(paddle_cuda ${external_project_dependencies}) add_style_check_target(paddle_cuda ${CUDA_SOURCES} From fc47492f4189ead6be33b6512a46663dc2541b69 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 01:06:16 +0800 Subject: [PATCH 18/22] Fix merge conflict bug and glog --- paddle/function/CMakeLists.txt | 2 - paddle/function/ContextProjectionOp.cpp | 52 ++++++++++----------- paddle/function/Function.cpp | 12 +++-- paddle/gserver/layers/ContextProjection.cpp | 6 ++- warp-ctc | 1 - 5 files changed, 38 insertions(+), 35 deletions(-) delete mode 160000 warp-ctc diff --git a/paddle/function/CMakeLists.txt b/paddle/function/CMakeLists.txt index cfa45e117c1d4..b133d2419a9ab 100644 --- a/paddle/function/CMakeLists.txt +++ b/paddle/function/CMakeLists.txt @@ -12,8 +12,6 @@ endif() add_library(paddle_function STATIC ${cpp_files} ${cu_objs}) add_dependencies(paddle_function ${external_project_dependencies}) -add_library(paddle_test_main STATIC TestMain.cpp) -add_dependencies(paddle_test_main ${external_project_dependencies}) if(WITH_GPU) if(WITH_TESTING) diff --git a/paddle/function/ContextProjectionOp.cpp b/paddle/function/ContextProjectionOp.cpp index bd367a859e10c..07907fc1ba797 100644 --- a/paddle/function/ContextProjectionOp.cpp +++ b/paddle/function/ContextProjectionOp.cpp @@ -85,15 +85,15 @@ class ContextProjectionForwardFunc : public FunctionBase { void calc(const Arguments& inputs, const Arguments& outputs, const Arguments& inouts) override { - CHECK_EQ(3, inputs.size()); - CHECK_EQ(1, outputs.size()); - CHECK_EQ(0, inouts.size()); + CHECK_EQ(3, static_cast(inputs.size())); + CHECK_EQ(1, static_cast(outputs.size())); + CHECK_EQ(0, static_cast(inouts.size())); CHECK(outputs[0].getData() && inputs[0].getData() && inputs[2].getData()); - CHECK_EQ(outputs[0].dims_.size(), 2); - CHECK_EQ(inputs[0].dims_.size(), 2); - CHECK_EQ(inputs[1].dims_.size(), 2); - CHECK_EQ(inputs[2].dims_.size(), 1); + CHECK_EQ(static_cast(outputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[1].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[2].dims_.size()), 1); /// dim of output = dim of input * context_length CHECK_EQ(outputs[0].dims_[1], inputs[0].dims_[1] * context_length_); /// dim of input == dim of weight @@ -202,15 +202,15 @@ class ContextProjectionBackwardFunc : public FunctionBase { void calc(const Arguments& inputs, const Arguments& outputs, const Arguments& inouts) override { - CHECK_EQ(3, inputs.size()); - CHECK_EQ(1, outputs.size()); - CHECK_EQ(0, inouts.size()); + CHECK_EQ(3, static_cast(inputs.size())); + CHECK_EQ(1, static_cast(outputs.size())); + CHECK_EQ(0, static_cast(inouts.size())); CHECK(outputs[0].getData() && inputs[2].getData()); - CHECK_EQ(outputs[0].dims_.size(), 2); - CHECK_EQ(inputs[0].dims_.size(), 2); - CHECK_EQ(inputs[1].dims_.size(), 2); - CHECK_EQ(inputs[2].dims_.size(), 1); + CHECK_EQ(static_cast(outputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[1].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[2].dims_.size()), 1); /// dim of input == dim of weight CHECK_EQ(inputs[0].dims_[1], inputs[1].dims_[1]); @@ -269,13 +269,13 @@ class ContextProjectionBackwardDataFunc : public FunctionBase { void calc(const Arguments& inputs, const Arguments& outputs, const Arguments& inouts) override { - CHECK_EQ(2, inputs.size()); - CHECK_EQ(1, outputs.size()); - CHECK_EQ(0, inouts.size()); + CHECK_EQ(2, static_cast(inputs.size())); + CHECK_EQ(1, static_cast(outputs.size())); + CHECK_EQ(0, static_cast(inouts.size())); CHECK(inputs[0].getData() && outputs[0].getData() && inputs[1].getData()); - CHECK_EQ(outputs[0].dims_.size(), 2); - CHECK_EQ(inputs[0].dims_.size(), 2); - CHECK_EQ(inputs[1].dims_.size(), 1); + CHECK_EQ(static_cast(outputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[1].dims_.size()), 1); CHECK_EQ(outputs[0].dims_[1], inputs[0].dims_[1] * context_length_); /// input and output has the same batch_size CHECK_EQ(inputs[0].dims_[0], outputs[0].dims_[0]); @@ -317,14 +317,14 @@ class ContextProjectionBackwardWeightFunc : public FunctionBase { void calc(const Arguments& inputs, const Arguments& outputs, const Arguments& inouts) override { - CHECK_EQ(2, inputs.size()); - CHECK_EQ(1, outputs.size()); - CHECK_EQ(0, inouts.size()); + CHECK_EQ(2, static_cast(inputs.size())); + CHECK_EQ(1, static_cast(outputs.size())); + CHECK_EQ(0, static_cast(inouts.size())); CHECK(inputs[0].getData() && outputs[0].getData() && inputs[1].getData()); - CHECK_EQ(outputs[0].dims_.size(), 2); - CHECK_EQ(inputs[0].dims_.size(), 2); - CHECK_EQ(inputs[1].dims_.size(), 1); + CHECK_EQ(static_cast(outputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[0].dims_.size()), 2); + CHECK_EQ(static_cast(inputs[1].dims_.size()), 1); CHECK_EQ(outputs[0].dims_[1], inputs[0].dims_[1] * context_length_); auto out_grad_mat = std::make_shared::type>( diff --git a/paddle/function/Function.cpp b/paddle/function/Function.cpp index 6f82a8d053bc2..614e76b8ac0c9 100644 --- a/paddle/function/Function.cpp +++ b/paddle/function/Function.cpp @@ -46,28 +46,32 @@ bool FuncConfig::get(const std::string& key) const { template <> FuncConfig& FuncConfig::set(const std::string& key, size_t v) { - CHECK_EQ(valueMap_.count(key), 0) << "Duplicated value: " << key; + CHECK_EQ(static_cast(valueMap_.count(key)), 0) << "Duplicated value: " + << key; valueMap_[key].s = v; return *this; } template <> FuncConfig& FuncConfig::set(const std::string& key, real v) { - CHECK_EQ(valueMap_.count(key), 0) << "Duplicated value: " << key; + CHECK_EQ(static_cast(valueMap_.count(key)), 0) << "Duplicated value: " + << key; valueMap_[key].r = v; return *this; } template <> FuncConfig& FuncConfig::set(const std::string& key, int v) { - CHECK_EQ(valueMap_.count(key), 0) << "Duplicated value: " << key; + CHECK_EQ(static_cast(valueMap_.count(key)), 0) << "Duplicated value: " + << key; valueMap_[key].i = v; return *this; } template <> FuncConfig& FuncConfig::set(const std::string& key, bool v) { - CHECK_EQ(valueMap_.count(key), 0) << "Duplicated value: " << key; + CHECK_EQ(static_cast(valueMap_.count(key)), 0) << "Duplicated value: " + << key; valueMap_[key].b = v; return *this; } diff --git a/paddle/gserver/layers/ContextProjection.cpp b/paddle/gserver/layers/ContextProjection.cpp index e947b2b9ecbeb..ee4db219890a1 100644 --- a/paddle/gserver/layers/ContextProjection.cpp +++ b/paddle/gserver/layers/ContextProjection.cpp @@ -111,7 +111,8 @@ void ContextProjection::forward() { size_t dim = out_->value->getWidth(); CHECK_EQ(dim, input_dim * config_.context_length()); size_t batch_size = in_->value->getHeight(); - CHECK_EQ(forward_.size(), 1) << "Only one forward function here"; + CHECK_EQ(static_cast(forward_.size()), 1) + << "Only one forward function here"; REGISTER_TIMER_INFO("ContextProjectionForward", getName().c_str()); bool is_padding = config_.trainable_padding(); @@ -154,7 +155,8 @@ void ContextProjection::backward(const UpdateCallback& callback) { CHECK_EQ(dim, input_dim * config_.context_length()); size_t batch_size = in_->value->getHeight(); CHECK_EQ(batch_size, out_->value->getHeight()); - CHECK_EQ(backward_.size(), 1) << "Only one backward function here"; + CHECK_EQ(static_cast(backward_.size()), 1) + << "Only one backward function here"; REGISTER_TIMER_INFO("ContextProjectionBackward", getName().c_str()); bool is_padding = config_.trainable_padding(); diff --git a/warp-ctc b/warp-ctc deleted file mode 160000 index bd535c8d44e03..0000000000000 --- a/warp-ctc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit bd535c8d44e03c8ebd2d768e06c8c05fdccd11d2 From 3ecc63ad165913b4b62f41a009c9e1c2406a3441 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 11:08:22 +0800 Subject: [PATCH 19/22] Add python in travis ci for paddle version command --- .travis.yml | 2 ++ paddle/api/CMakeLists.txt | 24 ++++++++++++------------ paddle/api/test/CMakeLists.txt | 2 +- paddle/api/test/run_tests.sh | 4 ---- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 61d31132b73be..27a4be38f5456 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,9 @@ addons: - git - build-essential - libatlas-base-dev + - python - python-pip + - python2.7-dev - curl - graphviz - clang-format-3.8 diff --git a/paddle/api/CMakeLists.txt b/paddle/api/CMakeLists.txt index dd617e32681e7..3ac50e34bb434 100644 --- a/paddle/api/CMakeLists.txt +++ b/paddle/api/CMakeLists.txt @@ -94,17 +94,17 @@ add_dependencies(python_api_wheel python_swig_sources paddle_cuda) if(WITH_TESTING) - SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip) - ExternalProject_Add(pip - ${EXTERNAL_PROJECT_LOG_ARGS} - GIT_REPOSITORY https://github.com/pypa/pip.git - GIT_TAG 9.0.1 - PREFIX ${PIP_SOURCES_DIR} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install - BUILD_IN_SOURCE 1 - DEPENDS python setuptools python_api_wheel - ) + SET(PIP_SOURCES_DIR ${PYTHON_SOURCES_DIR}/pip) + ExternalProject_Add(pip + ${EXTERNAL_PROJECT_LOG_ARGS} + GIT_REPOSITORY https://github.com/pypa/pip.git + GIT_TAG 9.0.1 + PREFIX ${PIP_SOURCES_DIR} + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py install + BUILD_IN_SOURCE 1 + DEPENDS python setuptools python_api_wheel + ) add_subdirectory(test) endif() diff --git a/paddle/api/test/CMakeLists.txt b/paddle/api/test/CMakeLists.txt index 985df6f56ea52..a2fa623c80087 100644 --- a/paddle/api/test/CMakeLists.txt +++ b/paddle/api/test/CMakeLists.txt @@ -1,2 +1,2 @@ add_test(NAME test_swig_api - COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE} ${PYTHON_INSTALL_DIR}/bin/pip) + COMMAND bash ${PROJ_ROOT}/paddle/api/test/run_tests.sh ${PYTHON_EXECUTABLE}) diff --git a/paddle/api/test/run_tests.sh b/paddle/api/test/run_tests.sh index f00ec2c967d8c..bcf06afa86aaa 100755 --- a/paddle/api/test/run_tests.sh +++ b/paddle/api/test/run_tests.sh @@ -20,10 +20,6 @@ popd > /dev/null cd $SCRIPTPATH -# rm -rf .test_env -# virtualenv .test_env -# source .test_env/bin/activate - $1 -m pip install ../../dist/*.whl test_list="testArguments.py testGradientMachine.py testMatrix.py testVector.py testTrain.py testTrainer.py" From 0407902592d61addceeb5b82313e63225911e6df Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 11:59:09 +0800 Subject: [PATCH 20/22] Add wheel for paddle version command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 27a4be38f5456..4c65b8c7e9fa4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ before_install: fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi - - pip install protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker + - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker script: - paddle/scripts/travis/main.sh notifications: From 19dc8df41d69396c9810402f830cbc01dd49f19e Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 14:47:21 +0800 Subject: [PATCH 21/22] Update travis ci pip --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4c65b8c7e9fa4..ba2f8482b5502 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,6 +47,7 @@ before_install: fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi + - pip install --upgrade pip - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker script: - paddle/scripts/travis/main.sh From d0a5ce290a4f9ecdb984a465ed90a790e30d47a5 Mon Sep 17 00:00:00 2001 From: liaogang Date: Thu, 5 Jan 2017 18:06:00 +0800 Subject: [PATCH 22/22] Clean travis ci --- .travis.yml | 2 +- cmake/external/python.cmake | 35 ++++++------------------- paddle/scripts/travis/build_and_test.sh | 8 +++--- python/CMakeLists.txt | 2 +- 4 files changed, 15 insertions(+), 32 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba2f8482b5502..426f0eb746d6f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ before_install: fi - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then paddle/scripts/travis/before_install.osx.sh; fi - if [[ "$JOB" == "PRE_COMMIT" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi - - pip install --upgrade pip + - pip install --upgrade pip - pip install wheel protobuf sphinx recommonmark sphinx_rtd_theme virtualenv pre-commit requests==2.9.2 LinkChecker script: - paddle/scripts/travis/main.sh diff --git a/cmake/external/python.cmake b/cmake/external/python.cmake index 479ec301b72f3..e4c570479f682 100644 --- a/cmake/external/python.cmake +++ b/cmake/external/python.cmake @@ -138,41 +138,22 @@ SET(NUMPY_SOURCES_DIR ${PYTHON_SOURCES_DIR}/numpy) SET(NUMPY_TAG_VERSION "v1.11.3") SET(NUMPY_VERSION "1.11.3") +SET(EGG_NAME "") +SET(PYTHON_NUMPY_INCLUDE_DIR "") IF(WIN32) SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}.egg") ELSE(WIN32) IF(APPLE) - SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}-x86_64.egg") + SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-${HOST_SYSTEM}-${MACOS_VERSION}") ELSE(APPLE) - SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux-x86_64.egg") + SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux") + SET(EGG_NAME "numpy-${NUMPY_VERSION}-py2.7-linux") ENDIF(APPLE) -ENDIF(WIN32) -SET(PYTHON_NUMPY_INCLUDE_DIR "${PY_SITE_PACKAGES_PATH}/${EGG_NAME}/numpy/core/include") -IF(${PYTHON_FOUND}) # local python - SET(PYTHON_NUMPY_INCLUDE_DIR - "${PY_SITE_PACKAGES_PATH}/${EGG_NAME}/numpy/core/include") -ELSE(${PYTHON_FOUND}) # global python - SET(PYTHON_NUMPY_INCLUDE_DIR "") - SET(PY_SITE_PACKAGES_DIR "") - FILE(WRITE ${PROJECT_BINARY_DIR}/FindNumpyPath.py - "try: import site; print(site.getsitepackages())\nexcept:pass\n") - EXEC_PROGRAM("env ${py_env} ${PYTHON_EXECUTABLE}" ${PROJECT_BINARY_DIR} - ARGS "FindNumpyPath.py" OUTPUT_VARIABLE NUMPY_PATH) - - STRING(REPLACE "[" "" NUMPY_PATH "${NUMPY_PATH}") - STRING(REPLACE "]" "" NUMPY_PATH "${NUMPY_PATH}") - STRING(REPLACE "'" "" NUMPY_PATH "${NUMPY_PATH}") - STRING(REPLACE ", " ";" SITE_DIRS "${NUMPY_PATH}") - - FOREACH(SITE_DIR ${SITE_DIRS}) - IF(EXISTS ${SITE_DIR}) - LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR - "${SITE_DIR}/${EGG_NAME}/numpy/core/include") - SET(PY_SITE_PACKAGES_DIR "${SITE_DIR}") - ENDIF() + FOREACH(suffix x86_64 intel fat64 fat32 universal) + LIST(APPEND PYTHON_NUMPY_INCLUDE_DIR ${PY_SITE_PACKAGES_PATH}/${EGG_NAME}-${suffix}.egg/numpy/core/include) ENDFOREACH() -ENDIF(${PYTHON_FOUND}) +ENDIF(WIN32) INCLUDE_DIRECTORIES(${PYTHON_NUMPY_INCLUDE_DIR}) diff --git a/paddle/scripts/travis/build_and_test.sh b/paddle/scripts/travis/build_and_test.sh index fb21712188def..ffc48eae66aa6 100755 --- a/paddle/scripts/travis/build_and_test.sh +++ b/paddle/scripts/travis/build_and_test.sh @@ -1,6 +1,8 @@ #!/bin/bash source ./common.sh +python -c 'import pip; print(pip.pep425tags.get_supported())' + if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then CMAKE_EXTRA="-DWITH_SWIG_PY=OFF" else @@ -14,11 +16,11 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then NRPOC=`nproc` make -j $NPROC make coveralls + sudo make install elif [[ "$TRAVIS_OS_NAME" == "osx" ]]; then NPROC=`sysctl -n hw.ncpu` make -j $NPROC env CTEST_OUTPUT_ON_FAILURE=1 make test ARGS="-j $NPROC" + sudo make install + sudo paddle version fi - -sudo make install -sudo paddle version diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 0a3599a47aee8..1cda4762eb2a5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -16,7 +16,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in add_custom_command(OUTPUT ${OUTPUT_DIR}/.timestamp COMMAND env ${py_env} ${PYTHON_EXECUTABLE} setup.py bdist_wheel COMMAND ${CMAKE_COMMAND} -E touch ${OUTPUT_DIR}/.timestamp - DEPENDS gen_proto_py ${PY_FILES}) + DEPENDS gen_proto_py ${PY_FILES} ${external_project_dependencies}) add_custom_target(paddle_python ALL DEPENDS ${OUTPUT_DIR}/.timestamp)