Upstream updates + data set merging + cost effective gradient boosting

.appveyor.yml

@@ -1,4 +1,4 @@
-version: 2.2.3.{build}
+version: 2.2.4.{build}
 
 image: Visual Studio 2015
 platform: x64
@@ -30,7 +30,7 @@ install:
   - ps: $env:LGB_VER = (Get-Content VERSION.txt).trim()
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
-  - conda create -q -n test-env python=%PYTHON_VERSION% matplotlib nose numpy pandas pytest python-graphviz scikit-learn scipy
+  - conda create -q -n test-env python=%PYTHON_VERSION% matplotlib nose numpy pandas psutil pytest python-graphviz scikit-learn scipy
   - activate test-env
 
 build_script:
@@ -48,8 +48,9 @@ test_script:
       (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
   - ps: >-
       foreach ($file in @(Get-ChildItem *.py)) {
+        @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
         python $file
-        if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
+        if (!$?) { $host.SetShouldExit(-1) }
       }  # run all examples
   - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide\notebooks
   - conda install -y -n test-env ipywidgets notebook

.ci/test.sh

@@ -58,7 +58,7 @@ if [[ $TASK == "if-else" ]]; then
     exit 0
 fi
 
-conda install -q -y -n $CONDA_ENV matplotlib nose numpy pandas pytest python-graphviz scikit-learn scipy
+conda install -q -y -n $CONDA_ENV matplotlib nose numpy pandas psutil pytest python-graphviz scikit-learn scipy
 
 if [[ $OS_NAME == "macos" ]] && [[ $COMPILER == "clang" ]]; then
     sudo ln -sf `ls -d "$(brew --cellar libomp)"/*/lib`/* $CONDA_PREFIX/lib || exit -1  # fix "OMP: Error #15: Initializing libiomp5.dylib, but found libomp.dylib already initialized." (OpenMP library conflict due to conda's MKL)

.ci/test_windows.ps1

@@ -1,42 +1,43 @@
 function Check-Output {
-  param( [int]$ExitCode )
-  if ($ExitCode -ne 0) {
-    $host.SetShouldExit($ExitCode)
+  param( [bool]$success )
+  if (!$success) {
+    $host.SetShouldExit(-1)
     Exit -1
   }
 }
 
 if ($env:TASK -eq "regular") {
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
-  cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $LastExitCode
+  cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
   cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py install --precompile ; Check-Output $LastExitCode
+  python setup.py install --precompile ; Check-Output $?
   cp $env:BUILD_SOURCESDIRECTORY/Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
   cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 }
 elseif ($env:TASK -eq "sdist") {
   cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py sdist --formats gztar ; Check-Output $LastExitCode
-  cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $LastExitCode
+  python setup.py sdist --formats gztar ; Check-Output $?
+  cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
 }
 elseif ($env:TASK -eq "bdist") {
   cd $env:BUILD_SOURCESDIRECTORY/python-package
-  python setup.py bdist_wheel --plat-name=win-amd64 --universal ; Check-Output $LastExitCode
-  cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $LastExitCode
+  python setup.py bdist_wheel --plat-name=win-amd64 --universal ; Check-Output $?
+  cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $?
   cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
 }
 
 $tests = $env:BUILD_SOURCESDIRECTORY + $(If ($env:TASK -eq "sdist") {"/tests/python_package_test"} Else {"/tests"})  # cannot test C API with "sdist" task
-pytest $tests ; Check-Output $LastExitCode
+pytest $tests ; Check-Output $?
 
 if ($env:TASK -eq "regular") {
   cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
   @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
   (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
   foreach ($file in @(Get-ChildItem *.py)) {
-    python $file ; Check-Output $LastExitCode
+    @("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
+    python $file ; Check-Output $?
   }  # run all examples
   cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide/notebooks
   conda install -y -n $env:CONDA_ENV ipywidgets notebook
-  jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Check-Output $LastExitCode  # run all notebooks
+  jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Check-Output $?  # run all notebooks
 }

.github/ISSUE_TEMPLATE.md

@@ -14,6 +14,8 @@ CPU/GPU model:
 
 C++/Python/R version:
 
+LightGBM version or commit hash:
+
 ## Error message
 
 <!-- Paste error log here -->

.gitignore

@@ -390,3 +390,6 @@ R-package/src/src/
 lightgbm_r/*
 lightgbm*.tar.gz
 lightgbm.Rcheck/
+
+# Files generated by aspell
+**/*.bak

.travis.yml

@@ -1,5 +1,4 @@
 language: cpp
-sudo: required
 dist: trusty
 
 git:
@@ -38,7 +37,6 @@ matrix:
 before_install:
   - test -n $CC  && unset CC
   - test -n $CXX && unset CXX
-  - export HOMEBREW_LOGS="$HOME/brew_logs"  # brew hotfix 
   - export HOME_DIRECTORY="$HOME"
   - export BUILD_DIRECTORY="$TRAVIS_BUILD_DIR"
   - if [[ $TRAVIS_OS_NAME == "osx" ]]; then

.vsts-ci.yml

@@ -107,7 +107,6 @@ jobs:
     inputs:
       updateConda: false
   - script: |
-      echo "##vso[task.setvariable variable=HOMEBREW_LOGS]$AGENT_HOMEDIRECTORY/brew_logs"
       echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY"
       echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
       echo "##vso[task.setvariable variable=OS_NAME]macos"
@@ -147,7 +146,7 @@ jobs:
       createCustomEnvironment: true
       updateConda: true
       environmentName: $(CONDA_ENV)
-      packageSpecs: 'python=$(PYTHON_VERSION) matplotlib nose numpy pandas pytest python-graphviz scikit-learn scipy'
+      packageSpecs: 'python=$(PYTHON_VERSION) matplotlib nose numpy pandas psutil pytest python-graphviz scikit-learn scipy'
       createOptions: '-q'
   - powershell: $(Build.SourcesDirectory)/.ci/test_windows.ps1
     displayName: Test

CMakeLists.txt

@@ -211,8 +211,8 @@ endif(USE_MPI)
 
 if(USE_OPENMP)
     if(CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang")
-        TARGET_LINK_LIBRARIES(lightgbm ${OpenMP_libomp_LIBRARY})
-        TARGET_LINK_LIBRARIES(_lightgbm ${OpenMP_libomp_LIBRARY})
+        TARGET_LINK_LIBRARIES(lightgbm OpenMP::OpenMP_CXX)
+        TARGET_LINK_LIBRARIES(_lightgbm OpenMP::OpenMP_CXX)
     endif()
 endif(USE_OPENMP)
 

R-package/DESCRIPTION

@@ -1,8 +1,8 @@
 Package: lightgbm
 Type: Package
 Title: Light Gradient Boosting Machine
-Version: 2.2.3
-Date: 2018-11-6
+Version: 2.2.4
+Date: 2019-02-05
 Authors@R: c(
 	person("Guolin", "Ke", email = "[email protected]", role = c("aut", "cre")),
 	person("Damien", "Soukhavong", email = "[email protected]", role = c("ctb")),

R-package/R/lgb.Dataset.R

@@ -492,6 +492,10 @@ Dataset <- R6::R6Class(
     update_params = function(params) {
 
       # Parameter updating
+      if (!lgb.is.null.handle(private$handle)) {
+        lgb.call("LGBM_DatasetUpdateParam_R", ret = NULL, private$handle, lgb.params2str(params))
+        return(invisible(self))
+      }
       private$params <- modifyList(private$params, params)
       return(invisible(self))
 

R-package/demo/basic_walkthrough.R

@@ -95,7 +95,7 @@ print(paste("sum(abs(pred2-pred))=", sum(abs(pred2 - pred))))
 #--------------------Advanced features ---------------------------
 # To use advanced features, we need to put data in lgb.Dataset
 dtrain <- lgb.Dataset(data = train$data, label = train$label, free_raw_data = FALSE)
-dtest <- lgb.Dataset(data = test$data, label = test$label, free_raw_data = FALSE)
+dtest <- lgb.Dataset.create.valid(dtrain, data = test$data, label = test$label)
 
 #--------------------Using validation set-------------------------
 # valids is a list of lgb.Dataset, each of them is tagged with name

R-package/demo/boost_from_prediction.R

@@ -5,7 +5,7 @@ require(methods)
 data(agaricus.train, package = "lightgbm")
 data(agaricus.test, package = "lightgbm")
 dtrain <- lgb.Dataset(agaricus.train$data, label = agaricus.train$label)
-dtest <- lgb.Dataset(agaricus.test$data, label = agaricus.test$label)
+dtest <- lgb.Dataset.create.valid(dtrain, data = agaricus.test$data, label = agaricus.test$label)
 
 valids <- list(eval = dtest, train = dtrain)
 #--------------------Advanced features ---------------------------

R-package/demo/categorical_features_prepare.R

@@ -31,24 +31,24 @@ str(bank)
 # For this task, we use lgb.prepare
 # The function transforms the data into a fittable data
 #
-# Classes 'data.table' and 'data.frame':	4521 obs. of  17 variables:
+# Classes ‘data.table’ and 'data.frame':	4521 obs. of  17 variables:
 # $ age      : int  30 33 35 30 59 35 36 39 41 43 ...
-# $ job      : chr  "unemployed" "services" "management" "management" ...
-# $ marital  : chr  "married" "married" "single" "married" ...
-# $ education: chr  "primary" "secondary" "tertiary" "tertiary" ...
-# $ default  : chr  "no" "no" "no" "no" ...
+# $ job      : num  11 8 5 5 2 5 7 10 3 8 ...
+# $ marital  : num  2 2 3 2 2 3 2 2 2 2 ...
+# $ education: num  1 2 3 3 2 3 3 2 3 1 ...
+# $ default  : num  1 1 1 1 1 1 1 1 1 1 ...
 # $ balance  : int  1787 4789 1350 1476 0 747 307 147 221 -88 ...
-# $ housing  : chr  "no" "yes" "yes" "yes" ...
-# $ loan     : chr  "no" "yes" "no" "yes" ...
-# $ contact  : chr  "cellular" "cellular" "cellular" "unknown" ...
+# $ housing  : num  1 2 2 2 2 1 2 2 2 2 ...
+# $ loan     : num  1 2 1 2 1 1 1 1 1 2 ...
+# $ contact  : num  1 1 1 3 3 1 1 1 3 1 ...
 # $ day      : int  19 11 16 3 5 23 14 6 14 17 ...
-# $ month    : chr  "oct" "may" "apr" "jun" ...
+# $ month    : num  11 9 1 7 9 4 9 9 9 1 ...
 # $ duration : int  79 220 185 199 226 141 341 151 57 313 ...
 # $ campaign : int  1 1 1 4 1 2 1 2 2 1 ...
 # $ pdays    : int  -1 339 330 -1 -1 176 330 -1 -1 147 ...
 # $ previous : int  0 4 1 0 0 3 2 0 0 2 ...
-# $ poutcome : chr  "unknown" "failure" "failure" "unknown" ...
-# $ y        : chr  "no" "no" "no" "no" ...
+# $ poutcome : num  4 1 1 4 4 1 2 4 4 1 ...
+# $ y        : num  1 1 1 1 1 1 1 1 1 1 ...
 bank <- lgb.prepare(data = bank)
 str(bank)
 

R-package/demo/categorical_features_rules.R

@@ -70,8 +70,9 @@ my_data_test <- as.matrix(bank_test[, 1:16, with = FALSE])
 # The categorical features can be passed to lgb.train to not copy and paste a lot
 dtrain <- lgb.Dataset(data = my_data_train,
                       label = bank_train$y)
-dtest <- lgb.Dataset(data = my_data_test,
-                     label = bank_test$y)
+dtest <- lgb.Dataset.create.valid(dtrain,
+                                  data = my_data_test,
+                                  label = bank_test$y)
 
 # We can now train a model
 model <- lgb.train(list(objective = "binary",

R-package/demo/cross_validation.R

@@ -3,7 +3,7 @@ require(lightgbm)
 data(agaricus.train, package = "lightgbm")
 data(agaricus.test, package = "lightgbm")
 dtrain <- lgb.Dataset(agaricus.train$data, label = agaricus.train$label)
-dtest <- lgb.Dataset(agaricus.test$data, label = agaricus.test$label)
+dtest <- lgb.Dataset.create.valid(dtrain, data = agaricus.test$data, label = agaricus.test$label)
 
 nrounds <- 2
 param <- list(num_leaves = 4,

R-package/demo/early_stopping.R

@@ -6,7 +6,7 @@ data(agaricus.train, package = "lightgbm")
 data(agaricus.test, package = "lightgbm")
 
 dtrain <- lgb.Dataset(agaricus.train$data, label = agaricus.train$label)
-dtest <- lgb.Dataset(agaricus.test$data, label = agaricus.test$label)
+dtest <- lgb.Dataset.create.valid(dtrain, data = agaricus.test$data, label = agaricus.test$label)
 
 # Note: for customized objective function, we leave objective as default
 # Note: what we are getting is margin value in prediction

R-package/demo/multiclass_custom_objective.R

@@ -8,18 +8,21 @@ data(iris)
 # For instance: 0, 1, 2, 3, 4, 5...
 iris$Species <- as.numeric(as.factor(iris$Species)) - 1
 
-# We cut the data set into 80% train and 20% validation
+# Create imbalanced training data (20, 30, 40 examples for classes 0, 1, 2)
+train <- as.matrix(iris[c(1:20, 51:80, 101:140), ])
 # The 10 last samples of each class are for validation
-
-train <- as.matrix(iris[c(1:40, 51:90, 101:140), ])
 test <- as.matrix(iris[c(41:50, 91:100, 141:150), ])
+
 dtrain <- lgb.Dataset(data = train[, 1:4], label = train[, 5])
 dtest <- lgb.Dataset.create.valid(dtrain, data = test[, 1:4], label = test[, 5])
-valids <- list(test = dtest)
+valids <- list(train = dtrain, test = dtest)
 
 # Method 1 of training with built-in multiclass objective
+# Note: need to turn off boost from average to match custom objective
+# (https://github.com/Microsoft/LightGBM/issues/1846)
 model_builtin <- lgb.train(list(),
                            dtrain,
+                           boost_from_average = FALSE,
                            100,
                            valids,
                            min_data = 1,
@@ -29,7 +32,8 @@ model_builtin <- lgb.train(list(),
                            metric = "multi_logloss",
                            num_class = 3)
 
-preds_builtin <- predict(model_builtin, test[, 1:4], rawscore = TRUE)
+preds_builtin <- predict(model_builtin, test[, 1:4], rawscore = TRUE, reshape = TRUE)
+probs_builtin <- exp(preds_builtin) / rowSums(exp(preds_builtin))
 
 # Method 2 of training with custom objective function
 
@@ -64,7 +68,6 @@ custom_multiclass_metric = function(preds, dtrain) {
     return(list(name = "error",
                 value = -mean(log(prob[cbind(1:length(labels), labels + 1)])),
                 higher_better = FALSE))
-
 }
 
 model_custom <- lgb.train(list(),
@@ -78,8 +81,10 @@ model_custom <- lgb.train(list(),
                           eval = custom_multiclass_metric,
                           num_class = 3)
 
-preds_custom <- predict(model_custom, test[, 1:4], rawscore = TRUE)
+preds_custom <- predict(model_custom, test[, 1:4], rawscore = TRUE, reshape = TRUE)
+probs_custom <- exp(preds_custom) / rowSums(exp(preds_custom))
 
 # compare predictions
-identical(preds_builtin, preds_custom)
+stopifnot(identical(probs_builtin, probs_custom))
+stopifnot(identical(preds_builtin, preds_custom))
 

R-package/src/install.libs.R

@@ -53,7 +53,7 @@ if (!use_precompile) {
     } else {
       try_vs <- 0
       local_vs_def <- ""
-      vs_versions <- c("Visual Studio 15 2017 Win64", "Visual Studio 14 2015 Win64")
+      vs_versions <- c("Visual Studio 15 2017 Win64", "Visual Studio 14 2015 Win64", "Visual Studio 16 2019")
       for(vs in vs_versions){
         vs_def <- paste0(" -G \"", vs, "\"")
         tmp_cmake_cmd <- paste0(cmake_cmd, vs_def)

README.md

@@ -130,4 +130,4 @@ Huan Zhang, Si Si and Cho-Jui Hsieh. "[GPU Acceleration for Large-scale Tree Boo
 License
 -------
 
-This project is licensed under the terms of the MIT license. See [LICENSE](https://github.com/Microsoft/LightGBM/blob/master/LICENSE) for addtional details.
+This project is licensed under the terms of the MIT license. See [LICENSE](https://github.com/Microsoft/LightGBM/blob/master/LICENSE) for additional details.

VERSION.txt

@@ -1 +1 @@
-2.2.3
+2.2.4

docs/Development-Guide.rst

@@ -73,7 +73,7 @@ Refer to `docs README <./README.rst>`__.
 C API
 -----
 
-Refere to the comments in `c\_api.h <https://github.com/Microsoft/LightGBM/blob/master/include/LightGBM/c_api.h>`__.
+Refer to the comments in `c\_api.h <https://github.com/Microsoft/LightGBM/blob/master/include/LightGBM/c_api.h>`__.
 
 High Level Language Package
 ---------------------------