Merge pull request #263048 from Madouura/pr/triton

Make openai-triton/pytorch free again (and various fixes)
NixOS · Nov 10, 2023 · 6cd5fb6 · 6cd5fb6
2 parents 83e0b3d + 3aab2bc
commit 6cd5fb6
Show file tree

Hide file tree

Showing 23 changed files with 420 additions and 96 deletions.
diff --git a/...ent/python-modules/openai-triton/llvm.nix → ...by-name/op/openai-triton-llvm/package.nix b/...ent/python-modules/openai-triton/llvm.nix → ...by-name/op/openai-triton-llvm/package.nix
@@ -22,7 +22,7 @@
 }:
 
 stdenv.mkDerivation (finalAttrs: {
-  pname = "triton-llvm";
+  pname = "openai-triton-llvm";
   version = "14.0.6-f28c006a5895";
 
   outputs = [

diff --git a/pkgs/development/libraries/rapidjson/0000-unstable-use-nixpkgs-gtest.patch b/pkgs/development/libraries/rapidjson/0000-unstable-use-nixpkgs-gtest.patch
@@ -0,0 +1,35 @@
+diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
+index 11c1b04c..762eaa75 100644
+--- a/test/CMakeLists.txt
++++ b/test/CMakeLists.txt
+@@ -1,20 +1,14 @@
+-find_package(GTestSrc)
++enable_testing()
+
+-IF(GTESTSRC_FOUND)
+-    enable_testing()
++if (WIN32 AND (NOT CYGWIN) AND (NOT MINGW))
++    set(gtest_disable_pthreads ON)
++    set(gtest_force_shared_crt ON)
++endif()
+
+-    if (WIN32 AND (NOT CYGWIN) AND (NOT MINGW))
+-        set(gtest_disable_pthreads ON)
+-        set(gtest_force_shared_crt ON)
+-    endif()
++include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
+
+-    add_subdirectory(${GTEST_SOURCE_DIR} ${CMAKE_BINARY_DIR}/googletest)
+-    include_directories(SYSTEM ${GTEST_INCLUDE_DIR})
++set(TEST_LIBRARIES gtest gtest_main)
+
+-    set(TEST_LIBRARIES gtest gtest_main)
+-
+-    add_custom_target(tests ALL)
+-    add_subdirectory(perftest)
+-    add_subdirectory(unittest)
+-
+-ENDIF(GTESTSRC_FOUND)
++add_custom_target(tests ALL)
++add_subdirectory(perftest)
++add_subdirectory(unittest)
diff --git a/pkgs/development/libraries/rapidjson/0001-unstable-valgrind-suppress-failures.patch b/pkgs/development/libraries/rapidjson/0001-unstable-valgrind-suppress-failures.patch
@@ -0,0 +1,36 @@
+diff --git a/test/valgrind.supp b/test/valgrind.supp
+index c9d3d226..63af7bf9 100644
+--- a/test/valgrind.supp
++++ b/test/valgrind.supp
+@@ -24,3 +24,31 @@
+    fun:*Uri*Parse_UTF16_Std*
+ }
+
++{
++   Suppress memcpy_chk valgrind report 5
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   fun:*dtoa_normal*
++}
++
++{
++   Suppress memcpy_chk valgrind report 6
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   fun:*dtoa_maxDecimalPlaces*
++}
++
++{
++   Suppress memcpy_chk valgrind report 7
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   ...
++   fun:*Reader*ParseDoubleHandler*
++}
++
++{
++   Suppress memcpy_chk valgrind report 8
++   Memcheck:Overlap
++   fun:__memcpy_chk
++   fun:*Reader*ParseNumber_NormalPrecisionError*
++}
diff --git a/pkgs/development/libraries/rapidjson/unstable.nix b/pkgs/development/libraries/rapidjson/unstable.nix
@@ -1,50 +1,82 @@
 { lib
 , stdenv
 , fetchFromGitHub
-, fetchpatch
-, pkg-config
 , cmake
+, doxygen
+, graphviz
 , gtest
 , valgrind
+# One of "11" or "17"; default in source is CXX 11
+, cxxStandard ? "11"
+, buildDocs ? true
+, buildTests ? !stdenv.hostPlatform.isStatic && !stdenv.isDarwin
+, buildExamples ? true
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "rapidjson";
   version = "unstable-2023-09-28";
 
+  outputs = [
+    "out"
+  ] ++ lib.optionals buildDocs [
+    "doc"
+  ] ++ lib.optionals buildExamples [
+    "example"
+  ];
+
   src = fetchFromGitHub {
     owner = "Tencent";
     repo = "rapidjson";
     rev = "f9d53419e912910fd8fa57d5705fa41425428c35";
     hash = "sha256-rl7iy14jn1K2I5U2DrcZnoTQVEGEDKlxmdaOCF/3hfY=";
   };
 
+  patches = lib.optionals buildTests [
+    ./0000-unstable-use-nixpkgs-gtest.patch
+    # https://github.com/Tencent/rapidjson/issues/2214
+    ./0001-unstable-valgrind-suppress-failures.patch
+  ];
+
   nativeBuildInputs = [
-    pkg-config
     cmake
+  ] ++ lib.optionals buildDocs [
+    doxygen
+    graphviz
   ];
 
-  patches = [
-    (fetchpatch {
-      name = "do-not-include-gtest-src-dir.patch";
-      url = "https://git.alpinelinux.org/aports/plain/community/rapidjson/do-not-include-gtest-src-dir.patch?id=9e5eefc7a5fcf5938a8dc8a3be8c75e9e6809909";
-      hash = "sha256-BjSZEwfCXA/9V+kxQ/2JPWbc26jQn35CfN8+8NW24s4=";
-    })
+  cmakeFlags = [
+    (lib.cmakeBool "RAPIDJSON_BUILD_DOC" buildDocs)
+    (lib.cmakeBool "RAPIDJSON_BUILD_TESTS" buildTests)
+    (lib.cmakeBool "RAPIDJSON_BUILD_EXAMPLES" buildExamples)
+    (lib.cmakeBool "RAPIDJSON_BUILD_CXX11" (cxxStandard == "11"))
+    (lib.cmakeBool "RAPIDJSON_BUILD_CXX17" (cxxStandard == "17"))
+  ] ++ lib.optionals buildTests [
+    (lib.cmakeFeature "GTEST_INCLUDE_DIR" "${lib.getDev gtest}")
+  ];
+
+  doCheck = buildTests;
+
+  nativeCheckInputs = [
+    gtest
+    valgrind
   ];
 
-  # for tests, adding gtest to checkInputs does not work
-  # https://github.com/NixOS/nixpkgs/pull/212200
-  buildInputs = [ gtest ];
-  cmakeFlags = [ "-DGTEST_SOURCE_DIR=${gtest.dev}/include" ];
+  postInstall = lib.optionalString buildExamples ''
+    mkdir -p $example/bin
 
-  nativeCheckInputs = [ valgrind ];
-  doCheck = !stdenv.hostPlatform.isStatic && !stdenv.isDarwin;
+    find bin -type f -executable \
+      -not -name "perftest" \
+      -not -name "unittest" \
+      -exec cp -a {} $example/bin \;
+  '';
 
   meta = with lib; {
     description = "Fast JSON parser/generator for C++ with both SAX/DOM style API";
     homepage = "http://rapidjson.org/";
     license = licenses.mit;
     platforms = platforms.unix;
     maintainers = with maintainers; [ Madouura ];
+    broken = (cxxStandard != "11" && cxxStandard != "17");
   };
-}
+})
diff --git a/pkgs/development/python-modules/arviz/default.nix b/pkgs/development/python-modules/arviz/default.nix
@@ -28,6 +28,7 @@
 , pyro-ppl
   #, pystan (not packaged)
 , numpyro
+, bokeh
 }:
 
 buildPythonPackage rec {
@@ -72,6 +73,7 @@ buildPythonPackage rec {
     pytestCheckHook
     torchvision
     zarr
+    bokeh
   ];
 
   preCheck = ''
@@ -97,6 +99,8 @@ buildPythonPackage rec {
     # An issue has been opened upstream: https://github.com/arviz-devs/arviz/issues/2282
     "test_plot_ppc_discrete"
     "test_plot_ppc_discrete_save_animation"
+    # Assertion error
+    "test_data_zarr"
   ];
 
   pythonImportsCheck = [

diff --git a/pkgs/development/python-modules/ax/default.nix b/pkgs/development/python-modules/ax/default.nix
@@ -69,6 +69,10 @@ buildPythonPackage rec {
   disabledTests = [
     # exact comparison of floating points
     "test_optimize_l0_homotopy"
+    # AssertionError: 5 != 2
+    "test_get_standard_plots_moo"
+    # AssertionError: Expected 'warning' to be called once. Called 3 times
+    "test_validate_kwarg_typing"
   ];
   pythonImportsCheck = [ "ax" ];
 

diff --git a/pkgs/development/python-modules/cleanlab/default.nix b/pkgs/development/python-modules/cleanlab/default.nix
@@ -6,23 +6,27 @@
 , termcolor
 , tqdm
 , pandas
+, setuptools
 # test dependencies
+, pytestCheckHook
+, pytest-lazy-fixture
 , tensorflow
 , torch
 , datasets
 , torchvision
 , keras
 , fasttext
+, hypothesis
+, wget
+, matplotlib
+, skorch
 }:
-let
+
+buildPythonPackage rec {
   pname = "cleanlab";
   version = "2.5.0";
-in
-buildPythonPackage {
-  inherit pname version;
-  format = "setuptools";
-
-  disabled = pythonOlder "3.8";
+  pyproject = true;
+  disabled = pythonOlder "3.7";
 
   src = fetchFromGitHub {
     owner = "cleanlab";
@@ -31,11 +35,7 @@ buildPythonPackage {
     hash = "sha256-5XQQVrhjpvjwtFM79DqttObmw/GQLkMQVXb5jhiC8e0=";
   };
 
-  # postPatch = ''
-  #   substituteInPlace pyproject.toml \
-  #     --replace '"rich <= 13.0.1"' '"rich"' \
-  #     --replace '"numpy < 1.24.0"' '"numpy"'
-  # '';
+  nativeBuildInputs = [ setuptools ];
 
   propagatedBuildInputs = [
     scikit-learn
@@ -44,13 +44,38 @@ buildPythonPackage {
     pandas
   ];
 
+  # This is ONLY turned off when we have testing enabled.
+  # The reason we do this is because of duplicate packages in the enclosure
+  # when using the packages in nativeCheckInputs.
+  # Affected packages: grpcio protobuf tensorboard tensorboard-plugin-profile
+  catchConflicts = (!doCheck);
+  doCheck = true;
+
   nativeCheckInputs = [
+    pytestCheckHook
+    pytest-lazy-fixture
     tensorflow
     torch
     datasets
     torchvision
     keras
     fasttext
+    hypothesis
+    wget
+    matplotlib
+    skorch
+  ];
+
+  disabledTests = [
+    # Requires the datasets we prevent from downloading
+    "test_create_imagelab"
+  ];
+
+  disabledTestPaths = [
+    # Requires internet
+    "tests/test_dataset.py"
+    # Requires the datasets we just prevented from downloading
+    "tests/datalab/test_cleanvision_integration.py"
   ];
 
   meta = with lib; {

diff --git a/pkgs/development/python-modules/cvxpy/default.nix b/pkgs/development/python-modules/cvxpy/default.nix
@@ -12,19 +12,20 @@
 , scs
 , setuptools
 , wheel
+, pybind11
 , useOpenmp ? (!stdenv.isDarwin)
 }:
 
 buildPythonPackage rec {
   pname = "cvxpy";
-  version = "1.3.2";
+  version = "1.4.1";
   format = "pyproject";
 
   disabled = pythonOlder "3.7";
 
   src = fetchPypi {
     inherit pname version;
-    hash = "sha256-C2heUEDxmfPXA/MPXSLR+GVZdiNFUVPR3ddwJFrvCXU=";
+    hash = "sha256-ep7zTjxX/4yETYbwo4NPtVda8ZIzlHY53guld8YSLj4=";
   };
 
   # we need to patch out numpy version caps from upstream
@@ -35,6 +36,7 @@ buildPythonPackage rec {
   nativeBuildInputs = [
     setuptools
     wheel
+    pybind11
   ];
 
   propagatedBuildInputs = [
@@ -44,7 +46,6 @@ buildPythonPackage rec {
     osqp
     scipy
     scs
-    setuptools
   ];
 
   nativeCheckInputs = [