Merge pull request #353 from glinscott/next

v0.7

.gitignore

@@ -30,3 +30,5 @@ training/tf/models/
 *.exe
 *.dll
 /go/src/client/settings.json
+*.vspx
+*.psess

CMakeLists.txt

@@ -29,7 +29,9 @@ set (Boost_USE_MULTITHREADED ON)
 find_package (Boost 1.54.0 REQUIRED program_options filesystem)
 find_package (Threads REQUIRED)
 find_package (ZLIB REQUIRED)
-find_package (OpenCL REQUIRED)
+if (NOT FEATURE_USE_CPU_ONLY)
+    find_package (OpenCL REQUIRED)
+endif()
 
 # We need OpenBLAS for now, because we make some specific
 # calls. Ideally we'd use OpenBLAS is possible and fall back to
@@ -84,7 +86,9 @@ set (SrcPath "${CMAKE_CURRENT_SOURCE_DIR}/src")
 
 include_directories (${IncludePath})
 include_directories (SYSTEM ${Boost_INCLUDE_DIRS})
-include_directories (SYSTEM ${OpenCL_INCLUDE_DIRS})
+if (NOT FEATURE_USE_CPU_ONLY)
+    include_directories (SYSTEM ${OpenCL_INCLUDE_DIRS})
+endif()
 include_directories (SYSTEM ${ZLIB_INCLUDE_DIRS})
 
 if ((UNIX AND NOT APPLE) OR WIN32)
@@ -106,7 +110,9 @@ add_executable (lczero $<TARGET_OBJECTS:objs> ${lczero_MAIN})
 
 target_link_libraries (lczero ${Boost_LIBRARIES})
 target_link_libraries (lczero ${BLAS_LIBRARIES})
-target_link_libraries (lczero ${OpenCL_LIBRARIES})
+if (NOT FEATURE_USE_CPU_ONLY)
+    target_link_libraries (lczero ${OpenCL_LIBRARIES})
+endif()
 target_link_libraries (lczero ${ZLIB_LIBRARIES})
 target_link_libraries (lczero ${CMAKE_THREAD_LIBS_INIT})
 install (TARGETS lczero DESTINATION bin)
@@ -117,6 +123,8 @@ add_executable (tests ${tests_SRC} $<TARGET_OBJECTS:objs>)
 
 target_link_libraries (tests ${Boost_LIBRARIES})
 target_link_libraries (tests ${BLAS_LIBRARIES})
-target_link_libraries (tests ${OpenCL_LIBRARIES})
+if (NOT FEATURE_USE_CPU_ONLY)
+    target_link_libraries (tests ${OpenCL_LIBRARIES})
+endif()
 target_link_libraries (tests ${ZLIB_LIBRARIES})
 target_link_libraries (tests gtest_main ${CMAKE_THREAD_LIBS_INIT})

go/src/client/http/http.go

@@ -2,7 +2,6 @@ package client
 
 import (
 	"bytes"
-	"compress/gzip"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -117,12 +116,6 @@ func DownloadNetwork(httpClient *http.Client, hostname string, networkPath strin
 		return err
 	}
 
-	// Copy while decompressing
-	zr, err := gzip.NewReader(r.Body)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	_, err = io.Copy(out, zr)
+	_, err = io.Copy(out, r.Body)
 	return err
 }

go/src/client/main.go

@@ -78,7 +78,7 @@ func getExtraParams() map[string]string {
 	return map[string]string{
 		"user":     *USER,
 		"password": *PASSWORD,
-		"version":  "6",
+		"version":  "7",
 	}
 }
 

go/src/server/main.go

@@ -225,7 +225,7 @@ func uploadNetwork(c *gin.Context) {
 	}
 
 	// TODO(gary): Make this more generic - upload to s3 for now
-	cmd := exec.Command("aws", "s3", "cp", network.Path, "s3://lczero/" + network.Path)
+	cmd := exec.Command("aws", "s3", "cp", network.Path, "s3://lczero/networks/")
 	err = cmd.Run()
 	if err != nil {
 		log.Println(err.Error())

lc0/build.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/bash
 
 rm -fr build
-CC=clang CXX=clang++ meson build --buildtype release
-# CC=clang CXX=clang++ meson build --buildtype debugoptimized
+CC=clang CXX=clang++ meson build --buildtype release # -Db_ndebug=true
+# CC=clang CXX=clang++ meson build --buildtype debugoptimized -Db_asneeded=false
+# CC=clang CXX=clang++ meson build --buildtype debug
 cd build
 ninja

lc0/meson.build

@@ -1,7 +1,7 @@
-project('lc0', 'cpp', 
-        default_options : ['c_std=c17', 'cpp_std=c++17'])
+project('lc0', 'cpp')
+  #        default_options : ['cpp_std=c++17'])
 
-# add_global_arguments('-Wno-macro-redefined', language : 'cpp')
+add_global_arguments('-std=c++17', language : 'cpp')
 cc = meson.get_compiler('cpp')
 
 # Installed from https://github.com/FloopCZ/tensorflow_cc
@@ -33,6 +33,8 @@ files = [
   'src/chess/board.cc',
   'src/neural/loader.cc',
   'src/neural/network_tf.cc',
+  'src/neural/network_random.cc',
+  'src/neural/cache.cc',
   'src/mcts/search.cc',
   'src/mcts/node.cc',
   'src/engine.cc',
@@ -62,3 +64,8 @@ test('Network',
   executable('network_test', 'src/neural/network_test.cc',
   files, include_directories: includes, dependencies: test_deps
 ))
+
+test('HashCat',
+  executable('hashcat_test', 'src/utils/hashcat_test.cc',
+  files, include_directories: includes, dependencies: test_deps
+))

lc0/src/chess/bitboard.cc

@@ -256,15 +256,7 @@ const Move kIdxToMove[] = {
     "e7f8r", "e7f8b", "f7e8q", "f7e8r", "f7e8b", "f7f8q", "f7f8r", "f7f8b",
     "f7g8q", "f7g8r", "f7g8b", "g7f8q", "g7f8r", "g7f8b", "g7g8q", "g7g8r",
     "g7g8b", "g7h8q", "g7h8r", "g7h8b", "h7g8q", "h7g8r", "h7g8b", "h7h8q",
-    "h7h8r", "h7h8b", "a2a1q", "a2a1r", "a2a1b", "a2b1q", "a2b1r", "a2b1b",
-    "b2a1q", "b2a1r", "b2a1b", "b2b1q", "b2b1r", "b2b1b", "b2c1q", "b2c1r",
-    "b2c1b", "c2b1q", "c2b1r", "c2b1b", "c2c1q", "c2c1r", "c2c1b", "c2d1q",
-    "c2d1r", "c2d1b", "d2c1q", "d2c1r", "d2c1b", "d2d1q", "d2d1r", "d2d1b",
-    "d2e1q", "d2e1r", "d2e1b", "e2d1q", "e2d1r", "e2d1b", "e2e1q", "e2e1r",
-    "e2e1b", "e2f1q", "e2f1r", "e2f1b", "f2e1q", "f2e1r", "f2e1b", "f2f1q",
-    "f2f1r", "f2f1b", "f2g1q", "f2g1r", "f2g1b", "g2f1q", "g2f1r", "g2f1b",
-    "g2g1q", "g2g1r", "g2g1b", "g2h1q", "g2h1r", "g2h1b", "h2g1q", "h2g1r",
-    "h2g1b", "h2h1q", "h2h1r", "h2h1b"};
+    "h7h8r", "h7h8b"};
 
 std::vector<unsigned short> BuildMoveIndices() {
   std::vector<unsigned short> res(4 * 64 * 64);

lc0/src/chess/bitboard.h

@@ -36,7 +36,7 @@ class BoardSquare {
   // From row(bottom to top), and col(left to right), 0-based.
   constexpr BoardSquare(int row, int col) : BoardSquare(row * 8 + col) {}
   // From Square name, e.g e4. Only lowercase.
-  constexpr BoardSquare(const std::string& str, bool black = false)
+  BoardSquare(const std::string& str, bool black = false)
       : BoardSquare(black ? '8' - str[1] : str[1] - '1', str[0] - 'a') {}
   constexpr std::uint8_t as_int() const { return square_; }
   void set(int row, int col) { square_ = row * 8 + col; }
@@ -198,11 +198,13 @@ class Move {
   BoardSquare from() const { return from_; }
   BoardSquare to() const { return to_; }
   Promotion promotion() const { return promotion_; }
+  bool IsCastling() const { return castling_; }
+  void SetCastling() { castling_ = true; }
 
   // 0 .. 16384, knight promotion and no promotion is the same.
   uint16_t as_packed_int() const;
 
-  // 0 .. 1923, to use in neural networks.
+  // 0 .. 1857, to use in neural networks.
   uint16_t as_nn_index() const;
 
   bool operator==(const Move& other) const {
@@ -219,7 +221,11 @@ class Move {
   }
 
   std::string as_string() const {
-    std::string res = from_.as_string() + to_.as_string();
+    BoardSquare to = to_;
+    if (castling_) {
+      to = BoardSquare(to.row(), (to.col() == 7) ? 6 : 2);
+    }
+    std::string res = from_.as_string() + to.as_string();
     switch (promotion_) {
       case Promotion::None:
         return res;
@@ -238,6 +244,7 @@ class Move {
   BoardSquare from_;
   BoardSquare to_;
   Promotion promotion_ = Promotion::None;
+  bool castling_ = false;
 };
 
 using MoveList = std::vector<Move>;

lc0/src/chess/board.cc

@@ -24,6 +24,10 @@
 #include <sstream>
 #include "utils/exception.h"
 
+#ifdef _MSC_VER
+#include <nmmintrin.h>
+#endif
+
 namespace lczero {
 
 using std::string;
@@ -199,7 +203,8 @@ MoveList ChessBoard::GeneratePseudovalidMoves() const {
           }
         }
         if (can_castle) {
-          result.emplace_back(source, BoardSquare(0, 6));
+          result.emplace_back(source, BoardSquare(0, 7));
+          result.back().SetCastling();
         }
       }
       if (castlings_.we_can_000()) {
@@ -219,7 +224,8 @@ MoveList ChessBoard::GeneratePseudovalidMoves() const {
           }
         }
         if (can_castle) {
-          result.emplace_back(source, BoardSquare(0, 2));
+          result.emplace_back(source, BoardSquare(0, 0));
+          result.back().SetCastling();
         }
       }
       continue;
@@ -303,7 +309,7 @@ MoveList ChessBoard::GeneratePseudovalidMoves() const {
               // Ordinary capture.
               result.emplace_back(source, destination);
             }
-          } else if (dst_row == 5 and pawns_.get(7, dst_col)) {
+          } else if (dst_row == 5 && pawns_.get(7, dst_col)) {
             // En passant.
             result.emplace_back(source, destination);
           }
@@ -330,9 +336,9 @@ bool ChessBoard::ApplyMove(Move move) {
   const auto to_row = to.row();
   const auto to_col = to.col();
 
-  // Move in our pieces.
+  // Remove our piece from old location, but not put to destination
+  // (for the case of castling).
   our_pieces_.reset(from);
-  our_pieces_.set(to);
 
   // Remove captured piece
   bool reset_50_moves = their_pieces_.get(to);
@@ -364,24 +370,33 @@ bool ChessBoard::ApplyMove(Move move) {
   if (from == our_king_) {
     castlings_.reset_we_can_00();
     castlings_.reset_we_can_000();
-    our_king_ = to;
     // Castling
-    if (to_col - from_col == 2) {
+    if (to_col - from_col > 1) {
       // 0-0
       our_pieces_.reset(7);
       rooks_.reset(7);
       our_pieces_.set(5);
       rooks_.set(5);
-    } else if (from_col - to_col == 2) {
+      our_king_ = BoardSquare(0, 6); /* g8 */
+      our_pieces_.set(our_king_);
+    } else if (from_col - to_col > 1) {
       // 0-0-0
       our_pieces_.reset(0);
       rooks_.reset(0);
       our_pieces_.set(3);
       rooks_.set(3);
+      our_king_ = BoardSquare(0, 2); /* c8 */
+      our_pieces_.set(our_king_);
+    } else {
+      our_king_ = to;
+      our_pieces_.set(to);
     }
     return reset_50_moves;
   }
 
+  // Now destination square for our piece is known.
+  our_pieces_.set(to);
+
   // Promotion
   if (move.promotion() != Move::Promotion::None) {
     switch (move.promotion()) {
@@ -418,7 +433,7 @@ bool ChessBoard::ApplyMove(Move move) {
   pawns_.reset(from);
 
   // Set en passant flag.
-  if (to_row - from_row == 2 and pawns_.get(to)) {
+  if (to_row - from_row == 2 && pawns_.get(to)) {
     pawns_.set(0, to_col);
   }
   return reset_50_moves;
@@ -600,8 +615,13 @@ bool ChessBoard::HasMatingMaterial() const {
     return true;
   }
 
+#ifdef _MSC_VER
+  int our = _mm_popcnt_u64(our_pieces_.as_int());
+  int their = _mm_popcnt_u64(their_pieces_.as_int());
+#else
   int our = __builtin_popcountll(our_pieces_.as_int());
   int their = __builtin_popcountll(their_pieces_.as_int());
+#endif
   if (our > 2 || their > 2) {
     return true;
   }
@@ -661,7 +681,8 @@ string ChessBoard::DebugString() const {
     }
     if (i == 0) {
       result += " " + castlings_.as_string();
-      result += flipped_ ? " (from black's eyes)" : "(from white's eyes)";
+      result += flipped_ ? " (from black's eyes)" : " (from white's eyes)";
+      result += " Hash: " + std::to_string(Hash());
     }
     result += '\n';
   }

lc0/src/chess/board.h

@@ -20,6 +20,7 @@
 
 #include <string>
 #include "chess/bitboard.h"
+#include "utils/hashcat.h"
 
 namespace lczero {
 
@@ -59,6 +60,13 @@ class ChessBoard {
   // Returns a list of valid moves and board positions after the move is made.
   std::vector<MoveExecution> GenerateValidMoves() const;
 
+  uint64_t Hash() const {
+    return HashCat({our_pieces_.as_int(), their_pieces_.as_int(),
+                    rooks_.as_int(), bishops_.as_int(), pawns_.as_int(),
+                    our_king_.as_int(), their_king_.as_int(),
+                    castlings_.as_int(), flipped_});
+  }
+
   class Castlings {
    public:
     void set_we_can_00() { data_ |= 1; }
@@ -88,6 +96,8 @@ class ChessBoard {
       return result;
     }
 
+    uint8_t as_int() const { return data_; }
+
     bool operator==(const Castlings& other) const {
       return data_ == other.data_;
     }

lc0/src/chess/board_test.cc

@@ -74,7 +74,6 @@ int Perft(const ChessBoard& board, int max_depth, bool dump = false,
     int count = Perft(new_board, max_depth, dump, depth + 1);
     if (dump && depth == 0) {
       Move m = move;
-      if (depth == 0) m.Mirror();
       std::cerr << m.as_string() << ' ' << count << '\n'
                 << new_board.DebugString();
     }
@@ -100,7 +99,7 @@ TEST(ChessBoard, MoveGenStartingPos) {
 TEST(ChessBoard, MoveGenKiwipete) {
   ChessBoard board;
   board.SetFromFen(
-      "r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq -");
+      "r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq - 1 1");
 
   EXPECT_EQ(Perft(board, 1), 48);
   EXPECT_EQ(Perft(board, 2), 2039);
@@ -111,7 +110,7 @@ TEST(ChessBoard, MoveGenKiwipete) {
 
 TEST(ChessBoard, MoveGenPosition3) {
   ChessBoard board;
-  board.SetFromFen("8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - -");
+  board.SetFromFen("8/2p5/3p4/KP5r/1R3p1k/8/4P1P1/8 w - - 1 1");
 
   EXPECT_EQ(Perft(board, 1), 14);
   EXPECT_EQ(Perft(board, 2), 191);