Update loop.cc

appveyor.yml

@@ -5,18 +5,19 @@ image:
 - Visual Studio 2017
 environment:
   matrix:
-  - NAME: cuda
-  - NAME: opencl
-  - NAME: blas
+  - NAME: gpu-nvidia-cuda
+  - NAME: gpu-opencl
+  - NAME: cpu-openblas
+clone_folder: c:\projects\lc0
 install:
 - cmd: set CUDA=false
 - cmd: set OPENCL=false
 - cmd: set BLAS=false
 - cmd: set GTEST=false
-- cmd: IF %NAME%==cuda set CUDA=true
-- cmd: IF %NAME%==opencl set OPENCL=true
-- cmd: IF %NAME%==blas set BLAS=true
-- cmd: IF %NAME%==blas set GTEST=true
+- cmd: IF %NAME%==gpu-nvidia-cuda set CUDA=true
+- cmd: IF %NAME%==gpu-opencl set OPENCL=true
+- cmd: IF %NAME%==cpu-openblas set BLAS=true
+- cmd: IF %NAME%==cpu-openblas set GTEST=true
 - cmd: IF %BLAS%==true IF NOT EXIST C:\cache\OpenBLAS appveyor DownloadFile https://sjeng.org/ftp/OpenBLAS-0.3.3-win-oldthread.zip
 - cmd: IF %BLAS%==true IF NOT EXIST C:\cache\OpenBLAS 7z x OpenBLAS-0.3.3-win-oldthread.zip -oC:\cache\OpenBLAS
 - cmd: IF %OPENCL%==true nuget install opencl-nug -Version 0.777.77 -OutputDirectory C:\cache
@@ -31,24 +32,24 @@ install:
 - cmd: IF %CUDA%==true IF NOT EXIST C:\cache\cuda appveyor DownloadFile http://developer.download.nvidia.com/compute/redist/cudnn/v7.4.2/cudnn-10.0-windows10-x64-v7.4.2.24.zip
 - cmd: IF %CUDA%==true IF NOT EXIST C:\cache\cuda 7z x cudnn-10.0-windows10-x64-v7.4.2.24.zip -oC:\cache
 - cmd: set PATH=C:\Python36;C:\Python36\scripts;%PATH%
-- cmd: pip3 install --upgrade meson
+- cmd: pip3 install --upgrade meson==0.51.2
 - cmd: call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsall.bat" amd64
 - cmd: set PKG_FOLDER="C:\cache"
 - cmd: IF NOT EXIST c:\cache\protobuf\ git clone -b v3.5.1 --single-branch --depth 1 https://github.com/google/protobuf.git
 - cmd: IF NOT EXIST c:\cache\protobuf\ mkdir protobuf\build_msvc
 - cmd: IF NOT EXIST c:\cache\protobuf\ cd protobuf\build_msvc
-- cmd: IF NOT EXIST c:\cache\protobuf\ cmake -G "Visual Studio 15 2017 Win64" -Dprotobuf_BUILD_SHARED_LIBS=NO -Dprotobuf_MSVC_STATIC_RUNTIME=NO -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=c:/cache/protobuf ../cmake
+- cmd: IF NOT EXIST c:\cache\protobuf\ cmake -G "Visual Studio 15 2017 Win64" -Dprotobuf_BUILD_SHARED_LIBS=NO -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=c:/cache/protobuf ../cmake
 - cmd: IF NOT EXIST c:\cache\protobuf\ msbuild INSTALL.vcxproj /p:Configuration=Release /p:Platform=x64 /m
 - cmd: set PATH=c:\cache\protobuf\bin;%PATH%
 - cmd: IF NOT EXIST c:\cache\testnet appveyor DownloadFile http://lczero.org/get_network?sha=7170f639ba1cdc407283b8e52377283e36845b954788c6ada8897937637ef032 -Filename c:\cache\testnet
-- cmd: IF %GTEST%==true IF NOT EXIST C:\cache\syzygy mkdir C:\cache\syzygy 
+- cmd: IF %GTEST%==true IF NOT EXIST C:\cache\syzygy mkdir C:\cache\syzygy
 - cmd: IF %GTEST%==true cd C:\cache\syzygy
 - cmd: IF %GTEST%==true IF NOT EXIST KQvK.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5/K{P,N,R,B,Q}vK.rtb{w,z}
 - cmd: IF %GTEST%==true IF NOT EXIST KQQvK.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5/K{P,N,R,B,Q}{P,N,R,B,Q}vK.rtb{w,z}
 - cmd: IF %GTEST%==true IF NOT EXIST KQvKQ.rtbz curl --remote-name-all https://tablebase.lichess.ovh/tables/standard/3-4-5/K{P,N,R,B,Q}vK{P,N,R,B,Q}.rtb{w,z}
 - cmd: cd C:\projects\lc0
 cache:
-  - C:\cache
+  - C:\cache -> appveyor.yml
   - 'C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.0'
   - C:\projects\lc0\subprojects\packagecache
 before_build:
@@ -79,7 +80,7 @@ after_build:
 - cmd: IF %APPVEYOR_REPO_TAG%==true IF %CUDA%==true type dist\README-cuda.txt |more /P > dist\README.txt
 - cmd: IF %APPVEYOR_REPO_TAG%==true IF %CUDA%==true 7z a lc0-%APPVEYOR_REPO_TAG_NAME%-windows-%NAME%.zip .\dist\README.txt .\dist\COPYING .\dist\CUDA.txt .\dist\CUDNN.txt
 artifacts:
-  - path: build/rescorer.exe
+  - path: build/lc0.exe
     name: lc0-$(NAME)
   - path: lc0-$(APPVEYOR_REPO_TAG_NAME)-windows-$(NAME).zip
     name: lc0-$(APPVEYOR_REPO_TAG_NAME)-windows-$(NAME)-zip

meson.build

@@ -160,7 +160,6 @@ if get_option('build_backends')
       tensorflow_include[0] + '/bazel-genfiles',
       tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads',
       tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/absl',
-      tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/eigen',
       tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/gemmlowp',
       tensorflow_include[0] + '/tensorflow/contrib/makefile/downloads/nsync/public',
       tensorflow_include[0] + '/tensorflow/contrib/makefile/gen/protobuf-host/include',
@@ -195,13 +194,12 @@ if get_option('build_backends')
   endif
 
   if get_option('blas')
-    if get_option('mkl') and mkl_lib.found()
       add_project_arguments('-DUSE_MKL', language : 'cpp')
       includes += include_directories(get_option('mkl_include'))
       deps += [ mkl_lib ]
       has_blas = true
 
-    elif get_option('accelerate') and accelerate_lib.found()
+    if get_option('accelerate') and accelerate_lib.found()
       includes += include_directories('/System/Library/Frameworks/Accelerate.framework/Frameworks/vecLib.framework/Headers')
       deps += [ accelerate_lib ]
       has_blas = true
@@ -373,7 +371,12 @@ if get_option('build_backends')
     cuda_arguments = ['@EXTRA_ARGS@', '-c', '@INPUT@', '-o', '@OUTPUT@',
                       '-I', meson.current_source_dir() + '/src']
     if host_machine.system() == 'windows'
-      cuda_arguments += ['-Xcompiler', '-MD']
+      # meson bug fixed in v0.51
+      if meson.version().version_compare('<0.51') or get_option('default_library') != 'static'
+        cuda_arguments += ['-Xcompiler', '-MD']
+      else
+        cuda_arguments += ['-Xcompiler', '-MT']
+      endif
     else
       cuda_arguments += ['--std=c++14', '-Xcompiler', '-fPIC']
     endif
@@ -437,14 +440,14 @@ if not get_option('pext')
   add_project_arguments('-DNO_PEXT', language : 'cpp')
 endif
 
-executable('rescorer', 'src/main.cc',
+executable('lc0', 'src/main.cc',
   files, include_directories: includes, dependencies: deps, install: true)
 
 
 ### Tests
-gtest = dependency('gtest', fallback: ['gtest', 'gtest_dep'], required: false)
 
-if get_option('gtest') and gtest.found()
+if get_option('gtest')
+  gtest = dependency('gtest', fallback: ['gtest', 'gtest_dep'])
   lc0_lib = library('lc0_lib', files, include_directories: includes, dependencies: deps)
 
   test('ChessBoard',

meson_options.txt

@@ -14,9 +14,9 @@ option('openblas_include',
        description: 'Paths to openblas include directories')
 
 option('opencl_include',
-       type: 'array',
-       value: ['/usr/include/'],
-       description: 'Paths to OpenCL include directories')
+       type: 'string',
+       value: '/usr/include/',
+       description: 'Path to OpenCL include directory')
 
 option('tensorflow_libdir',
        type: 'array',

src/selfplay/loop.cc

@@ -72,7 +72,8 @@ std::atomic<int> policy_bump_total_hist[11];
 
 void ProcessFile(const std::string& file, SyzygyTablebase* tablebase,
                  std::string outputDir, float distTemp, float distOffset,
-                 float dtzBoost, std::ofstream& draws, std::ofstream& wins, std::ofstream& losses) {
+                 float dtzBoost, std::ofstream& draws, std::ofstream& wins, std::ofstream& losses,
+                 std::unordered_map<std::string, WDLScore> known_positions) {
   // Scope to ensure reader and writer are closed before deleting source file.
   {
     TrainingDataReader reader(file);
@@ -109,14 +110,16 @@ void ProcessFile(const std::string& file, SyzygyTablebase* tablebase,
     for (int i = 0; i < moves.size(); i++) {
       history.Append(moves[i]);
       const auto& board = history.Last().GetBoard();
-      if ((board.ours() | board.theirs()).count() <= 5) {
+      const auto& count = (board.ours() | board.theirs()).count();
+      Position pos = history.Last();
+      std::string target_fen = pos.GetFen();
+
+      if (count <= 5) {
         fileContents.resize(i);
         max_i = i;
         break;
       }
-      if ((board.ours() | board.theirs()).count() == 8) {
-        Position pos = history.Last(); 
-        std::string target_fen = pos.GetFen();
+      if (count == 8) {
         if (fileContents[i].result == 0) {
           draws << target_fen << std::endl;
         } else if (fileContents[i].result == 1) {
@@ -125,12 +128,23 @@ void ProcessFile(const std::string& file, SyzygyTablebase* tablebase,
           losses << target_fen << std::endl;
         }
       }
-      if (board.castlings().no_legal_castle() &&
-          history.Last().GetNoCaptureNoPawnPly() == 0 &&
-          (board.ours() | board.theirs()).count() <=
-              tablebase->max_cardinality()) {
+
+      std::unordered_map<std::string, WDLScore>::iterator iter = known_positions.find(target_fen);
+
+      if ((board.castlings().no_legal_castle() && history.Last().GetNoCaptureNoPawnPly() == 0 &&
+          count <= tablebase->max_cardinality())
+          || iter != known_positions.end()
+         ) {
+
         ProbeState state;
-        WDLScore wdl = tablebase->probe_wdl(history.Last(), &state);
+        WDLScore wdl;
+        if (iter != known_positions.end()) {
+          wdl = iter->second;
+          state = OK;
+        }
+        else {
+          wdl = tablebase->probe_wdl(history.Last(), &state);
+        }
         // Only fail state means the WDL is wrong, probe_wdl may produce correct
         // result with a stat other than OK.
         if (state != FAIL) {
@@ -174,10 +188,9 @@ void ProcessFile(const std::string& file, SyzygyTablebase* tablebase,
     for (int i = 0; i < max_i; i++) {
       history.Append(moves[i]);
       const auto& board = history.Last().GetBoard();
-      if (board.castlings().no_legal_castle() &&
-          history.Last().GetNoCaptureNoPawnPly() != 0 &&
-          (board.ours() | board.theirs()).count() <=
-              tablebase->max_cardinality()) {
+      const auto& count = (board.ours() | board.theirs()).count();
+      if (board.castlings().no_legal_castle() && history.Last().GetNoCaptureNoPawnPly() != 0 &&
+          count <= tablebase->max_cardinality()) {
         ProbeState state;
         WDLScore wdl = tablebase->probe_wdl(history.Last(), &state);
         // Only fail state means the WDL is wrong, probe_wdl may produce correct
@@ -264,12 +277,12 @@ void ProcessFile(const std::string& file, SyzygyTablebase* tablebase,
       int move_index = 0;
       for (auto& chunk : fileContents) {
         const auto& board = history.Last().GetBoard();
+        const auto& count = (board.ours() | board.theirs()).count();
         std::vector<bool> boost_probs(1858, false);
         int boost_count = 0;
 
         if (dtzBoost != 0.0f && board.castlings().no_legal_castle() &&
-            (board.ours() | board.theirs()).count() <=
-                tablebase->max_cardinality()) {
+            count <= tablebase->max_cardinality()) {
           MoveList to_boost;
           tablebase->root_probe(history.Last(), true, true, &to_boost);
           for (auto& move : to_boost) {
@@ -396,9 +409,28 @@ void ProcessFiles(const std::vector<std::string>& files,
   draws.open(outputDir + "." + std::to_string(offset) + ".draws.txt");
   wins.open(outputDir + "." + std::to_string(offset) + ".wins.txt");
   losses.open(outputDir + "." + std::to_string(offset) + ".losses.txt");
+
+  std::unordered_map<std::string, WDLScore> known_positions;
+  std::string line;
+  std::ifstream in_win("wins.txt");
+  while (std::getline(in_win, line))
+  {
+    known_positions[line] = WDL_WIN;
+  }
+  std::ifstream in_draws("draws.txt");
+  while (std::getline(in_draws, line))
+  {
+    known_positions[line] = WDL_DRAW;
+  }
+  std::ifstream in_losses("losses.txt");
+  while (std::getline(in_losses, line))
+  {
+    known_positions[line] = WDL_LOSS;
+  }
+
   for (int i = offset; i < files.size(); i += mod) {
     try {
-      ProcessFile(files[i], tablebase, outputDir, distTemp, distOffset, dtzBoost, draws, wins, losses);      
+      ProcessFile(files[i], tablebase, outputDir, distTemp, distOffset, dtzBoost, draws, wins, losses, known_positions);      
     } catch (...) {
       std::cerr << "Caught error on: " << files[i] << std::endl;
       int error = rename( files[i].c_str(), std::string(std::string("G:\\old-lczero-training\\convert\\toConvert\\errors\\") + files[i]).c_str() );

subprojects/zlib.wrap

@@ -5,6 +5,6 @@ source_url = http://zlib.net/fossils/zlib-1.2.11.tar.gz
 source_filename = zlib-1.2.11.tar.gz
 source_hash = c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
 
-patch_url = https://wrapdb.mesonbuild.com/v1/projects/zlib/1.2.11/3/get_zip
+patch_url = https://github.com/mesonbuild/zlib/releases/download/1.2.11-3/zlib.zip
 patch_filename = zlib-1.2.11-3-wrap.zip
-patch_hash = f07dc491ab3d05daf00632a0591e2ae61b470615b5b73bcf9b3f061fff65cff0
+patch_hash = f07dc491ab3d05daf00632a0591e2ae61b470615b5b73bcf9b3f061fff65cff0