Randomly play opening moves based on priors

Author: Mardak

src/selfplay/game.cc

@@ -29,6 +29,7 @@
 #include <algorithm>
 
 #include "neural/writer.h"
+#include "utils/random.h"
 
 namespace lczero {
 
@@ -69,11 +70,12 @@ SelfPlayGame::SelfPlayGame(PlayerOptions player1, PlayerOptions player2,
 }
 
 void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
-                        bool enable_resign) {
-  bool blacks_move = false;
+                        int random_opening_plies, bool enable_resign) {
+  SearchLimits random_opening_limits;
+  random_opening_limits.visits = 1;
 
   // Do moves while not end of the game. (And while not abort_)
-  while (!abort_) {
+  for (auto blacks_move = false; !abort_; blacks_move = !blacks_move) {
     game_result_ = tree_[0]->GetPositionHistory().ComputeGameResult();
 
     // If endgame, stop.
@@ -94,7 +96,9 @@ void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
       if (abort_) break;
       search_ = std::make_unique<Search>(
           *tree_[idx], options_[idx].network, options_[idx].best_move_callback,
-          options_[idx].info_callback, options_[idx].search_limits,
+          options_[idx].info_callback,
+          random_opening_plies ? random_opening_limits
+                               : options_[idx].search_limits,
           *options_[idx].uci_options, options_[idx].cache, nullptr);
       // TODO: add Syzygy option for selfplay.
     }
@@ -103,8 +107,23 @@ void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
     search_->RunBlocking(blacks_move ? black_threads : white_threads);
     if (abort_) break;
 
+    // Override the best move if we're still picking a random opening.
+    auto move = search_->GetBestMove().first;
+    if (random_opening_plies) {
+      // Pick a move proportionally to its prior.
+      auto toss = Random::Get().GetFloat(1.0f);
+      for (const auto& edge : tree_[idx]->GetCurrentHead()->Edges()) {
+        toss -= edge.GetP();
+        if (toss < 0.0f) {
+          move = edge.GetMove(blacks_move);
+          break;
+        }
+      }
+    }
+
+    // Get data from the head before cleaning up the children with the move.
     auto best_eval = search_->GetBestEval();
-    if (training) {
+    if (!random_opening_plies && training) {
       // Append training data. The GameResult is later overwritten.
       auto best_q = best_eval.first;
       auto best_d = best_eval.second;
@@ -113,10 +132,28 @@ void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
           search_->GetParams().GetHistoryFill(), best_q, best_d));
     }
 
+    // Add move to the tree.
+    tree_[0]->MakeMove(move);
+    if (tree_[0] != tree_[1]) tree_[1]->MakeMove(move);
+
+    // Skip adjudication if random openings can still balance out.
+    if (random_opening_plies) {
+      random_opening_plies--;
+
+      // Randomly played into a decided result, so stop and treat as undecided.
+      if (tree_[0]->GetPositionHistory().ComputeGameResult() !=
+          GameResult::UNDECIDED) {
+        break;
+      }
+      continue;
+    }
+
+    // Adjudicate if evals exceed resignation thresholds.
     float eval = best_eval.first;
     eval = (eval + 1) / 2;
     if (eval < min_eval_[idx]) min_eval_[idx] = eval;
-    const int move_number = tree_[0]->GetPositionHistory().GetLength() / 2 + 1;
+    const int move_number =
+        (tree_[0]->GetPositionHistory().GetLength() + 1) / 2;
     if (enable_resign && move_number >= options_[idx].uci_options->Get<int>(
                                             kResignEarliestMoveId.GetId())) {
       const float resignpct =
@@ -149,12 +186,6 @@ void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
         }
       }
     }
-
-    // Add best move to the tree.
-    const Move move = search_->GetBestMove().first;
-    tree_[0]->MakeMove(move);
-    if (tree_[0] != tree_[1]) tree_[1]->MakeMove(move);
-    blacks_move = !blacks_move;
   }
 }
 
@@ -184,9 +215,8 @@ void SelfPlayGame::Abort() {
 
 void SelfPlayGame::WriteTrainingData(TrainingDataWriter* writer) const {
   assert(!training_data_.empty());
-  bool black_to_move =
-      tree_[0]->GetPositionHistory().Starting().IsBlackToMove();
   for (auto chunk : training_data_) {
+    auto black_to_move = chunk.side_to_move;
     if (game_result_ == GameResult::WHITE_WON) {
       chunk.result = black_to_move ? -1 : 1;
     } else if (game_result_ == GameResult::BLACK_WON) {
@@ -195,7 +225,6 @@ void SelfPlayGame::WriteTrainingData(TrainingDataWriter* writer) const {
       chunk.result = 0;
     }
     writer->WriteChunk(chunk);
-    black_to_move = !black_to_move;
   }
 }
 

src/selfplay/game.h

@@ -70,7 +70,7 @@ class SelfPlayGame {
 
   // Starts the game and blocks until the game is finished.
   void Play(int white_threads, int black_threads, bool training,
-            bool enable_resign = true);
+            int random_opening_plies, bool enable_resign = true);
   // Aborts the game currently played, doesn't matter if it's synchronous or
   // not.
   void Abort();

src/selfplay/tournament.cc

@@ -59,6 +59,10 @@ const OptionId kTrainingId{
     "temporary subdirectory that the engine creates."};
 const OptionId kVerboseThinkingId{"verbose-thinking", "VerboseThinking",
                                   "Show verbose thinking messages."};
+const OptionId kMaxRandomOpeningPliesId{
+    "max-random-opening-plies", "MaxRandomOpeningPlies",
+    "Maximum number of opening plies to randomly play based on move priors. "
+    "The actual number of opening plies will be uniformly randomly picked."};
 const OptionId kResignPlaythroughId{
     "resign-playthrough", "ResignPlaythrough",
     "The percentage of games which ignore resign."};
@@ -82,6 +86,7 @@ void SelfPlayTournament::PopulateOptions(OptionsParser* options) {
   options->Add<IntOption>(kTimeMsId, -1, 999999999) = -1;
   options->Add<BoolOption>(kTrainingId) = false;
   options->Add<BoolOption>(kVerboseThinkingId) = false;
+  options->Add<IntOption>(kMaxRandomOpeningPliesId, 0, 999) = 0;
   options->Add<FloatOption>(kResignPlaythroughId, 0.0f, 100.0f) = 0.0f;
 
   SelfPlayGame::PopulateUciParams(options);
@@ -124,6 +129,8 @@ SelfPlayTournament::SelfPlayTournament(const OptionsDict& options,
       kShareTree(options.Get<bool>(kShareTreesId.GetId())),
       kParallelism(options.Get<int>(kParallelGamesId.GetId())),
       kTraining(options.Get<bool>(kTrainingId.GetId())),
+      kMaxRandomOpeningPlies(
+          options.Get<int>(kMaxRandomOpeningPliesId.GetId())),
       kResignPlaythrough(options.Get<float>(kResignPlaythroughId.GetId())) {
   // If playing just one game, the player1 is white, otherwise randomize.
   if (kTotalGames != 1) {
@@ -243,7 +250,7 @@ void SelfPlayTournament::PlayOneGame(int game_number) {
 
   // PLAY GAME!
   game.Play(kThreads[color_idx[0]], kThreads[color_idx[1]], kTraining,
-            enable_resign);
+            Random::Get().GetInt(0, kMaxRandomOpeningPlies), enable_resign);
 
   // If game was aborted, it's still undecided.
   if (game.GetGameResult() != GameResult::UNDECIDED) {

src/selfplay/tournament.h

@@ -101,6 +101,7 @@ class SelfPlayTournament {
   const bool kShareTree;
   const size_t kParallelism;
   const bool kTraining;
+  const int kMaxRandomOpeningPlies;
   const float kResignPlaythrough;
 };