Additional reporting details for selfplay to support T50+ (#821)

* Hack to report average nodes per move for monitoring training.

* WDLstyle resign false positive reporting

* Make this less WIP and more submittable.

* Cast correctly.

* Fix typo in last commit...

* Add comment about unsupported combination.

Author: Tilps

src/chess/callbacks.h

@@ -118,6 +118,8 @@ struct TournamentInfo {
   // e.g. results[2][1] is how many times player 1 lost as black.
   int results[3][2] = {{0, 0}, {0, 0}, {0, 0}};
   using Callback = std::function<void(const TournamentInfo&)>;
+  int move_count_ = 0;
+  uint64_t nodes_total_ = 0;
 };
 
 }  // namespace lczero

src/selfplay/game.cc

@@ -101,6 +101,8 @@ void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
 
     // Do search.
     search_->RunBlocking(blacks_move ? black_threads : white_threads);
+    move_count_++;
+    nodes_total_ += search_->GetTotalPlayouts();
     if (abort_) break;
 
     auto best_eval = search_->GetBestEval();
@@ -117,15 +119,18 @@ void SelfPlayGame::Play(int white_threads, int black_threads, bool training,
     eval = (eval + 1) / 2;
     if (eval < min_eval_[idx]) min_eval_[idx] = eval;
     const int move_number = tree_[0]->GetPositionHistory().GetLength() / 2 + 1;
+    auto best_w = (best_eval.first + 1.0f - best_eval.second) / 2.0f;
+    auto best_d = best_eval.second;
+    auto best_l = best_w - best_eval.first;
+    max_eval_[0] = std::max(max_eval_[0], blacks_move ? best_l : best_w);
+    max_eval_[1] = std::max(max_eval_[1], best_d);
+    max_eval_[2] = std::max(max_eval_[2], blacks_move ? best_w : best_l);
     if (enable_resign && move_number >= options_[idx].uci_options->Get<int>(
                                             kResignEarliestMoveId.GetId())) {
       const float resignpct =
           options_[idx].uci_options->Get<float>(kResignPercentageId.GetId()) /
           100;
       if (options_[idx].uci_options->Get<bool>(kResignWDLStyleId.GetId())) {
-        auto best_w = (best_eval.first + 1.0f - best_eval.second) / 2.0f;
-        auto best_d = best_eval.second;
-        auto best_l = best_w - best_eval.first;
         auto threshold = 1.0f - resignpct;
         if (best_w > threshold) {
           game_result_ =
@@ -171,6 +176,17 @@ std::vector<Move> SelfPlayGame::GetMoves() const {
 }
 
 float SelfPlayGame::GetWorstEvalForWinnerOrDraw() const {
+  // TODO: This assumes both players have the same resign style.
+  // Supporting otherwise involves mixing the meaning of worst.
+  if (options_[0].uci_options->Get<bool>(kResignWDLStyleId.GetId())) {
+    if (game_result_ == GameResult::WHITE_WON) {
+      return std::max(max_eval_[1], max_eval_[2]);
+    } else if (game_result_ == GameResult::BLACK_WON) {
+      return std::max(max_eval_[1], max_eval_[0]);
+    } else {
+      return std::max(max_eval_[2], max_eval_[0]);
+    }
+  }
   if (game_result_ == GameResult::WHITE_WON) return min_eval_[0];
   if (game_result_ == GameResult::BLACK_WON) return min_eval_[1];
   return std::min(min_eval_[0], min_eval_[1]);

src/selfplay/game.h

@@ -83,6 +83,8 @@ class SelfPlayGame {
   // Gets the eval which required the biggest swing up to get the final outcome.
   // Eval is the expected outcome in the range 0<->1.
   float GetWorstEvalForWinnerOrDraw() const;
+  int move_count_ = 0;
+  uint64_t nodes_total_ = 0;
 
  private:
   // options_[0] is for white player, [1] for black.
@@ -99,6 +101,8 @@ class SelfPlayGame {
   // Track minimum eval for each player so that GetWorstEvalForWinnerOrDraw()
   // can be calculated after end of game.
   float min_eval_[2] = {1.0f, 1.0f};
+  // Track the maximum eval for white win, draw, black win for comparison to actual outcome.
+  float max_eval_[3] = {0.0f, 0.0f, 0.0f};
   std::mutex mutex_;
 
   // Training data to send.

src/selfplay/loop.cc

@@ -173,6 +173,9 @@ void SelfPlayLoop::SendTournament(const TournamentInfo& info) {
       << info.results[1][0];
   oss << " P1-B: +" << info.results[0][1] << " -" << info.results[2][1] << " ="
       << info.results[1][1];
+  oss << " npm " + std::to_string(static_cast<double>(info.nodes_total_) / info.move_count_);
+  oss << " nodes " + std::to_string(info.nodes_total_);
+  oss << " moves " + std::to_string(info.move_count_);
   SendResponse(oss.str());
 }
 

src/selfplay/tournament.cc

@@ -273,6 +273,8 @@ void SelfPlayTournament::PlayOneGame(int game_number) {
                        : game.GetGameResult() == GameResult::WHITE_WON ? 0 : 2;
       if (player1_black) result = 2 - result;
       ++tournament_info_.results[result][player1_black ? 1 : 0];
+      tournament_info_.move_count_ += game.move_count_;
+      tournament_info_.nodes_total_ += game.nodes_total_;
       tournament_callback_(tournament_info_);
     }
   }