Merge pull request #1 from glinscott/master

Update to v4

Author: jkiliani

go/src/client/main.go

@@ -76,11 +76,11 @@ func getExtraParams() map[string]string {
 	return map[string]string{
 		"user":     *USER,
 		"password": *PASSWORD,
-		"version":  "3",
+		"version":  "4",
 	}
 }
 
-func uploadGame(httpClient *http.Client, path string, pgn string, nextGame client.NextGameResponse) error {
+func uploadGame(httpClient *http.Client, path string, pgn string, nextGame client.NextGameResponse, retryCount uint) error {
 	extraParams := getExtraParams()
 	extraParams["training_id"] = strconv.Itoa(int(nextGame.TrainingId))
 	extraParams["network_id"] = strconv.Itoa(int(nextGame.NetworkId))
@@ -96,13 +96,33 @@ func uploadGame(httpClient *http.Client, path string, pgn string, nextGame clien
 	body := &bytes.Buffer{}
 	_, err = body.ReadFrom(resp.Body)
 	if err != nil {
+		log.Print(err)
+		log.Print("Error uploading, retrying...")
+		time.Sleep(time.Second * (2 << retryCount))
+		err = uploadGame(httpClient, path, pgn, nextGame, retryCount+1)
 		return err
 	}
 	resp.Body.Close()
 	fmt.Println(resp.StatusCode)
 	fmt.Println(resp.Header)
 	fmt.Println(body)
 
+	train_dir := filepath.Dir(path)
+	if _, err := os.Stat(train_dir); err == nil {
+		files, err := ioutil.ReadDir(train_dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("Cleanup training files:\n")
+		for _, f := range files {
+			fmt.Printf("%s/%s\n", train_dir, f.Name())
+		}
+		err = os.RemoveAll(train_dir)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
 	return nil
 }
 
@@ -252,36 +272,21 @@ func playMatch(baselinePath string, candidatePath string, params []string, flip
 	return result, game.String(), nil
 }
 
-func train(networkPath string, params []string) (string, string) {
+func train(networkPath string, count int, params []string) (string, string) {
 	// pid is intended for use in multi-threaded training
 	pid := os.Getpid()
 
 	dir, _ := os.Getwd()
-	train_dir := path.Join(dir, fmt.Sprintf("data-%v", pid))
-	if _, err := os.Stat(train_dir); err == nil {
-		files, err := ioutil.ReadDir(train_dir)
-		if err != nil {
-			log.Fatal(err)
-		}
-		fmt.Printf("Cleanup training files:\n")
-		for _, f := range files {
-			fmt.Printf("%s/%s\n", train_dir, f.Name())
-		}
-		err = os.RemoveAll(train_dir)
-		if err != nil {
-			log.Fatal(err)
-		}
-	}
-
+	train_dir := path.Join(dir, fmt.Sprintf("data-%v-%v", pid, count))
 	if *DEBUG {
 		logs_dir := path.Join(dir, fmt.Sprintf("logs-%v", pid))
 		os.MkdirAll(logs_dir, os.ModePerm)
 		logfile := path.Join(logs_dir, fmt.Sprintf("%s.log", time.Now().Format("20060102150405")))
-		params = append(params, "-l" + logfile)
+		params = append(params, "-l"+logfile)
 	}
 
 	num_games := 1
-	train_cmd := fmt.Sprintf("--start=train %v %v", pid, num_games)
+	train_cmd := fmt.Sprintf("--start=train %v-%v %v", pid, count, num_games)
 	params = append(params, train_cmd)
 
 	c := CmdWrapper{}
@@ -319,7 +324,7 @@ func getNetwork(httpClient *http.Client, sha string, clearOld bool) (string, err
 	return path, nil
 }
 
-func nextGame(httpClient *http.Client) error {
+func nextGame(httpClient *http.Client, count int) error {
 	nextGame, err := client.NextGame(httpClient, *HOSTNAME, getExtraParams())
 	if err != nil {
 		return err
@@ -343,15 +348,15 @@ func nextGame(httpClient *http.Client) error {
 		if err != nil {
 			return err
 		}
-		client.UploadMatchResult(httpClient, *HOSTNAME, nextGame.MatchGameId, result, pgn, getExtraParams())
+		go client.UploadMatchResult(httpClient, *HOSTNAME, nextGame.MatchGameId, result, pgn, getExtraParams())
 		return nil
 	} else if nextGame.Type == "train" {
 		networkPath, err := getNetwork(httpClient, nextGame.Sha, true)
 		if err != nil {
 			return err
 		}
-		trainFile, pgn := train(networkPath, params)
-		uploadGame(httpClient, trainFile, pgn, nextGame)
+		trainFile, pgn := train(networkPath, count, params)
+		go uploadGame(httpClient, trainFile, pgn, nextGame, 0)
 		return nil
 	}
 
@@ -373,13 +378,16 @@ func main() {
 	}
 
 	httpClient := &http.Client{}
-	for {
-		err := nextGame(httpClient)
+	start := time.Now()
+	for i := 0; ; i++ {
+		err := nextGame(httpClient, i)
 		if err != nil {
 			log.Print(err)
 			log.Print("Sleeping for 30 seconds...")
 			time.Sleep(30 * time.Second)
 			continue
 		}
+		elapsed := time.Since(start)
+		log.Printf("Completed %d games in %s time", i, elapsed)
 	}
 }

go/src/server/README.md

@@ -17,6 +17,9 @@ sudo apt-get install -y nginx
 sudo systemctl status nginx
 
 cp nginx/default /etc/nginx/sites-available/default
+
+# Create cache directory
+mkdir -p /home/web/nginx/cache/
 ```
 
 Installing postgres:
@@ -78,6 +81,11 @@ Connecting through psql:
 sudo -u postgres psql -d gorm
 ```
 
+Restarting nginx:
+```
+sudo service nginx restart
+```
+
 ### Setting up backup
 
 ```

go/src/server/nginx/default

@@ -15,6 +15,9 @@ upstream backend {
         server 127.0.0.1:8080;
 }
 
+proxy_cache_path /home/web/nginx/cache/ levels=1:2 keys_zone=cache:10m max_size=10g
+                 inactive=60m use_temp_path=off;
+
 # Default server configuration
 #
 server {
@@ -52,6 +55,12 @@ server {
 	proxy_set_header X-Forwarded-Proto $scheme;
 
 	location / {
+		proxy_cache cache;
+
+		# TODO(gary): Should set these from the server...
+		proxy_ignore_headers Cache-Control;
+		proxy_cache_valid any 1m;
+
 		proxy_pass http://backend;
 	}
 }

go/src/server/templates/index.tmpl

@@ -1,6 +1,6 @@
 {{define "content"}}
-<div class="alert alert-primary" role="alert">
-  Client v3 is now the required version.  Download <a href="https://github.com/glinscott/leela-chess/releases/tag/v0.2">here</a>.
+<div class="alert alert-warning" role="alert">
+  v0.4 released, please upgrade.  Download <a href="https://github.com/glinscott/leela-chess/releases/tag/v0.4">here</a>.
 </div>
 
 <div class="d-flex justify-content-between flex-wrap flex-md-nowrap align-items-center pb-2 mb-3 border-bottom">

src/UCTNode.cpp

@@ -384,7 +384,6 @@ UCTNode::node_ptr_t UCTNode::find_new_root(Key prevroot_full_key, BoardHistory&
     UCTNode::node_ptr_t new_root = nullptr;
     std::vector<Move> moves;
     for (auto pos : boost::adaptors::reverse(new_bh.positions)) {
-        auto move = pos.get_move();
         if (pos.full_key() == prevroot_full_key) {
             new_root = find_path(moves);
             break;

src/config.h

@@ -31,8 +31,10 @@
 #define USE_OPENBLAS
 #endif
 //#define USE_MKL
+#ifndef FEATURE_USE_CPU_ONLY
 #define USE_OPENCL
 #define USE_OPENCL_SELFCHECK
+#endif
 static constexpr int SELFCHECK_PROBABILITY = 2000;
 static constexpr int SELFCHECK_MIN_EXPANSIONS = 2'000'000;
 //#define USE_TUNER

training/tf/chunkparser.py

@@ -208,6 +208,8 @@ def convert_v2_to_tuple(self, content):
                 uint8 planes (120 * 8 * 8 bytes)
         """
         (ver, probs, planes, us_ooo, us_oo, them_ooo, them_oo, stm, rule50_count, move_count, winner) = self.v2_struct.unpack(content)
+        # Enforce move_count to 0
+        move_count = 0
         # Unpack planes.
         planes = np.unpackbits(np.frombuffer(planes, dtype=np.uint8))
         planes = planes.tobytes() + self.flat_planes[us_ooo] + self.flat_planes[us_oo] + self.flat_planes[them_ooo] + self.flat_planes[them_oo] + self.flat_planes[stm] + self.flat_planes[rule50_count] + self.flat_planes[move_count] + self.flat_planes[0]