open files on windows with shared delete mode

flatfs.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"math"
 	"math/rand"
 	"os"
@@ -608,7 +607,7 @@ func (fs *Datastore) Get(key datastore.Key) (value []byte, err error) {
 	}
 
 	_, path := fs.encode(key)
-	data, err := ioutil.ReadFile(path)
+	data, err := readFile(path)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, datastore.ErrNotFound
@@ -1013,7 +1012,7 @@ func (fs *Datastore) writeDiskUsageFile(du int64, doSync bool) {
 // readDiskUsageFile is only safe to call in Open()
 func (fs *Datastore) readDiskUsageFile() int64 {
 	fpath := filepath.Join(fs.path, DiskUsageFile)
-	duB, err := ioutil.ReadFile(fpath)
+	duB, err := readFile(fpath)
 	if err != nil {
 		return 0
 	}
@@ -1050,7 +1049,7 @@ func (fs *Datastore) Accuracy() string {
 }
 
 func (fs *Datastore) tempFile() (*os.File, error) {
-	file, err := ioutil.TempFile(fs.tempPath, "temp-")
+	file, err := tempFile(fs.tempPath, "temp-")
 	return file, err
 }
 
@@ -1093,7 +1092,7 @@ func (fs *Datastore) walk(path string, qrb *query.ResultBuilder) error {
 		var result query.Result
 		result.Key = key.String()
 		if !qrb.Query.KeysOnly {
-			value, err := ioutil.ReadFile(filepath.Join(path, fn))
+			value, err := readFile(filepath.Join(path, fn))
 			if err != nil {
 				result.Error = err
 			} else {

go.mod

@@ -1,6 +1,7 @@
 module github.com/ipfs/go-ds-flatfs
 
 require (
+	github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
 	github.com/ipfs/go-datastore v0.4.4
 	github.com/ipfs/go-log v1.0.3
 	github.com/jbenet/goprocess v0.1.4

go.sum

@@ -1,3 +1,5 @@
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM=
+github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=

util_unix.go

@@ -0,0 +1,16 @@
+// +build !windows
+
+package flatfs
+
+import (
+	"io/ioutil"
+	"os"
+)
+
+func tempFile(dir, pattern string) (f *os.File, err error) {
+	return ioutil.TempFile(dir, pattern)
+}
+
+func readFile(filename string) ([]byte, error) {
+	return ioutil.ReadFile(filename)
+}

util_windows.go

@@ -0,0 +1,93 @@
+// +build windows
+
+package flatfs
+
+import (
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	goissue34681 "github.com/alexbrainman/goissue34681"
+)
+
+var tmpRand uint32
+var randmu sync.Mutex
+
+func reseed() uint32 {
+	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
+}
+
+func nextRandom() string {
+	randmu.Lock()
+	r := tmpRand
+	if r == 0 {
+		r = reseed()
+	}
+	r = r*1664525 + 1013904223 // constants from Numerical Recipes
+	tmpRand = r
+	randmu.Unlock()
+	return strconv.Itoa(int(1e9 + r%1e9))[1:]
+}
+
+func prefixAndSuffix(pattern string) (prefix, suffix string) {
+	if pos := strings.LastIndex(pattern, "*"); pos != -1 {
+		prefix, suffix = pattern[:pos], pattern[pos+1:]
+	} else {
+		prefix = pattern
+	}
+	return
+}
+
+func tempFile(dir, pattern string) (f *os.File, err error) {
+	if dir == "" {
+		dir = os.TempDir()
+	}
+
+	prefix, suffix := prefixAndSuffix(pattern)
+
+	nconflict := 0
+	for i := 0; i < 10000; i++ {
+		name := filepath.Join(dir, prefix+nextRandom()+suffix)
+		f, err = goissue34681.OpenFile(name, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0600)
+		if os.IsExist(err) {
+			if nconflict++; nconflict > 10 {
+				randmu.Lock()
+				tmpRand = reseed()
+				randmu.Unlock()
+			}
+			continue
+		}
+		break
+	}
+	return
+}
+
+func readFile(filename string) ([]byte, error) {
+	f, err := goissue34681.Open(filename)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	// It's a good but not certain bet that FileInfo will tell us exactly how much to
+	// read, so let's try it but be prepared for the answer to be wrong.
+	var n int64 = bytes.MinRead
+
+	if fi, err := f.Stat(); err == nil {
+		// As initial capacity for readAll, use Size + a little extra in case Size
+		// is zero, and to avoid another allocation after Read has filled the
+		// buffer. The readAll call will read into its allocated internal buffer
+		// cheaply. If the size was wrong, we'll either waste some space off the end
+		// or reallocate as needed, but in the overwhelmingly common case we'll get
+		// it just right.
+		if size := fi.Size() + bytes.MinRead; size > n {
+			n = size
+		}
+	}
+
+	return ioutil.ReadAll(f)
+}