Adds word count example over Shakespear/Jane Austen works

examples/wordcount/go.mod

@@ -0,0 +1,10 @@
+module github.com/sourcegraph/conc/examples/shakespeare
+
+go 1.23.0
+
+require github.com/sourcegraph/conc v0.3.0
+
+require (
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+)

examples/wordcount/go.sum

@@ -0,0 +1,17 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

examples/wordcount/wordcount.go

@@ -0,0 +1,102 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/sourcegraph/conc/iter"
+	"github.com/sourcegraph/conc/pool"
+	"github.com/sourcegraph/conc/stream"
+)
+
+func main() {
+	start := time.Now()
+	files, err := os.ReadDir("./works")
+	if err != nil {
+		log.Fatalf("Error reading dir: %v", err)
+	}
+	maxGoRoutines := 3
+
+	p := pool.NewWithResults[workWordCounts]().WithMaxGoroutines(maxGoRoutines).WithErrors()
+	for _, dirEntry := range files {
+		p.Go(func() (workWordCounts, error) {
+			workStart := time.Now()
+			f, err := os.Open(filepath.Join("./works", dirEntry.Name()))
+			if err != nil {
+				return nil, fmt.Errorf("error opening file: %w", err)
+			}
+			wc, err := countWords(f)
+			slog.Info("Done counting words", "work", dirEntry.Name(), "duration", time.Since(workStart))
+			return wc, err
+		})
+	}
+	allWordCounts, err := p.Wait()
+	if err != nil {
+		slog.Error("error generating word counts", "err", err)
+	}
+
+	totalWordCounts := make(workWordCounts)
+	s := stream.New().WithMaxGoroutines(maxGoRoutines)
+	for _, wwc := range allWordCounts {
+		s.Go(func() stream.Callback {
+			comWord, comCount := wwc.mostCommonWord()
+			slog.Info("unique", "words", len(wwc), "mostCommonWord", comWord, "mostCommonCount", comCount)
+			return func() {
+				totalWordCounts.add(wwc)
+			}
+		})
+	}
+	s.Wait()
+	totalCommonWord, totalCommonCount := totalWordCounts.mostCommonWord()
+	slog.Info("Done with all word counts",
+		"in", time.Since(start),
+		"totalUniqueWords", len(totalWordCounts),
+		"mostCommonWord", totalCommonWord,
+		"mostCommonCount", totalCommonCount)
+}
+
+func countWords(in io.ReadCloser) (workWordCounts, error) {
+	defer in.Close()
+	inputBytes, err := io.ReadAll(in)
+	if err != nil {
+		return nil, err
+	}
+	splits := bytes.Fields(inputBytes)
+	result := make(workWordCounts)
+	mu := &sync.Mutex{}
+	iter.ForEach(splits, func(i *[]byte) {
+		mu.Lock()
+		defer mu.Unlock()
+		result[string(*i)]++
+	})
+
+	return result, nil
+}
+
+type workWordCounts map[string]int
+
+func (wwc workWordCounts) mostCommonWord() (string, int) {
+	word := ""
+	count := -1
+
+	for candidateWord, candiateCount := range wwc {
+		if candiateCount > count {
+			word = candidateWord
+			count = candiateCount
+		}
+	}
+	return word, count
+}
+
+func (wwc workWordCounts) add(other workWordCounts) {
+	for word, count := range other {
+		wwc[word] = wwc[word] + count
+	}
+}