55 "fmt"
66 "time"
77
8+ "github.com/ipfs/boxo/blockstore"
89 "github.com/ipfs/boxo/fetcher"
910 pin "github.com/ipfs/boxo/pinning/pinner"
1011 provider "github.com/ipfs/boxo/provider"
@@ -14,11 +15,87 @@ import (
1415)
1516
1617func ProviderSys (reprovideInterval time.Duration ) fx.Option {
17- return fx .Provide (func (lc fx.Lifecycle , cr irouting.ProvideManyRouter , keyProvider provider.KeyChanFunc , repo repo.Repo ) (provider.System , error ) {
18+ const magicThroughputReportCount = 128
19+ return fx .Provide (func (lc fx.Lifecycle , cr irouting.ProvideManyRouter , keyProvider provider.KeyChanFunc , repo repo.Repo , bs blockstore.Blockstore ) (provider.System , error ) {
1820 sys , err := provider .New (repo .Datastore (),
1921 provider .Online (cr ),
2022 provider .ReproviderInterval (reprovideInterval ),
2123 provider .KeyProvider (keyProvider ),
24+ provider .ThroughputReport (func (reprovide bool , complete bool , keysProvided uint , duration time.Duration ) bool {
25+ avgProvideSpeed := duration / time .Duration (keysProvided )
26+ count := uint64 (keysProvided )
27+
28+ if ! reprovide || ! complete {
29+ // We don't know how many CIDs we have to provide, try to fetch it from the blockstore.
30+ // But don't try for too long as this might be very expensive if you have a huge datastore.
31+ ctx , cancel := context .WithTimeout (context .Background (), time .Minute * 5 )
32+ defer cancel ()
33+
34+ // FIXME: I want a running counter of blocks so size of blockstore can be an O(1) lookup.
35+ ch , err := bs .AllKeysChan (ctx )
36+ if err != nil {
37+ logger .Errorf ("fetching AllKeysChain in provider ThroughputReport: %v" , err )
38+ return false
39+ }
40+ count = 0
41+ countLoop:
42+ for {
43+ select {
44+ case _ , ok := <- ch :
45+ if ! ok {
46+ break countLoop
47+ }
48+ count ++
49+ case <- ctx .Done ():
50+ // really big blockstore mode
51+
52+ // how many blocks would be in a 10TiB blockstore with 128KiB blocks.
53+ const probableBigBlockstore = (10 * 1024 * 1024 * 1024 * 1024 ) / (128 * 1024 )
54+ // How long per block that lasts us.
55+ expectedProvideSpeed := reprovideInterval / probableBigBlockstore
56+ if avgProvideSpeed > expectedProvideSpeed {
57+ // FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore.
58+ logger .Errorf (`
59+ 🔔🔔🔔 YOU MAY BE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔
60+
61+ ⚠️ Your system might be struggling to keep up with DHT reprovides!
62+ Meaning your content being partially or completely inacessible on the network.
63+ We observed that you recently provided %d keys at an average rate of %v per key.
64+
65+ 🕑 An attempt to estimate your blockstore size timed out after 5 minutes,
66+ implying your blockstore might be exceedingly large. Assuming a considerable
67+ size of 10TiB, it would take %v to provide the complete set.
68+
69+ ⏰ The total provide time needs to stay under 24 hours to prevent falling behind!
70+
71+ 💡 Consider enabling the Accelerated DHT to enhance your system performance.` ,
72+ keysProvided , avgProvideSpeed , avgProvideSpeed * probableBigBlockstore )
73+ return false
74+ }
75+ }
76+ }
77+ }
78+
79+ // How long per block that lasts us.
80+ expectedProvideSpeed := reprovideInterval / time .Duration (count )
81+ if avgProvideSpeed > expectedProvideSpeed {
82+ // FIXME(@Jorropo): add link to the accelerated DHT client docs once this isn't experimental anymore.
83+ logger .Errorf (`
84+ 🔔🔔🔔 YOU ARE FALLING BEHIND DHT REPROVIDES! 🔔🔔🔔
85+
86+ ⚠️ Your system is struggling to keep up with DHT reprovides!
87+ Meaning your content being partially or completely inacessible on the network.
88+ We observed that you recently provided %d keys at an average rate of %v per key.
89+
90+ 💾 Your total CID count is ~%d which would total at %v reprovide process.
91+
92+ ⏰ The total provide time needs to stay under 24 hours to prevent falling behind!
93+
94+ 💡 Consider enabling the Accelerated DHT to enhance your reprovide throughput.` ,
95+ keysProvided , avgProvideSpeed , count , avgProvideSpeed * time .Duration (count ))
96+ }
97+ return false
98+ }, magicThroughputReportCount ),
2299 )
23100 if err != nil {
24101 return nil , err
@@ -56,7 +133,7 @@ func OnlineProviders(useStrategicProviding bool, reprovideStrategy string, repro
56133
57134 return fx .Options (
58135 keyProvider ,
59- fx . Provide ( ProviderSys (reprovideInterval ) ),
136+ ProviderSys (reprovideInterval ),
60137 )
61138}
62139
0 commit comments