Skip to content

Commit 7c853c0

Browse files
feat: add multi-client support
1 parent a6be456 commit 7c853c0

File tree

11 files changed

+221
-22
lines changed

11 files changed

+221
-22
lines changed

README.md

+31-2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@
3939
<ul>
4040
<li><a href="#optional-vars">Optional environment variables</a></li>
4141
</ul>
42+
<ul>
43+
<li><a href="#use-multiple-bots-to-speed-up">Using multiple bots</a></li>
44+
</ul>
4245
</li>
4346
<li><a href="#contributing">Contributing</a></li>
4447
<li><a href="#contact-me">Contact me</a></li>
@@ -73,10 +76,13 @@ An example of `.env` file:
7376
```sh
7477
API_ID=452525
7578
API_HASH=esx576f8738x883f3sfzx83
76-
BOT_TOKEN=55838383:yourtbottokenhere
79+
BOT_TOKEN=55838383:yourbottokenhere
7780
BIN_CHANNEL=-10045145224562
7881
PORT=8080
7982
HOST=http://yourserverip
83+
# (if you want to set up multiple bots)
84+
MULTI_TOKEN1=55838373:yourworkerbottokenhere
85+
MULTI_TOKEN2=55838355:yourworkerbottokenhere
8086
```
8187

8288
### Required Vars
@@ -97,7 +103,30 @@ In addition to the mandatory variables, you can also set the following optional
97103

98104
- `HOST` : A Fully Qualified Domain Name if present or use your server IP. (eg. `https://example.com` or `http://14.1.154.2:8080`)
99105

100-
- `HASH_LENGTH` : This is the custom hash length for generated URLs. The hash length must be greater than 5 and less than or equal to 32. The default value is 6.
106+
- `HASH_LENGTH` : Custom hash length for generated URLs. The hash length must be greater than 5 and less than or equal to 32. The default value is 6.
107+
108+
- `USE_SESSION_FILE` : Use session files for worker client(s). This speeds up the worker bot startups. (default: `false`)
109+
110+
### Use Multiple Bots to speed up
111+
112+
> **Note**
113+
> What it multi-client feature and what it does? <br>
114+
> This feature shares the Telegram API requests between worker bots to speed up download speed when many users are using the server and to avoid the flood limits that are set by Telegram. <br>
115+
116+
> **Note**
117+
> You can add up to 50 bots since 50 is the max amount of bot admins you can set in a Telegram Channel.
118+
119+
To enable multi-client, generate new bot tokens and add it as your `.env` with the following key names.
120+
121+
`MULTI_TOKEN1`: Add your first bot token here.
122+
123+
`MULTI_TOKEN2`: Add your second bot token here.
124+
125+
you may also add as many as bots you want. (max limit is 50)
126+
`MULTI_TOKEN3`, `MULTI_TOKEN4`, etc.
127+
128+
> **Warning**
129+
> Don't forget to add all these worker bots to the `BIN_CHANNEL` for the proper functioning
101130
102131
## Contributing
103132

bot/client.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func StartClient(log *zap.Logger) (*gotgproto.Client, error) {
2020
BotToken: config.ValueOf.BotToken,
2121
},
2222
&gotgproto.ClientOpts{
23-
Session: sessionMaker.NewSession("fsb", sessionMaker.Session),
23+
Session: sessionMaker.SqliteSession("fsb"),
2424
DisableCopyright: true,
2525
},
2626
)

bot/workers.go

+144
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
package bot
2+
3+
import (
4+
"EverythingSuckz/fsb/config"
5+
"fmt"
6+
"os"
7+
"path/filepath"
8+
"sync"
9+
10+
"github.com/celestix/gotgproto"
11+
"github.com/celestix/gotgproto/sessionMaker"
12+
"github.com/gotd/td/tg"
13+
"go.uber.org/zap"
14+
)
15+
16+
type Worker struct {
17+
ID int
18+
Client *gotgproto.Client
19+
Self *tg.User
20+
log *zap.Logger
21+
}
22+
23+
func (w *Worker) String() string {
24+
return fmt.Sprintf("{Worker (%d|@%s)}", w.ID, w.Self.Username)
25+
}
26+
27+
type BotWorkers struct {
28+
Bots []*Worker
29+
starting int
30+
index int
31+
mut sync.Mutex
32+
log *zap.Logger
33+
}
34+
35+
var Workers *BotWorkers = &BotWorkers{
36+
log: nil,
37+
Bots: make([]*Worker, 0),
38+
}
39+
40+
func (w *BotWorkers) Init(log *zap.Logger) {
41+
w.log = log.Named("Workers")
42+
}
43+
44+
func (w *BotWorkers) AddDefaultClient(client *gotgproto.Client, self *tg.User) {
45+
if w.Bots == nil {
46+
w.Bots = make([]*Worker, 0)
47+
}
48+
w.incStarting()
49+
w.Bots = append(w.Bots, &Worker{
50+
Client: client,
51+
ID: w.starting,
52+
Self: self,
53+
})
54+
w.log.Sugar().Info("Default bot loaded")
55+
}
56+
57+
func (w *BotWorkers) incStarting() {
58+
w.mut.Lock()
59+
defer w.mut.Unlock()
60+
w.starting++
61+
}
62+
63+
func (w *BotWorkers) Add(token string) (err error) {
64+
w.incStarting()
65+
var botID int = w.starting
66+
client, err := startWorker(w.log, token, botID)
67+
if err != nil {
68+
return err
69+
}
70+
w.log.Sugar().Infof("Bot @%s loaded with ID %d", client.Self.Username, botID)
71+
w.Bots = append(w.Bots, &Worker{
72+
Client: client,
73+
ID: botID,
74+
Self: client.Self,
75+
log: w.log,
76+
})
77+
return nil
78+
}
79+
80+
func GetNextWorker() *Worker {
81+
Workers.mut.Lock()
82+
defer Workers.mut.Unlock()
83+
index := (Workers.index + 1) % len(Workers.Bots)
84+
Workers.index = index
85+
worker := Workers.Bots[index]
86+
Workers.log.Sugar().Infof("Using worker %d", worker.ID)
87+
return worker
88+
}
89+
90+
func StartWorkers(log *zap.Logger) {
91+
log.Sugar().Info("Starting workers")
92+
Workers.Init(log)
93+
if config.ValueOf.UseSessionFile {
94+
log.Sugar().Info("Using session file for workers")
95+
newpath := filepath.Join(".", "sessions")
96+
err := os.MkdirAll(newpath, os.ModePerm)
97+
if err != nil {
98+
log.Error("Failed to create sessions directory", zap.Error(err))
99+
return
100+
}
101+
}
102+
c := make(chan struct{})
103+
for i := 0; i < len(config.ValueOf.MultiTokens); i++ {
104+
go func(i int) {
105+
err := Workers.Add(config.ValueOf.MultiTokens[i])
106+
if err != nil {
107+
log.Error("Failed to start worker", zap.Error(err))
108+
return
109+
}
110+
c <- struct{}{}
111+
}(i)
112+
}
113+
// wait for all workers to start
114+
log.Sugar().Info("Waiting for all workers to start")
115+
for i := 0; i < len(config.ValueOf.MultiTokens); i++ {
116+
<-c
117+
}
118+
}
119+
120+
func startWorker(l *zap.Logger, botToken string, index int) (*gotgproto.Client, error) {
121+
log := l.Named("Worker").Sugar()
122+
log.Infof("Starting worker with index - %d", index)
123+
var sessionType *sessionMaker.SqliteSessionConstructor
124+
if config.ValueOf.UseSessionFile {
125+
sessionType = sessionMaker.SqliteSession(fmt.Sprintf("sessions/worker-%d", index))
126+
} else {
127+
sessionType = sessionMaker.SqliteSession(":memory:")
128+
}
129+
client, err := gotgproto.NewClient(
130+
int(config.ValueOf.ApiID),
131+
config.ValueOf.ApiHash,
132+
gotgproto.ClientType{
133+
BotToken: botToken,
134+
},
135+
&gotgproto.ClientOpts{
136+
Session: sessionType,
137+
DisableCopyright: true,
138+
},
139+
)
140+
if err != nil {
141+
return nil, err
142+
}
143+
return client, nil
144+
}

commands/stream.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,11 @@ func sendLink(ctx *ext.Context, u *ext.Update) error {
2727
return dispatcher.EndGroups
2828
}
2929
chatId := u.EffectiveChat().GetID()
30-
peerChatId := storage.GetPeerById(chatId)
30+
peerChatId := ctx.PeerStorage.GetPeerById(chatId)
3131
if peerChatId.Type != int(storage.TypeUser) {
3232
return dispatcher.EndGroups
3333
}
34-
peer := storage.GetPeerById(config.ValueOf.LogChannelID)
34+
peer := ctx.PeerStorage.GetPeerById(config.ValueOf.LogChannelID)
3535
switch storage.EntityType(peer.Type) {
3636
case storage.TypeChat:
3737
return dispatcher.EndGroups

config/config.go

+14
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
package config
22

33
import (
4+
"os"
5+
"reflect"
6+
"regexp"
47
"strconv"
58
"strings"
69

@@ -20,13 +23,24 @@ type config struct {
2023
Port int `envconfig:"PORT" default:"8080"`
2124
Host string `envconfig:"HOST" default:"http://localhost:8080"`
2225
HashLength int `envconfig:"HASH_LENGTH" default:"6"`
26+
UseSessionFile bool `envconfig:"USE_SESSION_FILE" default:"true"`
27+
MultiTokens []string
2328
}
2429

30+
var botTokenRegex = regexp.MustCompile(`MULTI\_TOKEN[\d+]=(.*)`)
31+
2532
func (c *config) setupEnvVars() {
2633
err := envconfig.Process("", c)
2734
if err != nil {
2835
panic(err)
2936
}
37+
val := reflect.ValueOf(c).Elem()
38+
for _, env := range os.Environ() {
39+
if strings.HasPrefix(env, "MULTI_TOKEN") {
40+
c.MultiTokens = append(c.MultiTokens, botTokenRegex.FindStringSubmatch(env)[1])
41+
}
42+
}
43+
val.FieldByName("MultiTokens").Set(reflect.ValueOf(c.MultiTokens))
3044
}
3145

3246
func Load(log *zap.Logger) {

go.mod

+4-4
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module EverythingSuckz/fsb
33
go 1.21.3
44

55
require (
6-
github.com/celestix/gotgproto v1.0.0-beta13
6+
github.com/celestix/gotgproto v1.0.0-beta13.0.20231124171805-6c04fae60b80
77
github.com/gin-gonic/gin v1.9.1
88
github.com/gotd/td v0.89.0
99
github.com/joho/godotenv v1.5.1
@@ -58,9 +58,9 @@ require (
5858
github.com/ugorji/go/codec v1.2.11 // indirect
5959
go.uber.org/zap v1.26.0
6060
golang.org/x/arch v0.3.0 // indirect
61-
golang.org/x/crypto v0.15.0 // indirect
62-
golang.org/x/net v0.18.0 // indirect
63-
golang.org/x/sys v0.14.0 // indirect
61+
golang.org/x/crypto v0.16.0 // indirect
62+
golang.org/x/net v0.19.0 // indirect
63+
golang.org/x/sys v0.15.0 // indirect
6464
golang.org/x/text v0.14.0 // indirect
6565
google.golang.org/protobuf v1.30.0 // indirect
6666
gopkg.in/natefinch/lumberjack.v2 v2.2.1

go.sum

+8
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ github.com/bytedance/sonic v1.9.1 h1:6iJ6NqdoxCDr6mbY8h18oSO+cShGSMRGCEo7F2h0x8s
55
github.com/bytedance/sonic v1.9.1/go.mod h1:i736AoUSYt75HyZLoJW9ERYxcy6eaN6h4BZXU064P/U=
66
github.com/celestix/gotgproto v1.0.0-beta13 h1:5BlGUJMwJmXrWD9RhBbHRuJhbPkv5CJd04x/sDCpYeg=
77
github.com/celestix/gotgproto v1.0.0-beta13/go.mod h1:WHwqFwgXEpFN/2ReP+vVnxCs2IvULaRK7n0N5ouVmDw=
8+
github.com/celestix/gotgproto v1.0.0-beta13.0.20231124171805-6c04fae60b80 h1:OhekKvJhPQx7jkPVomt7rD8tAlnel4l/sR6X7D9Pkpw=
9+
github.com/celestix/gotgproto v1.0.0-beta13.0.20231124171805-6c04fae60b80/go.mod h1:sPTsFAhN6apWcxCLc07LFEkb9wuoQa9L7JxXl6znLY4=
810
github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
911
github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
1012
github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
@@ -124,17 +126,23 @@ golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k=
124126
golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8=
125127
golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
126128
golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
129+
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
130+
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
127131
golang.org/x/exp v0.0.0-20230116083435-1de6713980de h1:DBWn//IJw30uYCgERoxCg84hWtA97F4wMiKOIh00Uf0=
128132
golang.org/x/exp v0.0.0-20230116083435-1de6713980de/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
129133
golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg=
130134
golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ=
135+
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
136+
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
131137
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
132138
golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
133139
golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
134140
golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
135141
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
136142
golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
137143
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
144+
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
145+
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
138146
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
139147
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
140148
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

main.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,14 @@ func main() {
2626
mainLogger.Info("Starting server")
2727
config.Load(log)
2828
router := getRouter(log)
29-
29+
3030
_, err := bot.StartClient(log)
3131
if err != nil {
3232
log.Info(err.Error())
3333
return
3434
}
3535
cache.InitCache(log)
36+
bot.StartWorkers(log)
3637
mainLogger.Info("Server started", zap.Int("port", config.ValueOf.Port))
3738
mainLogger.Info("File Stream Bot", zap.String("version", versionString))
3839
err = router.Run(fmt.Sprintf(":%d", config.ValueOf.Port))

routes/stream.go

+4-2
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,9 @@ func getStreamRoute(ctx *gin.Context) {
4343
var start, end int64
4444
rangeHeader := r.Header.Get("Range")
4545

46-
file, err := utils.FileFromMessage(ctx, bot.Bot.Client, messageID)
46+
worker := bot.GetNextWorker()
47+
48+
file, err := utils.FileFromMessage(ctx, worker.Client, messageID)
4749
if err != nil {
4850
http.Error(w, err.Error(), http.StatusBadRequest)
4951
return
@@ -100,7 +102,7 @@ func getStreamRoute(ctx *gin.Context) {
100102
http.Error(w, err.Error(), http.StatusInternalServerError)
101103
return
102104
}
103-
lr, _ := utils.NewTelegramReader(ctx, bot.Bot.Client, file.Location, start, end, contentLength)
105+
lr, _ := utils.NewTelegramReader(ctx, worker.Client, file.Location, start, end, contentLength)
104106
if _, err := io.CopyN(w, lr, contentLength); err != nil {
105107
log.WithOptions(zap.AddStacktrace(zap.DPanicLevel)).Error(err.Error())
106108
}

utils/helpers.go

+8-7
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@ import (
88
"errors"
99
"fmt"
1010

11-
"github.com/gotd/td/telegram"
11+
"github.com/celestix/gotgproto"
1212
"github.com/gotd/td/tg"
13+
"go.uber.org/zap"
1314
)
1415

15-
func GetTGMessage(ctx context.Context, client *telegram.Client, messageID int) (*tg.Message, error) {
16+
func GetTGMessage(ctx context.Context, client *gotgproto.Client, messageID int) (*tg.Message, error) {
1617
inputMessageID := tg.InputMessageClass(&tg.InputMessageID{ID: messageID})
1718
channel, err := GetChannelById(ctx, client)
1819
if err != nil {
@@ -58,16 +59,16 @@ func FileFromMedia(media tg.MessageMediaClass) (*types.File, error) {
5859
return nil, fmt.Errorf("unexpected type %T", media)
5960
}
6061

61-
func FileFromMessage(ctx context.Context, client *telegram.Client, messageID int) (*types.File, error) {
62-
key := fmt.Sprintf("file:%d", messageID)
62+
func FileFromMessage(ctx context.Context, client *gotgproto.Client, messageID int) (*types.File, error) {
63+
key := fmt.Sprintf("file:%d:%d", messageID, client.Self.ID)
6364
log := Logger.Named("GetMessageMedia")
6465
var cachedMedia types.File
6566
err := cache.GetCache().Get(key, &cachedMedia)
6667
if err == nil {
67-
log.Sugar().Debug("Using cached media message properties")
68+
log.Debug("Using cached media message properties", zap.Int("messageID", messageID), zap.Int64("clientID", client.Self.ID))
6869
return &cachedMedia, nil
6970
}
70-
log.Sugar().Debug("Fetching file properties from message ID")
71+
log.Debug("Fetching file properties from message ID", zap.Int("messageID", messageID), zap.Int64("clientID", client.Self.ID))
7172
message, err := GetTGMessage(ctx, client, messageID)
7273
if err != nil {
7374
return nil, err
@@ -88,7 +89,7 @@ func FileFromMessage(ctx context.Context, client *telegram.Client, messageID int
8889
// TODO: add photo support
8990
}
9091

91-
func GetChannelById(ctx context.Context, client *telegram.Client) (*tg.InputChannel, error) {
92+
func GetChannelById(ctx context.Context, client *gotgproto.Client) (*tg.InputChannel, error) {
9293
channel := &tg.InputChannel{}
9394
inputChannel := &tg.InputChannel{
9495
ChannelID: config.ValueOf.LogChannelID,

0 commit comments

Comments
 (0)