Skip to content

Commit 14c4d3c

Browse files
zhengkunwang223zhengkunwang223
authored
feat(file): Optimize file download progress tracking (#7186)
1 parent ff0606b commit 14c4d3c

File tree

18 files changed

+70
-334
lines changed

18 files changed

+70
-334
lines changed

‎agent/app/api/v2/file.go

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -751,11 +751,7 @@ func (b *BaseApi) Ws(c *gin.Context) {
751751

752752
func (b *BaseApi) Keys(c *gin.Context) {
753753
res := &response.FileProcessKeys{}
754-
keys, err := global.CACHE.PrefixScanKey("file-wget-")
755-
if err != nil {
756-
helper.InternalServer(c, err)
757-
return
758-
}
754+
keys := global.CACHE.PrefixScanKey("file-wget-")
759755
res.Keys = keys
760756
helper.SuccessWithData(c, res)
761757
}

‎agent/cron/cron.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,6 @@ func Run() {
4646
if _, err := global.Cron.AddJob(fmt.Sprintf("%v %v * * *", mathRand.Intn(60), mathRand.Intn(3)), job.NewAppStoreJob()); err != nil {
4747
global.LOG.Errorf("can not add appstore corn job: %s", err.Error())
4848
}
49-
if _, err := global.Cron.AddJob("@daily", job.NewCacheJob()); err != nil {
50-
global.LOG.Errorf("can not add cache corn job: %s", err.Error())
51-
}
5249

5350
var cronJobs []model.Cronjob
5451
if err := global.DB.Where("status = ?", constant.StatusEnable).Find(&cronJobs).Error; err != nil {

‎agent/cron/job/cache.go

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎agent/global/global.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@ package global
22

33
import (
44
"github.com/1Panel-dev/1Panel/agent/configs"
5-
"github.com/1Panel-dev/1Panel/agent/init/cache/badger_db"
6-
"github.com/dgraph-io/badger/v4"
5+
badger_db "github.com/1Panel-dev/1Panel/agent/init/cache/db"
76
"github.com/go-playground/validator/v10"
87
"github.com/nicksnyder/go-i18n/v2/i18n"
98
"github.com/robfig/cron/v3"
@@ -21,7 +20,6 @@ var (
2120
CONF configs.ServerConfig
2221
VALID *validator.Validate
2322
CACHE *badger_db.Cache
24-
CacheDb *badger.DB
2523
Viper *viper.Viper
2624

2725
Cron *cron.Cron

‎agent/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ require (
77
github.com/aws/aws-sdk-go v1.55.0
88
github.com/compose-spec/compose-go/v2 v2.1.4
99
github.com/creack/pty v1.1.21
10-
github.com/dgraph-io/badger/v4 v4.2.0
1110
github.com/docker/compose/v2 v2.29.0
1211
github.com/docker/docker v27.1.0+incompatible
1312
github.com/docker/go-connections v0.5.0
@@ -176,6 +175,7 @@ require (
176175
github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
177176
github.com/opencontainers/go-digest v1.0.0 // indirect
178177
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b // indirect
178+
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
179179
github.com/pelletier/go-toml v1.9.5 // indirect
180180
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
181181
github.com/pierrec/lz4/v4 v4.1.15 // indirect

‎agent/go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -601,6 +601,8 @@ github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M5
601601
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
602602
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b h1:FfH+VrHHk6Lxt9HdVS0PXzSXFyS2NbZKXv33FYPol0A=
603603
github.com/opentracing/opentracing-go v1.2.1-0.20220228012449-10b1cf09e00b/go.mod h1:AC62GU6hc0BrNm+9RK9VSiwa/EUe1bkIeFORAMcHvJU=
604+
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
605+
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
604606
github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
605607
github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
606608
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=

‎agent/init/cache/badger_db/badger_db.go

Lines changed: 0 additions & 79 deletions
This file was deleted.

‎agent/init/cache/cache.go

Lines changed: 2 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,10 @@
11
package cache
22

33
import (
4-
"time"
5-
64
"github.com/1Panel-dev/1Panel/agent/global"
7-
"github.com/1Panel-dev/1Panel/agent/init/cache/badger_db"
8-
"github.com/dgraph-io/badger/v4"
5+
cachedb "github.com/1Panel-dev/1Panel/agent/init/cache/db"
96
)
107

118
func Init() {
12-
c := global.CONF.System.Cache
13-
14-
options := badger.Options{
15-
Dir: c,
16-
ValueDir: c,
17-
ValueLogFileSize: 64 << 20,
18-
ValueLogMaxEntries: 10 << 20,
19-
VLogPercentile: 0.1,
20-
21-
MemTableSize: 32 << 20,
22-
BaseTableSize: 2 << 20,
23-
BaseLevelSize: 10 << 20,
24-
TableSizeMultiplier: 2,
25-
LevelSizeMultiplier: 10,
26-
MaxLevels: 7,
27-
NumGoroutines: 4,
28-
MetricsEnabled: true,
29-
NumCompactors: 2,
30-
NumLevelZeroTables: 5,
31-
NumLevelZeroTablesStall: 15,
32-
NumMemtables: 1,
33-
BloomFalsePositive: 0.01,
34-
BlockSize: 2 * 1024,
35-
SyncWrites: false,
36-
NumVersionsToKeep: 1,
37-
CompactL0OnClose: false,
38-
VerifyValueChecksum: false,
39-
BlockCacheSize: 32 << 20,
40-
IndexCacheSize: 0,
41-
ZSTDCompressionLevel: 1,
42-
EncryptionKey: []byte{},
43-
EncryptionKeyRotationDuration: 10 * 24 * time.Hour, // Default 10 days.
44-
DetectConflicts: true,
45-
NamespaceOffset: -1,
46-
}
47-
48-
cache, err := badger.Open(options)
49-
if err != nil {
50-
panic(err)
51-
}
52-
_ = cache.DropAll()
53-
global.CacheDb = cache
54-
global.CACHE = badger_db.NewCacheDB(cache)
55-
global.LOG.Info("init cache successfully")
9+
global.CACHE = cachedb.NewCacheDB()
5610
}

‎agent/init/cache/db/db.go

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package badger_db
2+
3+
import (
4+
"github.com/patrickmn/go-cache"
5+
"strings"
6+
"time"
7+
)
8+
9+
type Cache struct {
10+
db *cache.Cache
11+
}
12+
13+
func NewCacheDB() *Cache {
14+
db := cache.New(5*time.Minute, 10*time.Minute)
15+
return &Cache{
16+
db: db,
17+
}
18+
}
19+
20+
func (c *Cache) Set(key string, value interface{}) {
21+
c.db.Set(key, value, cache.DefaultExpiration)
22+
}
23+
24+
func (c *Cache) SetWithTTL(key string, value interface{}, d time.Duration) {
25+
c.db.Set(key, value, d)
26+
}
27+
func (c *Cache) Del(key string) {
28+
c.db.Delete(key)
29+
}
30+
31+
func (c *Cache) Clean() error {
32+
return nil
33+
}
34+
35+
func (c *Cache) Get(key string) string {
36+
obj, exist := c.db.Get(key)
37+
if !exist {
38+
return ""
39+
}
40+
return obj.(string)
41+
}
42+
43+
func (c *Cache) PrefixScanKey(prefixStr string) []string {
44+
var res []string
45+
values := c.db.Items()
46+
for key := range values {
47+
if strings.HasPrefix(key, prefixStr) {
48+
res = append(res, key)
49+
}
50+
}
51+
return res
52+
}

‎agent/server/server.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"crypto/tls"
55
"fmt"
66
"github.com/1Panel-dev/1Panel/agent/init/business"
7+
"github.com/1Panel-dev/1Panel/agent/init/cache"
78
"net"
89
"net/http"
910
"os"
@@ -30,6 +31,7 @@ func Start() {
3031
i18n.Init()
3132
log.Init()
3233
db.Init()
34+
cache.Init()
3335
migration.Init()
3436
app.Init()
3537
validator.Init()

‎agent/utils/files/file_op.go

Lines changed: 5 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -248,13 +248,9 @@ func (w *WriteCounter) SaveProcess() {
248248
}
249249
by, _ := json.Marshal(process)
250250
if percentValue < 100 {
251-
if err := global.CACHE.Set(w.Key, string(by)); err != nil {
252-
global.LOG.Errorf("save cache error, err %s", err.Error())
253-
}
251+
global.CACHE.Set(w.Key, string(by))
254252
} else {
255-
if err := global.CACHE.SetWithTTL(w.Key, string(by), time.Second*time.Duration(10)); err != nil {
256-
global.LOG.Errorf("save cache error, err %s", err.Error())
257-
}
253+
global.CACHE.SetWithTTL(w.Key, string(by), time.Second*time.Duration(10))
258254
}
259255
}
260256

@@ -293,20 +289,14 @@ func (f FileOp) DownloadFileWithProcess(url, dst, key string, ignoreCertificate
293289
out.Close()
294290
resp.Body.Close()
295291

296-
value, err := global.CACHE.Get(counter.Key)
297-
if err != nil {
298-
global.LOG.Errorf("get cache error,err %s", err.Error())
299-
return
300-
}
292+
value := global.CACHE.Get(counter.Key)
301293
process := &Process{}
302-
_ = json.Unmarshal(value, process)
294+
_ = json.Unmarshal([]byte(value), process)
303295
process.Percent = 100
304296
process.Name = counter.Name
305297
process.Total = process.Written
306298
by, _ := json.Marshal(process)
307-
if err := global.CACHE.SetWithTTL(counter.Key, string(by), time.Second*time.Duration(10)); err != nil {
308-
global.LOG.Errorf("save cache error, err %s", err.Error())
309-
}
299+
global.CACHE.Set(counter.Key, string(by))
310300
}()
311301
return nil
312302
}

‎agent/utils/websocket/process_data.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -134,13 +134,12 @@ func ProcessData(c *Client, inputMsg []byte) {
134134
func getDownloadProcess(progress DownloadProgress) (res []byte, err error) {
135135
var result []files.Process
136136
for _, k := range progress.Keys {
137-
value, err := global.CACHE.Get(k)
138-
if err != nil {
139-
global.LOG.Errorf("get cache error,err %s", err.Error())
140-
return nil, err
137+
value := global.CACHE.Get(k)
138+
if value == "" {
139+
return nil, fmt.Errorf("get cache error,err value is nil")
141140
}
142141
downloadProcess := &files.Process{}
143-
_ = json.Unmarshal(value, downloadProcess)
142+
_ = json.Unmarshal([]byte(value), downloadProcess)
144143
result = append(result, *downloadProcess)
145144
}
146145
res, err = json.Marshal(result)

‎core/global/global.go

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,7 @@ package global
22

33
import (
44
"github.com/1Panel-dev/1Panel/core/configs"
5-
"github.com/1Panel-dev/1Panel/core/init/cache/badger_db"
65
"github.com/1Panel-dev/1Panel/core/init/session/psession"
7-
"github.com/dgraph-io/badger/v4"
86
"github.com/go-playground/validator/v10"
97
"github.com/nicksnyder/go-i18n/v2/i18n"
108
"github.com/robfig/cron/v3"
@@ -20,8 +18,6 @@ var (
2018
CONF configs.ServerConfig
2119
VALID *validator.Validate
2220
SESSION *psession.PSession
23-
CACHE *badger_db.Cache
24-
CacheDb *badger.DB
2521
Viper *viper.Viper
2622

2723
I18n *i18n.Localizer

0 commit comments

Comments
 (0)