Skip to content

Commit 97060ae

Browse files
mudlermudler
committed
chore(refactor): do not expose internal backend Loader
Signed-off-by: Ettore Di Giacinto <[email protected]>
1 parent bae9939 commit 97060ae

File tree

13 files changed

+29
-55
lines changed

13 files changed

+29
-55
lines changed

‎core/backend/embeddings.go

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,9 @@ import (
1111

1212
func ModelEmbedding(s string, tokens []int, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() ([]float32, error), error) {
1313

14-
var inferenceModel interface{}
15-
var err error
16-
1714
opts := ModelOptions(backendConfig, appConfig)
1815

19-
if backendConfig.Backend == "" {
20-
inferenceModel, err = loader.GreedyLoader(opts...)
21-
} else {
22-
opts = append(opts, model.WithBackendString(backendConfig.Backend))
23-
inferenceModel, err = loader.BackendLoader(opts...)
24-
}
16+
inferenceModel, err := loader.Load(opts...)
2517
if err != nil {
2618
return nil, err
2719
}

‎core/backend/image.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,7 @@ import (
1010
func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negative_prompt, src, dst string, loader *model.ModelLoader, backendConfig config.BackendConfig, appConfig *config.ApplicationConfig) (func() error, error) {
1111

1212
opts := ModelOptions(backendConfig, appConfig)
13-
14-
inferenceModel, err := loader.BackendLoader(
13+
inferenceModel, err := loader.Load(
1514
opts...,
1615
)
1716
if err != nil {

‎core/backend/llm.go

Lines changed: 2 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@ import (
1616
"github.com/mudler/LocalAI/core/schema"
1717

1818
"github.com/mudler/LocalAI/core/gallery"
19-
"github.com/mudler/LocalAI/pkg/grpc"
2019
"github.com/mudler/LocalAI/pkg/grpc/proto"
2120
model "github.com/mudler/LocalAI/pkg/model"
2221
"github.com/mudler/LocalAI/pkg/utils"
@@ -35,15 +34,6 @@ type TokenUsage struct {
3534
func ModelInference(ctx context.Context, s string, messages []schema.Message, images, videos, audios []string, loader *model.ModelLoader, c config.BackendConfig, o *config.ApplicationConfig, tokenCallback func(string, TokenUsage) bool) (func() (LLMResponse, error), error) {
3635
modelFile := c.Model
3736

38-
var inferenceModel grpc.Backend
39-
var err error
40-
41-
opts := ModelOptions(c, o)
42-
43-
if c.Backend != "" {
44-
opts = append(opts, model.WithBackendString(c.Backend))
45-
}
46-
4737
// Check if the modelFile exists, if it doesn't try to load it from the gallery
4838
if o.AutoloadGalleries { // experimental
4939
if _, err := os.Stat(modelFile); os.IsNotExist(err) {
@@ -56,12 +46,8 @@ func ModelInference(ctx context.Context, s string, messages []schema.Message, im
5646
}
5747
}
5848

59-
if c.Backend == "" {
60-
inferenceModel, err = loader.GreedyLoader(opts...)
61-
} else {
62-
inferenceModel, err = loader.BackendLoader(opts...)
63-
}
64-
49+
opts := ModelOptions(c, o)
50+
inferenceModel, err := loader.Load(opts...)
6551
if err != nil {
6652
return nil, err
6753
}

‎core/backend/rerank.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212
func Rerank(modelFile string, request *proto.RerankRequest, loader *model.ModelLoader, appConfig *config.ApplicationConfig, backendConfig config.BackendConfig) (*proto.RerankResult, error) {
1313

1414
opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
15-
rerankModel, err := loader.BackendLoader(opts...)
15+
rerankModel, err := loader.Load(opts...)
1616
if err != nil {
1717
return nil, err
1818
}

‎core/backend/soundgeneration.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,7 @@ func SoundGeneration(
2626
) (string, *proto.Result, error) {
2727

2828
opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
29-
30-
soundGenModel, err := loader.BackendLoader(opts...)
29+
soundGenModel, err := loader.Load(opts...)
3130
if err != nil {
3231
return "", nil, err
3332
}

‎core/backend/stores.go

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -8,16 +8,15 @@ import (
88
)
99

1010
func StoreBackend(sl *model.ModelLoader, appConfig *config.ApplicationConfig, storeName string) (grpc.Backend, error) {
11-
if storeName == "" {
12-
storeName = "default"
13-
}
11+
if storeName == "" {
12+
storeName = "default"
13+
}
1414

15-
sc := []model.Option{
16-
model.WithBackendString(model.LocalStoreBackend),
17-
model.WithAssetDir(appConfig.AssetsDestination),
18-
model.WithModel(storeName),
19-
}
15+
sc := []model.Option{
16+
model.WithBackendString(model.LocalStoreBackend),
17+
model.WithAssetDir(appConfig.AssetsDestination),
18+
model.WithModel(storeName),
19+
}
2020

21-
return sl.BackendLoader(sc...)
21+
return sl.Load(sc...)
2222
}
23-

‎core/backend/token_metrics.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ func TokenMetrics(
1616
backendConfig config.BackendConfig) (*proto.MetricsResponse, error) {
1717

1818
opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
19-
model, err := loader.BackendLoader(opts...)
19+
model, err := loader.Load(opts...)
2020
if err != nil {
2121
return nil, err
2222
}

‎core/backend/tokenize.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ func ModelTokenize(s string, loader *model.ModelLoader, backendConfig config.Bac
1717
opts := ModelOptions(backendConfig, appConfig, model.WithModel(modelFile))
1818

1919
if backendConfig.Backend == "" {
20-
inferenceModel, err = loader.GreedyLoader(opts...)
20+
inferenceModel, err = loader.Load(opts...)
2121
} else {
2222
opts = append(opts, model.WithBackendString(backendConfig.Backend))
23-
inferenceModel, err = loader.BackendLoader(opts...)
23+
inferenceModel, err = loader.Load(opts...)
2424
}
2525
if err != nil {
2626
return schema.TokenizeResponse{}, err

‎core/backend/transcript.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ func ModelTranscription(audio, language string, translate bool, ml *model.ModelL
2020

2121
opts := ModelOptions(backendConfig, appConfig)
2222

23-
transcriptionModel, err := ml.BackendLoader(opts...)
23+
transcriptionModel, err := ml.Load(opts...)
2424
if err != nil {
2525
return nil, err
2626
}

‎core/backend/tts.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func ModelTTS(
2929
}
3030

3131
opts := ModelOptions(backendConfig, appConfig, model.WithBackendString(bb), model.WithModel(modelFile))
32-
ttsModel, err := loader.BackendLoader(opts...)
32+
ttsModel, err := loader.Load(opts...)
3333
if err != nil {
3434
return "", nil, err
3535
}

‎core/startup/startup.go

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -163,12 +163,7 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
163163
o := backend.ModelOptions(*cfg, options)
164164

165165
var backendErr error
166-
if cfg.Backend != "" {
167-
o = append(o, model.WithBackendString(cfg.Backend))
168-
_, backendErr = ml.BackendLoader(o...)
169-
} else {
170-
_, backendErr = ml.GreedyLoader(o...)
171-
}
166+
_, backendErr = ml.Load(o...)
172167
if backendErr != nil {
173168
return nil, nil, nil, err
174169
}

‎pkg/model/initializers.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -455,7 +455,7 @@ func (ml *ModelLoader) ListAvailableBackends(assetdir string) ([]string, error)
455455
return orderBackends(backends)
456456
}
457457

458-
func (ml *ModelLoader) BackendLoader(opts ...Option) (client grpc.Backend, err error) {
458+
func (ml *ModelLoader) backendLoader(opts ...Option) (client grpc.Backend, err error) {
459459
o := NewOptions(opts...)
460460

461461
log.Info().Msgf("Loading model '%s' with backend %s", o.modelID, o.backendString)
@@ -500,7 +500,7 @@ func (ml *ModelLoader) stopActiveBackends(modelID string, singleActiveBackend bo
500500
}
501501
}
502502

503-
func (ml *ModelLoader) GreedyLoader(opts ...Option) (grpc.Backend, error) {
503+
func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
504504
o := NewOptions(opts...)
505505

506506
// Return earlier if we have a model already loaded
@@ -513,6 +513,10 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (grpc.Backend, error) {
513513

514514
ml.stopActiveBackends(o.modelID, o.singleActiveBackend)
515515

516+
if o.backendString != "" {
517+
return ml.backendLoader(opts...)
518+
}
519+
516520
var err error
517521

518522
// get backends embedded in the binary
@@ -536,7 +540,7 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (grpc.Backend, error) {
536540
WithBackendString(key),
537541
}...)
538542

539-
model, modelerr := ml.BackendLoader(options...)
543+
model, modelerr := ml.backendLoader(options...)
540544
if modelerr == nil && model != nil {
541545
log.Info().Msgf("[%s] Loads OK", key)
542546
return model, nil

‎tests/integration/stores_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ var _ = Describe("Integration tests for the stores backend(s) and internal APIs"
5757
}
5858

5959
sl = model.NewModelLoader("")
60-
sc, err = sl.BackendLoader(storeOpts...)
60+
sc, err = sl.Load(storeOpts...)
6161
Expect(err).ToNot(HaveOccurred())
6262
Expect(sc).ToNot(BeNil())
6363
})

0 commit comments

Comments
 (0)