Skip to content

Commit a274a77

Browse files
sozercansozercan
authored
Merge branch 'master' into flux-singlefile2
2 parents 6c9aee4 + 5be2d22 commit a274a77

File tree

24 files changed

+211
-63
lines changed

24 files changed

+211
-63
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,15 @@ DETECT_LIBS?=true
88
# llama.cpp versions
99
GOLLAMA_REPO?=https://github.com/go-skynet/go-llama.cpp
1010
GOLLAMA_VERSION?=2b57a8ae43e4699d3dc5d1496a1ccd42922993be
11-
CPPLLAMA_VERSION?=45f097645efb11b6d09a5b4adbbfd7c312ac0126
11+
CPPLLAMA_VERSION?=958367bf530d943a902afa1ce1c342476098576b
1212

1313
# go-rwkv version
1414
RWKV_REPO?=https://github.com/donomii/go-rwkv.cpp
1515
RWKV_VERSION?=661e7ae26d442f5cfebd2a0881b44e8c55949ec6
1616

1717
# whisper.cpp version
1818
WHISPER_REPO?=https://github.com/ggerganov/whisper.cpp
19-
WHISPER_CPP_VERSION?=a5abfe6a90495f7bf19fe70d016ecc255e97359c
19+
WHISPER_CPP_VERSION?=0fbaac9c891055796456df7b9122a70c220f9ca1
2020

2121
# bert.cpp version
2222
BERT_REPO?=https://github.com/go-skynet/go-bert.cpp

‎backend/backend.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,7 @@ message ModelOptions {
219219
int32 SwapSpace = 53;
220220
int32 MaxModelLen = 54;
221221
int32 TensorParallelSize = 55;
222+
string LoadFormat = 58;
222223

223224
string MMProj = 41;
224225

‎backend/python/parler-tts/install.sh

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,12 +15,12 @@ installRequirements
1515

1616
# https://github.com/descriptinc/audiotools/issues/101
1717
# incompatible protobuf versions.
18-
PYDIR=python3.10
19-
pyenv="${MY_DIR}/venv/lib/${PYDIR}/site-packages/google/protobuf/internal/"
18+
# PYDIR=python3.10
19+
# pyenv="${MY_DIR}/venv/lib/${PYDIR}/site-packages/google/protobuf/internal/"
2020

21-
if [ ! -d ${pyenv} ]; then
22-
echo "(parler-tts/install.sh): Error: ${pyenv} does not exist"
23-
exit 1
24-
fi
21+
# if [ ! -d ${pyenv} ]; then
22+
# echo "(parler-tts/install.sh): Error: ${pyenv} does not exist"
23+
# exit 1
24+
# fi
2525

26-
curl -L https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -o ${pyenv}/builder.py
26+
# curl -L https://raw.githubusercontent.com/protocolbuffers/protobuf/main/python/google/protobuf/internal/builder.py -o ${pyenv}/builder.py

‎backend/python/parler-tts/requirements-after.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
git+https://github.com/huggingface/parler-tts.git@8e465f1b5fcd223478e07175cb40494d19ffbe17
22
llvmlite==0.43.0
33
numba==0.60.0
4+
git+https://github.com/descriptinc/audiotools

‎backend/python/vllm/backend.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ async def LoadModel(self, request, context):
9595

9696
if request.Quantization != "":
9797
engine_args.quantization = request.Quantization
98+
if request.LoadFormat != "":
99+
engine_args.load_format = request.LoadFormat
98100
if request.GPUMemoryUtilization != 0:
99101
engine_args.gpu_memory_utilization = request.GPUMemoryUtilization
100102
if request.TrustRemoteCode:

‎core/backend/options.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ func grpcModelOpts(c config.BackendConfig) *pb.ModelOptions {
139139
DraftModel: c.DraftModel,
140140
AudioPath: c.VallE.AudioPath,
141141
Quantization: c.Quantization,
142+
LoadFormat: c.LoadFormat,
142143
GPUMemoryUtilization: c.GPUMemoryUtilization,
143144
TrustRemoteCode: c.TrustRemoteCode,
144145
EnforceEager: c.EnforceEager,

‎core/cli/run.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ type RunCMD struct {
5353
OpaqueErrors bool `env:"LOCALAI_OPAQUE_ERRORS" default:"false" help:"If true, all error responses are replaced with blank 500 errors. This is intended only for hardening against information leaks and is normally not recommended." group:"hardening"`
5454
UseSubtleKeyComparison bool `env:"LOCALAI_SUBTLE_KEY_COMPARISON" default:"false" help:"If true, API Key validation comparisons will be performed using constant-time comparisons rather than simple equality. This trades off performance on each request for resiliancy against timing attacks." group:"hardening"`
5555
DisableApiKeyRequirementForHttpGet bool `env:"LOCALAI_DISABLE_API_KEY_REQUIREMENT_FOR_HTTP_GET" default:"false" help:"If true, a valid API key is not required to issue GET requests to portions of the web ui. This should only be enabled in secure testing environments" group:"hardening"`
56+
DisableMetricsEndpoint bool `env:"LOCALAI_DISABLE_METRICS_ENDPOINT,DISABLE_METRICS_ENDPOINT" default:"false" help:"Disable the /metrics endpoint" group:"api"`
5657
HttpGetExemptedEndpoints []string `env:"LOCALAI_HTTP_GET_EXEMPTED_ENDPOINTS" default:"^/$,^/browse/?$,^/talk/?$,^/p2p/?$,^/chat/?$,^/text2image/?$,^/tts/?$,^/static/.*$,^/swagger.*$" help:"If LOCALAI_DISABLE_API_KEY_REQUIREMENT_FOR_HTTP_GET is overriden to true, this is the list of endpoints to exempt. Only adjust this in case of a security incident or as a result of a personal security posture review" group:"hardening"`
5758
Peer2Peer bool `env:"LOCALAI_P2P,P2P" name:"p2p" default:"false" help:"Enable P2P mode" group:"p2p"`
5859
Peer2PeerDHTInterval int `env:"LOCALAI_P2P_DHT_INTERVAL,P2P_DHT_INTERVAL" default:"360" name:"p2p-dht-interval" help:"Interval for DHT refresh (used during token generation)" group:"p2p"`
@@ -108,6 +109,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
108109
config.WithLoadToMemory(r.LoadToMemory),
109110
}
110111

112+
if r.DisableMetricsEndpoint {
113+
opts = append(opts, config.DisableMetricsEndpoint)
114+
}
115+
111116
token := ""
112117
if r.Peer2Peer || r.Peer2PeerToken != "" {
113118
log.Info().Msg("P2P mode enabled")

‎core/config/application_config.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ type ApplicationConfig struct {
3939
OpaqueErrors bool
4040
UseSubtleKeyComparison bool
4141
DisableApiKeyRequirementForHttpGet bool
42+
DisableMetrics bool
4243
HttpGetExemptedEndpoints []*regexp.Regexp
4344
DisableGalleryEndpoint bool
4445
LoadToMemory []string
@@ -350,6 +351,10 @@ func WithDisableApiKeyRequirementForHttpGet(required bool) AppOption {
350351
}
351352
}
352353

354+
var DisableMetricsEndpoint AppOption = func(o *ApplicationConfig) {
355+
o.DisableMetrics = true
356+
}
357+
353358
func WithHttpGetExemptedEndpoints(endpoints []string) AppOption {
354359
return func(o *ApplicationConfig) {
355360
o.HttpGetExemptedEndpoints = []*regexp.Regexp{}

‎core/config/backend_config.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,7 @@ type LLMConfig struct {
143143
DraftModel string `yaml:"draft_model"`
144144
NDraft int32 `yaml:"n_draft"`
145145
Quantization string `yaml:"quantization"`
146+
LoadFormat string `yaml:"load_format"`
146147
GPUMemoryUtilization float32 `yaml:"gpu_memory_utilization"` // vLLM
147148
TrustRemoteCode bool `yaml:"trust_remote_code"` // vLLM
148149
EnforceEager bool `yaml:"enforce_eager"` // vLLM
@@ -197,9 +198,7 @@ type TemplateConfig struct {
197198
// It defaults to \n
198199
JoinChatMessagesByCharacter *string `yaml:"join_chat_messages_by_character"`
199200

200-
Video string `yaml:"video"`
201-
Image string `yaml:"image"`
202-
Audio string `yaml:"audio"`
201+
Multimodal string `yaml:"multimodal"`
203202
}
204203

205204
func (c *BackendConfig) UnmarshalYAML(value *yaml.Node) error {

‎core/http/app.go

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -109,19 +109,21 @@ func App(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *confi
109109
app.Use(recover.New())
110110
}
111111

112-
metricsService, err := services.NewLocalAIMetricsService()
113-
if err != nil {
114-
return nil, err
115-
}
112+
if !appConfig.DisableMetrics {
113+
metricsService, err := services.NewLocalAIMetricsService()
114+
if err != nil {
115+
return nil, err
116+
}
116117

117-
if metricsService != nil {
118-
app.Use(localai.LocalAIMetricsAPIMiddleware(metricsService))
119-
app.Hooks().OnShutdown(func() error {
120-
return metricsService.Shutdown()
121-
})
122-
}
118+
if metricsService != nil {
119+
app.Use(localai.LocalAIMetricsAPIMiddleware(metricsService))
120+
app.Hooks().OnShutdown(func() error {
121+
return metricsService.Shutdown()
122+
})
123+
}
123124

124-
// Health Checks should always be exempt from auth, so register these first
125+
}
126+
// Health Checks should always be exempt from auth, so register these first
125127
routes.HealthRoutes(app)
126128

127129
kaConfig, err := middleware.GetKeyAuthConfig(appConfig)

‎core/http/endpoints/openai/request.go

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -149,18 +149,27 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
149149
// Decode each request's message content
150150
imgIndex, vidIndex, audioIndex := 0, 0, 0
151151
for i, m := range input.Messages {
152+
nrOfImgsInMessage := 0
153+
nrOfVideosInMessage := 0
154+
nrOfAudiosInMessage := 0
155+
152156
switch content := m.Content.(type) {
153157
case string:
154158
input.Messages[i].StringContent = content
155159
case []interface{}:
156160
dat, _ := json.Marshal(content)
157161
c := []schema.Content{}
158162
json.Unmarshal(dat, &c)
163+
164+
textContent := ""
165+
// we will template this at the end
166+
159167
CONTENT:
160168
for _, pp := range c {
161169
switch pp.Type {
162170
case "text":
163-
input.Messages[i].StringContent = pp.Text
171+
textContent += pp.Text
172+
//input.Messages[i].StringContent = pp.Text
164173
case "video", "video_url":
165174
// Decode content as base64 either if it's an URL or base64 text
166175
base64, err := utils.GetContentURIAsBase64(pp.VideoURL.URL)
@@ -169,14 +178,8 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
169178
continue CONTENT
170179
}
171180
input.Messages[i].StringVideos = append(input.Messages[i].StringVideos, base64) // TODO: make sure that we only return base64 stuff
172-
173-
t := "[vid-{{.ID}}]{{.Text}}"
174-
if config.TemplateConfig.Video != "" {
175-
t = config.TemplateConfig.Video
176-
}
177-
// set a placeholder for each image
178-
input.Messages[i].StringContent, _ = templates.TemplateMultiModal(t, vidIndex, input.Messages[i].StringContent)
179181
vidIndex++
182+
nrOfVideosInMessage++
180183
case "audio_url", "audio":
181184
// Decode content as base64 either if it's an URL or base64 text
182185
base64, err := utils.GetContentURIAsBase64(pp.AudioURL.URL)
@@ -185,13 +188,8 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
185188
continue CONTENT
186189
}
187190
input.Messages[i].StringAudios = append(input.Messages[i].StringAudios, base64) // TODO: make sure that we only return base64 stuff
188-
// set a placeholder for each image
189-
t := "[audio-{{.ID}}]{{.Text}}"
190-
if config.TemplateConfig.Audio != "" {
191-
t = config.TemplateConfig.Audio
192-
}
193-
input.Messages[i].StringContent, _ = templates.TemplateMultiModal(t, audioIndex, input.Messages[i].StringContent)
194191
audioIndex++
192+
nrOfAudiosInMessage++
195193
case "image_url", "image":
196194
// Decode content as base64 either if it's an URL or base64 text
197195
base64, err := utils.GetContentURIAsBase64(pp.ImageURL.URL)
@@ -200,16 +198,21 @@ func updateRequestConfig(config *config.BackendConfig, input *schema.OpenAIReque
200198
continue CONTENT
201199
}
202200

203-
t := "[img-{{.ID}}]{{.Text}}"
204-
if config.TemplateConfig.Image != "" {
205-
t = config.TemplateConfig.Image
206-
}
207201
input.Messages[i].StringImages = append(input.Messages[i].StringImages, base64) // TODO: make sure that we only return base64 stuff
208-
// set a placeholder for each image
209-
input.Messages[i].StringContent, _ = templates.TemplateMultiModal(t, imgIndex, input.Messages[i].StringContent)
202+
210203
imgIndex++
204+
nrOfImgsInMessage++
211205
}
212206
}
207+
208+
input.Messages[i].StringContent, _ = templates.TemplateMultiModal(config.TemplateConfig.Multimodal, templates.MultiModalOptions{
209+
TotalImages: imgIndex,
210+
TotalVideos: vidIndex,
211+
TotalAudios: audioIndex,
212+
ImagesInMessage: nrOfImgsInMessage,
213+
VideosInMessage: nrOfVideosInMessage,
214+
AudiosInMessage: nrOfAudiosInMessage,
215+
}, textContent)
213216
}
214217
}
215218

‎core/http/routes/localai.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,9 @@ func RegisterLocalAIRoutes(app *fiber.App,
4242
app.Post("/stores/get", localai.StoresGetEndpoint(sl, appConfig))
4343
app.Post("/stores/find", localai.StoresFindEndpoint(sl, appConfig))
4444

45-
app.Get("/metrics", localai.LocalAIMetricsEndpoint())
45+
if !appConfig.DisableMetrics {
46+
app.Get("/metrics", localai.LocalAIMetricsEndpoint())
47+
}
4648

4749
// Experimental Backend Statistics Module
4850
backendMonitorService := services.NewBackendMonitorService(ml, cl, appConfig) // Split out for now

‎docs/content/docs/getting-started/quickstart.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,19 @@ For a full list of options, refer to the [Installer Options]({{% relref "docs/ad
3030

3131
Binaries can also be [manually downloaded]({{% relref "docs/reference/binaries" %}}).
3232

33+
## Using Homebrew on MacOS
34+
35+
{{% alert icon="⚠️" %}}
36+
The Homebrew formula currently doesn't have the same options than the bash script
37+
{{% /alert %}}
38+
39+
You can install Homebrew's [LocalAI](https://formulae.brew.sh/formula/localai) with the following command:
40+
41+
```
42+
brew install localai
43+
```
44+
45+
3346
## Using Container Images or Kubernetes
3447

3548
LocalAI is available as a container image compatible with various container engines such as Docker, Podman, and Kubernetes. Container images are published on [quay.io](https://quay.io/repository/go-skynet/local-ai?tab=tags&tag=latest) and [Docker Hub](https://hub.docker.com/r/localai/localai).

‎docs/data/version.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"version": "v2.22.0"
2+
"version": "v2.22.1"
33
}

‎docs/themes/hugo-theme-relearn

‎examples/chainlit/requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
llama_index==0.11.17
1+
llama_index==0.11.19
22
requests==2.32.3
3-
weaviate_client==4.8.1
3+
weaviate_client==4.9.0
44
transformers
55
torch
66
chainlit

‎examples/functions/requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
langchain==0.3.3
2-
openai==1.51.2
1+
langchain==0.3.4
2+
openai==1.52.0

‎examples/langchain-chroma/requirements.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
langchain==0.3.3
2-
openai==1.51.2
2+
openai==1.52.0
33
chromadb==0.5.13
4-
llama-index==0.11.17
4+
llama-index==0.11.19

‎examples/langchain/langchainpy-localai-example/requirements.txt

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ frozenlist==1.4.1
1111
greenlet==3.1.1
1212
idna==3.10
1313
langchain==0.3.3
14-
langchain-community==0.3.2
15-
marshmallow==3.22.0
14+
langchain-community==0.3.3
15+
marshmallow==3.23.0
1616
marshmallow-enum==1.5.1
1717
multidict==6.1.0
1818
mypy-extensions==1.0.0
@@ -24,10 +24,10 @@ packaging>=23.2
2424
pydantic==2.9.2
2525
PyYAML==6.0.2
2626
requests==2.32.3
27-
SQLAlchemy==2.0.35
27+
SQLAlchemy==2.0.36
2828
tenacity==8.5.0
2929
tqdm==4.66.5
3030
typing-inspect==0.9.0
3131
typing_extensions==4.12.2
3232
urllib3==2.2.3
33-
yarl==1.15.2
33+
yarl==1.16.0

‎gallery/phi-3-vision.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,6 @@ config_file: |
2020
completion: |
2121
{{.Input}}
2222
use_tokenizer_template: false
23+
multimodal: "{{ range .Images }}<|image_{{ add1 .ID}}|>{{end}}\n{{.Text}}"
24+
# XXX: The one below can be dropped after a new release is out
2325
image: "<|image_{{ add1 .ID }}|>\n{{.Text}}"

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ require (
231231
github.com/moby/sys/sequential v0.5.0 // indirect
232232
github.com/moby/term v0.5.0 // indirect
233233
github.com/mr-tron/base58 v1.2.0 // indirect
234-
github.com/mudler/go-piper v0.0.0-20240315144837-9d0100873a7d
234+
github.com/mudler/go-piper v0.0.0-20241022074816-3854e0221ffb
235235
github.com/mudler/water v0.0.0-20221010214108-8c7313014ce0 // indirect
236236
github.com/muesli/reflow v0.3.0 // indirect
237237
github.com/muesli/termenv v0.15.2 // indirect

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -498,6 +498,8 @@ github.com/mudler/edgevpn v0.28.3 h1:yIuoMExwKHy/mNMBXIsm6FUFbnB9ELIxw9KXrK9KHDk
498498
github.com/mudler/edgevpn v0.28.3/go.mod h1:HWcdIwj5zBgOD04Hn3I+J5E5Yb3kK1CwwWaEe6/QERo=
499499
github.com/mudler/go-piper v0.0.0-20240315144837-9d0100873a7d h1:8udOFrDf/I83JL0/u22j6U6Q9z9LoSdby2a/DWdd0/s=
500500
github.com/mudler/go-piper v0.0.0-20240315144837-9d0100873a7d/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig=
501+
github.com/mudler/go-piper v0.0.0-20241022074816-3854e0221ffb h1:5qcuxQEpAqeV4ftV5nUt3/hB/RoTXq3MaaauOAedyXo=
502+
github.com/mudler/go-piper v0.0.0-20241022074816-3854e0221ffb/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig=
501503
github.com/mudler/go-processmanager v0.0.0-20240820160718-8b802d3ecf82 h1:FVT07EI8njvsD4tC2Hw8Xhactp5AWhsQWD4oTeQuSAU=
502504
github.com/mudler/go-processmanager v0.0.0-20240820160718-8b802d3ecf82/go.mod h1:Urp7LG5jylKoDq0663qeBh0pINGcRl35nXdKx82PSoU=
503505
github.com/mudler/go-stable-diffusion v0.0.0-20240429204715-4a3cd6aeae6f h1:cxtMSRkUfy+mjIQ3yMrU0txwQ4It913NEN4m1H8WWgo=

0 commit comments

Comments
 (0)