Skip to content

Commit 69e51f9

Browse files
mudlermudler
authored
Merge branch 'master' into fix-dynamicconfigsdir-root
2 parents 2c1202b + c9451cb commit 69e51f9

File tree

16 files changed

+70
-34
lines changed

16 files changed

+70
-34
lines changed

‎.github/workflows/image-pr.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ jobs:
6868
- build-type: 'sycl_f16'
6969
platforms: 'linux/amd64'
7070
tag-latest: 'false'
71-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
71+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
7272
grpc-base-image: "ubuntu:22.04"
7373
tag-suffix: 'sycl-f16-ffmpeg'
7474
ffmpeg: 'true'
@@ -110,7 +110,7 @@ jobs:
110110
- build-type: 'sycl_f16'
111111
platforms: 'linux/amd64'
112112
tag-latest: 'false'
113-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
113+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
114114
grpc-base-image: "ubuntu:22.04"
115115
tag-suffix: 'sycl-f16-ffmpeg-core'
116116
ffmpeg: 'true'

‎.github/workflows/image.yml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ jobs:
148148
- build-type: 'sycl_f16'
149149
platforms: 'linux/amd64'
150150
tag-latest: 'auto'
151-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
151+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
152152
grpc-base-image: "ubuntu:22.04"
153153
tag-suffix: '-sycl-f16-ffmpeg'
154154
ffmpeg: 'true'
@@ -161,7 +161,7 @@ jobs:
161161
- build-type: 'sycl_f32'
162162
platforms: 'linux/amd64'
163163
tag-latest: 'auto'
164-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
164+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
165165
grpc-base-image: "ubuntu:22.04"
166166
tag-suffix: '-sycl-f32-ffmpeg'
167167
ffmpeg: 'true'
@@ -175,7 +175,7 @@ jobs:
175175
- build-type: 'sycl_f16'
176176
platforms: 'linux/amd64'
177177
tag-latest: 'false'
178-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
178+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
179179
grpc-base-image: "ubuntu:22.04"
180180
tag-suffix: '-sycl-f16-core'
181181
ffmpeg: 'false'
@@ -185,7 +185,7 @@ jobs:
185185
- build-type: 'sycl_f32'
186186
platforms: 'linux/amd64'
187187
tag-latest: 'false'
188-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
188+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
189189
grpc-base-image: "ubuntu:22.04"
190190
tag-suffix: '-sycl-f32-core'
191191
ffmpeg: 'false'
@@ -195,7 +195,7 @@ jobs:
195195
- build-type: 'sycl_f16'
196196
platforms: 'linux/amd64'
197197
tag-latest: 'false'
198-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
198+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
199199
grpc-base-image: "ubuntu:22.04"
200200
tag-suffix: '-sycl-f16-ffmpeg-core'
201201
ffmpeg: 'true'
@@ -205,7 +205,7 @@ jobs:
205205
- build-type: 'sycl_f32'
206206
platforms: 'linux/amd64'
207207
tag-latest: 'false'
208-
base-image: "intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04"
208+
base-image: "intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04"
209209
grpc-base-image: "ubuntu:22.04"
210210
tag-suffix: '-sycl-f32-ffmpeg-core'
211211
ffmpeg: 'true'

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -707,15 +707,15 @@ docker-aio-all:
707707

708708
docker-image-intel:
709709
docker build \
710-
--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04 \
710+
--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04 \
711711
--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
712712
--build-arg GO_TAGS="none" \
713713
--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \
714714
--build-arg BUILD_TYPE=sycl_f32 -t $(DOCKER_IMAGE) .
715715

716716
docker-image-intel-xpu:
717717
docker build \
718-
--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.0.1-devel-ubuntu22.04 \
718+
--build-arg BASE_IMAGE=intel/oneapi-basekit:2024.1.0-devel-ubuntu22.04 \
719719
--build-arg IMAGE_TYPE=$(IMAGE_TYPE) \
720720
--build-arg GO_TAGS="none" \
721721
--build-arg MAKEFLAGS="$(DOCKER_MAKEFLAGS)" \

‎backend/go/transcribe/transcript.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ import (
1111
"github.com/go-skynet/LocalAI/core/schema"
1212
)
1313

14-
func runCommand(command []string) (string, error) {
15-
cmd := exec.Command(command[0], command[1:]...)
14+
func ffmpegCommand(args []string) (string, error) {
15+
cmd := exec.Command("ffmpeg", args...) // Constrain this to ffmpeg to permit security scanner to see that the command is safe.
1616
cmd.Env = os.Environ()
1717
out, err := cmd.CombinedOutput()
1818
return string(out), err
@@ -21,8 +21,8 @@ func runCommand(command []string) (string, error) {
2121
// AudioToWav converts audio to wav for transcribe.
2222
// TODO: use https://github.com/mccoyst/ogg?
2323
func audioToWav(src, dst string) error {
24-
command := []string{"ffmpeg", "-i", src, "-format", "s16le", "-ar", "16000", "-ac", "1", "-acodec", "pcm_s16le", dst}
25-
out, err := runCommand(command)
24+
commandArgs := []string{"-i", src, "-format", "s16le", "-ar", "16000", "-ac", "1", "-acodec", "pcm_s16le", dst}
25+
out, err := ffmpegCommand(commandArgs)
2626
if err != nil {
2727
return fmt.Errorf("error: %w out: %s", err, out)
2828
}

‎backend/python/common-env/transformers/transformers.yml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,9 +60,10 @@ dependencies:
6060
- networkx
6161
- numpy==1.26.0
6262
- onnx==1.15.0
63-
- openvino==2024.0.0
64-
- openvino-telemetry==2023.2.1
65-
- optimum[openvino]==1.17.1
63+
- openvino==2024.1.0
64+
- openvino-telemetry==2024.1.0
65+
- optimum[openvino]==1.19.1
66+
- optimum-intel==1.16.1
6667
- packaging==23.2
6768
- pandas
6869
- peft==0.5.0

‎backend/python/transformers/transformers_server.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ def LoadModel(self, request, context):
150150
self.model = OVModelForCausalLM.from_pretrained(model_name,
151151
compile=True,
152152
trust_remote_code=request.TrustRemoteCode,
153-
ov_config={"PERFORMANCE_HINT": "LATENCY"},
153+
ov_config={"PERFORMANCE_HINT": "CUMULATIVE_THROUGHPUT"},
154154
device=device_map)
155155
self.OV = True
156156
else:

‎core/backend/options.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@ import (
77

88
"github.com/go-skynet/LocalAI/core/config"
99
pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
10-
model "github.com/go-skynet/LocalAI/pkg/model"
10+
"github.com/go-skynet/LocalAI/pkg/model"
11+
"github.com/rs/zerolog/log"
1112
)
1213

1314
func modelOpts(c config.BackendConfig, so *config.ApplicationConfig, opts []model.Option) []model.Option {
@@ -109,8 +110,12 @@ func gRPCPredictOpts(c config.BackendConfig, modelPath string) *pb.PredictOption
109110
promptCachePath := ""
110111
if c.PromptCachePath != "" {
111112
p := filepath.Join(modelPath, c.PromptCachePath)
112-
os.MkdirAll(filepath.Dir(p), 0750)
113-
promptCachePath = p
113+
err := os.MkdirAll(filepath.Dir(p), 0750)
114+
if err == nil {
115+
promptCachePath = p
116+
} else {
117+
log.Error().Err(err).Str("promptCachePath", promptCachePath).Msg("error creating prompt cache folder")
118+
}
114119
}
115120

116121
return &pb.PredictOptions{

‎core/http/render.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/go-skynet/LocalAI/core/schema"
1111
"github.com/gofiber/fiber/v2"
1212
fiberhtml "github.com/gofiber/template/html/v2"
13+
"github.com/microcosm-cc/bluemonday"
1314
"github.com/russross/blackfriday"
1415
)
1516

@@ -39,5 +40,5 @@ func renderEngine() *fiberhtml.Engine {
3940

4041
func markDowner(args ...interface{}) template.HTML {
4142
s := blackfriday.MarkdownCommon([]byte(fmt.Sprintf("%s", args...)))
42-
return template.HTML(s)
43+
return template.HTML(bluemonday.UGCPolicy().Sanitize(string(s)))
4344
}

‎core/startup/startup.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,10 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
122122
// Watch the configuration directory
123123
// If the directory does not exist, we don't watch it
124124
configHandler := newConfigFileHandler(options)
125-
configHandler.Watch()
125+
err = configHandler.Watch()
126+
if err != nil {
127+
log.Error().Err(err).Msg("error establishing configuration directory watcher")
128+
}
126129

127130
log.Info().Msg("core/startup process completed!")
128131
return cl, ml, options, nil

‎embedded/embedded.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strings"
88

99
"github.com/go-skynet/LocalAI/pkg/downloader"
10+
"github.com/rs/zerolog/log"
1011

1112
"github.com/go-skynet/LocalAI/pkg/assets"
1213
"gopkg.in/yaml.v3"
@@ -29,7 +30,10 @@ func ModelShortURL(s string) string {
2930
}
3031

3132
func init() {
32-
yaml.Unmarshal(modelLibrary, &modelShorteners)
33+
err := yaml.Unmarshal(modelLibrary, &modelShorteners)
34+
if err != nil {
35+
log.Error().Err(err).Msg("error while unmarshalling embedded modelLibrary")
36+
}
3337
}
3438

3539
func GetRemoteLibraryShorteners(url string) (map[string]string, error) {

‎examples/semantic-todo/main.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,10 @@ func (app *App) updateUI() {
239239
task := Task{Description: inputField.GetText()}
240240
app.tasks = append(app.tasks, task)
241241
app.state = StateRoot
242-
postTasksToExternalService([]Task{task})
242+
err := postTasksToExternalService([]Task{task})
243+
if err != nil {
244+
panic(err)
245+
}
243246
}
244247
app.updateUI()
245248
})

‎go.mod

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ require (
9393
github.com/golang/protobuf v1.5.3 // indirect
9494
github.com/golang/snappy v0.0.2 // indirect
9595
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
96-
github.com/gorilla/css v1.0.0 // indirect
96+
github.com/gorilla/css v1.0.1 // indirect
9797
github.com/huandu/xstrings v1.3.3 // indirect
9898
github.com/josharian/intern v1.0.0 // indirect
9999
github.com/klauspost/pgzip v1.2.5 // indirect
@@ -133,9 +133,9 @@ require (
133133
github.com/yuin/goldmark-emoji v1.0.2 // indirect
134134
go.opentelemetry.io/otel/sdk v1.19.0 // indirect
135135
go.opentelemetry.io/otel/trace v1.19.0 // indirect
136-
golang.org/x/crypto v0.21.0 // indirect
136+
golang.org/x/crypto v0.22.0 // indirect
137137
golang.org/x/mod v0.16.0 // indirect
138-
golang.org/x/term v0.18.0 // indirect
138+
golang.org/x/term v0.19.0 // indirect
139139
google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect
140140
gopkg.in/fsnotify.v1 v1.4.7 // indirect
141141
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
@@ -161,8 +161,8 @@ require (
161161
github.com/rivo/uniseg v0.2.0 // indirect
162162
github.com/valyala/bytebufferpool v1.0.0 // indirect
163163
github.com/valyala/tcplisten v1.0.0 // indirect
164-
golang.org/x/net v0.22.0 // indirect
165-
golang.org/x/sys v0.18.0 // indirect
164+
golang.org/x/net v0.24.0 // indirect
165+
golang.org/x/sys v0.19.0 // indirect
166166
golang.org/x/text v0.14.0 // indirect
167167
golang.org/x/tools v0.19.0 // indirect
168168
)

‎go.sum

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,8 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
146146
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
147147
github.com/gorilla/css v1.0.0 h1:BQqNyPTi50JCFMTw/b67hByjMVXZRwGha6wxVGkeihY=
148148
github.com/gorilla/css v1.0.0/go.mod h1:Dn721qIggHpt4+EFCcTLTU/vk5ySda2ReITrtgBl60c=
149+
github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8=
150+
github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0=
149151
github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
150152
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
151153
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
@@ -377,6 +379,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
377379
golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
378380
golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
379381
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
382+
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
383+
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
380384
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
381385
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
382386
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -395,6 +399,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
395399
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
396400
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
397401
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
402+
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
403+
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
398404
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
399405
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
400406
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -431,12 +437,16 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
431437
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
432438
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
433439
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
440+
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
441+
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
434442
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
435443
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
436444
golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
437445
golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
438446
golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
439447
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
448+
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
449+
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
440450
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
441451
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
442452
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

‎main.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,11 @@ func main() {
4343
for _, envFile := range envFiles {
4444
if _, err := os.Stat(envFile); err == nil {
4545
log.Info().Str("envFile", envFile).Msg("loading environment variables from file")
46-
godotenv.Load(envFile)
46+
err = godotenv.Load(envFile)
47+
if err != nil {
48+
log.Error().Err(err).Str("envFile", envFile).Msg("failed to load environment variables from file")
49+
continue
50+
}
4751
}
4852
}
4953

‎pkg/assets/list.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,12 @@ package assets
33
import (
44
"embed"
55
"io/fs"
6+
7+
"github.com/rs/zerolog/log"
68
)
79

810
func ListFiles(content embed.FS) (files []string) {
9-
fs.WalkDir(content, ".", func(path string, d fs.DirEntry, err error) error {
11+
err := fs.WalkDir(content, ".", func(path string, d fs.DirEntry, err error) error {
1012
if err != nil {
1113
return err
1214
}
@@ -18,5 +20,8 @@ func ListFiles(content embed.FS) (files []string) {
1820
files = append(files, path)
1921
return nil
2022
})
23+
if err != nil {
24+
log.Error().Err(err).Msg("error walking the embedded filesystem")
25+
}
2126
return
2227
}

‎pkg/grpc/server.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,10 @@ func (s *server) PredictStream(in *pb.PredictOptions, stream pb.Backend_PredictS
131131
done <- true
132132
}()
133133

134-
s.llm.PredictStream(in, resultChan)
134+
err := s.llm.PredictStream(in, resultChan)
135135
<-done
136136

137-
return nil
137+
return err
138138
}
139139

140140
func (s *server) TokenizeString(ctx context.Context, in *pb.PredictOptions) (*pb.TokenizationResponse, error) {

0 commit comments

Comments
 (0)