|
| 1 | +package cli |
| 2 | + |
| 3 | +import ( |
| 4 | + "fmt" |
| 5 | + "os" |
| 6 | + "strings" |
| 7 | + "time" |
| 8 | + |
| 9 | + "github.com/go-skynet/LocalAI/core/config" |
| 10 | + "github.com/go-skynet/LocalAI/core/http" |
| 11 | + "github.com/go-skynet/LocalAI/core/startup" |
| 12 | + "github.com/rs/zerolog/log" |
| 13 | +) |
| 14 | + |
| 15 | +type RunCMD struct { |
| 16 | + ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"` |
| 17 | + |
| 18 | + ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"` |
| 19 | + BackendAssetsPath string `env:"LOCALAI_BACKEND_ASSETS_PATH,BACKEND_ASSETS_PATH" type:"path" default:"/tmp/localai/backend_data" help:"Path used to extract libraries that are required by some of the backends in runtime" group:"storage"` |
| 20 | + ImagePath string `env:"LOCALAI_IMAGE_PATH,IMAGE_PATH" type:"path" default:"/tmp/generated/images" help:"Location for images generated by backends (e.g. stablediffusion)" group:"storage"` |
| 21 | + AudioPath string `env:"LOCALAI_AUDIO_PATH,AUDIO_PATH" type:"path" default:"/tmp/generated/audio" help:"Location for audio generated by backends (e.g. piper)" group:"storage"` |
| 22 | + UploadPath string `env:"LOCALAI_UPLOAD_PATH,UPLOAD_PATH" type:"path" default:"/tmp/localai/upload" help:"Path to store uploads from files api" group:"storage"` |
| 23 | + ConfigPath string `env:"LOCALAI_CONFIG_PATH,CONFIG_PATH" default:"/tmp/localai/config" group:"storage"` |
| 24 | + LocalaiConfigDir string `env:"LOCALAI_CONFIG_DIR" type:"path" default:"${basepath}/configuration" help:"Directory for dynamic loading of certain configuration files (currently api_keys.json and external_backends.json)" group:"storage"` |
| 25 | + // The alias on this option is there to preserve functionality with the old `--config-file` parameter |
| 26 | + ModelsConfigFile string `env:"LOCALAI_MODELS_CONFIG_FILE,CONFIG_FILE" aliases:"config-file" help:"YAML file containing a list of model backend configs" group:"storage"` |
| 27 | + |
| 28 | + Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models"` |
| 29 | + AutoloadGalleries bool `env:"LOCALAI_AUTOLOAD_GALLERIES,AUTOLOAD_GALLERIES" group:"models"` |
| 30 | + RemoteLibrary string `env:"LOCALAI_REMOTE_LIBRARY,REMOTE_LIBRARY" default:"${remoteLibraryURL}" help:"A LocalAI remote library URL" group:"models"` |
| 31 | + PreloadModels string `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"` |
| 32 | + Models []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"` |
| 33 | + PreloadModelsConfig string `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"` |
| 34 | + |
| 35 | + F16 bool `name:"f16" env:"LOCALAI_F16,F16" help:"Enable GPU acceleration" group:"performance"` |
| 36 | + Threads int `env:"LOCALAI_THREADS,THREADS" short:"t" default:"4" help:"Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested" group:"performance"` |
| 37 | + ContextSize int `env:"LOCALAI_CONTEXT_SIZE,CONTEXT_SIZE" default:"512" help:"Default context size for models" group:"performance"` |
| 38 | + |
| 39 | + Address string `env:"LOCALAI_ADDRESS,ADDRESS" default:":8080" help:"Bind address for the API server" group:"api"` |
| 40 | + CORS bool `env:"LOCALAI_CORS,CORS" help:"" group:"api"` |
| 41 | + CORSAllowOrigins string `env:"LOCALAI_CORS_ALLOW_ORIGINS,CORS_ALLOW_ORIGINS" group:"api"` |
| 42 | + UploadLimit int `env:"LOCALAI_UPLOAD_LIMIT,UPLOAD_LIMIT" default:"15" help:"Default upload-limit in MB" group:"api"` |
| 43 | + APIKeys []string `env:"LOCALAI_API_KEY,API_KEY" help:"List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys" group:"api"` |
| 44 | + DisableWelcome bool `env:"LOCALAI_DISABLE_WELCOME,DISABLE_WELCOME" default:"false" help:"Disable welcome pages" group:"api"` |
| 45 | + |
| 46 | + ParallelRequests bool `env:"LOCALAI_PARALLEL_REQUESTS,PARALLEL_REQUESTS" help:"Enable backends to handle multiple requests in parallel if they support it (e.g.: llama.cpp or vllm)" group:"backends"` |
| 47 | + SingleActiveBackend bool `env:"LOCALAI_SINGLE_ACTIVE_BACKEND,SINGLE_ACTIVE_BACKEND" help:"Allow only one backend to be run at a time" group:"backends"` |
| 48 | + PreloadBackendOnly bool `env:"LOCALAI_PRELOAD_BACKEND_ONLY,PRELOAD_BACKEND_ONLY" default:"false" help:"Do not launch the API services, only the preloaded models / backends are started (useful for multi-node setups)" group:"backends"` |
| 49 | + ExternalGRPCBackends []string `env:"LOCALAI_EXTERNAL_GRPC_BACKENDS,EXTERNAL_GRPC_BACKENDS" help:"A list of external grpc backends" group:"backends"` |
| 50 | + EnableWatchdogIdle bool `env:"LOCALAI_WATCHDOG_IDLE,WATCHDOG_IDLE" default:"false" help:"Enable watchdog for stopping backends that are idle longer than the watchdog-idle-timeout" group:"backends"` |
| 51 | + WatchdogIdleTimeout string `env:"LOCALAI_WATCHDOG_IDLE_TIMEOUT,WATCHDOG_IDLE_TIMEOUT" default:"15m" help:"Threshold beyond which an idle backend should be stopped" group:"backends"` |
| 52 | + EnableWatchdogBusy bool `env:"LOCALAI_WATCHDOG_BUSY,WATCHDOG_BUSY" default:"false" help:"Enable watchdog for stopping backends that are busy longer than the watchdog-busy-timeout" group:"backends"` |
| 53 | + WatchdogBusyTimeout string `env:"LOCALAI_WATCHDOG_BUSY_TIMEOUT,WATCHDOG_BUSY_TIMEOUT" default:"5m" help:"Threshold beyond which a busy backend should be stopped" group:"backends"` |
| 54 | +} |
| 55 | + |
| 56 | +func (r *RunCMD) Run(ctx *Context) error { |
| 57 | + opts := []config.AppOption{ |
| 58 | + config.WithConfigFile(r.ModelsConfigFile), |
| 59 | + config.WithJSONStringPreload(r.PreloadModels), |
| 60 | + config.WithYAMLConfigPreload(r.PreloadModelsConfig), |
| 61 | + config.WithModelPath(r.ModelsPath), |
| 62 | + config.WithContextSize(r.ContextSize), |
| 63 | + config.WithDebug(ctx.Debug), |
| 64 | + config.WithImageDir(r.ImagePath), |
| 65 | + config.WithAudioDir(r.AudioPath), |
| 66 | + config.WithUploadDir(r.UploadPath), |
| 67 | + config.WithConfigsDir(r.ConfigPath), |
| 68 | + config.WithF16(r.F16), |
| 69 | + config.WithStringGalleries(r.Galleries), |
| 70 | + config.WithModelLibraryURL(r.RemoteLibrary), |
| 71 | + config.WithDisableMessage(false), |
| 72 | + config.WithCors(r.CORS), |
| 73 | + config.WithCorsAllowOrigins(r.CORSAllowOrigins), |
| 74 | + config.WithThreads(r.Threads), |
| 75 | + config.WithBackendAssets(ctx.BackendAssets), |
| 76 | + config.WithBackendAssetsOutput(r.BackendAssetsPath), |
| 77 | + config.WithUploadLimitMB(r.UploadLimit), |
| 78 | + config.WithApiKeys(r.APIKeys), |
| 79 | + config.WithModelsURL(append(r.Models, r.ModelArgs...)...), |
| 80 | + } |
| 81 | + |
| 82 | + idleWatchDog := r.EnableWatchdogIdle |
| 83 | + busyWatchDog := r.EnableWatchdogBusy |
| 84 | + |
| 85 | + if r.DisableWelcome { |
| 86 | + opts = append(opts, config.DisableWelcomePage) |
| 87 | + } |
| 88 | + |
| 89 | + if idleWatchDog || busyWatchDog { |
| 90 | + opts = append(opts, config.EnableWatchDog) |
| 91 | + if idleWatchDog { |
| 92 | + opts = append(opts, config.EnableWatchDogIdleCheck) |
| 93 | + dur, err := time.ParseDuration(r.WatchdogIdleTimeout) |
| 94 | + if err != nil { |
| 95 | + return err |
| 96 | + } |
| 97 | + opts = append(opts, config.SetWatchDogIdleTimeout(dur)) |
| 98 | + } |
| 99 | + if busyWatchDog { |
| 100 | + opts = append(opts, config.EnableWatchDogBusyCheck) |
| 101 | + dur, err := time.ParseDuration(r.WatchdogBusyTimeout) |
| 102 | + if err != nil { |
| 103 | + return err |
| 104 | + } |
| 105 | + opts = append(opts, config.SetWatchDogBusyTimeout(dur)) |
| 106 | + } |
| 107 | + } |
| 108 | + if r.ParallelRequests { |
| 109 | + opts = append(opts, config.EnableParallelBackendRequests) |
| 110 | + } |
| 111 | + if r.SingleActiveBackend { |
| 112 | + opts = append(opts, config.EnableSingleBackend) |
| 113 | + } |
| 114 | + |
| 115 | + // split ":" to get backend name and the uri |
| 116 | + for _, v := range r.ExternalGRPCBackends { |
| 117 | + backend := v[:strings.IndexByte(v, ':')] |
| 118 | + uri := v[strings.IndexByte(v, ':')+1:] |
| 119 | + opts = append(opts, config.WithExternalBackend(backend, uri)) |
| 120 | + } |
| 121 | + |
| 122 | + if r.AutoloadGalleries { |
| 123 | + opts = append(opts, config.EnableGalleriesAutoload) |
| 124 | + } |
| 125 | + |
| 126 | + if r.PreloadBackendOnly { |
| 127 | + _, _, _, err := startup.Startup(opts...) |
| 128 | + return err |
| 129 | + } |
| 130 | + |
| 131 | + cl, ml, options, err := startup.Startup(opts...) |
| 132 | + |
| 133 | + if err != nil { |
| 134 | + return fmt.Errorf("failed basic startup tasks with error %s", err.Error()) |
| 135 | + } |
| 136 | + |
| 137 | + // Watch the configuration directory |
| 138 | + // If the directory does not exist, we don't watch it |
| 139 | + if _, err := os.Stat(r.LocalaiConfigDir); err == nil { |
| 140 | + closeConfigWatcherFn, err := startup.WatchConfigDirectory(r.LocalaiConfigDir, options) |
| 141 | + defer closeConfigWatcherFn() |
| 142 | + |
| 143 | + if err != nil { |
| 144 | + return fmt.Errorf("failed while watching configuration directory %s", r.LocalaiConfigDir) |
| 145 | + } |
| 146 | + } |
| 147 | + |
| 148 | + appHTTP, err := http.App(cl, ml, options) |
| 149 | + if err != nil { |
| 150 | + log.Error().Err(err).Msg("error during HTTP App construction") |
| 151 | + return err |
| 152 | + } |
| 153 | + |
| 154 | + return appHTTP.Listen(r.Address) |
| 155 | +} |
0 commit comments