Basic ASP.NET Core website example

LLama.Web/Hubs/ISessionClient.cs

@@ -0,0 +1,11 @@
+using LLama.Web.Models;
+
+namespace LLama.Web.Hubs
+{
+    public interface ISessionClient
+    {
+        Task OnStatus(string status, string data = null);
+        Task OnResponse(ResponseFragment fragment);
+        Task OnError(string error);
+    }
+}

LLama.Web/Hubs/InteractiveHub.cs

@@ -0,0 +1,97 @@
+using LLama.Web.Models;
+using LLama.Web.Services;
+using Microsoft.AspNetCore.SignalR;
+using Microsoft.Extensions.Options;
+using System.Diagnostics;
+
+namespace LLama.Web.Hubs
+{
+    public class InteractiveHub : Hub<ISessionClient>
+    {
+        private readonly LLamaOptions _options;
+        private readonly ILogger<InteractiveHub> _logger;
+        private readonly IModelSessionService _modelSessionService;
+
+        public InteractiveHub(ILogger<InteractiveHub> logger, IOptions<LLamaOptions> options, IModelSessionService modelSessionService)
+        {
+            _logger = logger;
+            _options = options.Value;
+            _modelSessionService = modelSessionService;
+        }
+
+
+        public override async Task OnConnectedAsync()
+        {
+            _logger.Log(LogLevel.Information, "OnConnectedAsync, Id: {0}", Context.ConnectionId);
+            await base.OnConnectedAsync();
+            await Clients.Caller.OnStatus("Connected", Context.ConnectionId);
+        }
+
+
+        public override async Task OnDisconnectedAsync(Exception? exception)
+        {
+            _logger.Log(LogLevel.Information, "[OnDisconnectedAsync], Id: {0}", Context.ConnectionId);
+           await _modelSessionService.RemoveAsync(Context.ConnectionId);
+            await base.OnDisconnectedAsync(exception);
+        }
+
+
+        [HubMethodName("LoadModel")]
+        public async Task OnLoadModel(string modelName, string promptName, string parameterName)
+        {
+            _logger.Log(LogLevel.Information, "[OnLoadModel] - Load new model, Connection: {0}, Model: {1}, Prompt: {2}, Parameter: {3}", Context.ConnectionId, modelName, promptName, parameterName);
+            await _modelSessionService.RemoveAsync(Context.ConnectionId);
+
+            var modelOption = _options.Models.First(x => x.Name == modelName);
+            var promptOption = _options.Prompts.First(x => x.Name == promptName);
+            var parameterOption = _options.Parameters.First(x => x.Name == parameterName);
+            var interactiveExecutor = new InteractiveExecutor(new LLamaModel(modelOption));
+            var modelSession = await _modelSessionService.CreateAsync(Context.ConnectionId, interactiveExecutor,  modelOption, promptOption, parameterOption);
+            if (modelSession is null)
+            {
+                _logger.Log(LogLevel.Error, "[OnLoadModel] - Failed to add new model session, Connection: {0}", Context.ConnectionId);
+                await Clients.Caller.OnError("No model has been loaded");
+                return;
+
+            }
+            _logger.Log(LogLevel.Information, "[OnLoadModel] - New model session added, Connection: {0}", Context.ConnectionId);
+            await Clients.Caller.OnStatus("Loaded", Context.ConnectionId);
+        }
+
+
+        [HubMethodName("SendPrompt")]
+        public async Task OnSendPrompt(string prompt)
+        {
+            var stopwatch = Stopwatch.GetTimestamp();
+            _logger.Log(LogLevel.Information, "[OnSendPrompt] - New prompt received, Connection: {0}", Context.ConnectionId);
+            var modelSession = await _modelSessionService.GetAsync(Context.ConnectionId);
+            if (modelSession is null)
+            {
+                _logger.Log(LogLevel.Warning, "[OnSendPrompt] -  No model has been loaded for this connection, Connection: {0}", Context.ConnectionId);
+                await Clients.Caller.OnError("No model has been loaded");
+                return;
+            }
+
+            // Create unique response id
+            var responseId = Guid.NewGuid().ToString();
+
+            // Send begin of response
+            await Clients.Caller.OnResponse(new ResponseFragment(responseId, isFirst: true));
+
+            // Send content of response
+            await foreach (var fragment in modelSession.InferAsync(prompt, CancellationTokenSource.CreateLinkedTokenSource(Context.ConnectionAborted)))
+            {
+                await Clients.Caller.OnResponse(new ResponseFragment(responseId, fragment));
+            }
+
+            // Send end of response
+            var elapsedTime = Stopwatch.GetElapsedTime(stopwatch);
+            var signature = modelSession.IsInferCanceled()
+                ? $"Inference cancelled after {elapsedTime.TotalSeconds:F0} seconds"
+                : $"Inference completed in {elapsedTime.TotalSeconds:F0} seconds";
+            await Clients.Caller.OnResponse(new ResponseFragment(responseId, signature, isLast: true));
+            _logger.Log(LogLevel.Information, "[OnSendPrompt] - Inference complete, Connection: {0}, Elapsed: {1}, Canceled: {2}", Context.ConnectionId, elapsedTime, modelSession.IsInferCanceled());
+        }
+
+    }
+}

LLama.Web/LLama.Web.csproj

@@ -0,0 +1,17 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+  <PropertyGroup>
+    <TargetFramework>net7.0</TargetFramework>
+    <Nullable>disable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\LLama\LLamaSharp.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Folder Include="wwwroot\image\" />
+  </ItemGroup>
+
+</Project>

LLama.Web/Models/CancelModel.cs

@@ -0,0 +1,7 @@
+namespace LLama.Web.Models
+{
+    public class CancelModel
+    {
+        public string ConnectionId { get; set; }
+    }
+}

LLama.Web/Models/LLamaOptions.cs

@@ -0,0 +1,20 @@
+namespace LLama.Web.Models
+{
+    public class LLamaOptions
+    {
+        public List<ModelOptions> Models { get; set; }
+        public List<PromptOptions> Prompts { get; set; } = new List<PromptOptions>();
+        public List<ParameterOptions> Parameters { get; set; } = new List<ParameterOptions>();
+
+        public void Initialize()
+        {
+            foreach (var prompt in Prompts)
+            {
+                if (File.Exists(prompt.Path))
+                {
+                    prompt.Prompt = File.ReadAllText(prompt.Path).Trim();
+                }
+            }
+        }
+    }
+}

LLama.Web/Models/ModelOptions.cs

@@ -0,0 +1,15 @@
+using LLama.Common;
+
+namespace LLama.Web.Models
+{
+    public class ModelOptions : ModelParams
+    {
+        public ModelOptions() : base("", 512, 20, 1337, true, true, false, false, "", "", -1, 512, false, false)
+        {
+        }
+
+        public string Name { get; set; }
+        public int MaxInstances { get; set; }
+
+    }
+}

LLama.Web/Models/ModelSession.cs

@@ -0,0 +1,64 @@
+using LLama.Abstractions;
+
+namespace LLama.Web.Models
+{
+    public class ModelSession : IDisposable
+    {
+        private bool _isFirstInteraction = true;
+        private ModelOptions _modelOptions;
+        private PromptOptions _promptOptions;
+        private ParameterOptions _inferenceOptions;
+        private ITextStreamTransform _outputTransform;
+        private ILLamaExecutor _executor;
+        private CancellationTokenSource _cancellationTokenSource;
+
+        public ModelSession(string connectionId, ILLamaExecutor executor, ModelOptions modelOptions, PromptOptions promptOptions, ParameterOptions parameterOptions)
+        {
+            ConnectionId = connectionId;
+            _executor = executor;
+            _modelOptions = modelOptions;
+            _promptOptions = promptOptions;
+            _inferenceOptions = parameterOptions;
+
+            _inferenceOptions.AntiPrompts = _promptOptions.AntiPrompt?.Concat(_inferenceOptions.AntiPrompts ?? Enumerable.Empty<string>()).Distinct() ?? _inferenceOptions.AntiPrompts;
+            if (_promptOptions.OutputFilter?.Count > 0)
+                _outputTransform = new LLamaTransforms.KeywordTextOutputStreamTransform(_promptOptions.OutputFilter, redundancyLength: 5);
+        }
+
+        public string ConnectionId { get; }
+
+        public IAsyncEnumerable<string> InferAsync(string message, CancellationTokenSource cancellationTokenSource)
+        {
+            _cancellationTokenSource = cancellationTokenSource;
+            if (_isFirstInteraction)
+            {
+                _isFirstInteraction = false;
+                message = _promptOptions.Prompt + message;
+            }
+
+            if (_outputTransform is not null)
+                return _outputTransform.TransformAsync(_executor.InferAsync(message, _inferenceOptions, _cancellationTokenSource.Token));
+
+            return _executor.InferAsync(message, _inferenceOptions, _cancellationTokenSource.Token);
+        }
+
+
+        public void CancelInfer()
+        {
+            _cancellationTokenSource?.Cancel();
+        }
+
+        public bool IsInferCanceled()
+        {
+            return _cancellationTokenSource.IsCancellationRequested;
+        }
+
+        public void Dispose()
+        {
+            _inferenceOptions = null;
+            _outputTransform = null;
+            _executor.Model?.Dispose();
+            _executor = null;
+        }
+    }
+}

LLama.Web/Models/ParameterOptions.cs

@@ -0,0 +1,9 @@
+using LLama.Common;
+
+namespace LLama.Web.Models
+{
+    public class ParameterOptions : InferenceParams
+    {
+        public string Name { get; set; }
+    }
+}

LLama.Web/Models/PromptOptions.cs

@@ -0,0 +1,11 @@
+namespace LLama.Web.Models
+{
+    public class PromptOptions
+    {
+        public string Name { get; set; }
+        public string Path { get; set; }
+        public string Prompt { get; set; }
+        public List<string> AntiPrompt { get; set; }
+        public List<string> OutputFilter { get; set; }
+    }
+}

LLama.Web/Models/ResponseFragment.cs

@@ -0,0 +1,18 @@
+namespace LLama.Web.Models
+{
+    public class ResponseFragment
+    {
+        public ResponseFragment(string id, string content = null, bool isFirst = false, bool isLast = false)
+        {
+            Id = id;
+            IsLast = isLast;
+            IsFirst = isFirst;
+            Content = content;
+        }
+
+        public string Id { get; set; }
+        public string Content { get; set; }
+        public bool IsLast { get; set; }
+        public bool IsFirst { get; set; }
+    }
+}

LLama.Web/Pages/Error.cshtml

@@ -0,0 +1,26 @@
+@page
+@model ErrorModel
+@{
+    ViewData["Title"] = "Error";
+}
+
+<h1 class="text-danger">Error.</h1>
+<h2 class="text-danger">An error occurred while processing your request.</h2>
+
+@if (Model.ShowRequestId)
+{
+    <p>
+        <strong>Request ID:</strong> <code>@Model.RequestId</code>
+    </p>
+}
+
+<h3>Development Mode</h3>
+<p>
+    Swapping to the <strong>Development</strong> environment displays detailed information about the error that occurred.
+</p>
+<p>
+    <strong>The Development environment shouldn't be enabled for deployed applications.</strong>
+    It can result in displaying sensitive information from exceptions to end users.
+    For local debugging, enable the <strong>Development</strong> environment by setting the <strong>ASPNETCORE_ENVIRONMENT</strong> environment variable to <strong>Development</strong>
+    and restarting the app.
+</p>

LLama.Web/Pages/Error.cshtml.cs

@@ -0,0 +1,27 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+using System.Diagnostics;
+
+namespace LLama.Web.Pages
+{
+    [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)]
+    [IgnoreAntiforgeryToken]
+    public class ErrorModel : PageModel
+    {
+        public string? RequestId { get; set; }
+
+        public bool ShowRequestId => !string.IsNullOrEmpty(RequestId);
+
+        private readonly ILogger<ErrorModel> _logger;
+
+        public ErrorModel(ILogger<ErrorModel> logger)
+        {
+            _logger = logger;
+        }
+
+        public void OnGet()
+        {
+            RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier;
+        }
+    }
+}

LLama.Web/Pages/Index.cshtml

@@ -0,0 +1,10 @@
+@page
+@model IndexModel
+@{
+    ViewData["Title"] = "Home page";
+}
+
+<div class="text-center">
+    <h1 class="display-4">Welcome</h1>
+    <p>Learn about <a href="https://docs.microsoft.com/aspnet/core">building Web apps with ASP.NET Core</a>.</p>
+</div>

LLama.Web/Pages/Index.cshtml.cs

@@ -0,0 +1,20 @@
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Mvc.RazorPages;
+
+namespace LLama.Web.Pages
+{
+    public class IndexModel : PageModel
+    {
+        private readonly ILogger<IndexModel> _logger;
+
+        public IndexModel(ILogger<IndexModel> logger)
+        {
+            _logger = logger;
+        }
+
+        public void OnGet()
+        {
+
+        }
+    }
+}