Skip to content

Commit 596d511

Browse files
Add DecodeSpecialTokens property to IInferenceParams - implement it where applicable. Use the new property in StatelessExecutor
1 parent ac902ad commit 596d511

File tree

4 files changed

+18
-1
lines changed

4 files changed

+18
-1
lines changed

LLama.Web/Common/InferenceOptions.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,8 @@ public class InferenceOptions
2121

2222
/// <inheritdoc />
2323
public ISamplingPipeline SamplingPipeline { get; set; } = new DefaultSamplingPipeline();
24+
25+
/// <inheritdoc />
26+
public bool DecodeSpecialTokens { get; set; }
2427
}
2528
}

LLama/Abstractions/IInferenceParams.cs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,5 +28,13 @@ public interface IInferenceParams
2828
/// Set a custom sampling pipeline to use.
2929
/// </summary>
3030
ISamplingPipeline SamplingPipeline { get; set; }
31+
32+
/// <summary>
33+
/// If true, special characters will be converted to text. If false they will be invisible.
34+
/// </summary>
35+
/// <remark>
36+
/// Controls the behavior of decoders like <see cref="StreamingTokenDecoder" />
37+
/// </remark>
38+
public bool DecodeSpecialTokens { get; set; }
3139
}
3240
}

LLama/Common/InferenceParams.cs

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ public record InferenceParams
3030

3131
/// <inheritdoc />
3232
public ISamplingPipeline SamplingPipeline { get; set; } = new DefaultSamplingPipeline();
33+
34+
/// <inheritdoc />
35+
public bool DecodeSpecialTokens { get; set; }
3336
}
3437

3538
/// <summary>

LLama/LLamaStatelessExecutor.cs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,10 @@ public async IAsyncEnumerable<string> InferAsync(string prompt, IInferenceParams
8888
throw new ArgumentOutOfRangeException(nameof(inferenceParams), $"TokensKeep ({inferenceParams.TokensKeep}) cannot be larger than ContextSize ({Context.ContextSize})");
8989

9090
// Create decoders for the token stream
91-
var decoder = new StreamingTokenDecoder(Context);
91+
var decoder = new StreamingTokenDecoder(Context)
92+
{
93+
DecodeSpecialTokens = inferenceParams.DecodeSpecialTokens,
94+
};
9295
var antiprocessor = new AntipromptProcessor(inferenceParams.AntiPrompts);
9396

9497
if (ApplyTemplate)

0 commit comments

Comments
 (0)