|
| 1 | +/* |
| 2 | + * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. |
| 3 | + * SPDX-License-Identifier: Apache-2.0 |
| 4 | + * |
| 5 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | + * you may not use this file except in compliance with the License. |
| 7 | + * You may obtain a copy of the License at |
| 8 | + * |
| 9 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | + * |
| 11 | + * Unless required by applicable law or agreed to in writing, software |
| 12 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | + * See the License for the specific language governing permissions and |
| 15 | + * limitations under the License. |
| 16 | + */ |
| 17 | + |
| 18 | +#pragma once |
| 19 | + |
| 20 | +#include "tensorrt_llm/batch_manager/common.h" |
| 21 | +#include "tensorrt_llm/runtime/bufferManager.h" |
| 22 | +#include "tensorrt_llm/runtime/iTensor.h" |
| 23 | +#include "tensorrt_llm/runtime/modelConfig.h" |
| 24 | +#include "tensorrt_llm/runtime/promptTuningParams.h" |
| 25 | +#include "tensorrt_llm/runtime/worldConfig.h" |
| 26 | + |
| 27 | +namespace tensorrt_llm::batch_manager |
| 28 | +{ |
| 29 | + |
| 30 | +class PromptTuningBuffers |
| 31 | +{ |
| 32 | + |
| 33 | +public: |
| 34 | + using SizeType32 = tensorrt_llm::runtime::SizeType32; |
| 35 | + using ITensor = tensorrt_llm::runtime::ITensor; |
| 36 | + using TensorPtr = runtime::ITensor::SharedPtr; |
| 37 | + |
| 38 | + runtime::PromptTuningParams mPromptTuningParams; |
| 39 | + SizeType32 mMaxPromptVocabSize; |
| 40 | + |
| 41 | + PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, |
| 42 | + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig); |
| 43 | + |
| 44 | + PromptTuningBuffers(SizeType32 maxBatchSize, runtime::BufferManager const& manager, |
| 45 | + runtime::ModelConfig const& modelConfig, runtime::WorldConfig const& worldConfig, bool promptTableOffloading); |
| 46 | + |
| 47 | + void validate(std::optional<TensorPtr> const& optReqPromptEmbeddingTable, |
| 48 | + std::optional<SizeType32> const& optReqPromptVocabSize); |
| 49 | + |
| 50 | + void fill(RequestVector const& contextRequests, RequestVector const& genRequests, |
| 51 | + runtime::BufferManager const& manager, bool packed); |
| 52 | + |
| 53 | + /* |
| 54 | + * The below functions are specific for Chunked Prefill mode |
| 55 | + * Chunk Ptable with Ping-Pong Buffer Implementation |
| 56 | + * ----------------------------------------------- |
| 57 | + * |
| 58 | + * Overview: |
| 59 | + * The chunk ptable (prompt tuning table) system uses a ping-pong buffer mechanism to efficiently |
| 60 | + * manage large embedding tables when operating in context Prefill mode. This allows |
| 61 | + * for processing of large embedding tables by loading them in chunks from CPU to GPU memory, |
| 62 | + * enabling support for tables that exceed available GPU memory. |
| 63 | + * |
| 64 | + * Key Components: |
| 65 | + * 1. Ping-Pong Buffers (mChunkPtableBuffers): |
| 66 | + * - Two alternating GPU buffers that store chunks of the embedding table |
| 67 | + * - While the current buffer is being processed by the model, |
| 68 | + * the next chunk can be asynchronously loaded into the other buffer |
| 69 | + * - Managed through mChunkPtableCurrentIndex (toggles between 0 and 1) |
| 70 | + * 2. Start Positions Tracking (mChunkPtableBufferStartPositions): |
| 71 | + * - Mainly used for multi-batch processing |
| 72 | + * - Maintains the starting position of each batch's data within each buffer |
| 73 | + * - Maintained separately for each ping-pong buffer |
| 74 | + * |
| 75 | + * Memory Optimization: |
| 76 | + * - Only two GPU buffers are maintained regardless of total embedding table size |
| 77 | + * - Each buffer size is limited to contextChunkSize * hiddenSize |
| 78 | + * - Efficient memory usage through chunk-based processing |
| 79 | + */ |
| 80 | + |
| 81 | + bool mPromptTableOffloading; |
| 82 | + |
| 83 | + bool mChunkPtableInitialized{false}; |
| 84 | + std::optional<std::array<TensorPtr, 2>> mChunkPtableBuffers; |
| 85 | + std::optional<std::vector<std::vector<SizeType32>>> mChunkPtableBufferStartPositions; |
| 86 | + size_t mChunkPtableCurrentIndex{0}; |
| 87 | + |
| 88 | + void initializeChunkPtableBuffers(runtime::BufferManager const& manager, runtime::ModelConfig const& modelConfig, |
| 89 | + SizeType32 contextChunkSize, std::shared_ptr<LlmRequest> const& llmReq); |
| 90 | + |
| 91 | + void switchChunkPtableBuffer(); |
| 92 | + |
| 93 | + size_t getChunkPtableCurrentIndex(); |
| 94 | + |
| 95 | + [[nodiscard]] TensorPtr& getChunkPtableBuffer(size_t index); |
| 96 | + |
| 97 | + [[nodiscard]] SizeType32 getChunkPtableBufferSliceSize(size_t index, size_t batchIdx); |
| 98 | + |
| 99 | + [[nodiscard]] SizeType32 getChunkPtableBufferStartPosition(size_t index, size_t batchIdx); |
| 100 | + |
| 101 | + void updateBufferStartPosition(size_t index, SizeType32 numRows); |
| 102 | + |
| 103 | + void clearBufferStartPositions(size_t index); |
| 104 | +}; |
| 105 | + |
| 106 | +} // namespace tensorrt_llm::batch_manager |
0 commit comments