Skip to content

Commit 8ef2e9c

Browse files
lpawelczlpawelcz
committed
modules/zstd: Add block decoder module
This adds a decoder of block data. It decodes block header and demuxes remaining input data into one of specific block decoders depending on the type of the parsed block. Then it muxes outputs from those decoders into single output channel. Internal-tag: [#51873] Signed-off-by: Pawel Czarnecki <[email protected]>
1 parent bc16a91 commit 8ef2e9c

File tree

2 files changed

+188
-0
lines changed

2 files changed

+188
-0
lines changed

‎xls/modules/zstd/BUILD

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,3 +220,22 @@ xls_dslx_test(
220220
name = "dec_demux_dslx_test",
221221
library = ":dec_demux_dslx",
222222
)
223+
224+
xls_dslx_library(
225+
name = "block_dec_dslx",
226+
srcs = [
227+
"block_dec.x",
228+
],
229+
deps = [
230+
":common_dslx",
231+
":dec_demux_dslx",
232+
":raw_block_dec_dslx",
233+
":rle_block_dec_dslx",
234+
":dec_mux_dslx",
235+
],
236+
)
237+
238+
xls_dslx_test(
239+
name = "block_dec_dslx_test",
240+
library = ":block_dec_dslx",
241+
)

‎xls/modules/zstd/block_dec.x

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
// Copyright 2023 The XLS Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import xls.modules.zstd.common;
16+
import xls.modules.zstd.dec_demux as demux;
17+
import xls.modules.zstd.raw_block_dec as raw;
18+
import xls.modules.zstd.rle_block_dec as rle;
19+
import xls.modules.zstd.dec_mux as mux;
20+
21+
type BlockDataPacket = common::BlockDataPacket;
22+
type BlockData = common::BlockData;
23+
type BlockPacketLength = common::BlockPacketLength;
24+
type ExtendedBlockDataPacket = common::ExtendedBlockDataPacket;
25+
type CopyOrMatchContent = common::CopyOrMatchContent;
26+
type CopyOrMatchLength = common::CopyOrMatchLength;
27+
type SequenceExecutorPacket = common::SequenceExecutorPacket;
28+
type SequenceExecutorMessageType = common::SequenceExecutorMessageType;
29+
30+
// Proc responsible for connecting internal procs used in Block data decoding.
31+
// It handles incoming block data packets by redirecting those to demuxer which passes those to
32+
// block decoder procs specific for given block type. Results are then gathered by mux which
33+
// transfers decoded data further. The connections are visualised on the following diagram:
34+
//
35+
// Block Decoder
36+
// ┌───────────────────────────────────────┐
37+
// │ Raw Block Decoder │
38+
// │ ┌───────────────────┐ │
39+
// │ ┌─► ├┐ │
40+
// │ Demux │ └───────────────────┘│ Mux │
41+
// │┌─────┐│ Rle Block Decoder │ ┌─────┐│
42+
// ││ ├┘ ┌───────────────────┐└─► ││
43+
// ──┼► ├──► ├──► ├┼─►
44+
// ││ ├┐ └───────────────────┘┌─► ││
45+
// │└─────┘│ Cmp Block Decoder │ └─────┘│
46+
// │ │ ┌───────────────────┐│ │
47+
// │ └─► ├┘ │
48+
// │ └───────────────────┘ │
49+
// └───────────────────────────────────────┘
50+
51+
proc BlockDecoder {
52+
input_r: chan<BlockDataPacket> in;
53+
output_s: chan<SequenceExecutorPacket> out;
54+
55+
config (input_r: chan<BlockDataPacket> in, output_s: chan<SequenceExecutorPacket> out) {
56+
let (demux_raw_s, demux_raw_r) = chan<BlockDataPacket>;
57+
let (demux_rle_s, demux_rle_r) = chan<BlockDataPacket>;
58+
let (demux_cmp_s, demux_cmp_r) = chan<BlockDataPacket>;
59+
let (mux_raw_s, mux_raw_r) = chan<ExtendedBlockDataPacket>;
60+
let (mux_rle_s, mux_rle_r) = chan<ExtendedBlockDataPacket>;
61+
let (mux_cmp_s, mux_cmp_r) = chan<ExtendedBlockDataPacket>;
62+
63+
spawn demux::DecoderDemux(input_r, demux_raw_s, demux_rle_s, demux_cmp_s);
64+
spawn raw::RawBlockDecoder(demux_raw_r, mux_raw_s);
65+
spawn rle::RleBlockDecoder(demux_rle_r, mux_rle_s);
66+
// TODO(lpawelcz): 2023-11-28 change to compressed block decoder proc
67+
spawn raw::RawBlockDecoder(demux_cmp_r, mux_cmp_s);
68+
spawn mux::DecoderMux(mux_raw_r, mux_rle_r, mux_cmp_r, output_s);
69+
70+
(input_r, output_s)
71+
}
72+
73+
init { }
74+
75+
next(tok: token, state: ()) { }
76+
}
77+
78+
#[test_proc]
79+
proc BlockDecoderTest {
80+
terminator: chan<bool> out;
81+
input_s: chan<BlockDataPacket> out;
82+
output_r: chan<SequenceExecutorPacket> in;
83+
84+
init {}
85+
86+
config (terminator: chan<bool> out) {
87+
let (input_s, input_r) = chan<BlockDataPacket>;
88+
let (output_s, output_r) = chan<SequenceExecutorPacket>;
89+
90+
spawn BlockDecoder(input_r, output_s);
91+
92+
(terminator, input_s, output_r)
93+
}
94+
95+
next(tok: token, state: ()) {
96+
let EncodedDataBlocksPackets: BlockDataPacket[13] = [
97+
// RAW Block 1 byte
98+
BlockDataPacket { id: u32:0, last: true, last_block: false, data: BlockData:0xDE000008, length: BlockPacketLength:32 },
99+
// RAW Block 2 bytes
100+
BlockDataPacket { id: u32:1, last: true, last_block: false, data: BlockData:0xDEAD000010, length: BlockPacketLength:40 },
101+
// RAW Block 4 bytes
102+
BlockDataPacket { id: u32:2, last: true, last_block: false, data: BlockData:0xDEADBEEF000020, length: BlockPacketLength:56 },
103+
// RAW Block 5 bytes (block header takes one full packet)
104+
BlockDataPacket { id: u32:3, last: true, last_block: false, data: BlockData:0xDEADBEEFEF000028, length: BlockPacketLength:64 },
105+
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
106+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x12345678900000C0, length: BlockPacketLength:64 },
107+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x1234567890ABCDEF, length: BlockPacketLength:64 },
108+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0xFEDCBA0987654321, length: BlockPacketLength:64 },
109+
BlockDataPacket { id: u32:4, last: true, last_block: false, data: BlockData:0xF0F0F0, length: BlockPacketLength:24 },
110+
111+
// RLE Block 1 byte
112+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x6700000a, length: BlockPacketLength:32 },
113+
// RLE Block 2 bytes
114+
BlockDataPacket { id: u32:6, last: true, last_block: false, data: BlockData:0x45000012, length: BlockPacketLength:32 },
115+
// RLE Block 4 bytes
116+
BlockDataPacket { id: u32:7, last: true, last_block: false, data: BlockData:0x23000022, length: BlockPacketLength:32 },
117+
// RLE Block 8 bytes (block takes one full packet)
118+
BlockDataPacket { id: u32:8, last: true, last_block: false, data: BlockData:0x10000042, length: BlockPacketLength:32 },
119+
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
120+
BlockDataPacket { id: u32:9, last: true, last_block: true, data: BlockData:0xDE0000d2, length: BlockPacketLength:32 },
121+
];
122+
123+
let tok = for ((counter, block_packet), tok): ((u32, BlockDataPacket), token) in enumerate(EncodedDataBlocksPackets) {
124+
let tok = send(tok, input_s, block_packet);
125+
trace_fmt!("Sent #{} encoded block packet, {:#x}", counter + u32:1, block_packet);
126+
(tok)
127+
}(tok);
128+
129+
let DecodedDataBlocksPackets: SequenceExecutorPacket[16] = [
130+
// RAW Block 1 byte
131+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDE, length: CopyOrMatchLength:8 },
132+
// RAW Block 2 bytes
133+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEAD, length: CopyOrMatchLength:16 },
134+
// RAW Block 4 bytes
135+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEADBEEF, length: CopyOrMatchLength:32 },
136+
// RAW Block 5 bytes (block header takes one full packet)
137+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEADBEEFEF, length: CopyOrMatchLength:40 },
138+
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
139+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0x1234567890, length: CopyOrMatchLength:40 },
140+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0x1234567890ABCDEF, length: CopyOrMatchLength:64 },
141+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xFEDCBA0987654321, length: CopyOrMatchLength:64 },
142+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xF0F0F0, length: CopyOrMatchLength:24 },
143+
144+
// RLE Block 1 byte
145+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0x67, length: CopyOrMatchLength:8 },
146+
// RLE Block 2 bytes
147+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0x4545, length: CopyOrMatchLength:16 },
148+
// RLE Block 4 bytes
149+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0x23232323, length: CopyOrMatchLength:32 },
150+
// RLE Block 8 bytes (block takes one full packet)
151+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0x1010101010101010, length: CopyOrMatchLength:64 },
152+
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
153+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEDEDEDEDEDEDEDE, length: CopyOrMatchLength:64 },
154+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEDEDEDEDEDEDEDE, length: CopyOrMatchLength:64 },
155+
SequenceExecutorPacket { last: false, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEDEDEDEDEDEDEDE, length: CopyOrMatchLength:64 },
156+
SequenceExecutorPacket { last: true, msg_type: SequenceExecutorMessageType::LITERAL, content: CopyOrMatchContent:0xDEDE, length: CopyOrMatchLength:16 },
157+
];
158+
159+
let tok = for ((counter, expected_block_packet), tok): ((u32, SequenceExecutorPacket), token) in enumerate(DecodedDataBlocksPackets) {
160+
let (tok, decoded_block_packet) = recv(tok, output_r);
161+
trace_fmt!("Received #{} decoded block packet, data: 0x{:x}", counter + u32:1, decoded_block_packet);
162+
trace_fmt!("Expected #{} decoded block packet, data: 0x{:x}", counter + u32:1, expected_block_packet);
163+
assert_eq(decoded_block_packet, expected_block_packet);
164+
(tok)
165+
}(tok);
166+
167+
send(tok, terminator, true);
168+
}
169+
}

0 commit comments

Comments
 (0)