Skip to content

Commit be431b3

Browse files
lpawelczlpawelcz
committed
modules/zstd: Add block decoder module
This adds a decoder of block data. It decodes block header and demuxes remaining input data into one of specific block decoders depending on the type of the parsed block. Then it muxes outputs from those decoders into single output channel. Internal-tag: [#51873] Signed-off-by: Pawel Czarnecki <[email protected]>
1 parent 7ebb50f commit be431b3

File tree

2 files changed

+183
-0
lines changed

2 files changed

+183
-0
lines changed

‎xls/modules/zstd/BUILD

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,3 +220,22 @@ xls_dslx_test(
220220
name = "dec_demux_dslx_test",
221221
library = ":dec_demux_dslx",
222222
)
223+
224+
xls_dslx_library(
225+
name = "block_dec_dslx",
226+
srcs = [
227+
"block_dec.x",
228+
],
229+
deps = [
230+
":common_dslx",
231+
":dec_demux_dslx",
232+
":raw_block_dec_dslx",
233+
":rle_block_dec_dslx",
234+
":dec_mux_dslx",
235+
],
236+
)
237+
238+
xls_dslx_test(
239+
name = "block_dec_dslx_test",
240+
library = ":block_dec_dslx",
241+
)

‎xls/modules/zstd/block_dec.x

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
// Copyright 2023 The XLS Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import xls.modules.zstd.common
16+
import xls.modules.zstd.dec_demux as demux
17+
import xls.modules.zstd.raw_block_dec as raw
18+
import xls.modules.zstd.rle_block_dec as rle
19+
import xls.modules.zstd.dec_mux as mux
20+
21+
type BlockDataPacket = common::BlockDataPacket;
22+
type BlockData = common::BlockData;
23+
type BlockPacketLength = common::BlockPacketLength;
24+
25+
// Proc responsible for connecting internal procs used in Block data decoding.
26+
// It handles incoming block data packets by redirecting those to demuxer which passes those to
27+
// block decoder procs specific for given block type. Results are then gathered by mux which
28+
// transfers decoded data further. The connections are visualised on the following diagram:
29+
//
30+
// Block Decoder
31+
// ┌───────────────────────────────────────┐
32+
// │ Raw Block Decoder │
33+
// │ ┌───────────────────┐ │
34+
// │ ┌─► ├┐ │
35+
// │ Demux │ └───────────────────┘│ Mux │
36+
// │┌─────┐│ Rle Block Decoder │ ┌─────┐│
37+
// ││ ├┘ ┌───────────────────┐└─► ││
38+
// ──┼► ├──► ├──► ├┼─►
39+
// ││ ├┐ └───────────────────┘┌─► ││
40+
// │└─────┘│ Cmp Block Decoder │ └─────┘│
41+
// │ │ ┌───────────────────┐│ │
42+
// │ └─► ├┘ │
43+
// │ └───────────────────┘ │
44+
// └───────────────────────────────────────┘
45+
46+
proc BlockDecoder {
47+
input_r: chan<BlockDataPacket> in;
48+
output_s: chan<BlockData> out;
49+
50+
config (input_r: chan<BlockDataPacket> in, output_s: chan<BlockData> out) {
51+
let (demux_raw_s, demux_raw_r) = chan<BlockDataPacket>;
52+
let (demux_rle_s, demux_rle_r) = chan<BlockDataPacket>;
53+
let (demux_cmp_s, demux_cmp_r) = chan<BlockDataPacket>;
54+
let (mux_raw_s, mux_raw_r) = chan<BlockDataPacket>;
55+
let (mux_rle_s, mux_rle_r) = chan<BlockDataPacket>;
56+
let (mux_cmp_s, mux_cmp_r) = chan<BlockDataPacket>;
57+
58+
spawn demux::DecoderDemux(input_r, demux_raw_s, demux_rle_s, demux_cmp_s);
59+
spawn raw::RawBlockDecoder(demux_raw_r, mux_raw_s);
60+
spawn rle::RleBlockDecoder(demux_rle_r, mux_rle_s);
61+
// TODO(lpawelcz): 2023-11-28 change to compressed block decoder proc
62+
spawn raw::RawBlockDecoder(demux_cmp_r, mux_cmp_s);
63+
spawn mux::DecoderMux(mux_raw_r, mux_rle_r, mux_cmp_r, output_s);
64+
65+
(input_r, output_s)
66+
}
67+
68+
init { }
69+
70+
next(tok: token, state: ()) { }
71+
}
72+
73+
#[test_proc]
74+
proc BlockDecoderTest {
75+
terminator: chan<bool> out;
76+
input_s: chan<BlockDataPacket> out;
77+
output_r: chan<BlockData> in;
78+
79+
init {}
80+
81+
config (terminator: chan<bool> out) {
82+
let (input_s, input_r) = chan<BlockDataPacket>;
83+
let (output_s, output_r) = chan<BlockData>;
84+
85+
spawn BlockDecoder(input_r, output_s);
86+
87+
(terminator, input_s, output_r)
88+
}
89+
90+
next(tok: token, state: ()) {
91+
let EncodedDataBlocksPackets: BlockDataPacket[13] = [
92+
// RAW Block 1 byte
93+
BlockDataPacket { id: u32:0, last: true, last_block: false, data: BlockData:0xDE000008, length: BlockPacketLength:32 },
94+
// RAW Block 2 bytes
95+
BlockDataPacket { id: u32:1, last: true, last_block: false, data: BlockData:0xDEAD000010, length: BlockPacketLength:40 },
96+
// RAW Block 4 bytes
97+
BlockDataPacket { id: u32:2, last: true, last_block: false, data: BlockData:0xDEADBEEF000020, length: BlockPacketLength:56 },
98+
// RAW Block 5 bytes (block header takes one full packet)
99+
BlockDataPacket { id: u32:3, last: true, last_block: false, data: BlockData:0xDEADBEEFEF000028, length: BlockPacketLength:64 },
100+
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
101+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x12345678900000C0, length: BlockPacketLength:64 },
102+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x1234567890ABCDEF, length: BlockPacketLength:64 },
103+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0xFEDCBA0987654321, length: BlockPacketLength:64 },
104+
BlockDataPacket { id: u32:4, last: true, last_block: false, data: BlockData:0xF0F0F0, length: BlockPacketLength:24 },
105+
106+
// RLE Block 1 byte
107+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x6700000a, length: BlockPacketLength:32 },
108+
// RLE Block 2 bytes
109+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x45000012, length: BlockPacketLength:32 },
110+
// RLE Block 4 bytes
111+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x23000022, length: BlockPacketLength:32 },
112+
// RLE Block 8 bytes (block takes one full packet)
113+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x10000042, length: BlockPacketLength:32 },
114+
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
115+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0xDE0000d2, length: BlockPacketLength:32 },
116+
];
117+
118+
let tok = for ((counter, block_packet), tok): ((u32, BlockDataPacket), token) in enumerate(EncodedDataBlocksPackets) {
119+
let tok = send(tok, input_s, block_packet);
120+
trace_fmt!("Sent #{} encoded block packet, {:#x}", counter + u32:1, block_packet);
121+
(tok)
122+
}(tok);
123+
124+
let DecodedDataBlocksPackets: BlockData[16] = [
125+
// RAW Block 1 byte
126+
BlockData:0xDE,
127+
// RAW Block 2 bytes
128+
BlockData:0xDEAD,
129+
// RAW Block 4 bytes
130+
BlockData:0xDEADBEEF,
131+
// RAW Block 5 bytes (block header takes one full packet)
132+
BlockData:0xDEADBEEFEF,
133+
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
134+
BlockData:0x1234567890,
135+
BlockData:0x1234567890ABCDEF,
136+
BlockData:0xFEDCBA0987654321,
137+
BlockData:0xF0F0F0,
138+
139+
// RLE Block 1 byte
140+
BlockData:0x67,
141+
// RLE Block 2 bytes
142+
BlockData:0x4545,
143+
// RLE Block 4 bytes
144+
BlockData:0x23232323,
145+
// RLE Block 8 bytes (block takes one full packet)
146+
BlockData:0x1010101010101010,
147+
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
148+
BlockData:0xDEDEDEDEDEDEDEDE,
149+
BlockData:0xDEDEDEDEDEDEDEDE,
150+
BlockData:0xDEDEDEDEDEDEDEDE,
151+
BlockData:0xDEDE,
152+
];
153+
154+
let tok = for ((counter, expected_block_packet), tok): ((u32, BlockData), token) in enumerate(DecodedDataBlocksPackets) {
155+
let (tok, decoded_block_packet) = recv(tok, output_r);
156+
trace_fmt!("Received #{} decoded block packet, data: 0x{:x}", counter + u32:1, decoded_block_packet);
157+
trace_fmt!("Expected #{} decoded block packet, data: 0x{:x}", counter + u32:1, expected_block_packet);
158+
assert_eq(decoded_block_packet, expected_block_packet);
159+
(tok)
160+
}(tok);
161+
162+
send(tok, terminator, true);
163+
}
164+
}

0 commit comments

Comments
 (0)