Skip to content

Commit 51e9672

Browse files
lpawelczlpawelcz
committed
modules/zstd: Add block decoder module
This adds a decoder of block data. It decodes block header and demuxes remaining input data into one of specific block decoders depending on the type of the parsed block. Then it muxes outputs from those decoders into single output channel. Internal-tag: [#51873] Signed-off-by: Pawel Czarnecki <[email protected]>
1 parent 1d96053 commit 51e9672

File tree

2 files changed

+162
-0
lines changed

2 files changed

+162
-0
lines changed

‎xls/modules/zstd/BUILD

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,3 +219,22 @@ xls_dslx_test(
219219
name = "dec_demux_dslx_test",
220220
library = ":dec_demux_dslx",
221221
)
222+
223+
xls_dslx_library(
224+
name = "block_dec_dslx",
225+
srcs = [
226+
"block_dec.x",
227+
],
228+
deps = [
229+
":common_dslx",
230+
":dec_demux_dslx",
231+
":raw_block_dec_dslx",
232+
":rle_block_dec_dslx",
233+
":dec_mux_dslx",
234+
],
235+
)
236+
237+
xls_dslx_test(
238+
name = "block_dec_dslx_test",
239+
library = ":block_dec_dslx",
240+
)

‎xls/modules/zstd/block_dec.x

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Copyright 2023 The XLS Authors
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import xls.modules.zstd.common
16+
import xls.modules.zstd.dec_demux as demux
17+
import xls.modules.zstd.raw_block_dec as raw
18+
import xls.modules.zstd.rle_block_dec as rle
19+
import xls.modules.zstd.dec_mux as mux
20+
21+
type BlockDataPacket = common::BlockDataPacket;
22+
type BlockData = common::BlockData;
23+
type BlockPacketLength = common::BlockPacketLength;
24+
25+
proc BlockDecoder {
26+
input_r: chan<BlockDataPacket> in;
27+
output_s: chan<BlockData> out;
28+
29+
config (input_r: chan<BlockDataPacket> in, output_s: chan<BlockData> out) {
30+
let (demux_raw_s, demux_raw_r) = chan<BlockDataPacket>;
31+
let (demux_rle_s, demux_rle_r) = chan<BlockDataPacket>;
32+
let (demux_cmp_s, demux_cmp_r) = chan<BlockDataPacket>;
33+
let (mux_raw_s, mux_raw_r) = chan<BlockDataPacket>;
34+
let (mux_rle_s, mux_rle_r) = chan<BlockDataPacket>;
35+
let (mux_cmp_s, mux_cmp_r) = chan<BlockDataPacket>;
36+
37+
spawn demux::DecoderDemux(input_r, demux_raw_s, demux_rle_s, demux_cmp_s);
38+
spawn raw::RawBlockDecoder(demux_raw_r, mux_raw_s);
39+
spawn rle::RleBlockDecoder(demux_rle_r, mux_rle_s);
40+
// TODO(lpawelcz): 2023-11-28 change to compressed block decoder proc
41+
spawn raw::RawBlockDecoder(demux_cmp_r, mux_cmp_s);
42+
spawn mux::DecoderMux(mux_raw_r, mux_rle_r, mux_cmp_r, output_s);
43+
44+
(input_r, output_s)
45+
}
46+
47+
init { }
48+
49+
next(tok: token, state: ()) { }
50+
}
51+
52+
#[test_proc]
53+
proc BlockDecoderTest {
54+
terminator: chan<bool> out;
55+
input_s: chan<BlockDataPacket> out;
56+
output_r: chan<BlockData> in;
57+
58+
init {}
59+
60+
config (terminator: chan<bool> out) {
61+
let (input_s, input_r) = chan<BlockDataPacket>;
62+
let (output_s, output_r) = chan<BlockData>;
63+
64+
spawn BlockDecoder(input_r, output_s);
65+
66+
(terminator, input_s, output_r)
67+
}
68+
69+
next(tok: token, state: ()) {
70+
let EncodedDataBlocksPackets: BlockDataPacket[13] = [
71+
// RAW Block 1 byte
72+
BlockDataPacket { id: u32:0, last: true, last_block: false, data: BlockData:0xDE000008, length: BlockPacketLength:32 },
73+
// RAW Block 2 bytes
74+
BlockDataPacket { id: u32:1, last: true, last_block: false, data: BlockData:0xDEAD000010, length: BlockPacketLength:40 },
75+
// RAW Block 4 bytes
76+
BlockDataPacket { id: u32:2, last: true, last_block: false, data: BlockData:0xDEADBEEF000020, length: BlockPacketLength:56 },
77+
// RAW Block 5 bytes (block header takes one full packet)
78+
BlockDataPacket { id: u32:3, last: true, last_block: false, data: BlockData:0xDEADBEEFEF000028, length: BlockPacketLength:64 },
79+
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
80+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x12345678900000C0, length: BlockPacketLength:64 },
81+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0x1234567890ABCDEF, length: BlockPacketLength:64 },
82+
BlockDataPacket { id: u32:4, last: false, last_block: false, data: BlockData:0xFEDCBA0987654321, length: BlockPacketLength:64 },
83+
BlockDataPacket { id: u32:4, last: true, last_block: false, data: BlockData:0xF0F0F0, length: BlockPacketLength:24 },
84+
85+
// RLE Block 1 byte
86+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x6700000a, length: BlockPacketLength:32 },
87+
// RLE Block 2 bytes
88+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x45000012, length: BlockPacketLength:32 },
89+
// RLE Block 4 bytes
90+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x23000022, length: BlockPacketLength:32 },
91+
// RLE Block 8 bytes (block takes one full packet)
92+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0x10000042, length: BlockPacketLength:32 },
93+
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
94+
BlockDataPacket { id: u32:5, last: true, last_block: false, data: BlockData:0xDE0000d2, length: BlockPacketLength:32 },
95+
];
96+
97+
let tok = for ((counter, block_packet), tok): ((u32, BlockDataPacket), token) in enumerate(EncodedDataBlocksPackets) {
98+
let tok = send(tok, input_s, block_packet);
99+
trace_fmt!("Sent #{} encoded block packet, {:#x}", counter + u32:1, block_packet);
100+
(tok)
101+
}(tok);
102+
103+
let DecodedDataBlocksPackets: BlockData[16] = [
104+
// RAW Block 1 byte
105+
BlockData:0xDE,
106+
// RAW Block 2 bytes
107+
BlockData:0xDEAD,
108+
// RAW Block 4 bytes
109+
BlockData:0xDEADBEEF,
110+
// RAW Block 5 bytes (block header takes one full packet)
111+
BlockData:0xDEADBEEFEF,
112+
// RAW Block 24 bytes (multi-packet block header with unaligned data in the last packet)
113+
BlockData:0x1234567890,
114+
BlockData:0x1234567890ABCDEF,
115+
BlockData:0xFEDCBA0987654321,
116+
BlockData:0xF0F0F0,
117+
118+
// RLE Block 1 byte
119+
BlockData:0x67,
120+
// RLE Block 2 bytes
121+
BlockData:0x4545,
122+
// RLE Block 4 bytes
123+
BlockData:0x23232323,
124+
// RLE Block 8 bytes (block takes one full packet)
125+
BlockData:0x1010101010101010,
126+
// RLE Block 26 bytes (multi-packet block header with unaligned data in the last packet)
127+
BlockData:0xDEDEDEDEDEDEDEDE,
128+
BlockData:0xDEDEDEDEDEDEDEDE,
129+
BlockData:0xDEDEDEDEDEDEDEDE,
130+
BlockData:0xDEDE,
131+
];
132+
133+
let tok = for ((counter, expected_block_packet), tok): ((u32, BlockData), token) in enumerate(DecodedDataBlocksPackets) {
134+
let (tok, decoded_block_packet) = recv(tok, output_r);
135+
trace_fmt!("Received #{} decoded block packet, data: 0x{:x}", counter + u32:1, decoded_block_packet);
136+
trace_fmt!("Expected #{} decoded block packet, data: 0x{:x}", counter + u32:1, expected_block_packet);
137+
assert_eq(decoded_block_packet, expected_block_packet);
138+
(tok)
139+
}(tok);
140+
141+
send(tok, terminator, true);
142+
}
143+
}

0 commit comments

Comments
 (0)