Parsing Shreds

This guide shows you how to parse the data shreds delivered by ShredStream.com, detect batch and slot boundaries, and reassemble complete block data. For the full binary layout, see Packet Format.

Extracting Fields

Every UDP packet is exactly 1203 bytes with variant 0xA5. The key fields you need for parsing are:

Field	Offset	How to Read
Slot	0x41	8-byte little-endian unsigned integer
Index	0x49	4-byte little-endian unsigned integer
Flags	0x55	Single byte, interpreted as a bitmask
Size	0x56	2-byte little-endian unsigned integer
Payload	0x58	Read exactly Size bytes starting at this offset

Detecting Batch Boundaries

The DATA_COMPLETE flag (0x40) in the flags byte signals the end of the current FEC set or entry batch. When you see this flag, you know that all data shreds for the current batch have been sent.

flags = packet[0x55]
is_batch_complete = (flags & 0x40) != 0

This is useful if you want to process entries incrementally rather than waiting for the entire slot to finish.

Detecting Slot Boundaries

The LAST_IN_SLOT flag (0x20) signals that this shred is the final data shred for the slot. Once you receive a shred with this flag set, the slot is complete and you can reassemble it.

is_last_in_slot = (flags & 0x20) != 0

Reassembling Entries

To reconstruct the full block data for a slot:

1. Collect all shreds for the same slot number.

2. Order them by the Index field (ascending).

3. Concatenate their payloads in order.

The resulting byte stream is the slot's entry data.

Tip: Use a sorted data structure (like a BTreeMap in Rust or sorted keys in Python) so that ordering is handled automatically as shreds arrive.

---

Complete Examples

The examples below show end-to-end receivers that listen on a UDP socket, parse every incoming shred, accumulate payloads per slot, and print a summary when each slot is complete.

Python

import socket
import struct
from collections import defaultdict

SHRED_SIZE = 1203
VARIANT_OFFSET = 0x40
SLOT_OFFSET = 0x41
INDEX_OFFSET = 0x49
FLAGS_OFFSET = 0x55
SIZE_OFFSET = 0x56
PAYLOAD_OFFSET = 0x58

DATA_COMPLETE = 0x40
LAST_IN_SLOT = 0x20
EXPECTED_VARIANT = 0xA5


def parse_shred(data: bytes) -> dict | None:
    """Parse a raw UDP packet into its shred fields.
    Returns None if the packet is malformed or not a data shred."""
    if len(data) != SHRED_SIZE:
        return None

    variant = data[VARIANT_OFFSET]
    if variant != EXPECTED_VARIANT:
        return None

    slot = struct.unpack_from("<Q", data, SLOT_OFFSET)[0]
    index = struct.unpack_from("<I", data, INDEX_OFFSET)[0]
    flags = data[FLAGS_OFFSET]
    size = struct.unpack_from("<H", data, SIZE_OFFSET)[0]
    payload = data[PAYLOAD_OFFSET : PAYLOAD_OFFSET + size]

    return {
        "slot": slot,
        "index": index,
        "flags": flags,
        "size": size,
        "payload": payload,
        "is_data_complete": bool(flags & DATA_COMPLETE),
        "is_last_in_slot": bool(flags & LAST_IN_SLOT),
    }


# Accumulate shreds per slot
slots = defaultdict(dict)

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 26_214_400)
sock.bind(("0.0.0.0", 8001))

while True:
    data, _ = sock.recvfrom(2048)
    shred = parse_shred(data)
    if not shred:
        continue

    slots[shred["slot"]][shred["index"]] = shred["payload"]

    if shred["is_last_in_slot"]:
        ordered = [
            slots[shred["slot"]][i]
            for i in sorted(slots[shred["slot"]].keys())
        ]
        block_data = b"".join(ordered)
        print(
            f"Slot {shred['slot']}: "
            f"{len(ordered)} shreds, {len(block_data)} bytes"
        )
        del slots[shred["slot"]]

Rust

use std::collections::BTreeMap;
use std::collections::HashMap;
use std::net::UdpSocket;

const SHRED_SIZE: usize = 1203;
const VARIANT_OFFSET: usize = 0x40;
const SLOT_OFFSET: usize = 0x41;
const INDEX_OFFSET: usize = 0x49;
const FLAGS_OFFSET: usize = 0x55;
const SIZE_OFFSET: usize = 0x56;
const PAYLOAD_OFFSET: usize = 0x58;

const DATA_COMPLETE: u8 = 0x40;
const LAST_IN_SLOT: u8 = 0x20;
const EXPECTED_VARIANT: u8 = 0xA5;

struct ParsedShred {
    slot: u64,
    index: u32,
    flags: u8,
    payload: Vec<u8>,
}

fn parse_shred(data: &[u8]) -> Option<ParsedShred> {
    if data.len() != SHRED_SIZE {
        return None;
    }
    if data[VARIANT_OFFSET] != EXPECTED_VARIANT {
        return None;
    }

    let slot = u64::from_le_bytes(
        data[SLOT_OFFSET..SLOT_OFFSET + 8].try_into().ok()?,
    );
    let index = u32::from_le_bytes(
        data[INDEX_OFFSET..INDEX_OFFSET + 4].try_into().ok()?,
    );
    let flags = data[FLAGS_OFFSET];
    let size = u16::from_le_bytes(
        data[SIZE_OFFSET..SIZE_OFFSET + 2].try_into().ok()?,
    ) as usize;
    let payload = data[PAYLOAD_OFFSET..PAYLOAD_OFFSET + size].to_vec();

    Some(ParsedShred { slot, index, flags, payload })
}

fn main() -> std::io::Result<()> {
    let socket = UdpSocket::bind("0.0.0.0:8001")?;
    socket.set_recv_buffer_size(26_214_400)?;

    let mut buf = [0u8; 2048];
    let mut slots: HashMap<u64, BTreeMap<u32, Vec<u8>>> = HashMap::new();

    loop {
        let (len, _) = socket.recv_from(&mut buf)?;
        if let Some(shred) = parse_shred(&buf[..len]) {
            let entry = slots.entry(shred.slot).or_default();
            entry.insert(shred.index, shred.payload);

            if shred.flags & LAST_IN_SLOT != 0 {
                let block: Vec<u8> =
                    entry.values().flatten().copied().collect();
                println!(
                    "Slot {}: {} shreds, {} bytes",
                    shred.slot,
                    entry.len(),
                    block.len()
                );
                slots.remove(&shred.slot);
            }
        }
    }
}

TypeScript

import dgram from "dgram";

const SHRED_SIZE = 1203;
const VARIANT_OFFSET = 0x40;
const SLOT_OFFSET = 0x41;
const INDEX_OFFSET = 0x49;
const FLAGS_OFFSET = 0x55;
const SIZE_OFFSET = 0x56;
const PAYLOAD_OFFSET = 0x58;

const DATA_COMPLETE = 0x40;
const LAST_IN_SLOT = 0x20;
const EXPECTED_VARIANT = 0xa5;

interface ParsedShred {
  slot: bigint;
  index: number;
  flags: number;
  size: number;
  payload: Buffer;
  isDataComplete: boolean;
  isLastInSlot: boolean;
}

function parseShred(data: Buffer): ParsedShred | null {
  if (data.length !== SHRED_SIZE) return null;
  if (data[VARIANT_OFFSET] !== EXPECTED_VARIANT) return null;

  const slot = data.readBigUInt64LE(SLOT_OFFSET);
  const index = data.readUInt32LE(INDEX_OFFSET);
  const flags = data[FLAGS_OFFSET];
  const size = data.readUInt16LE(SIZE_OFFSET);
  const payload = data.subarray(PAYLOAD_OFFSET, PAYLOAD_OFFSET + size);

  return {
    slot,
    index,
    flags,
    size,
    payload,
    isDataComplete: (flags & DATA_COMPLETE) !== 0,
    isLastInSlot: (flags & LAST_IN_SLOT) !== 0,
  };
}

const slots = new Map<bigint, Map<number, Buffer>>();
const server = dgram.createSocket("udp4");

server.on("message", (msg) => {
  const shred = parseShred(msg);
  if (!shred) return;

  if (!slots.has(shred.slot)) slots.set(shred.slot, new Map());
  slots.get(shred.slot)!.set(shred.index, shred.payload);

  if (shred.isLastInSlot) {
    const entry = slots.get(shred.slot)!;
    const ordered = [...entry.keys()].sort((a, b) => a - b);
    const blockData = Buffer.concat(ordered.map((i) => entry.get(i)!));
    console.log(
      `Slot ${shred.slot}: ${entry.size} shreds, ${blockData.length} bytes`
    );
    slots.delete(shred.slot);
  }
});

server.bind(8001, () => console.log("Listening on port 8001..."));

---

Next Steps

• Review Best Practices for tips on buffer sizing, loss detection, and production hardening.

• See UDP Setup for network configuration and firewall requirements.

• If packets are not arriving, check Troubleshooting.