Skip to main content

apache_avro/
codec.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Logic for all supported compression codecs in Avro.
19
20use crate::{AvroResult, Error, error::Details, types::Value};
21use strum::{EnumIter, EnumString, IntoStaticStr};
22
23/// Settings for the `Deflate` codec.
24#[derive(Clone, Copy, Eq, PartialEq, Debug)]
25pub struct DeflateSettings {
26    compression_level: miniz_oxide::deflate::CompressionLevel,
27}
28
29impl DeflateSettings {
30    pub fn new(compression_level: miniz_oxide::deflate::CompressionLevel) -> Self {
31        DeflateSettings { compression_level }
32    }
33
34    /// Get the compression level as a `u8`, note that this means the [`miniz_oxide::deflate::CompressionLevel::DefaultCompression`] variant
35    /// will appear as `255`, this is normalized by the [`miniz_oxide`] crate later.
36    pub fn compression_level(&self) -> u8 {
37        self.compression_level as u8
38    }
39}
40
41impl Default for DeflateSettings {
42    /// Default compression level is `miniz_oxide::deflate::CompressionLevel::DefaultCompression`.
43    fn default() -> Self {
44        Self::new(miniz_oxide::deflate::CompressionLevel::DefaultCompression)
45    }
46}
47
48/// The compression codec used to compress blocks.
49#[derive(Clone, Copy, Debug, Eq, PartialEq, EnumIter, EnumString, IntoStaticStr)]
50#[strum(serialize_all = "kebab_case")]
51pub enum Codec {
52    /// The `Null` codec simply passes through data uncompressed.
53    Null,
54    /// The `Deflate` codec writes the data block using the deflate algorithm
55    /// as specified in RFC 1951, and typically implemented using the zlib library.
56    /// Note that this format (unlike the "zlib format" in RFC 1950) does not have a checksum.
57    Deflate(DeflateSettings),
58    #[cfg(feature = "snappy")]
59    /// The `Snappy` codec uses Google's [Snappy](http://google.github.io/snappy/)
60    /// compression library. Each compressed block is followed by the 4-byte, big-endian
61    /// CRC32 checksum of the uncompressed data in the block.
62    Snappy,
63    #[cfg(feature = "zstandard")]
64    /// The `Zstandard` codec uses Facebook's [Zstandard](https://facebook.github.io/zstd/)
65    Zstandard(zstandard::ZstandardSettings),
66    #[cfg(feature = "bzip")]
67    /// The `BZip2` codec uses [BZip2](https://sourceware.org/bzip2/)
68    /// compression library.
69    Bzip2(bzip::Bzip2Settings),
70    #[cfg(feature = "xz")]
71    /// The `Xz` codec uses [Xz utils](https://tukaani.org/xz/)
72    /// compression library.
73    Xz(xz::XzSettings),
74}
75
76impl From<Codec> for Value {
77    fn from(value: Codec) -> Self {
78        Self::Bytes(<&str>::from(value).as_bytes().to_vec())
79    }
80}
81
82impl Codec {
83    /// Compress a stream of bytes in-place.
84    pub fn compress(self, stream: &mut Vec<u8>) -> AvroResult<()> {
85        match self {
86            Codec::Null => (),
87            Codec::Deflate(settings) => {
88                let compressed =
89                    miniz_oxide::deflate::compress_to_vec(stream, settings.compression_level());
90                *stream = compressed;
91            }
92            #[cfg(feature = "snappy")]
93            Codec::Snappy => {
94                let mut encoded: Vec<u8> = vec![0; snap::raw::max_compress_len(stream.len())];
95                let compressed_size = snap::raw::Encoder::new()
96                    .compress(&stream[..], &mut encoded[..])
97                    .map_err(Details::SnappyCompress)?;
98
99                let checksum = crc_fast::crc32_iso_hdlc(&stream[..]);
100                let checksum_as_bytes = checksum.to_be_bytes();
101                let checksum_len = checksum_as_bytes.len();
102                encoded.truncate(compressed_size + checksum_len);
103                encoded[compressed_size..].copy_from_slice(&checksum_as_bytes);
104
105                *stream = encoded;
106            }
107            #[cfg(feature = "zstandard")]
108            Codec::Zstandard(settings) => {
109                use std::io::Write;
110                let mut encoder = zstd::Encoder::new(Vec::new(), settings.compression_level as i32)
111                    .map_err(Details::ZstdCompress)?;
112                encoder.write_all(stream).map_err(Details::ZstdCompress)?;
113                *stream = encoder.finish().map_err(Details::ZstdCompress)?;
114            }
115            #[cfg(feature = "bzip")]
116            Codec::Bzip2(settings) => {
117                use bzip2::read::BzEncoder;
118                use std::io::Read;
119
120                let mut encoder = BzEncoder::new(&stream[..], settings.compression());
121                let mut buffer = Vec::new();
122                encoder
123                    .read_to_end(&mut buffer)
124                    .unwrap_or_else(|_| unreachable!("No I/O errors possible with Vec<u8>"));
125                *stream = buffer;
126            }
127            #[cfg(feature = "xz")]
128            Codec::Xz(settings) => {
129                use liblzma::read::XzEncoder;
130                use std::io::Read;
131
132                let mut encoder = XzEncoder::new(&stream[..], settings.compression_level as u32);
133                let mut buffer = Vec::new();
134                encoder
135                    .read_to_end(&mut buffer)
136                    .unwrap_or_else(|_| unreachable!("No I/O errors possible with Vec<u8>"));
137                *stream = buffer;
138            }
139        };
140
141        Ok(())
142    }
143
144    /// Decompress a stream of bytes in-place.
145    pub fn decompress(self, stream: &mut Vec<u8>) -> AvroResult<()> {
146        // Cap the decompressed output at the configured allocation budget so a
147        // small compressed block cannot inflate to an enormous buffer (a
148        // "decompression bomb") and exhaust memory.
149        let max_bytes =
150            crate::util::max_allocation_bytes(crate::util::DEFAULT_MAX_ALLOCATION_BYTES);
151        *stream = match self {
152            Codec::Null => return Ok(()),
153            Codec::Deflate(_settings) => miniz_oxide::inflate::decompress_to_vec_with_limit(stream, max_bytes).map_err(|e| {
154                use std::io::ErrorKind;
155                use miniz_oxide::inflate::TINFLStatus;
156
157                let details = match e.status {
158                    TINFLStatus::FailedCannotMakeProgress | TINFLStatus::NeedsMoreInput => Details::DeflateDecompress(ErrorKind::UnexpectedEof.into()),
159                    TINFLStatus::Adler32Mismatch | TINFLStatus::Failed | TINFLStatus::BadParam => Details::DeflateDecompress(ErrorKind::InvalidData.into()),
160                    TINFLStatus::Done => Details::DeflateDecompress(std::io::Error::other("Unexpected error: miniz_oxide reported an error with a success status. Please report this to avro-rs developers.")),
161                    // Output is larger than max allocation allowed
162                    TINFLStatus::HasMoreOutput => Details::MemoryAllocation {
163                        desired: None,
164                        maximum: max_bytes,
165                    },
166                    other => Details::DeflateDecompress(std::io::Error::other(format!("Unexpected error: {other:?}")))
167                };
168                Error::new(details)
169            })?,
170            #[cfg(feature = "snappy")]
171            Codec::Snappy => {
172                // The block ends with a 4-byte CRC32; a truncated/corrupt block
173                // shorter than that must error rather than underflow the slice.
174                let data_end = stream
175                    .len()
176                    .checked_sub(4)
177                    .ok_or(Details::BadSnappyLength(stream.len()))?;
178                let decompressed_size = snap::raw::decompress_len(&stream[..data_end])
179                    .map_err(Details::GetSnappyDecompressLen)?;
180                // The decompressed size is taken from the (untrusted) block
181                // header, so bound it before allocating for it.
182                let decompressed_size = crate::util::safe_len(decompressed_size)?;
183                let mut decoded = vec![0; decompressed_size];
184                snap::raw::Decoder::new()
185                    .decompress(&stream[..data_end], &mut decoded[..])
186                    .map_err(Details::SnappyDecompress)?;
187
188                let mut last_four: [u8; 4] = [0; 4];
189                last_four.copy_from_slice(&stream[data_end..]);
190                let expected: u32 = u32::from_be_bytes(last_four);
191
192                let actual = crc_fast::crc32_iso_hdlc(&decoded);
193
194                if expected != actual {
195                    return Err(Details::SnappyCrc32{expected, actual}.into());
196                }
197                decoded
198            }
199            #[cfg(feature = "zstandard")]
200            Codec::Zstandard(_settings) => {
201                use std::io::{BufReader, Read};
202                use zstd::zstd_safe;
203
204                let mut decoded = Vec::new();
205                let buffer_size = zstd_safe::DCtx::in_size();
206                let buffer = BufReader::with_capacity(buffer_size, &stream[..]);
207                let decoder = zstd::Decoder::new(buffer).map_err(Details::ZstdDecompress)?;
208                // Read one byte past the budget so an output that exactly fills
209                // it is allowed, while a larger (bomb) output is detected.
210                decoder
211                    .take((max_bytes as u64).saturating_add(1))
212                    .read_to_end(&mut decoded)
213                    .map_err(Details::ZstdDecompress)?;
214                if decoded.len() > max_bytes {
215                    return Err(Details::MemoryAllocation { desired: None, maximum: max_bytes }.into());
216                }
217                decoded
218            }
219            #[cfg(feature = "bzip")]
220            Codec::Bzip2(_) => {
221                use bzip2::read::BzDecoder;
222                use std::io::Read;
223
224                let mut decoded = Vec::new();
225                BzDecoder::new(&stream[..])
226                    .take((max_bytes as u64).saturating_add(1))
227                    .read_to_end(&mut decoded)
228                    .map_err(Details::Bzip2Decompress)?;
229                if decoded.len() > max_bytes {
230                    return Err(Details::MemoryAllocation { desired: None, maximum: max_bytes }.into());
231                }
232                decoded
233            }
234            #[cfg(feature = "xz")]
235            Codec::Xz(_) => {
236                use liblzma::read::XzDecoder;
237                use std::io::Read;
238
239                let mut decoded: Vec<u8> = Vec::new();
240                XzDecoder::new(&stream[..])
241                    .take((max_bytes as u64).saturating_add(1))
242                    .read_to_end(&mut decoded)
243                    .map_err(Details::XzDecompress)?;
244                if decoded.len() > max_bytes {
245                    return Err(Details::MemoryAllocation { desired: None, maximum: max_bytes }.into());
246                }
247                decoded
248            }
249        };
250        Ok(())
251    }
252}
253
254#[cfg(feature = "bzip")]
255pub mod bzip {
256    use bzip2::Compression;
257
258    /// Settings for configuring bzip2 compression.
259    #[derive(Clone, Copy, Eq, PartialEq, Debug)]
260    pub struct Bzip2Settings {
261        pub compression_level: u8,
262    }
263
264    impl Bzip2Settings {
265        pub fn new(compression_level: u8) -> Self {
266            Self { compression_level }
267        }
268
269        pub(crate) fn compression(&self) -> Compression {
270            Compression::new(self.compression_level as u32)
271        }
272    }
273
274    impl Default for Bzip2Settings {
275        fn default() -> Self {
276            Bzip2Settings::new(Compression::best().level() as u8)
277        }
278    }
279}
280
281#[cfg(feature = "zstandard")]
282pub mod zstandard {
283    /// Settings for configuring zstandard compression.
284    #[derive(Clone, Copy, Eq, PartialEq, Debug)]
285    pub struct ZstandardSettings {
286        pub compression_level: u8,
287    }
288
289    impl ZstandardSettings {
290        pub fn new(compression_level: u8) -> Self {
291            Self { compression_level }
292        }
293    }
294
295    impl Default for ZstandardSettings {
296        fn default() -> Self {
297            Self::new(0)
298        }
299    }
300}
301
302#[cfg(feature = "xz")]
303pub mod xz {
304    /// Settings for configuring xz compression.
305    #[derive(Clone, Copy, Eq, PartialEq, Debug)]
306    pub struct XzSettings {
307        pub compression_level: u8,
308    }
309
310    impl XzSettings {
311        pub fn new(compression_level: u8) -> Self {
312            Self { compression_level }
313        }
314    }
315
316    impl Default for XzSettings {
317        fn default() -> Self {
318            XzSettings::new(9)
319        }
320    }
321}
322
323#[cfg(test)]
324mod tests {
325    use super::*;
326    use apache_avro_test_helper::TestResult;
327    use miniz_oxide::deflate::CompressionLevel;
328    use pretty_assertions::{assert_eq, assert_ne};
329
330    const INPUT: &[u8] = b"theanswertolifetheuniverseandeverythingis42theanswertolifetheuniverseandeverythingis4theanswertolifetheuniverseandeverythingis2";
331
332    #[test]
333    fn null_compress_and_decompress() -> TestResult {
334        let codec = Codec::Null;
335        let mut stream = INPUT.to_vec();
336        codec.compress(&mut stream)?;
337        assert_eq!(INPUT, stream.as_slice());
338        codec.decompress(&mut stream)?;
339        assert_eq!(INPUT, stream.as_slice());
340        Ok(())
341    }
342
343    #[test]
344    fn deflate_compress_and_decompress() -> TestResult {
345        compress_and_decompress(Codec::Deflate(DeflateSettings::new(
346            CompressionLevel::BestCompression,
347        )))
348    }
349
350    #[cfg(feature = "snappy")]
351    #[test]
352    fn snappy_compress_and_decompress() -> TestResult {
353        compress_and_decompress(Codec::Snappy)
354    }
355
356    #[cfg(feature = "snappy")]
357    #[test]
358    fn avro_rs_660_snappy_compress_writes_crc32_iso_hdlc_trailer() -> TestResult {
359        let mut stream = b"123456789".to_vec();
360        Codec::Snappy.compress(&mut stream)?;
361
362        assert_eq!(stream[stream.len() - 4..], 0xcbf4_3926_u32.to_be_bytes());
363
364        Ok(())
365    }
366
367    #[cfg(feature = "snappy")]
368    #[test]
369    fn snappy_decompress_short_block_errors_without_panicking() {
370        // A block shorter than the trailing 4-byte CRC must return an error
371        // rather than underflowing `stream.len() - 4` and panicking.
372        for len in 0..4usize {
373            let mut stream = vec![0u8; len];
374            let result = Codec::Snappy.decompress(&mut stream);
375            assert!(result.is_err(), "len={len} should error, got {result:?}");
376        }
377    }
378
379    #[cfg(feature = "zstandard")]
380    #[test]
381    fn zstd_compress_and_decompress() -> TestResult {
382        compress_and_decompress(Codec::Zstandard(zstandard::ZstandardSettings::default()))
383    }
384
385    #[cfg(feature = "bzip")]
386    #[test]
387    fn bzip_compress_and_decompress() -> TestResult {
388        compress_and_decompress(Codec::Bzip2(bzip::Bzip2Settings::default()))
389    }
390
391    #[cfg(feature = "xz")]
392    #[test]
393    fn xz_compress_and_decompress() -> TestResult {
394        compress_and_decompress(Codec::Xz(xz::XzSettings::default()))
395    }
396
397    fn compress_and_decompress(codec: Codec) -> TestResult {
398        let mut stream = INPUT.to_vec();
399        codec.compress(&mut stream)?;
400        assert_ne!(INPUT, stream.as_slice());
401        assert!(INPUT.len() > stream.len());
402        codec.decompress(&mut stream)?;
403        assert_eq!(INPUT, stream.as_slice());
404        Ok(())
405    }
406
407    #[test]
408    fn codec_to_str() {
409        assert_eq!(<&str>::from(Codec::Null), "null");
410        assert_eq!(
411            <&str>::from(Codec::Deflate(DeflateSettings::default())),
412            "deflate"
413        );
414
415        #[cfg(feature = "snappy")]
416        assert_eq!(<&str>::from(Codec::Snappy), "snappy");
417
418        #[cfg(feature = "zstandard")]
419        assert_eq!(
420            <&str>::from(Codec::Zstandard(zstandard::ZstandardSettings::default())),
421            "zstandard"
422        );
423
424        #[cfg(feature = "bzip")]
425        assert_eq!(
426            <&str>::from(Codec::Bzip2(bzip::Bzip2Settings::default())),
427            "bzip2"
428        );
429
430        #[cfg(feature = "xz")]
431        assert_eq!(<&str>::from(Codec::Xz(xz::XzSettings::default())), "xz");
432    }
433
434    #[test]
435    fn codec_from_str() {
436        use std::str::FromStr;
437
438        assert_eq!(Codec::from_str("null").unwrap(), Codec::Null);
439        assert_eq!(
440            Codec::from_str("deflate").unwrap(),
441            Codec::Deflate(DeflateSettings::default())
442        );
443
444        #[cfg(feature = "snappy")]
445        assert_eq!(Codec::from_str("snappy").unwrap(), Codec::Snappy);
446
447        #[cfg(feature = "zstandard")]
448        assert_eq!(
449            Codec::from_str("zstandard").unwrap(),
450            Codec::Zstandard(zstandard::ZstandardSettings::default())
451        );
452
453        #[cfg(feature = "bzip")]
454        assert_eq!(
455            Codec::from_str("bzip2").unwrap(),
456            Codec::Bzip2(bzip::Bzip2Settings::default())
457        );
458
459        #[cfg(feature = "xz")]
460        assert_eq!(
461            Codec::from_str("xz").unwrap(),
462            Codec::Xz(xz::XzSettings::default())
463        );
464
465        assert!(Codec::from_str("not a codec").is_err());
466    }
467}