Skip to main content

apache_avro/reader/
single_object.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::{io::Read, marker::PhantomData};
19
20use bon::bon;
21use serde::de::DeserializeOwned;
22
23use crate::{
24    AvroResult, AvroSchema, Schema,
25    decode::{DecodeContext, decode_internal},
26    error::Details,
27    headers::{HeaderBuilder, RabinFingerprintHeader},
28    schema::ResolvedOwnedSchema,
29    serde::deser_schema::{Config, SchemaAwareDeserializer},
30    types::Value,
31    util::is_human_readable,
32};
33
34/// Read `Value`s from a single object encoding.
35///
36/// Use [`SpecificSingleObjectReader`] if you need to deserialize into Rust types.
37pub struct GenericSingleObjectReader {
38    write_schema: ResolvedOwnedSchema,
39    expected_header: Vec<u8>,
40    human_readable: bool,
41}
42
43#[bon]
44impl GenericSingleObjectReader {
45    #[builder]
46    pub fn new(
47        schema: Schema,
48        /// The expected header.
49        #[builder(default = RabinFingerprintHeader::from_schema(&schema).build_header())]
50        header: Vec<u8>,
51        /// Was the data serialized with `human_readable`.
52        #[builder(default = is_human_readable())]
53        human_readable: bool,
54    ) -> AvroResult<GenericSingleObjectReader> {
55        Ok(Self {
56            write_schema: schema.try_into()?,
57            expected_header: header,
58            human_readable,
59        })
60    }
61}
62
63impl GenericSingleObjectReader {
64    pub fn read_value<R: Read>(&self, reader: &mut R) -> AvroResult<Value> {
65        self.read_header(reader)?;
66        decode_internal(
67            self.write_schema.get_root_schema(),
68            self.write_schema.get_names(),
69            None,
70            reader,
71            &mut DecodeContext::new(),
72        )
73    }
74
75    pub fn read_deser<T: DeserializeOwned>(&self, reader: &mut impl Read) -> AvroResult<T> {
76        self.read_header(reader)?;
77        let config = Config {
78            names: self.write_schema.get_names(),
79            human_readable: self.human_readable,
80            recursion_depth: 0,
81        };
82        T::deserialize(SchemaAwareDeserializer::new(
83            reader,
84            self.write_schema.get_root_schema(),
85            config,
86        )?)
87    }
88
89    fn read_header(&self, reader: &mut impl Read) -> AvroResult<()> {
90        let mut header = vec![0; self.expected_header.len()];
91        reader
92            .read_exact(&mut header)
93            .map_err(Details::ReadHeader)?;
94        if self.expected_header == header {
95            Ok(())
96        } else {
97            Err(Details::SingleObjectHeaderMismatch(self.expected_header.clone(), header).into())
98        }
99    }
100}
101
102/// Read Rust types from a single object encoding.
103///
104/// Use [`GenericSingleObjectReader`] if you need `Value`s that are not linked to a type.
105pub struct SpecificSingleObjectReader<T>
106where
107    T: AvroSchema,
108{
109    inner: GenericSingleObjectReader,
110    _model: PhantomData<T>,
111}
112
113impl<T> SpecificSingleObjectReader<T>
114where
115    T: AvroSchema,
116{
117    pub fn new() -> AvroResult<SpecificSingleObjectReader<T>> {
118        Ok(SpecificSingleObjectReader {
119            inner: GenericSingleObjectReader::builder()
120                .schema(T::get_schema())
121                .build()?,
122            _model: PhantomData,
123        })
124    }
125}
126
127impl<T> SpecificSingleObjectReader<T>
128where
129    T: AvroSchema + From<Value>,
130{
131    pub fn read_from_value<R: Read>(&self, reader: &mut R) -> AvroResult<T> {
132        self.inner.read_value(reader).map(|v| v.into())
133    }
134}
135
136impl<T> SpecificSingleObjectReader<T>
137where
138    T: AvroSchema + DeserializeOwned,
139{
140    pub fn read<R: Read>(&self, reader: &mut R) -> AvroResult<T> {
141        self.inner.read_deser(reader)
142    }
143}
144
145#[cfg(test)]
146mod tests {
147    use apache_avro_test_helper::TestResult;
148    use serde::Deserialize;
149    use uuid::Uuid;
150
151    use super::*;
152    use crate::{AvroSchema, Schema, encode::encode, headers::GlueSchemaUuidHeader, rabin::Rabin};
153
154    #[derive(Deserialize, Clone, PartialEq, Debug)]
155    struct TestSingleObjectReader {
156        a: i64,
157        b: f64,
158        c: Vec<String>,
159    }
160
161    impl AvroSchema for TestSingleObjectReader {
162        fn get_schema() -> Schema {
163            let schema = r#"
164            {
165                "type":"record",
166                "name":"TestSingleObjectReader",
167                "fields":[
168                    {
169                        "name":"a",
170                        "type":"long"
171                    },
172                    {
173                        "name":"b",
174                        "type":"double"
175                    },
176                    {
177                        "name":"c",
178                        "type":{
179                            "type":"array",
180                            "items":"string"
181                        }
182                    }
183                ]
184            }
185            "#;
186            Schema::parse_str(schema).unwrap()
187        }
188    }
189
190    impl From<Value> for TestSingleObjectReader {
191        fn from(obj: Value) -> TestSingleObjectReader {
192            if let Value::Record(fields) = obj {
193                let mut a = None;
194                let mut b = None;
195                let mut c = vec![];
196                for (field_name, v) in fields {
197                    match (field_name.as_str(), v) {
198                        ("a", Value::Long(i)) => a = Some(i),
199                        ("b", Value::Double(d)) => b = Some(d),
200                        ("c", Value::Array(v)) => {
201                            for inner_val in v {
202                                if let Value::String(s) = inner_val {
203                                    c.push(s);
204                                }
205                            }
206                        }
207                        (key, value) => panic!("Unexpected pair: {key:?} -> {value:?}"),
208                    }
209                }
210                TestSingleObjectReader {
211                    a: a.unwrap(),
212                    b: b.unwrap(),
213                    c,
214                }
215            } else {
216                panic!("Expected a Value::Record but was {obj:?}")
217            }
218        }
219    }
220
221    impl From<TestSingleObjectReader> for Value {
222        fn from(obj: TestSingleObjectReader) -> Value {
223            Value::Record(vec![
224                ("a".into(), obj.a.into()),
225                ("b".into(), obj.b.into()),
226                (
227                    "c".into(),
228                    Value::Array(obj.c.into_iter().map(|s| s.into()).collect()),
229                ),
230            ])
231        }
232    }
233
234    #[test]
235    fn test_avro_3507_single_object_reader() -> TestResult {
236        let obj = TestSingleObjectReader {
237            a: 42,
238            b: 3.33,
239            c: vec!["cat".into(), "dog".into()],
240        };
241        let mut to_read = Vec::<u8>::new();
242        to_read.extend_from_slice(&[0xC3, 0x01]);
243        to_read.extend_from_slice(
244            &TestSingleObjectReader::get_schema()
245                .fingerprint::<Rabin>()
246                .bytes[..],
247        );
248        encode(
249            &obj.clone().into(),
250            &TestSingleObjectReader::get_schema(),
251            &mut to_read,
252        )
253        .expect("Encode should succeed");
254        let mut to_read = &to_read[..];
255        let generic_reader = GenericSingleObjectReader::builder()
256            .schema(TestSingleObjectReader::get_schema())
257            .build()
258            .expect("Schema should resolve");
259        let val = generic_reader
260            .read_value(&mut to_read)
261            .expect("Should read");
262        let expected_value: Value = obj.into();
263        pretty_assertions::assert_eq!(expected_value, val);
264
265        Ok(())
266    }
267
268    #[test]
269    fn avro_3642_test_single_object_reader_incomplete_reads() -> TestResult {
270        let obj = TestSingleObjectReader {
271            a: 42,
272            b: 3.33,
273            c: vec!["cat".into(), "dog".into()],
274        };
275        // The two-byte marker, to show that the message uses this single-record format
276        let to_read_1 = [0xC3, 0x01];
277        let mut to_read_2 = Vec::<u8>::new();
278        to_read_2.extend_from_slice(
279            &TestSingleObjectReader::get_schema()
280                .fingerprint::<Rabin>()
281                .bytes[..],
282        );
283        let mut to_read_3 = Vec::<u8>::new();
284        encode(
285            &obj.clone().into(),
286            &TestSingleObjectReader::get_schema(),
287            &mut to_read_3,
288        )
289        .expect("Encode should succeed");
290        let mut to_read = (&to_read_1[..]).chain(&to_read_2[..]).chain(&to_read_3[..]);
291        let generic_reader = GenericSingleObjectReader::builder()
292            .schema(TestSingleObjectReader::get_schema())
293            .build()
294            .expect("Schema should resolve");
295        let val = generic_reader
296            .read_value(&mut to_read)
297            .expect("Should read");
298        let expected_value: Value = obj.into();
299        pretty_assertions::assert_eq!(expected_value, val);
300
301        Ok(())
302    }
303
304    #[test]
305    fn test_avro_3507_reader_parity() -> TestResult {
306        let obj = TestSingleObjectReader {
307            a: 42,
308            b: 3.33,
309            c: vec!["cat".into(), "dog".into()],
310        };
311
312        let mut to_read = Vec::<u8>::new();
313        to_read.extend_from_slice(&[0xC3, 0x01]);
314        to_read.extend_from_slice(
315            &TestSingleObjectReader::get_schema()
316                .fingerprint::<Rabin>()
317                .bytes[..],
318        );
319        encode(
320            &obj.clone().into(),
321            &TestSingleObjectReader::get_schema(),
322            &mut to_read,
323        )
324        .expect("Encode should succeed");
325        let generic_reader = GenericSingleObjectReader::builder()
326            .schema(TestSingleObjectReader::get_schema())
327            .build()
328            .expect("Schema should resolve");
329        let specific_reader = SpecificSingleObjectReader::<TestSingleObjectReader>::new()
330            .expect("schema should resolve");
331        let mut to_read1 = &to_read[..];
332        let mut to_read2 = &to_read[..];
333        let mut to_read3 = &to_read[..];
334
335        let val = generic_reader
336            .read_value(&mut to_read1)
337            .expect("Should read");
338        let read_obj1 = specific_reader
339            .read_from_value(&mut to_read2)
340            .expect("Should read from value");
341        let read_obj2 = specific_reader
342            .read(&mut to_read3)
343            .expect("Should read from deserilize");
344        let expected_value: Value = obj.clone().into();
345        pretty_assertions::assert_eq!(obj, read_obj1);
346        pretty_assertions::assert_eq!(obj, read_obj2);
347        pretty_assertions::assert_eq!(val, expected_value);
348
349        Ok(())
350    }
351
352    #[test]
353    fn avro_rs_164_generic_reader_alternate_header() -> TestResult {
354        let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?;
355        let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid);
356        let generic_reader = GenericSingleObjectReader::builder()
357            .schema(TestSingleObjectReader::get_schema())
358            .header(header_builder.build_header())
359            .build()
360            .expect("failed to build reader");
361        // First 18 bytes are the header, then it's a varint 0, double 0 and an empty list (0)
362        let data_to_read: Vec<u8> = vec![
363            3, 0, 178, 241, 207, 0, 4, 52, 1, 62, 67, 154, 18, 94, 184, 72, 90, 95, 0, 0, 0, 0, 0,
364            0, 0, 0, 0, 0,
365        ];
366        let mut to_read = &data_to_read[..];
367        let read_result = generic_reader.read_value(&mut to_read)?;
368        assert_eq!(
369            read_result,
370            Value::Record(vec![
371                ("a".to_string(), Value::Long(0)),
372                ("b".to_string(), Value::Double(0.0)),
373                ("c".to_string(), Value::Array(vec![]))
374            ])
375        );
376        Ok(())
377    }
378}