Skip to main content

apache_avro/reader/
datum.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::{io::Read, marker::PhantomData};
19
20use bon::bon;
21use serde::de::DeserializeOwned;
22
23use crate::{
24    AvroResult, AvroSchema, Schema,
25    decode::{DecodeContext, decode_internal},
26    schema::{ResolvedOwnedSchema, ResolvedSchema},
27    serde::deser_schema::{Config, SchemaAwareDeserializer},
28    types::Value,
29    util::is_human_readable,
30};
31
32/// Reader for reading raw Avro data.
33///
34/// This is most likely not what you need. Most users should use [`Reader`][crate::Reader],
35/// [`GenericSingleObjectReader`][crate::GenericSingleObjectReader], or
36/// [`SpecificSingleObjectReader`][crate::SpecificSingleObjectReader] instead.
37pub struct GenericDatumReader<'s> {
38    writer: &'s Schema,
39    resolved: ResolvedSchema<'s>,
40    reader: Option<(&'s Schema, ResolvedSchema<'s>)>,
41    human_readable: bool,
42}
43
44#[bon]
45impl<'s> GenericDatumReader<'s> {
46    /// Build a [`GenericDatumReader`].
47    ///
48    /// This is most likely not what you need. Most users should use [`Reader`][crate::Reader],
49    /// [`GenericSingleObjectReader`][crate::GenericSingleObjectReader], or
50    /// [`SpecificSingleObjectReader`][crate::SpecificSingleObjectReader] instead.
51    #[builder]
52    pub fn new(
53        /// The schema that was used to write the Avro datum.
54        #[builder(start_fn)]
55        writer_schema: &'s Schema,
56        /// Already resolved schemata that will be used to resolve references in the writer's schema.
57        resolved_writer_schemata: Option<ResolvedSchema<'s>>,
58        /// The schema that will be used to resolve the value to conform the the new schema.
59        reader_schema: Option<&'s Schema>,
60        /// Already resolved schemata that will be used to resolve references in the reader's schema.
61        resolved_reader_schemata: Option<ResolvedSchema<'s>>,
62        /// Was the data serialized with `human_readable`.
63        #[builder(default = is_human_readable())]
64        human_readable: bool,
65    ) -> AvroResult<Self> {
66        let resolved_writer_schemata = if let Some(resolved) = resolved_writer_schemata {
67            resolved
68        } else {
69            ResolvedSchema::try_from(writer_schema)?
70        };
71
72        let reader = if let Some(reader) = reader_schema {
73            if let Some(resolved) = resolved_reader_schemata {
74                Some((reader, resolved))
75            } else {
76                Some((reader, ResolvedSchema::try_from(reader)?))
77            }
78        } else {
79            None
80        };
81
82        Ok(Self {
83            writer: writer_schema,
84            resolved: resolved_writer_schemata,
85            reader,
86            human_readable,
87        })
88    }
89}
90
91impl<'s, S: generic_datum_reader_builder::State> GenericDatumReaderBuilder<'s, S> {
92    /// Set the schemata that will be used to resolve any references in the writer's schema.
93    ///
94    /// This is equivalent to `.resolved_writer_schemata(ResolvedSchema::new_with_schemata(schemata)?)`.
95    /// If you already have a [`ResolvedSchema`], use that function instead.
96    pub fn writer_schemata(
97        self,
98        schemata: Vec<&'s Schema>,
99    ) -> AvroResult<
100        GenericDatumReaderBuilder<'s, generic_datum_reader_builder::SetResolvedWriterSchemata<S>>,
101    >
102    where
103        S::ResolvedWriterSchemata: generic_datum_reader_builder::IsUnset,
104    {
105        let resolved = ResolvedSchema::new_with_schemata(schemata)?;
106        Ok(self.resolved_writer_schemata(resolved))
107    }
108
109    /// Set the schemata that will be used to resolve any references in the reader's schema.
110    ///
111    /// This is equivalent to `.resolved_reader_schemata(ResolvedSchema::new_with_schemata(schemata)?)`.
112    /// If you already have a [`ResolvedSchema`], use that function instead.
113    ///
114    /// This function can only be called after the reader schema is set.
115    pub fn reader_schemata(
116        self,
117        schemata: Vec<&'s Schema>,
118    ) -> AvroResult<
119        GenericDatumReaderBuilder<'s, generic_datum_reader_builder::SetResolvedReaderSchemata<S>>,
120    >
121    where
122        S::ResolvedReaderSchemata: generic_datum_reader_builder::IsUnset,
123        S::ReaderSchema: generic_datum_reader_builder::IsSet,
124    {
125        let resolved = ResolvedSchema::new_with_schemata(schemata)?;
126        Ok(self.resolved_reader_schemata(resolved))
127    }
128}
129
130impl<'s> GenericDatumReader<'s> {
131    /// Read a Avro datum from the reader.
132    pub fn read_value<R: Read>(&self, reader: &mut R) -> AvroResult<Value> {
133        let value = decode_internal(
134            self.writer,
135            self.resolved.get_names(),
136            None,
137            reader,
138            &mut DecodeContext::new(),
139        )?;
140        if let Some((reader, resolved)) = &self.reader {
141            value.resolve_internal(reader, resolved.get_names(), None, None)
142        } else {
143            Ok(value)
144        }
145    }
146
147    /// Read a Avro datum from the reader.
148    ///
149    /// # Panics
150    /// Will panic if a reader schema has been configured, this is a WIP.
151    pub fn read_deser<T: DeserializeOwned>(&self, reader: &mut impl Read) -> AvroResult<T> {
152        // `reader` is `impl Read` instead of a generic on the function like T so it's easier to
153        // specify the type wanted (`read_deser<String>` vs `read_deser<String, _>`)
154        if let Some((_, _)) = &self.reader {
155            // TODO: Implement SchemaAwareResolvingDeserializer
156            panic!("Schema aware deserialisation does not resolve schemas yet");
157        } else {
158            T::deserialize(SchemaAwareDeserializer::new(
159                reader,
160                self.writer,
161                Config {
162                    names: self.resolved.get_names(),
163                    human_readable: self.human_readable,
164                    recursion_depth: 0,
165                },
166            )?)
167        }
168    }
169}
170
171/// Reader for reading raw Avro data.
172///
173/// This is most likely not what you need. Most users should use [`Reader`][crate::Reader],
174/// [`GenericSingleObjectReader`][crate::GenericSingleObjectReader], or
175/// [`SpecificSingleObjectReader`][crate::SpecificSingleObjectReader] instead.
176pub struct SpecificDatumReader<T: AvroSchema> {
177    resolved: ResolvedOwnedSchema,
178    human_readable: bool,
179    phantom: PhantomData<T>,
180}
181
182#[bon]
183impl<T: AvroSchema> SpecificDatumReader<T> {
184    /// Build a [`SpecificDatumReader`].
185    ///
186    /// This is most likely not what you need. Most users should use [`Reader`][crate::Reader],
187    /// [`GenericSingleObjectReader`][crate::GenericSingleObjectReader], or
188    /// [`SpecificSingleObjectReader`][crate::SpecificSingleObjectReader] instead.
189    #[builder]
190    pub fn new(#[builder(default = is_human_readable())] human_readable: bool) -> AvroResult<Self> {
191        Ok(Self {
192            resolved: T::get_schema().try_into()?,
193            human_readable,
194            phantom: PhantomData,
195        })
196    }
197}
198
199impl<T: AvroSchema + DeserializeOwned> SpecificDatumReader<T> {
200    pub fn read<R: Read>(&self, reader: &mut R) -> AvroResult<T> {
201        T::deserialize(SchemaAwareDeserializer::new(
202            reader,
203            self.resolved.get_root_schema(),
204            Config {
205                names: self.resolved.get_names(),
206                human_readable: self.human_readable,
207                recursion_depth: 0,
208            },
209        )?)
210    }
211}
212
213#[cfg(test)]
214mod tests {
215    use apache_avro_test_helper::TestResult;
216    use serde::Deserialize;
217
218    use crate::{
219        Schema,
220        reader::datum::GenericDatumReader,
221        types::{Record, Value},
222    };
223
224    #[test]
225    fn test_from_avro_datum() -> TestResult {
226        let schema = Schema::parse_str(
227            r#"{
228            "type": "record",
229            "name": "test",
230            "fields": [
231                {
232                    "name": "a",
233                    "type": "long",
234                    "default": 42
235                },
236                {
237                    "name": "b",
238                    "type": "string"
239                }
240            ]
241        }"#,
242        )?;
243        let mut encoded: &'static [u8] = &[54, 6, 102, 111, 111];
244
245        let mut record = Record::new(&schema).unwrap();
246        record.put("a", 27i64);
247        record.put("b", "foo");
248        let expected = record.into();
249
250        let avro_datum = GenericDatumReader::builder(&schema)
251            .build()?
252            .read_value(&mut encoded)?;
253
254        assert_eq!(avro_datum, expected);
255
256        Ok(())
257    }
258
259    #[test]
260    fn test_from_avro_datum_with_union_to_struct() -> TestResult {
261        const TEST_RECORD_SCHEMA_3240: &str = r#"
262    {
263      "type": "record",
264      "name": "TestRecord3240",
265      "fields": [
266        {
267          "name": "a",
268          "type": "long",
269          "default": 42
270        },
271        {
272          "name": "b",
273          "type": "string"
274        },
275        {
276            "name": "a_nullable_array",
277            "type": ["null", {"type": "array", "items": {"type": "string"}}],
278            "default": null
279        },
280        {
281            "name": "a_nullable_boolean",
282            "type": ["null", {"type": "boolean"}],
283            "default": null
284        },
285        {
286            "name": "a_nullable_string",
287            "type": ["null", {"type": "string"}],
288            "default": null
289        }
290      ]
291    }
292    "#;
293        #[derive(Default, Debug, Deserialize, PartialEq, Eq)]
294        struct TestRecord3240 {
295            a: i64,
296            b: String,
297            a_nullable_array: Option<Vec<String>>,
298            // we are missing the 'a_nullable_boolean' field to simulate missing keys
299            // a_nullable_boolean: Option<bool>,
300            a_nullable_string: Option<String>,
301        }
302
303        let schema = Schema::parse_str(TEST_RECORD_SCHEMA_3240)?;
304        let mut encoded: &[u8] = &[54, 6, 102, 111, 111];
305
306        let error = GenericDatumReader::builder(&schema)
307            .build()?
308            .read_deser::<TestRecord3240>(&mut encoded)
309            .unwrap_err();
310        // TODO: Create a version of this test that does schema resolution
311        assert_eq!(
312            error.to_string(),
313            "Failed to read bytes for decoding variable length integer: failed to fill whole buffer"
314        );
315
316        Ok(())
317    }
318
319    #[test]
320    fn test_null_union() -> TestResult {
321        let schema = Schema::parse_str(r#"["null", "long"]"#)?;
322        let mut encoded: &'static [u8] = &[2, 0];
323
324        let avro_datum = GenericDatumReader::builder(&schema)
325            .build()?
326            .read_value(&mut encoded)?;
327        assert_eq!(avro_datum, Value::Union(1, Box::new(Value::Long(0))));
328
329        Ok(())
330    }
331}