Skip to main content

apache_avro/writer/
single_object.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use std::{io::Write, marker::PhantomData, ops::RangeInclusive};
19
20use bon::Builder;
21use serde::Serialize;
22
23use crate::Error;
24use crate::encode::encode_internal;
25use crate::serde::ser_schema::{Config, SchemaAwareSerializer};
26use crate::types::{SchemaPath, ValuePath};
27use crate::util::is_human_readable;
28use crate::{
29    AvroResult, AvroSchema, Schema,
30    error::Details,
31    headers::{HeaderBuilder, RabinFingerprintHeader},
32    schema::ResolvedOwnedSchema,
33    types::Value,
34};
35
36/// Encode `Value`s to the single object encoding.
37///
38/// Use [`SpecificSingleObjectWriter`] if you need to serialize Rust types.
39pub struct GenericSingleObjectWriter {
40    buffer: Vec<u8>,
41    resolved: ResolvedOwnedSchema,
42}
43
44impl GenericSingleObjectWriter {
45    pub fn new_with_capacity(
46        schema: &Schema,
47        initial_buffer_cap: usize,
48    ) -> AvroResult<GenericSingleObjectWriter> {
49        let header_builder = RabinFingerprintHeader::from_schema(schema);
50        Self::new_with_capacity_and_header_builder(schema, initial_buffer_cap, &header_builder)
51    }
52
53    pub fn new_with_capacity_and_header_builder<HB: HeaderBuilder>(
54        schema: &Schema,
55        initial_buffer_cap: usize,
56        header_builder: &HB,
57    ) -> AvroResult<GenericSingleObjectWriter> {
58        let mut buffer = Vec::with_capacity(initial_buffer_cap);
59        let header = header_builder.build_header();
60        buffer.extend_from_slice(&header);
61
62        Ok(GenericSingleObjectWriter {
63            buffer,
64            resolved: ResolvedOwnedSchema::try_from(schema.clone())?,
65        })
66    }
67
68    const HEADER_LENGTH_RANGE: RangeInclusive<usize> = 10_usize..=20_usize;
69
70    /// Write the referenced Value to the provided Write object. Returns a result with the number of bytes written including the header
71    pub fn write_value_ref<W: Write>(&mut self, v: &Value, writer: &mut W) -> AvroResult<usize> {
72        let original_length = self.buffer.len();
73        if !Self::HEADER_LENGTH_RANGE.contains(&original_length) {
74            Err(Details::IllegalSingleObjectWriterState.into())
75        } else {
76            write_value_ref_owned_resolved(&self.resolved, v, &mut self.buffer)?;
77            writer
78                .write_all(&self.buffer)
79                .map_err(Details::WriteBytes)?;
80            let len = self.buffer.len();
81            self.buffer.truncate(original_length);
82            Ok(len)
83        }
84    }
85
86    /// Write the Value to the provided Write object. Returns a result with the number of bytes written including the header
87    #[expect(
88        clippy::needless_pass_by_value,
89        reason = "That's the whole point of this wrapper function"
90    )]
91    pub fn write_value<W: Write>(&mut self, v: Value, writer: &mut W) -> AvroResult<usize> {
92        self.write_value_ref(&v, writer)
93    }
94}
95
96/// Encode Rust types to the single object encoding.
97///
98/// Use [`GenericSingleObjectWriter`] if you need to serialize `Value`s that are not linked to a type.
99#[derive(Builder)]
100pub struct SpecificSingleObjectWriter<T>
101where
102    T: AvroSchema,
103{
104    #[builder(
105        with = |schema: Schema| -> Result<_, Error> { ResolvedOwnedSchema::new(schema) },
106        default = ResolvedOwnedSchema::new(T::get_schema()).expect("AvroSchema implementation should create valid schemas")
107    )]
108    resolved: ResolvedOwnedSchema,
109    #[builder(
110        default = RabinFingerprintHeader::from_schema(resolved.get_root_schema()).build_header(),
111        with = |header_builder: impl HeaderBuilder| header_builder.build_header(),
112    )]
113    header: Vec<u8>,
114    /// Should [`Serialize`] implementations pick a human readable representation.
115    ///
116    /// It is recommended to set this to `false`.
117    #[builder(default = is_human_readable())]
118    human_readable: bool,
119    /// At what block size to start a new block (for arrays and maps).
120    ///
121    /// This is a minimum value, the block size will always be larger than this except for the last
122    /// block.
123    ///
124    /// When set to `None` all values will be written in a single block. This can be faster as no
125    /// intermediate buffer is used, but seeking through written data will be slower.
126    target_block_size: Option<usize>,
127    #[builder(skip)]
128    _model: PhantomData<T>,
129}
130
131impl<T> SpecificSingleObjectWriter<T>
132where
133    T: AvroSchema,
134{
135    pub fn new() -> AvroResult<Self> {
136        let schema = T::get_schema();
137        let header = RabinFingerprintHeader::from_schema(&schema).build_header();
138        let resolved = ResolvedOwnedSchema::new(schema)?;
139        // We don't use Self::new_with_header_builder as that would mean calling T::get_schema() twice
140        Ok(Self {
141            resolved,
142            header,
143            human_readable: is_human_readable(),
144            target_block_size: None,
145            _model: PhantomData,
146        })
147    }
148
149    pub fn new_with_header_builder(header_builder: &impl HeaderBuilder) -> AvroResult<Self> {
150        let header = header_builder.build_header();
151        let resolved = ResolvedOwnedSchema::new(T::get_schema())?;
152        Ok(Self {
153            resolved,
154            header,
155            human_readable: is_human_readable(),
156            target_block_size: None,
157            _model: PhantomData,
158        })
159    }
160}
161
162impl<T> SpecificSingleObjectWriter<T>
163where
164    T: AvroSchema + Into<Value>,
165{
166    /// Write the value to the writer
167    ///
168    /// Returns the number of bytes written.
169    ///
170    /// Each call writes a complete single-object encoded message (header + data),
171    /// making each message independently decodable.
172    pub fn write_value<W: Write>(&self, data: T, writer: &mut W) -> AvroResult<usize> {
173        writer
174            .write_all(&self.header)
175            .map_err(Details::WriteBytes)?;
176        let value: Value = data.into();
177        let bytes = write_value_ref_owned_resolved(&self.resolved, &value, writer)?;
178        Ok(bytes + self.header.len())
179    }
180}
181
182impl<T> SpecificSingleObjectWriter<T>
183where
184    T: AvroSchema + Serialize,
185{
186    /// Write the object to the writer.
187    ///
188    /// Returns the number of bytes written.
189    ///
190    /// Each call writes a complete single-object encoded message (header + data),
191    /// making each message independently decodable.
192    pub fn write_ref<W: Write>(&self, data: &T, writer: &mut W) -> AvroResult<usize> {
193        writer
194            .write_all(&self.header)
195            .map_err(Details::WriteBytes)?;
196
197        let config = Config {
198            names: self.resolved.get_names(),
199            target_block_size: self.target_block_size,
200            human_readable: self.human_readable,
201        };
202
203        let bytes = data.serialize(SchemaAwareSerializer::new(
204            writer,
205            self.resolved.get_root_schema(),
206            config,
207        )?)?;
208
209        Ok(bytes + self.header.len())
210    }
211
212    /// Write the object to the writer.
213    ///
214    /// Returns the number of bytes written.
215    ///
216    /// Each call writes a complete single-object encoded message (header + data),
217    /// making each message independently decodable.
218    #[expect(
219        clippy::needless_pass_by_value,
220        reason = "That's the whole point of this wrapper function"
221    )]
222    pub fn write<W: Write>(&self, data: T, writer: &mut W) -> AvroResult<usize> {
223        self.write_ref(&data, writer)
224    }
225}
226
227fn write_value_ref_owned_resolved<W: Write>(
228    resolved_schema: &ResolvedOwnedSchema,
229    value: &Value,
230    writer: &mut W,
231) -> AvroResult<usize> {
232    let root_schema = resolved_schema.get_root_schema();
233    if let Some(reason) = value.validate_internal(
234        root_schema,
235        resolved_schema.get_names(),
236        root_schema.namespace(),
237        &ValuePath::Start,
238        &SchemaPath::Start,
239    ) {
240        return Err(Details::ValidationWithReason {
241            value: value.clone(),
242            schema: root_schema.clone(),
243            reason,
244        }
245        .into());
246    }
247    encode_internal(
248        value,
249        root_schema,
250        resolved_schema.get_names(),
251        root_schema.namespace(),
252        writer,
253    )
254}
255
256#[cfg(test)]
257mod tests {
258    use apache_avro_test_helper::TestResult;
259    use uuid::Uuid;
260
261    use crate::{encode::encode, headers::GlueSchemaUuidHeader, rabin::Rabin};
262
263    use super::*;
264
265    #[derive(Serialize, Clone)]
266    struct TestSingleObjectWriter {
267        a: i64,
268        b: f64,
269        c: Vec<String>,
270    }
271
272    impl AvroSchema for TestSingleObjectWriter {
273        fn get_schema() -> Schema {
274            let schema = r#"
275            {
276                "type":"record",
277                "name":"TestSingleObjectWrtierSerialize",
278                "fields":[
279                    {
280                        "name":"a",
281                        "type":"long"
282                    },
283                    {
284                        "name":"b",
285                        "type":"double"
286                    },
287                    {
288                        "name":"c",
289                        "type":{
290                            "type":"array",
291                            "items":"string"
292                        }
293                    }
294                ]
295            }
296            "#;
297            Schema::parse_str(schema).unwrap()
298        }
299    }
300
301    impl From<TestSingleObjectWriter> for Value {
302        fn from(obj: TestSingleObjectWriter) -> Value {
303            Value::Record(vec![
304                ("a".into(), obj.a.into()),
305                ("b".into(), obj.b.into()),
306                (
307                    "c".into(),
308                    Value::Array(obj.c.into_iter().map(|s| s.into()).collect()),
309                ),
310            ])
311        }
312    }
313
314    #[test]
315    fn test_single_object_writer() -> TestResult {
316        let mut buf: Vec<u8> = Vec::new();
317        let obj = TestSingleObjectWriter {
318            a: 300,
319            b: 34.555,
320            c: vec!["cat".into(), "dog".into()],
321        };
322        let mut writer = GenericSingleObjectWriter::new_with_capacity(
323            &TestSingleObjectWriter::get_schema(),
324            1024,
325        )
326        .expect("Should resolve schema");
327        let value = obj.into();
328        let written_bytes = writer
329            .write_value_ref(&value, &mut buf)
330            .expect("Error serializing properly");
331
332        assert!(buf.len() > 10, "no bytes written");
333        assert_eq!(buf.len(), written_bytes);
334        assert_eq!(buf[0], 0xC3);
335        assert_eq!(buf[1], 0x01);
336        assert_eq!(
337            &buf[2..10],
338            &TestSingleObjectWriter::get_schema()
339                .fingerprint::<Rabin>()
340                .bytes[..]
341        );
342        let mut msg_binary = Vec::new();
343        encode(
344            &value,
345            &TestSingleObjectWriter::get_schema(),
346            &mut msg_binary,
347        )
348        .expect("encode should have failed by here as a dependency of any writing");
349        assert_eq!(&buf[10..], &msg_binary[..]);
350
351        Ok(())
352    }
353
354    #[test]
355    fn test_single_object_writer_with_header_builder() -> TestResult {
356        let mut buf: Vec<u8> = Vec::new();
357        let obj = TestSingleObjectWriter {
358            a: 300,
359            b: 34.555,
360            c: vec!["cat".into(), "dog".into()],
361        };
362        let schema_uuid = Uuid::parse_str("b2f1cf00-0434-013e-439a-125eb8485a5f")?;
363        let header_builder = GlueSchemaUuidHeader::from_uuid(schema_uuid);
364        let mut writer = GenericSingleObjectWriter::new_with_capacity_and_header_builder(
365            &TestSingleObjectWriter::get_schema(),
366            1024,
367            &header_builder,
368        )
369        .expect("Should resolve schema");
370        let value = obj.into();
371        writer
372            .write_value_ref(&value, &mut buf)
373            .expect("Error serializing properly");
374
375        assert_eq!(buf[0], 0x03);
376        assert_eq!(buf[1], 0x00);
377        assert_eq!(buf[2..18], schema_uuid.into_bytes()[..]);
378        Ok(())
379    }
380
381    #[test]
382    fn test_writer_parity() -> TestResult {
383        let obj1 = TestSingleObjectWriter {
384            a: 300,
385            b: 34.555,
386            c: vec!["cat".into(), "dog".into()],
387        };
388
389        let mut buf1: Vec<u8> = Vec::new();
390        let mut buf2: Vec<u8> = Vec::new();
391        let mut buf3: Vec<u8> = Vec::new();
392        let mut buf4: Vec<u8> = Vec::new();
393
394        let mut generic_writer = GenericSingleObjectWriter::new_with_capacity(
395            &TestSingleObjectWriter::get_schema(),
396            1024,
397        )
398        .expect("Should resolve schema");
399        let specific_writer = SpecificSingleObjectWriter::<TestSingleObjectWriter>::new()
400            .expect("Resolved should pass");
401        specific_writer
402            .write_ref(&obj1, &mut buf1)
403            .expect("Serialization expected");
404        specific_writer
405            .write_ref(&obj1, &mut buf2)
406            .expect("Serialization expected");
407        specific_writer
408            .write_value(obj1.clone(), &mut buf3)
409            .expect("Serialization expected");
410
411        generic_writer
412            .write_value(obj1.into(), &mut buf4)
413            .expect("Serialization expected");
414
415        assert_eq!(buf1, buf2);
416        assert_eq!(buf2, buf3);
417        assert_eq!(buf3, buf4);
418
419        Ok(())
420    }
421
422    #[test]
423    fn avro_rs_439_specific_single_object_writer_ref() -> TestResult {
424        #[derive(Serialize)]
425        struct Recursive {
426            field: bool,
427            recurse: Option<Box<Recursive>>,
428        }
429
430        impl AvroSchema for Recursive {
431            fn get_schema() -> Schema {
432                Schema::parse_str(
433                    r#"{
434                    "name": "Recursive",
435                    "type": "record",
436                    "fields": [
437                        { "name": "field", "type": "boolean" },
438                        { "name": "recurse", "type": ["null", "Recursive"] }
439                    ]
440                }"#,
441                )
442                .unwrap()
443            }
444        }
445
446        let mut buffer = Vec::new();
447        let writer = SpecificSingleObjectWriter::new()?;
448
449        writer.write(
450            Recursive {
451                field: true,
452                recurse: Some(Box::new(Recursive {
453                    field: false,
454                    recurse: None,
455                })),
456            },
457            &mut buffer,
458        )?;
459        assert_eq!(
460            buffer,
461            &[195, 1, 83, 223, 43, 26, 181, 179, 227, 224, 1, 2, 0, 0][..]
462        );
463
464        Ok(())
465    }
466}