Skip to main content

apache_avro/writer/
datum.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use bon::bon;
19use serde::Serialize;
20use std::io::Write;
21
22use crate::types::{SchemaPath, ValuePath};
23use crate::{
24    AvroResult, Schema,
25    encode::encode_internal,
26    error::Details,
27    schema::{NamesRef, ResolvedSchema},
28    serde::ser_schema::{Config, SchemaAwareSerializer},
29    types::Value,
30    util::is_human_readable,
31};
32
33/// Writer for writing raw Avro data.
34///
35/// This is most likely not what you need. Most users should use [`Writer`][crate::Writer],
36/// [`GenericSingleObjectWriter`][crate::GenericSingleObjectWriter], or
37/// [`SpecificSingleObjectWriter`][crate::SpecificSingleObjectWriter] instead.
38pub struct GenericDatumWriter<'s> {
39    schema: &'s Schema,
40    resolved: ResolvedSchema<'s>,
41    validate: bool,
42    human_readable: bool,
43    target_block_size: Option<usize>,
44}
45
46#[bon]
47impl<'s> GenericDatumWriter<'s> {
48    /// Configure a new writer.
49    #[builder]
50    pub fn new(
51        /// The schema for the data that will be written
52        #[builder(start_fn)]
53        schema: &'s Schema,
54        /// Already resolved schemata that will be used to resolve references in the writer's schema.
55        ///
56        /// You can also use [`Self::schemata`] instead.
57        resolved_schemata: Option<ResolvedSchema<'s>>,
58        /// Validate values against the writer schema before writing them.
59        ///
60        /// Defaults to `true`.
61        ///
62        /// Setting this to `false` and writing values that don't match the schema will make the
63        /// written data unreadable.
64        #[builder(default = true)]
65        validate: bool,
66        /// At what block size to start a new block (for arrays and maps).
67        ///
68        /// This is a minimum value, the block size will always be larger than this except for the last
69        /// block.
70        ///
71        /// When set to `None` all values will be written in a single block. This can be faster as no
72        /// intermediate buffer is used, but seeking through written data will be slower.
73        target_block_size: Option<usize>,
74        /// Should [`Serialize`] implementations pick a human readable representation.
75        ///
76        /// It is recommended to set this to `false`.
77        #[builder(default = is_human_readable())]
78        human_readable: bool,
79    ) -> AvroResult<Self> {
80        let resolved = if let Some(resolved) = resolved_schemata {
81            resolved
82        } else {
83            ResolvedSchema::try_from(schema)?
84        };
85        Ok(Self {
86            schema,
87            resolved,
88            validate,
89            human_readable,
90            target_block_size,
91        })
92    }
93}
94
95impl<'s, S: generic_datum_writer_builder::State> GenericDatumWriterBuilder<'s, S> {
96    /// Set the schemata that will be used to resolve any references in the schema.
97    ///
98    /// This is equivalent to `.resolved_schemata(ResolvedSchema::new_with_schemata(schemata)?)`.
99    /// If you already have a [`ResolvedSchema`], use that function instead.
100    pub fn schemata(
101        self,
102        schemata: Vec<&'s Schema>,
103    ) -> AvroResult<
104        GenericDatumWriterBuilder<'s, generic_datum_writer_builder::SetResolvedSchemata<S>>,
105    >
106    where
107        S::ResolvedSchemata: generic_datum_writer_builder::IsUnset,
108    {
109        let resolved = ResolvedSchema::new_with_schemata(schemata)?;
110        Ok(self.resolved_schemata(resolved))
111    }
112}
113
114impl GenericDatumWriter<'_> {
115    /// Write a value to the writer.
116    pub fn write_value<W: Write, V: Into<Value>>(
117        &self,
118        writer: &mut W,
119        value: V,
120    ) -> AvroResult<usize> {
121        let value = value.into();
122        self.write_value_ref(writer, &value)
123    }
124
125    /// Write a value to the writer.
126    pub fn write_value_ref<W: Write>(&self, writer: &mut W, value: &Value) -> AvroResult<usize> {
127        if self.validate
128            && value
129                .validate_internal(
130                    self.schema,
131                    self.resolved.get_names(),
132                    None,
133                    &ValuePath::Start,
134                    &SchemaPath::Start,
135                )
136                .is_some()
137        {
138            return Err(Details::Validation.into());
139        }
140        encode_internal(value, self.schema, self.resolved.get_names(), None, writer)
141    }
142
143    /// Write a value to a [`Vec`].
144    pub fn write_value_to_vec<V: Into<Value>>(&self, value: V) -> AvroResult<Vec<u8>> {
145        let mut vec = Vec::new();
146        self.write_value(&mut vec, value)?;
147        Ok(vec)
148    }
149
150    /// Serialize `T` to the writer.
151    pub fn write_ser<W: Write, T: Serialize>(
152        &self,
153        writer: &mut W,
154        value: &T,
155    ) -> AvroResult<usize> {
156        let config = Config {
157            names: self.resolved.get_names(),
158            target_block_size: self.target_block_size,
159            human_readable: self.human_readable,
160        };
161        value.serialize(SchemaAwareSerializer::new(writer, self.schema, config)?)
162    }
163
164    /// Serialize `T` to a [`Vec`].
165    pub fn write_ser_to_vec<T: Serialize>(&self, value: &T) -> AvroResult<Vec<u8>> {
166        let mut vec = Vec::new();
167        self.write_ser(&mut vec, value)?;
168        Ok(vec)
169    }
170}
171
172/// Write the referenced [Serialize]able object to the provided [Write] object.
173///
174/// It is recommended to use [`GenericDatumWriter`] instead.
175///
176/// Returns a result with the number of bytes written.
177///
178/// **NOTE**: This function has a quite small niche of usage and does **NOT** generate headers and sync
179/// markers; use [`Writer::append_ser`] to be fully Avro-compatible
180/// if you don't know what you are doing, instead.
181///
182/// [`Writer::append_ser`]: crate::Writer::append_ser
183pub fn write_avro_datum_ref<T: Serialize, W: Write>(
184    schema: &Schema,
185    names: &NamesRef,
186    data: &T,
187    writer: &mut W,
188) -> AvroResult<usize> {
189    let config = Config {
190        names,
191        target_block_size: None,
192        human_readable: is_human_readable(),
193    };
194    data.serialize(SchemaAwareSerializer::new(writer, schema, config)?)
195}
196
197#[cfg(test)]
198mod tests {
199    use apache_avro_test_helper::TestResult;
200    use std::num::NonZero;
201
202    use super::*;
203    use crate::reader::datum::GenericDatumReader;
204    use crate::{
205        Days, Decimal, Duration, Millis, Months,
206        schema::{DecimalSchema, FixedSchema, InnerDecimalSchema, Name},
207        types::Record,
208        util::zig_i64,
209    };
210
211    const SCHEMA: &str = r#"
212    {
213      "type": "record",
214      "name": "test",
215      "fields": [
216        {
217          "name": "a",
218          "type": "long",
219          "default": 42
220        },
221        {
222          "name": "b",
223          "type": "string"
224        }
225      ]
226    }
227    "#;
228
229    const UNION_SCHEMA: &str = r#"["null", "long"]"#;
230
231    #[test]
232    fn test_to_avro_datum() -> TestResult {
233        let schema = Schema::parse_str(SCHEMA)?;
234        let mut record = Record::new(&schema).unwrap();
235        record.put("a", 27i64);
236        record.put("b", "foo");
237
238        let mut expected = Vec::new();
239        zig_i64(27, &mut expected)?;
240        zig_i64(3, &mut expected)?;
241        expected.extend(b"foo");
242
243        let written = GenericDatumWriter::builder(&schema)
244            .build()?
245            .write_value_to_vec(record)?;
246
247        assert_eq!(written, expected);
248
249        Ok(())
250    }
251
252    #[test]
253    fn avro_rs_193_write_avro_datum_ref() -> TestResult {
254        #[derive(Serialize)]
255        struct TestStruct {
256            a: i64,
257            b: String,
258        }
259
260        let schema = Schema::parse_str(SCHEMA)?;
261        let mut writer: Vec<u8> = Vec::new();
262        let data = TestStruct {
263            a: 27,
264            b: "foo".to_string(),
265        };
266
267        let mut expected = Vec::new();
268        zig_i64(27, &mut expected)?;
269        zig_i64(3, &mut expected)?;
270        expected.extend(b"foo");
271
272        let bytes = GenericDatumWriter::builder(&schema)
273            .build()?
274            .write_ser(&mut writer, &data)?;
275
276        assert_eq!(bytes, expected.len());
277        assert_eq!(writer, expected);
278
279        Ok(())
280    }
281
282    #[test]
283    fn test_union_not_null() -> TestResult {
284        let schema = Schema::parse_str(UNION_SCHEMA)?;
285        let union = Value::Union(1, Box::new(Value::Long(3)));
286
287        let mut expected = Vec::new();
288        zig_i64(1, &mut expected)?;
289        zig_i64(3, &mut expected)?;
290
291        let written = GenericDatumWriter::builder(&schema)
292            .build()?
293            .write_value_to_vec(union)?;
294        assert_eq!(written, expected);
295
296        Ok(())
297    }
298
299    #[test]
300    fn test_union_null() -> TestResult {
301        let schema = Schema::parse_str(UNION_SCHEMA)?;
302        let union = Value::Union(0, Box::new(Value::Null));
303
304        let mut expected = Vec::new();
305        zig_i64(0, &mut expected)?;
306
307        let written = GenericDatumWriter::builder(&schema)
308            .build()?
309            .write_value_to_vec(union)?;
310        assert_eq!(written, expected);
311
312        Ok(())
313    }
314
315    #[expect(
316        clippy::needless_pass_by_value,
317        reason = "Value does not implement PartialEq<&Value>"
318    )]
319    fn logical_type_test<T: Into<Value> + Clone>(
320        schema_str: &'static str,
321
322        expected_schema: &Schema,
323        value: Value,
324
325        raw_schema: &Schema,
326        raw_value: T,
327    ) -> TestResult {
328        let schema = Schema::parse_str(schema_str)?;
329        assert_eq!(&schema, expected_schema);
330        // The serialized format should be the same as the schema.
331        let ser = GenericDatumWriter::builder(&schema)
332            .build()?
333            .write_value_to_vec(value.clone())?;
334        let raw_ser = GenericDatumWriter::builder(raw_schema)
335            .build()?
336            .write_value_to_vec(raw_value)?;
337        assert_eq!(ser, raw_ser);
338
339        // Should deserialize from the schema into the logical type.
340        let mut r = ser.as_slice();
341        let de = GenericDatumReader::builder(&schema)
342            .build()?
343            .read_value(&mut r)?;
344        assert_eq!(de, value);
345        Ok(())
346    }
347
348    #[test]
349    fn date() -> TestResult {
350        logical_type_test(
351            r#"{"type": "int", "logicalType": "date"}"#,
352            &Schema::Date,
353            Value::Date(1_i32),
354            &Schema::Int,
355            1_i32,
356        )
357    }
358
359    #[test]
360    fn time_millis() -> TestResult {
361        logical_type_test(
362            r#"{"type": "int", "logicalType": "time-millis"}"#,
363            &Schema::TimeMillis,
364            Value::TimeMillis(1_i32),
365            &Schema::Int,
366            1_i32,
367        )
368    }
369
370    #[test]
371    fn time_micros() -> TestResult {
372        logical_type_test(
373            r#"{"type": "long", "logicalType": "time-micros"}"#,
374            &Schema::TimeMicros,
375            Value::TimeMicros(1_i64),
376            &Schema::Long,
377            1_i64,
378        )
379    }
380
381    #[test]
382    fn timestamp_millis() -> TestResult {
383        logical_type_test(
384            r#"{"type": "long", "logicalType": "timestamp-millis"}"#,
385            &Schema::TimestampMillis,
386            Value::TimestampMillis(1_i64),
387            &Schema::Long,
388            1_i64,
389        )
390    }
391
392    #[test]
393    fn timestamp_micros() -> TestResult {
394        logical_type_test(
395            r#"{"type": "long", "logicalType": "timestamp-micros"}"#,
396            &Schema::TimestampMicros,
397            Value::TimestampMicros(1_i64),
398            &Schema::Long,
399            1_i64,
400        )
401    }
402
403    #[test]
404    fn decimal_fixed() -> TestResult {
405        let size = 30;
406        let fixed = FixedSchema {
407            name: Name::new("decimal")?,
408            aliases: None,
409            doc: None,
410            size,
411            attributes: Default::default(),
412        };
413        let inner = InnerDecimalSchema::Fixed(fixed.clone());
414        let value = vec![0u8; size];
415        logical_type_test(
416            r#"{"type": {"type": "fixed", "size": 30, "name": "decimal"}, "logicalType": "decimal", "precision": 20, "scale": 5}"#,
417            &Schema::Decimal(DecimalSchema {
418                precision: NonZero::new(20).unwrap(),
419                scale: 5,
420                inner,
421            }),
422            Value::Decimal(Decimal::new(&value)?),
423            &Schema::Fixed(fixed),
424            Value::Fixed(size, value),
425        )
426    }
427
428    #[test]
429    fn decimal_bytes() -> TestResult {
430        let value = vec![0u8; 10];
431        logical_type_test(
432            r#"{"type": "bytes", "logicalType": "decimal", "precision": 4, "scale": 3}"#,
433            &Schema::Decimal(DecimalSchema {
434                precision: NonZero::new(4).unwrap(),
435                scale: 3,
436                inner: InnerDecimalSchema::Bytes,
437            }),
438            Value::Decimal(Decimal::new(&value)?),
439            &Schema::Bytes,
440            value,
441        )
442    }
443
444    #[test]
445    fn duration() -> TestResult {
446        let inner = Schema::Fixed(FixedSchema {
447            name: Name::new("duration")?,
448            aliases: None,
449            doc: None,
450            size: 12,
451            attributes: Default::default(),
452        });
453        let value = Value::Duration(Duration::new(
454            Months::new(256),
455            Days::new(512),
456            Millis::new(1024),
457        ));
458        logical_type_test(
459            r#"{"type": {"type": "fixed", "name": "duration", "size": 12}, "logicalType": "duration"}"#,
460            &Schema::Duration(FixedSchema {
461                name: Name::try_from("duration").expect("Name is valid"),
462                aliases: None,
463                doc: None,
464                size: 12,
465                attributes: Default::default(),
466            }),
467            value,
468            &inner,
469            Value::Fixed(12, vec![0, 1, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0]),
470        )
471    }
472}