Skip to main content

apache_avro/writer/
mod.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18//! Logic handling writing in Avro format at user level.
19use crate::types::{SchemaPath, ValuePath};
20use crate::{
21    AvroResult, Codec, Error,
22    encode::{encode, encode_internal, encode_to_vec},
23    error::Details,
24    schema::{ResolvedSchema, Schema},
25    serde::ser_schema::{Config, SchemaAwareSerializer},
26    types::Value,
27    util::is_human_readable,
28};
29use serde::Serialize;
30use std::{collections::HashMap, io::Write, mem::ManuallyDrop};
31
32pub mod datum;
33pub mod single_object;
34
35const DEFAULT_BLOCK_SIZE: usize = 16000;
36const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01";
37
38/// Main interface for writing Avro formatted values.
39///
40/// It is critical to call flush before `Writer<W>` is dropped. Though dropping will attempt to flush
41/// the contents of the buffer, any errors that happen in the process of dropping will be ignored.
42/// Calling flush ensures that the buffer is empty and thus dropping will not even attempt file operations.
43pub struct Writer<'a, W: Write> {
44    schema: &'a Schema,
45    writer: W,
46    resolved_schema: ResolvedSchema<'a>,
47    codec: Codec,
48    block_size: usize,
49    buffer: Vec<u8>,
50    num_values: usize,
51    marker: [u8; 16],
52    has_header: bool,
53    user_metadata: HashMap<String, Value>,
54    human_readable: bool,
55    map_array_target_block_size: Option<usize>,
56}
57
58#[bon::bon]
59impl<'a, W: Write> Writer<'a, W> {
60    #[builder]
61    pub fn builder(
62        schema: &'a Schema,
63        schemata: Option<Vec<&'a Schema>>,
64        writer: W,
65        #[builder(default = Codec::Null)] codec: Codec,
66        #[builder(default = DEFAULT_BLOCK_SIZE)] block_size: usize,
67        #[builder(default = generate_sync_marker())] marker: [u8; 16],
68        /// Has the header already been written.
69        ///
70        /// To disable writing the header, this can be set to `true`.
71        #[builder(default = false)]
72        has_header: bool,
73        #[builder(default)] user_metadata: HashMap<String, Value>,
74        /// Should [`Serialize`] implementations pick a human readable representation.
75        ///
76        /// It is recommended to set this to `false`.
77        #[builder(default = is_human_readable())]
78        human_readable: bool,
79        /// At what block size to start a new block (for arrays and maps).
80        ///
81        /// This is a minimum value, the block size will always be larger than this except for the last
82        /// block.
83        ///
84        /// When set to `None` all values will be written in a single block. This can be faster as no
85        /// intermediate buffer is used, but seeking through written data will be slower.
86        map_array_target_block_size: Option<usize>,
87    ) -> AvroResult<Self> {
88        let resolved_schema = if let Some(schemata) = schemata {
89            ResolvedSchema::try_from(schemata)?
90        } else {
91            ResolvedSchema::try_from(schema)?
92        };
93        Ok(Self {
94            schema,
95            writer,
96            resolved_schema,
97            codec,
98            block_size,
99            buffer: Vec::with_capacity(block_size),
100            num_values: 0,
101            marker,
102            has_header,
103            user_metadata,
104            human_readable,
105            map_array_target_block_size,
106        })
107    }
108}
109
110impl<'a, W: Write> Writer<'a, W> {
111    /// Creates a `Writer` given a `Schema` and something implementing the `io::Write` trait to write
112    /// to.
113    /// No compression `Codec` will be used.
114    pub fn new(schema: &'a Schema, writer: W) -> AvroResult<Self> {
115        Writer::with_codec(schema, writer, Codec::Null)
116    }
117
118    /// Creates a `Writer` with a specific `Codec` given a `Schema` and something implementing the
119    /// `io::Write` trait to write to.
120    pub fn with_codec(schema: &'a Schema, writer: W, codec: Codec) -> AvroResult<Self> {
121        Self::builder()
122            .schema(schema)
123            .writer(writer)
124            .codec(codec)
125            .build()
126    }
127
128    /// Creates a `Writer` with a specific `Codec` given a `Schema` and something implementing the
129    /// `io::Write` trait to write to.
130    /// If the `schema` is incomplete, i.e. contains `Schema::Ref`s then all dependencies must
131    /// be provided in `schemata`.
132    pub fn with_schemata(
133        schema: &'a Schema,
134        schemata: Vec<&'a Schema>,
135        writer: W,
136        codec: Codec,
137    ) -> AvroResult<Self> {
138        Self::builder()
139            .schema(schema)
140            .schemata(schemata)
141            .writer(writer)
142            .codec(codec)
143            .build()
144    }
145
146    /// Creates a `Writer` that will append values to already populated
147    /// `std::io::Write` using the provided `marker`
148    /// No compression `Codec` will be used.
149    pub fn append_to(schema: &'a Schema, writer: W, marker: [u8; 16]) -> AvroResult<Self> {
150        Writer::append_to_with_codec(schema, writer, Codec::Null, marker)
151    }
152
153    /// Creates a `Writer` that will append values to already populated
154    /// `std::io::Write` using the provided `marker`
155    pub fn append_to_with_codec(
156        schema: &'a Schema,
157        writer: W,
158        codec: Codec,
159        marker: [u8; 16],
160    ) -> AvroResult<Self> {
161        Self::builder()
162            .schema(schema)
163            .writer(writer)
164            .codec(codec)
165            .marker(marker)
166            .has_header(true)
167            .build()
168    }
169
170    /// Creates a `Writer` that will append values to already populated
171    /// `std::io::Write` using the provided `marker`
172    pub fn append_to_with_codec_schemata(
173        schema: &'a Schema,
174        schemata: Vec<&'a Schema>,
175        writer: W,
176        codec: Codec,
177        marker: [u8; 16],
178    ) -> AvroResult<Self> {
179        Self::builder()
180            .schema(schema)
181            .schemata(schemata)
182            .writer(writer)
183            .codec(codec)
184            .marker(marker)
185            .has_header(true)
186            .build()
187    }
188
189    /// Get a reference to the `Schema` associated to a `Writer`.
190    pub fn schema(&self) -> &'a Schema {
191        self.schema
192    }
193
194    /// Append a value to the `Writer`, also performs schema validation.
195    ///
196    /// Returns the number of bytes written (it might be 0, see below).
197    ///
198    /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on
199    /// internal buffering for performance reasons. If you want to be sure the value has been
200    /// written, then call [`flush`](Writer::flush).
201    pub fn append_value<T: Into<Value>>(&mut self, value: T) -> AvroResult<usize> {
202        let avro = value.into();
203        self.append_value_ref(&avro)
204    }
205
206    /// Append a compatible value to a `Writer`, also performs schema validation.
207    ///
208    /// Returns the number of bytes written (it might be 0, see below).
209    ///
210    /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on
211    /// internal buffering for performance reasons. If you want to be sure the value has been
212    /// written, then call [`flush`](Writer::flush).
213    pub fn append_value_ref(&mut self, value: &Value) -> AvroResult<usize> {
214        if let Some(reason) = value.validate_internal(
215            self.schema,
216            self.resolved_schema.get_names(),
217            self.schema.namespace(),
218            &ValuePath::Start,
219            &SchemaPath::Start,
220        ) {
221            return Err(Details::ValidationWithReason {
222                value: value.clone(),
223                schema: self.schema.clone(),
224                reason,
225            }
226            .into());
227        }
228        self.unvalidated_append_value_ref(value)
229    }
230
231    /// Append a compatible value to a `Writer`.
232    ///
233    /// This function does **not** validate that the provided value matches the schema. If it does
234    /// not match, the file will contain corrupt data. Use [`Writer::append_value`] to have the
235    /// value validated during write or use [`Value::validate`] to validate the value.
236    ///
237    /// Returns the number of bytes written (it might be 0, see below).
238    ///
239    /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on
240    /// internal buffering for performance reasons. If you want to be sure the value has been
241    /// written, then call [`flush`](Writer::flush).
242    pub fn unvalidated_append_value<T: Into<Value>>(&mut self, value: T) -> AvroResult<usize> {
243        let value = value.into();
244        self.unvalidated_append_value_ref(&value)
245    }
246
247    /// Append a compatible value to a `Writer`.
248    ///
249    /// This function does **not** validate that the provided value matches the schema. If it does
250    /// not match, the file will contain corrupt data. Use [`Writer::append_value_ref`] to have the
251    /// value validated during write or use [`Value::validate`] to validate the value.
252    ///
253    /// Returns the number of bytes written (it might be 0, see below).
254    ///
255    /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on
256    /// internal buffering for performance reasons. If you want to be sure the value has been
257    /// written, then call [`flush`](Writer::flush).
258    pub fn unvalidated_append_value_ref(&mut self, value: &Value) -> AvroResult<usize> {
259        let n = self.maybe_write_header()?;
260        encode_internal(
261            value,
262            self.schema,
263            self.resolved_schema.get_names(),
264            self.schema.namespace(),
265            &mut self.buffer,
266        )?;
267
268        self.num_values += 1;
269
270        if self.buffer.len() >= self.block_size {
271            return self.flush().map(|b| b + n);
272        }
273
274        Ok(n)
275    }
276
277    /// Append anything implementing the `Serialize` trait to a `Writer` for
278    /// [`serde`](https://docs.serde.rs/serde/index.html) compatibility, also performing schema
279    /// validation.
280    ///
281    /// Returns the number of bytes written.
282    ///
283    /// **NOTE**: This function is not guaranteed to perform any actual write, since it relies on
284    /// internal buffering for performance reasons. If you want to be sure the value has been
285    /// written, then call [`flush`](Writer::flush).
286    pub fn append_ser<S: Serialize>(&mut self, value: S) -> AvroResult<usize> {
287        let n = self.maybe_write_header()?;
288
289        let config = Config {
290            names: self.resolved_schema.get_names(),
291            target_block_size: self.map_array_target_block_size,
292            human_readable: self.human_readable,
293        };
294
295        value.serialize(SchemaAwareSerializer::new(
296            &mut self.buffer,
297            self.schema,
298            config,
299        )?)?;
300        self.num_values += 1;
301
302        if self.buffer.len() >= self.block_size {
303            return self.flush().map(|b| b + n);
304        }
305
306        Ok(n)
307    }
308
309    /// Extend a `Writer` with an `Iterator` of values, also performs schema validation.
310    ///
311    /// Returns the number of bytes written.
312    ///
313    /// **NOTE**: This function forces the written data to be flushed (an implicit
314    /// call to [`flush`](Writer::flush) is performed).
315    pub fn extend<I, T: Into<Value>>(&mut self, values: I) -> AvroResult<usize>
316    where
317        I: IntoIterator<Item = T>,
318    {
319        /*
320        https://github.com/rust-lang/rfcs/issues/811 :(
321        let mut stream = values
322            .filter_map(|value| value.serialize(&mut self.serializer).ok())
323            .map(|value| value.encode(self.schema))
324            .collect::<Option<Vec<_>>>()
325            .ok_or_else(|| err_msg("value does not match given schema"))?
326            .into_iter()
327            .fold(Vec::new(), |mut acc, stream| {
328                num_values += 1;
329                acc.extend(stream); acc
330            });
331        */
332
333        let mut num_bytes = 0;
334        for value in values {
335            num_bytes += self.append_value(value)?;
336        }
337        num_bytes += self.flush()?;
338
339        Ok(num_bytes)
340    }
341
342    /// Extend a `Writer` with an `Iterator` of anything implementing the `Serialize` trait for
343    /// [`serde`](https://docs.serde.rs/serde/index.html) compatibility, also performing schema
344    /// validation.
345    ///
346    /// Returns the number of bytes written.
347    ///
348    /// **NOTE**: This function forces the written data to be flushed (an implicit
349    /// call to [`flush`](Writer::flush) is performed).
350    pub fn extend_ser<I, T: Serialize>(&mut self, values: I) -> AvroResult<usize>
351    where
352        I: IntoIterator<Item = T>,
353    {
354        /*
355        https://github.com/rust-lang/rfcs/issues/811 :(
356        let mut stream = values
357            .filter_map(|value| value.serialize(&mut self.serializer).ok())
358            .map(|value| value.encode(self.schema))
359            .collect::<Option<Vec<_>>>()
360            .ok_or_else(|| err_msg("value does not match given schema"))?
361            .into_iter()
362            .fold(Vec::new(), |mut acc, stream| {
363                num_values += 1;
364                acc.extend(stream); acc
365            });
366        */
367
368        let mut num_bytes = 0;
369        for value in values {
370            num_bytes += self.append_ser(value)?;
371        }
372        num_bytes += self.flush()?;
373
374        Ok(num_bytes)
375    }
376
377    /// Extend a `Writer` by appending each `Value` from a slice, while also performing schema
378    /// validation on each value appended.
379    ///
380    /// Returns the number of bytes written.
381    ///
382    /// **NOTE**: This function forces the written data to be flushed (an implicit
383    /// call to [`flush`](Writer::flush) is performed).
384    pub fn extend_from_slice(&mut self, values: &[Value]) -> AvroResult<usize> {
385        let mut num_bytes = 0;
386        for value in values {
387            num_bytes += self.append_value_ref(value)?;
388        }
389        num_bytes += self.flush()?;
390
391        Ok(num_bytes)
392    }
393
394    /// Flush the content to the inner `Writer`.
395    ///
396    /// Call this function to make sure all the content has been written before releasing the `Writer`.
397    /// This will also write the header if it wasn't written yet and hasn't been disabled using
398    /// [`WriterBuilder::has_header`].
399    ///
400    /// Returns the number of bytes written.
401    pub fn flush(&mut self) -> AvroResult<usize> {
402        let mut num_bytes = self.maybe_write_header()?;
403        if self.num_values == 0 {
404            return Ok(num_bytes);
405        }
406
407        self.codec.compress(&mut self.buffer)?;
408
409        let num_values = self.num_values;
410        let stream_len = self.buffer.len();
411
412        num_bytes += self.append_raw(&num_values.try_into()?, &Schema::Long)?
413            + self.append_raw(&stream_len.try_into()?, &Schema::Long)?
414            + self
415                .writer
416                .write(self.buffer.as_ref())
417                .map_err(Details::WriteBytes)?
418            + self.append_marker()?;
419
420        self.buffer.clear();
421        self.num_values = 0;
422
423        self.writer.flush().map_err(Details::FlushWriter)?;
424
425        Ok(num_bytes)
426    }
427
428    /// Return what the `Writer` is writing to, consuming the `Writer` itself.
429    ///
430    /// **NOTE**: This function forces the written data to be flushed (an implicit
431    /// call to [`flush`](Writer::flush) is performed).
432    pub fn into_inner(mut self) -> AvroResult<W> {
433        self.maybe_write_header()?;
434        self.flush()?;
435
436        let mut this = ManuallyDrop::new(self);
437
438        // Extract every member that is not Copy and therefore should be dropped
439        let _buffer = std::mem::take(&mut this.buffer);
440        let _user_metadata = std::mem::take(&mut this.user_metadata);
441        // SAFETY: resolved schema is not accessed after this and won't be dropped because of ManuallyDrop
442        unsafe { std::ptr::drop_in_place(&mut this.resolved_schema) };
443
444        // SAFETY: double-drops are prevented by putting `this` in a ManuallyDrop that is never dropped
445        let writer = unsafe { std::ptr::read(&this.writer) };
446
447        Ok(writer)
448    }
449
450    /// Gets a reference to the underlying writer.
451    ///
452    /// **NOTE**: There is likely data still in the buffer. To have all the data
453    /// in the writer call [`flush`](Writer::flush) first.
454    pub fn get_ref(&self) -> &W {
455        &self.writer
456    }
457
458    /// Gets a mutable reference to the underlying writer.
459    ///
460    /// It is inadvisable to directly write to the underlying writer.
461    ///
462    /// **NOTE**: There is likely data still in the buffer. To have all the data
463    /// in the writer call [`flush`](Writer::flush) first.
464    pub fn get_mut(&mut self) -> &mut W {
465        &mut self.writer
466    }
467
468    /// Generate and append synchronization marker to the payload.
469    fn append_marker(&mut self) -> AvroResult<usize> {
470        // using .writer.write directly to avoid mutable borrow of self
471        // with ref borrowing of self.marker
472        self.writer
473            .write(&self.marker)
474            .map_err(|e| Details::WriteMarker(e).into())
475    }
476
477    /// Append a raw Avro Value to the payload avoiding to encode it again.
478    fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult<usize> {
479        self.append_bytes(encode_to_vec(value, schema)?.as_ref())
480    }
481
482    /// Append pure bytes to the payload.
483    fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult<usize> {
484        self.writer
485            .write(bytes)
486            .map_err(|e| Details::WriteBytes(e).into())
487    }
488
489    /// Adds custom metadata to the file.
490    /// This method could be used only before adding the first record to the writer.
491    pub fn add_user_metadata<T: AsRef<[u8]>>(&mut self, key: String, value: T) -> AvroResult<()> {
492        if !self.has_header {
493            if key.starts_with("avro.") {
494                return Err(Details::InvalidMetadataKey(key).into());
495            }
496            self.user_metadata
497                .insert(key, Value::Bytes(value.as_ref().to_vec()));
498            Ok(())
499        } else {
500            Err(Details::FileHeaderAlreadyWritten.into())
501        }
502    }
503
504    /// Create an Avro header based on schema, codec and sync marker.
505    fn header(&self) -> Result<Vec<u8>, Error> {
506        let schema_bytes = serde_json::to_string(self.schema)
507            .map_err(Details::ConvertJsonToString)?
508            .into_bytes();
509
510        let mut metadata = HashMap::with_capacity(2);
511        metadata.insert("avro.schema", Value::Bytes(schema_bytes));
512        if self.codec != Codec::Null {
513            metadata.insert("avro.codec", self.codec.into());
514        }
515        match self.codec {
516            #[cfg(feature = "bzip")]
517            Codec::Bzip2(settings) => {
518                metadata.insert(
519                    "avro.codec.compression_level",
520                    Value::Bytes(vec![settings.compression_level]),
521                );
522            }
523            #[cfg(feature = "xz")]
524            Codec::Xz(settings) => {
525                metadata.insert(
526                    "avro.codec.compression_level",
527                    Value::Bytes(vec![settings.compression_level]),
528                );
529            }
530            #[cfg(feature = "zstandard")]
531            Codec::Zstandard(settings) => {
532                metadata.insert(
533                    "avro.codec.compression_level",
534                    Value::Bytes(vec![settings.compression_level]),
535                );
536            }
537            _ => {}
538        }
539
540        for (k, v) in &self.user_metadata {
541            metadata.insert(k.as_str(), v.clone());
542        }
543
544        let mut header = Vec::new();
545        header.extend_from_slice(AVRO_OBJECT_HEADER);
546        encode(
547            &metadata.into(),
548            &Schema::map(Schema::Bytes).build(),
549            &mut header,
550        )?;
551        header.extend_from_slice(&self.marker);
552
553        Ok(header)
554    }
555
556    fn maybe_write_header(&mut self) -> AvroResult<usize> {
557        if !self.has_header {
558            let header = self.header()?;
559            let n = self.append_bytes(header.as_ref())?;
560            self.has_header = true;
561            Ok(n)
562        } else {
563            Ok(0)
564        }
565    }
566}
567
568/// A buffer that can be cleared.
569pub trait Clearable {
570    /// Clear the buffer.
571    fn clear(&mut self);
572}
573
574impl Clearable for Vec<u8> {
575    fn clear(&mut self) {
576        Vec::clear(self);
577    }
578}
579
580impl<'a, W: Clearable + Write> Writer<'a, W> {
581    /// Reset the writer.
582    ///
583    /// This will clear the underlying writer, the internal buffer, and the user metadata.
584    /// It will also generate a new sync marker.
585    ///
586    /// # Example
587    /// ```
588    /// # use apache_avro::{Writer, Schema, Error};
589    /// # let schema = Schema::Boolean;
590    /// # let values = [true, false];
591    /// # fn send(_: &Vec<u8>) {}
592    /// let mut writer = Writer::new(&schema, Vec::new())?;
593    ///
594    /// // Write some values
595    /// for value in values {
596    ///     writer.append_value(value)?;
597    /// }
598    ///
599    /// // Flush the buffer and only then do something with buffer
600    /// writer.flush()?;
601    /// send(writer.get_ref());
602    ///
603    /// // Reset the writer
604    /// writer.reset();
605    ///
606    /// // Write some values again
607    /// for value in values {
608    ///     writer.append_value(value)?;
609    /// }
610    ///
611    /// # Ok::<(), Error>(())
612    /// ```
613    ///
614    /// # Warning
615    /// Any data that has been appended but not yet flushed will be silently
616    /// discarded. Call [`flush`](Writer::flush) before `reset()` if you need
617    /// to preserve in-flight records.
618    pub fn reset(&mut self) {
619        self.buffer.clear();
620        self.writer.clear();
621        self.has_header = false;
622        self.num_values = 0;
623        self.user_metadata.clear();
624        self.marker = generate_sync_marker();
625    }
626}
627
628impl<W: Write> Drop for Writer<'_, W> {
629    /// Drop the writer, will try to flush ignoring any errors.
630    fn drop(&mut self) {
631        let _ = self.maybe_write_header();
632        let _ = self.flush();
633    }
634}
635
636#[cfg(not(target_arch = "wasm32"))]
637fn generate_sync_marker() -> [u8; 16] {
638    rand::random()
639}
640
641#[cfg(target_arch = "wasm32")]
642fn generate_sync_marker() -> [u8; 16] {
643    let mut marker = [0_u8; 16];
644    std::iter::repeat_with(quad_rand::rand)
645        .take(4)
646        .flat_map(|i| i.to_be_bytes())
647        .enumerate()
648        .for_each(|(i, n)| marker[i] = n);
649    marker
650}
651
652#[cfg(test)]
653mod tests {
654    use std::{cell::RefCell, rc::Rc};
655
656    use super::*;
657    use crate::{Reader, types::Record, util::zig_i64};
658    use pretty_assertions::assert_eq;
659    use serde::{Deserialize, Serialize};
660
661    use crate::{codec::DeflateSettings, error::Details};
662    use apache_avro_test_helper::TestResult;
663
664    const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len();
665
666    const SCHEMA: &str = r#"
667    {
668      "type": "record",
669      "name": "test",
670      "fields": [
671        {
672          "name": "a",
673          "type": "long",
674          "default": 42
675        },
676        {
677          "name": "b",
678          "type": "string"
679        }
680      ]
681    }
682    "#;
683
684    #[test]
685    fn avro_rs_220_flush_write_header() -> TestResult {
686        let schema = Schema::parse_str(SCHEMA)?;
687
688        // By default flush should write the header even if nothing was added yet
689        let mut writer = Writer::new(&schema, Vec::new())?;
690        writer.flush()?;
691        let result = writer.into_inner()?;
692        assert_eq!(result.len(), 147);
693
694        // Unless the user indicates via the builder that the header has already been written
695        let mut writer = Writer::builder()
696            .has_header(true)
697            .schema(&schema)
698            .writer(Vec::new())
699            .build()?;
700        writer.flush()?;
701        let result = writer.into_inner()?;
702        assert_eq!(result.len(), 0);
703
704        Ok(())
705    }
706
707    #[test]
708    fn test_writer_append() -> TestResult {
709        let schema = Schema::parse_str(SCHEMA)?;
710        let mut writer = Writer::new(&schema, Vec::new())?;
711
712        let mut record = Record::new(&schema).unwrap();
713        record.put("a", 27i64);
714        record.put("b", "foo");
715
716        let n1 = writer.append_value(record.clone())?;
717        let n2 = writer.append_value(record.clone())?;
718        let n3 = writer.flush()?;
719        let result = writer.into_inner()?;
720
721        assert_eq!(n1 + n2 + n3, result.len());
722
723        let mut data = Vec::new();
724        zig_i64(27, &mut data)?;
725        zig_i64(3, &mut data)?;
726        data.extend(b"foo");
727        data.extend(data.clone());
728
729        // starts with magic
730        assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
731        // ends with data and sync marker
732        let last_data_byte = result.len() - 16;
733        assert_eq!(
734            &result[last_data_byte - data.len()..last_data_byte],
735            data.as_slice()
736        );
737
738        Ok(())
739    }
740
741    #[test]
742    fn test_writer_extend() -> TestResult {
743        let schema = Schema::parse_str(SCHEMA)?;
744        let mut writer = Writer::new(&schema, Vec::new())?;
745
746        let mut record = Record::new(&schema).unwrap();
747        record.put("a", 27i64);
748        record.put("b", "foo");
749        let record_copy = record.clone();
750        let records = vec![record, record_copy];
751
752        let n1 = writer.extend(records)?;
753        let n2 = writer.flush()?;
754        let result = writer.into_inner()?;
755
756        assert_eq!(n1 + n2, result.len());
757
758        let mut data = Vec::new();
759        zig_i64(27, &mut data)?;
760        zig_i64(3, &mut data)?;
761        data.extend(b"foo");
762        data.extend(data.clone());
763
764        // starts with magic
765        assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
766        // ends with data and sync marker
767        let last_data_byte = result.len() - 16;
768        assert_eq!(
769            &result[last_data_byte - data.len()..last_data_byte],
770            data.as_slice()
771        );
772
773        Ok(())
774    }
775
776    #[derive(Debug, Clone, Deserialize, Serialize)]
777    struct TestSerdeSerialize {
778        a: i64,
779        b: String,
780    }
781
782    #[test]
783    fn test_writer_append_ser() -> TestResult {
784        let schema = Schema::parse_str(SCHEMA)?;
785        let mut writer = Writer::new(&schema, Vec::new())?;
786
787        let record = TestSerdeSerialize {
788            a: 27,
789            b: "foo".to_owned(),
790        };
791
792        let n1 = writer.append_ser(record)?;
793        let n2 = writer.flush()?;
794        let result = writer.into_inner()?;
795
796        assert_eq!(n1 + n2, result.len());
797
798        let mut data = Vec::new();
799        zig_i64(27, &mut data)?;
800        zig_i64(3, &mut data)?;
801        data.extend(b"foo");
802
803        // starts with magic
804        assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
805        // ends with data and sync marker
806        let last_data_byte = result.len() - 16;
807        assert_eq!(
808            &result[last_data_byte - data.len()..last_data_byte],
809            data.as_slice()
810        );
811
812        Ok(())
813    }
814
815    #[test]
816    fn test_writer_extend_ser() -> TestResult {
817        let schema = Schema::parse_str(SCHEMA)?;
818        let mut writer = Writer::new(&schema, Vec::new())?;
819
820        let record = TestSerdeSerialize {
821            a: 27,
822            b: "foo".to_owned(),
823        };
824        let record_copy = record.clone();
825        let records = vec![record, record_copy];
826
827        let n1 = writer.extend_ser(records)?;
828        let n2 = writer.flush()?;
829        let result = writer.into_inner()?;
830
831        assert_eq!(n1 + n2, result.len());
832
833        let mut data = Vec::new();
834        zig_i64(27, &mut data)?;
835        zig_i64(3, &mut data)?;
836        data.extend(b"foo");
837        data.extend(data.clone());
838
839        // starts with magic
840        assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
841        // ends with data and sync marker
842        let last_data_byte = result.len() - 16;
843        assert_eq!(
844            &result[last_data_byte - data.len()..last_data_byte],
845            data.as_slice()
846        );
847
848        Ok(())
849    }
850
851    fn make_writer_with_codec(schema: &Schema) -> AvroResult<Writer<'_, Vec<u8>>> {
852        Writer::with_codec(
853            schema,
854            Vec::new(),
855            Codec::Deflate(DeflateSettings::default()),
856        )
857    }
858
859    fn make_writer_with_builder(schema: &Schema) -> AvroResult<Writer<'_, Vec<u8>>> {
860        Writer::builder()
861            .writer(Vec::new())
862            .schema(schema)
863            .codec(Codec::Deflate(DeflateSettings::default()))
864            .block_size(100)
865            .build()
866    }
867
868    fn check_writer(mut writer: Writer<'_, Vec<u8>>, schema: &Schema) -> TestResult {
869        let mut record = Record::new(schema).unwrap();
870        record.put("a", 27i64);
871        record.put("b", "foo");
872
873        let n1 = writer.append_value(record.clone())?;
874        let n2 = writer.append_value(record.clone())?;
875        let n3 = writer.flush()?;
876        let result = writer.into_inner()?;
877
878        assert_eq!(n1 + n2 + n3, result.len());
879
880        let mut data = Vec::new();
881        zig_i64(27, &mut data)?;
882        zig_i64(3, &mut data)?;
883        data.extend(b"foo");
884        data.extend(data.clone());
885        Codec::Deflate(DeflateSettings::default()).compress(&mut data)?;
886
887        // starts with magic
888        assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
889        // ends with data and sync marker
890        let last_data_byte = result.len() - 16;
891        assert_eq!(
892            &result[last_data_byte - data.len()..last_data_byte],
893            data.as_slice()
894        );
895
896        Ok(())
897    }
898
899    #[test]
900    fn test_writer_with_codec() -> TestResult {
901        let schema = Schema::parse_str(SCHEMA)?;
902        let writer = make_writer_with_codec(&schema)?;
903        check_writer(writer, &schema)
904    }
905
906    #[test]
907    fn test_writer_with_builder() -> TestResult {
908        let schema = Schema::parse_str(SCHEMA)?;
909        let writer = make_writer_with_builder(&schema)?;
910        check_writer(writer, &schema)
911    }
912
913    #[test]
914    fn test_logical_writer() -> TestResult {
915        const LOGICAL_TYPE_SCHEMA: &str = r#"
916        {
917          "type": "record",
918          "name": "logical_type_test",
919          "fields": [
920            {
921              "name": "a",
922              "type": [
923                "null",
924                {
925                  "type": "long",
926                  "logicalType": "timestamp-micros"
927                }
928              ]
929            }
930          ]
931        }
932        "#;
933        let codec = Codec::Deflate(DeflateSettings::default());
934        let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA)?;
935        let mut writer = Writer::builder()
936            .schema(&schema)
937            .codec(codec)
938            .writer(Vec::new())
939            .build()?;
940
941        let mut record1 = Record::new(&schema).unwrap();
942        record1.put(
943            "a",
944            Value::Union(1, Box::new(Value::TimestampMicros(1234_i64))),
945        );
946
947        let mut record2 = Record::new(&schema).unwrap();
948        record2.put("a", Value::Union(0, Box::new(Value::Null)));
949
950        let n1 = writer.append_value(record1)?;
951        let n2 = writer.append_value(record2)?;
952        let n3 = writer.flush()?;
953        let result = writer.into_inner()?;
954
955        assert_eq!(n1 + n2 + n3, result.len());
956
957        let mut data = Vec::new();
958        // byte indicating not null
959        zig_i64(1, &mut data)?;
960        zig_i64(1234, &mut data)?;
961
962        // byte indicating null
963        zig_i64(0, &mut data)?;
964        codec.compress(&mut data)?;
965
966        // starts with magic
967        assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
968        // ends with data and sync marker
969        let last_data_byte = result.len() - 16;
970        assert_eq!(
971            &result[last_data_byte - data.len()..last_data_byte],
972            data.as_slice()
973        );
974
975        Ok(())
976    }
977
978    #[test]
979    fn test_avro_3405_writer_add_metadata_success() -> TestResult {
980        let schema = Schema::parse_str(SCHEMA)?;
981        let mut writer = Writer::new(&schema, Vec::new())?;
982
983        writer.add_user_metadata("stringKey".to_string(), String::from("stringValue"))?;
984        writer.add_user_metadata("strKey".to_string(), "strValue")?;
985        writer.add_user_metadata("bytesKey".to_string(), b"bytesValue")?;
986        writer.add_user_metadata("vecKey".to_string(), vec![1, 2, 3])?;
987
988        let mut record = Record::new(&schema).unwrap();
989        record.put("a", 27i64);
990        record.put("b", "foo");
991
992        writer.append_value(record.clone())?;
993        writer.append_value(record.clone())?;
994        writer.flush()?;
995        let result = writer.into_inner()?;
996
997        assert_eq!(result.len(), 244);
998
999        Ok(())
1000    }
1001
1002    #[test]
1003    fn test_avro_3881_metadata_empty_body() -> TestResult {
1004        let schema = Schema::parse_str(SCHEMA)?;
1005        let mut writer = Writer::new(&schema, Vec::new())?;
1006        writer.add_user_metadata("a".to_string(), "b")?;
1007        let result = writer.into_inner()?;
1008
1009        let reader = Reader::builder(&result[..])
1010            .reader_schema(&schema)
1011            .build()?;
1012        let mut expected = HashMap::new();
1013        expected.insert("a".to_string(), vec![b'b']);
1014        assert_eq!(reader.user_metadata(), &expected);
1015        assert_eq!(reader.into_iter().count(), 0);
1016
1017        Ok(())
1018    }
1019
1020    #[test]
1021    fn test_avro_3405_writer_add_metadata_failure() -> TestResult {
1022        let schema = Schema::parse_str(SCHEMA)?;
1023        let mut writer = Writer::new(&schema, Vec::new())?;
1024
1025        let mut record = Record::new(&schema).unwrap();
1026        record.put("a", 27i64);
1027        record.put("b", "foo");
1028        writer.append_value(record.clone())?;
1029
1030        match writer
1031            .add_user_metadata("stringKey".to_string(), String::from("value2"))
1032            .map_err(Error::into_details)
1033        {
1034            Err(e @ Details::FileHeaderAlreadyWritten) => {
1035                assert_eq!(e.to_string(), "The file metadata is already flushed.");
1036            }
1037            Err(e) => panic!("Unexpected error occurred while writing user metadata: {e:?}"),
1038            Ok(_) => panic!("Expected an error that metadata cannot be added after adding data"),
1039        }
1040
1041        Ok(())
1042    }
1043
1044    #[test]
1045    fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult {
1046        let schema = Schema::parse_str(SCHEMA)?;
1047        let mut writer = Writer::new(&schema, Vec::new())?;
1048
1049        let key = "avro.stringKey".to_string();
1050        match writer
1051            .add_user_metadata(key.clone(), "value")
1052            .map_err(Error::into_details)
1053        {
1054            Err(ref e @ Details::InvalidMetadataKey(_)) => {
1055                assert_eq!(
1056                    e.to_string(),
1057                    format!(
1058                        "Metadata keys starting with 'avro.' are reserved for internal usage: {key}."
1059                    )
1060                );
1061            }
1062            Err(e) => panic!(
1063                "Unexpected error occurred while writing user metadata with reserved prefix ('avro.'): {e:?}"
1064            ),
1065            Ok(_) => {
1066                panic!("Expected an error that the metadata key cannot be prefixed with 'avro.'")
1067            }
1068        }
1069
1070        Ok(())
1071    }
1072
1073    #[test]
1074    fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult {
1075        let schema = Schema::parse_str(SCHEMA)?;
1076
1077        let mut user_meta_data: HashMap<String, Value> = HashMap::new();
1078        user_meta_data.insert(
1079            "stringKey".to_string(),
1080            Value::String("stringValue".to_string()),
1081        );
1082        user_meta_data.insert("bytesKey".to_string(), Value::Bytes(b"bytesValue".to_vec()));
1083        user_meta_data.insert("vecKey".to_string(), Value::Bytes(vec![1, 2, 3]));
1084
1085        let writer: Writer<'_, Vec<u8>> = Writer::builder()
1086            .writer(Vec::new())
1087            .schema(&schema)
1088            .user_metadata(user_meta_data.clone())
1089            .build()?;
1090
1091        assert_eq!(writer.user_metadata, user_meta_data);
1092
1093        Ok(())
1094    }
1095
1096    #[test]
1097    fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult {
1098        const SCHEMA: &str = r#"
1099  {
1100      "type": "record",
1101      "name": "Conference",
1102      "fields": [
1103          {"type": "string", "name": "name"},
1104          {"type": ["null", "long"], "name": "date", "aliases" : [ "time2", "time" ]}
1105      ]
1106  }"#;
1107
1108        #[derive(Debug, PartialEq, Eq, Clone, Serialize)]
1109        pub struct Conference {
1110            pub name: String,
1111            pub time: Option<i64>,
1112        }
1113
1114        let conf = Conference {
1115            name: "RustConf".to_string(),
1116            time: Some(1234567890),
1117        };
1118
1119        let schema = Schema::parse_str(SCHEMA)?;
1120        let mut writer = Writer::new(&schema, Vec::new())?;
1121
1122        let bytes = writer.append_ser(conf)?;
1123
1124        assert_eq!(182, bytes);
1125
1126        Ok(())
1127    }
1128
1129    #[test]
1130    fn avro_4014_validation_returns_a_detailed_error() -> TestResult {
1131        const SCHEMA: &str = r#"
1132  {
1133      "type": "record",
1134      "name": "Conference",
1135      "fields": [
1136          {"type": "string", "name": "name"},
1137          {"type": ["null", "long"], "name": "date", "aliases" : [ "time2", "time" ]}
1138      ]
1139  }"#;
1140
1141        #[derive(Debug, PartialEq, Clone, Serialize)]
1142        pub struct Conference {
1143            pub name: String,
1144            pub time: Option<f64>, // wrong type: f64 instead of i64
1145        }
1146
1147        let conf = Conference {
1148            name: "RustConf".to_string(),
1149            time: Some(12345678.90),
1150        };
1151
1152        let schema = Schema::parse_str(SCHEMA)?;
1153        let mut writer = Writer::new(&schema, Vec::new())?;
1154
1155        match writer.append_ser(conf) {
1156            Ok(bytes) => panic!("Expected an error, but got {bytes} bytes written"),
1157            Err(e) => {
1158                assert_eq!(
1159                    e.to_string(),
1160                    r#"Failed to serialize field 'date' of record RecordSchema { name: Name { name: "Conference", .. }, fields: [RecordField { name: "name", schema: String, .. }, RecordField { name: "date", aliases: ["time2", "time"], schema: Union(UnionSchema { schemas: [Null, Long] }), .. }], .. }: Failed to serialize value of type `f64`: Expected Schema::Double"#
1161                );
1162            }
1163        }
1164        Ok(())
1165    }
1166
1167    #[test]
1168    fn avro_4063_flush_applies_to_inner_writer() -> TestResult {
1169        const SCHEMA: &str = r#"
1170        {
1171            "type": "record",
1172            "name": "ExampleSchema",
1173            "fields": [
1174                {"name": "exampleField", "type": "string"}
1175            ]
1176        }
1177        "#;
1178
1179        #[derive(Clone, Default)]
1180        struct TestBuffer(Rc<RefCell<Vec<u8>>>);
1181
1182        impl TestBuffer {
1183            fn len(&self) -> usize {
1184                self.0.borrow().len()
1185            }
1186        }
1187
1188        impl Write for TestBuffer {
1189            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
1190                self.0.borrow_mut().write(buf)
1191            }
1192
1193            fn flush(&mut self) -> std::io::Result<()> {
1194                Ok(())
1195            }
1196        }
1197
1198        let shared_buffer = TestBuffer::default();
1199
1200        let buffered_writer = std::io::BufWriter::new(shared_buffer.clone());
1201
1202        let schema = Schema::parse_str(SCHEMA)?;
1203
1204        let mut writer = Writer::new(&schema, buffered_writer)?;
1205
1206        let mut record = Record::new(writer.schema()).unwrap();
1207        record.put("exampleField", "value");
1208
1209        writer.append_value(record)?;
1210        writer.flush()?;
1211
1212        assert_eq!(
1213            shared_buffer.len(),
1214            151,
1215            "the test buffer was not fully written to after Writer::flush was called"
1216        );
1217
1218        Ok(())
1219    }
1220
1221    #[test]
1222    fn avro_rs_310_append_unvalidated_value() -> TestResult {
1223        let schema = Schema::String;
1224        let value = Value::Int(1);
1225
1226        let mut writer = Writer::new(&schema, Vec::new())?;
1227        writer.unvalidated_append_value_ref(&value)?;
1228        writer.unvalidated_append_value(value)?;
1229        let buffer = writer.into_inner()?;
1230
1231        // Check the last two bytes for the sync marker
1232        assert_eq!(&buffer[buffer.len() - 18..buffer.len() - 16], &[2, 2]);
1233
1234        let mut writer = Writer::new(&schema, Vec::new())?;
1235        let value = Value::Int(1);
1236        let err = writer.append_value_ref(&value).unwrap_err();
1237        assert_eq!(
1238            err.to_string(),
1239            "Value Int(1) does not match schema String: Reason: Unsupported value-schema combination! Value: Int, schema: String"
1240        );
1241        let err = writer.append_value(value).unwrap_err();
1242        assert_eq!(
1243            err.to_string(),
1244            "Value Int(1) does not match schema String: Reason: Unsupported value-schema combination! Value: Int, schema: String"
1245        );
1246
1247        Ok(())
1248    }
1249
1250    #[test]
1251    fn avro_rs_469_reset_writer() -> TestResult {
1252        let schema = Schema::Boolean;
1253        let values = [true, false, true, false];
1254        let mut writer = Writer::new(&schema, Vec::new())?;
1255
1256        for value in values {
1257            writer.append_value(value)?;
1258        }
1259
1260        writer.flush()?;
1261        let first_buffer = writer.get_ref().clone();
1262
1263        writer.reset();
1264        assert_eq!(writer.get_ref().len(), 0);
1265
1266        for value in values {
1267            writer.append_value(value)?;
1268        }
1269
1270        writer.flush()?;
1271        let second_buffer = writer.get_ref().clone();
1272        assert_eq!(first_buffer.len(), second_buffer.len());
1273        // File structure:
1274        // Header: ? bytes
1275        // Sync marker: 16 bytes
1276        // Data: 6 bytes
1277        // Sync marker: 16 bytes
1278        let len = first_buffer.len();
1279        let header = len - 16 - 6 - 16;
1280        let data = header + 16;
1281        assert_eq!(
1282            first_buffer[..header],
1283            second_buffer[..header],
1284            "Written header must be the same, excluding sync marker"
1285        );
1286        assert_ne!(
1287            first_buffer[header..data],
1288            second_buffer[header..data],
1289            "Sync markers should be different"
1290        );
1291        assert_eq!(
1292            first_buffer[data..data + 6],
1293            second_buffer[data..data + 6],
1294            "Written data must be the same"
1295        );
1296        assert_ne!(
1297            first_buffer[len - 16..],
1298            second_buffer[len - 16..],
1299            "Sync markers should be different"
1300        );
1301
1302        Ok(())
1303    }
1304}