1use crate::types::{SchemaPath, ValuePath};
20use crate::{
21 AvroResult, Codec, Error,
22 encode::{encode, encode_internal, encode_to_vec},
23 error::Details,
24 schema::{ResolvedSchema, Schema},
25 serde::ser_schema::{Config, SchemaAwareSerializer},
26 types::Value,
27 util::is_human_readable,
28};
29use serde::Serialize;
30use std::{collections::HashMap, io::Write, mem::ManuallyDrop};
31
32pub mod datum;
33pub mod single_object;
34
35const DEFAULT_BLOCK_SIZE: usize = 16000;
36const AVRO_OBJECT_HEADER: &[u8] = b"Obj\x01";
37
38pub struct Writer<'a, W: Write> {
44 schema: &'a Schema,
45 writer: W,
46 resolved_schema: ResolvedSchema<'a>,
47 codec: Codec,
48 block_size: usize,
49 buffer: Vec<u8>,
50 num_values: usize,
51 marker: [u8; 16],
52 has_header: bool,
53 user_metadata: HashMap<String, Value>,
54 human_readable: bool,
55 map_array_target_block_size: Option<usize>,
56}
57
58#[bon::bon]
59impl<'a, W: Write> Writer<'a, W> {
60 #[builder]
61 pub fn builder(
62 schema: &'a Schema,
63 schemata: Option<Vec<&'a Schema>>,
64 writer: W,
65 #[builder(default = Codec::Null)] codec: Codec,
66 #[builder(default = DEFAULT_BLOCK_SIZE)] block_size: usize,
67 #[builder(default = generate_sync_marker())] marker: [u8; 16],
68 #[builder(default = false)]
72 has_header: bool,
73 #[builder(default)] user_metadata: HashMap<String, Value>,
74 #[builder(default = is_human_readable())]
78 human_readable: bool,
79 map_array_target_block_size: Option<usize>,
87 ) -> AvroResult<Self> {
88 let resolved_schema = if let Some(schemata) = schemata {
89 ResolvedSchema::try_from(schemata)?
90 } else {
91 ResolvedSchema::try_from(schema)?
92 };
93 Ok(Self {
94 schema,
95 writer,
96 resolved_schema,
97 codec,
98 block_size,
99 buffer: Vec::with_capacity(block_size),
100 num_values: 0,
101 marker,
102 has_header,
103 user_metadata,
104 human_readable,
105 map_array_target_block_size,
106 })
107 }
108}
109
110impl<'a, W: Write> Writer<'a, W> {
111 pub fn new(schema: &'a Schema, writer: W) -> AvroResult<Self> {
115 Writer::with_codec(schema, writer, Codec::Null)
116 }
117
118 pub fn with_codec(schema: &'a Schema, writer: W, codec: Codec) -> AvroResult<Self> {
121 Self::builder()
122 .schema(schema)
123 .writer(writer)
124 .codec(codec)
125 .build()
126 }
127
128 pub fn with_schemata(
133 schema: &'a Schema,
134 schemata: Vec<&'a Schema>,
135 writer: W,
136 codec: Codec,
137 ) -> AvroResult<Self> {
138 Self::builder()
139 .schema(schema)
140 .schemata(schemata)
141 .writer(writer)
142 .codec(codec)
143 .build()
144 }
145
146 pub fn append_to(schema: &'a Schema, writer: W, marker: [u8; 16]) -> AvroResult<Self> {
150 Writer::append_to_with_codec(schema, writer, Codec::Null, marker)
151 }
152
153 pub fn append_to_with_codec(
156 schema: &'a Schema,
157 writer: W,
158 codec: Codec,
159 marker: [u8; 16],
160 ) -> AvroResult<Self> {
161 Self::builder()
162 .schema(schema)
163 .writer(writer)
164 .codec(codec)
165 .marker(marker)
166 .has_header(true)
167 .build()
168 }
169
170 pub fn append_to_with_codec_schemata(
173 schema: &'a Schema,
174 schemata: Vec<&'a Schema>,
175 writer: W,
176 codec: Codec,
177 marker: [u8; 16],
178 ) -> AvroResult<Self> {
179 Self::builder()
180 .schema(schema)
181 .schemata(schemata)
182 .writer(writer)
183 .codec(codec)
184 .marker(marker)
185 .has_header(true)
186 .build()
187 }
188
189 pub fn schema(&self) -> &'a Schema {
191 self.schema
192 }
193
194 pub fn append_value<T: Into<Value>>(&mut self, value: T) -> AvroResult<usize> {
202 let avro = value.into();
203 self.append_value_ref(&avro)
204 }
205
206 pub fn append_value_ref(&mut self, value: &Value) -> AvroResult<usize> {
214 if let Some(reason) = value.validate_internal(
215 self.schema,
216 self.resolved_schema.get_names(),
217 self.schema.namespace(),
218 &ValuePath::Start,
219 &SchemaPath::Start,
220 ) {
221 return Err(Details::ValidationWithReason {
222 value: value.clone(),
223 schema: self.schema.clone(),
224 reason,
225 }
226 .into());
227 }
228 self.unvalidated_append_value_ref(value)
229 }
230
231 pub fn unvalidated_append_value<T: Into<Value>>(&mut self, value: T) -> AvroResult<usize> {
243 let value = value.into();
244 self.unvalidated_append_value_ref(&value)
245 }
246
247 pub fn unvalidated_append_value_ref(&mut self, value: &Value) -> AvroResult<usize> {
259 let n = self.maybe_write_header()?;
260 encode_internal(
261 value,
262 self.schema,
263 self.resolved_schema.get_names(),
264 self.schema.namespace(),
265 &mut self.buffer,
266 )?;
267
268 self.num_values += 1;
269
270 if self.buffer.len() >= self.block_size {
271 return self.flush().map(|b| b + n);
272 }
273
274 Ok(n)
275 }
276
277 pub fn append_ser<S: Serialize>(&mut self, value: S) -> AvroResult<usize> {
287 let n = self.maybe_write_header()?;
288
289 let config = Config {
290 names: self.resolved_schema.get_names(),
291 target_block_size: self.map_array_target_block_size,
292 human_readable: self.human_readable,
293 };
294
295 value.serialize(SchemaAwareSerializer::new(
296 &mut self.buffer,
297 self.schema,
298 config,
299 )?)?;
300 self.num_values += 1;
301
302 if self.buffer.len() >= self.block_size {
303 return self.flush().map(|b| b + n);
304 }
305
306 Ok(n)
307 }
308
309 pub fn extend<I, T: Into<Value>>(&mut self, values: I) -> AvroResult<usize>
316 where
317 I: IntoIterator<Item = T>,
318 {
319 let mut num_bytes = 0;
334 for value in values {
335 num_bytes += self.append_value(value)?;
336 }
337 num_bytes += self.flush()?;
338
339 Ok(num_bytes)
340 }
341
342 pub fn extend_ser<I, T: Serialize>(&mut self, values: I) -> AvroResult<usize>
351 where
352 I: IntoIterator<Item = T>,
353 {
354 let mut num_bytes = 0;
369 for value in values {
370 num_bytes += self.append_ser(value)?;
371 }
372 num_bytes += self.flush()?;
373
374 Ok(num_bytes)
375 }
376
377 pub fn extend_from_slice(&mut self, values: &[Value]) -> AvroResult<usize> {
385 let mut num_bytes = 0;
386 for value in values {
387 num_bytes += self.append_value_ref(value)?;
388 }
389 num_bytes += self.flush()?;
390
391 Ok(num_bytes)
392 }
393
394 pub fn flush(&mut self) -> AvroResult<usize> {
402 let mut num_bytes = self.maybe_write_header()?;
403 if self.num_values == 0 {
404 return Ok(num_bytes);
405 }
406
407 self.codec.compress(&mut self.buffer)?;
408
409 let num_values = self.num_values;
410 let stream_len = self.buffer.len();
411
412 num_bytes += self.append_raw(&num_values.try_into()?, &Schema::Long)?
413 + self.append_raw(&stream_len.try_into()?, &Schema::Long)?
414 + self
415 .writer
416 .write(self.buffer.as_ref())
417 .map_err(Details::WriteBytes)?
418 + self.append_marker()?;
419
420 self.buffer.clear();
421 self.num_values = 0;
422
423 self.writer.flush().map_err(Details::FlushWriter)?;
424
425 Ok(num_bytes)
426 }
427
428 pub fn into_inner(mut self) -> AvroResult<W> {
433 self.maybe_write_header()?;
434 self.flush()?;
435
436 let mut this = ManuallyDrop::new(self);
437
438 let _buffer = std::mem::take(&mut this.buffer);
440 let _user_metadata = std::mem::take(&mut this.user_metadata);
441 unsafe { std::ptr::drop_in_place(&mut this.resolved_schema) };
443
444 let writer = unsafe { std::ptr::read(&this.writer) };
446
447 Ok(writer)
448 }
449
450 pub fn get_ref(&self) -> &W {
455 &self.writer
456 }
457
458 pub fn get_mut(&mut self) -> &mut W {
465 &mut self.writer
466 }
467
468 fn append_marker(&mut self) -> AvroResult<usize> {
470 self.writer
473 .write(&self.marker)
474 .map_err(|e| Details::WriteMarker(e).into())
475 }
476
477 fn append_raw(&mut self, value: &Value, schema: &Schema) -> AvroResult<usize> {
479 self.append_bytes(encode_to_vec(value, schema)?.as_ref())
480 }
481
482 fn append_bytes(&mut self, bytes: &[u8]) -> AvroResult<usize> {
484 self.writer
485 .write(bytes)
486 .map_err(|e| Details::WriteBytes(e).into())
487 }
488
489 pub fn add_user_metadata<T: AsRef<[u8]>>(&mut self, key: String, value: T) -> AvroResult<()> {
492 if !self.has_header {
493 if key.starts_with("avro.") {
494 return Err(Details::InvalidMetadataKey(key).into());
495 }
496 self.user_metadata
497 .insert(key, Value::Bytes(value.as_ref().to_vec()));
498 Ok(())
499 } else {
500 Err(Details::FileHeaderAlreadyWritten.into())
501 }
502 }
503
504 fn header(&self) -> Result<Vec<u8>, Error> {
506 let schema_bytes = serde_json::to_string(self.schema)
507 .map_err(Details::ConvertJsonToString)?
508 .into_bytes();
509
510 let mut metadata = HashMap::with_capacity(2);
511 metadata.insert("avro.schema", Value::Bytes(schema_bytes));
512 if self.codec != Codec::Null {
513 metadata.insert("avro.codec", self.codec.into());
514 }
515 match self.codec {
516 #[cfg(feature = "bzip")]
517 Codec::Bzip2(settings) => {
518 metadata.insert(
519 "avro.codec.compression_level",
520 Value::Bytes(vec![settings.compression_level]),
521 );
522 }
523 #[cfg(feature = "xz")]
524 Codec::Xz(settings) => {
525 metadata.insert(
526 "avro.codec.compression_level",
527 Value::Bytes(vec![settings.compression_level]),
528 );
529 }
530 #[cfg(feature = "zstandard")]
531 Codec::Zstandard(settings) => {
532 metadata.insert(
533 "avro.codec.compression_level",
534 Value::Bytes(vec![settings.compression_level]),
535 );
536 }
537 _ => {}
538 }
539
540 for (k, v) in &self.user_metadata {
541 metadata.insert(k.as_str(), v.clone());
542 }
543
544 let mut header = Vec::new();
545 header.extend_from_slice(AVRO_OBJECT_HEADER);
546 encode(
547 &metadata.into(),
548 &Schema::map(Schema::Bytes).build(),
549 &mut header,
550 )?;
551 header.extend_from_slice(&self.marker);
552
553 Ok(header)
554 }
555
556 fn maybe_write_header(&mut self) -> AvroResult<usize> {
557 if !self.has_header {
558 let header = self.header()?;
559 let n = self.append_bytes(header.as_ref())?;
560 self.has_header = true;
561 Ok(n)
562 } else {
563 Ok(0)
564 }
565 }
566}
567
568pub trait Clearable {
570 fn clear(&mut self);
572}
573
574impl Clearable for Vec<u8> {
575 fn clear(&mut self) {
576 Vec::clear(self);
577 }
578}
579
580impl<'a, W: Clearable + Write> Writer<'a, W> {
581 pub fn reset(&mut self) {
619 self.buffer.clear();
620 self.writer.clear();
621 self.has_header = false;
622 self.num_values = 0;
623 self.user_metadata.clear();
624 self.marker = generate_sync_marker();
625 }
626}
627
628impl<W: Write> Drop for Writer<'_, W> {
629 fn drop(&mut self) {
631 let _ = self.maybe_write_header();
632 let _ = self.flush();
633 }
634}
635
636#[cfg(not(target_arch = "wasm32"))]
637fn generate_sync_marker() -> [u8; 16] {
638 rand::random()
639}
640
641#[cfg(target_arch = "wasm32")]
642fn generate_sync_marker() -> [u8; 16] {
643 let mut marker = [0_u8; 16];
644 std::iter::repeat_with(quad_rand::rand)
645 .take(4)
646 .flat_map(|i| i.to_be_bytes())
647 .enumerate()
648 .for_each(|(i, n)| marker[i] = n);
649 marker
650}
651
652#[cfg(test)]
653mod tests {
654 use std::{cell::RefCell, rc::Rc};
655
656 use super::*;
657 use crate::{Reader, types::Record, util::zig_i64};
658 use pretty_assertions::assert_eq;
659 use serde::{Deserialize, Serialize};
660
661 use crate::{codec::DeflateSettings, error::Details};
662 use apache_avro_test_helper::TestResult;
663
664 const AVRO_OBJECT_HEADER_LEN: usize = AVRO_OBJECT_HEADER.len();
665
666 const SCHEMA: &str = r#"
667 {
668 "type": "record",
669 "name": "test",
670 "fields": [
671 {
672 "name": "a",
673 "type": "long",
674 "default": 42
675 },
676 {
677 "name": "b",
678 "type": "string"
679 }
680 ]
681 }
682 "#;
683
684 #[test]
685 fn avro_rs_220_flush_write_header() -> TestResult {
686 let schema = Schema::parse_str(SCHEMA)?;
687
688 let mut writer = Writer::new(&schema, Vec::new())?;
690 writer.flush()?;
691 let result = writer.into_inner()?;
692 assert_eq!(result.len(), 147);
693
694 let mut writer = Writer::builder()
696 .has_header(true)
697 .schema(&schema)
698 .writer(Vec::new())
699 .build()?;
700 writer.flush()?;
701 let result = writer.into_inner()?;
702 assert_eq!(result.len(), 0);
703
704 Ok(())
705 }
706
707 #[test]
708 fn test_writer_append() -> TestResult {
709 let schema = Schema::parse_str(SCHEMA)?;
710 let mut writer = Writer::new(&schema, Vec::new())?;
711
712 let mut record = Record::new(&schema).unwrap();
713 record.put("a", 27i64);
714 record.put("b", "foo");
715
716 let n1 = writer.append_value(record.clone())?;
717 let n2 = writer.append_value(record.clone())?;
718 let n3 = writer.flush()?;
719 let result = writer.into_inner()?;
720
721 assert_eq!(n1 + n2 + n3, result.len());
722
723 let mut data = Vec::new();
724 zig_i64(27, &mut data)?;
725 zig_i64(3, &mut data)?;
726 data.extend(b"foo");
727 data.extend(data.clone());
728
729 assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
731 let last_data_byte = result.len() - 16;
733 assert_eq!(
734 &result[last_data_byte - data.len()..last_data_byte],
735 data.as_slice()
736 );
737
738 Ok(())
739 }
740
741 #[test]
742 fn test_writer_extend() -> TestResult {
743 let schema = Schema::parse_str(SCHEMA)?;
744 let mut writer = Writer::new(&schema, Vec::new())?;
745
746 let mut record = Record::new(&schema).unwrap();
747 record.put("a", 27i64);
748 record.put("b", "foo");
749 let record_copy = record.clone();
750 let records = vec![record, record_copy];
751
752 let n1 = writer.extend(records)?;
753 let n2 = writer.flush()?;
754 let result = writer.into_inner()?;
755
756 assert_eq!(n1 + n2, result.len());
757
758 let mut data = Vec::new();
759 zig_i64(27, &mut data)?;
760 zig_i64(3, &mut data)?;
761 data.extend(b"foo");
762 data.extend(data.clone());
763
764 assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
766 let last_data_byte = result.len() - 16;
768 assert_eq!(
769 &result[last_data_byte - data.len()..last_data_byte],
770 data.as_slice()
771 );
772
773 Ok(())
774 }
775
776 #[derive(Debug, Clone, Deserialize, Serialize)]
777 struct TestSerdeSerialize {
778 a: i64,
779 b: String,
780 }
781
782 #[test]
783 fn test_writer_append_ser() -> TestResult {
784 let schema = Schema::parse_str(SCHEMA)?;
785 let mut writer = Writer::new(&schema, Vec::new())?;
786
787 let record = TestSerdeSerialize {
788 a: 27,
789 b: "foo".to_owned(),
790 };
791
792 let n1 = writer.append_ser(record)?;
793 let n2 = writer.flush()?;
794 let result = writer.into_inner()?;
795
796 assert_eq!(n1 + n2, result.len());
797
798 let mut data = Vec::new();
799 zig_i64(27, &mut data)?;
800 zig_i64(3, &mut data)?;
801 data.extend(b"foo");
802
803 assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
805 let last_data_byte = result.len() - 16;
807 assert_eq!(
808 &result[last_data_byte - data.len()..last_data_byte],
809 data.as_slice()
810 );
811
812 Ok(())
813 }
814
815 #[test]
816 fn test_writer_extend_ser() -> TestResult {
817 let schema = Schema::parse_str(SCHEMA)?;
818 let mut writer = Writer::new(&schema, Vec::new())?;
819
820 let record = TestSerdeSerialize {
821 a: 27,
822 b: "foo".to_owned(),
823 };
824 let record_copy = record.clone();
825 let records = vec![record, record_copy];
826
827 let n1 = writer.extend_ser(records)?;
828 let n2 = writer.flush()?;
829 let result = writer.into_inner()?;
830
831 assert_eq!(n1 + n2, result.len());
832
833 let mut data = Vec::new();
834 zig_i64(27, &mut data)?;
835 zig_i64(3, &mut data)?;
836 data.extend(b"foo");
837 data.extend(data.clone());
838
839 assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
841 let last_data_byte = result.len() - 16;
843 assert_eq!(
844 &result[last_data_byte - data.len()..last_data_byte],
845 data.as_slice()
846 );
847
848 Ok(())
849 }
850
851 fn make_writer_with_codec(schema: &Schema) -> AvroResult<Writer<'_, Vec<u8>>> {
852 Writer::with_codec(
853 schema,
854 Vec::new(),
855 Codec::Deflate(DeflateSettings::default()),
856 )
857 }
858
859 fn make_writer_with_builder(schema: &Schema) -> AvroResult<Writer<'_, Vec<u8>>> {
860 Writer::builder()
861 .writer(Vec::new())
862 .schema(schema)
863 .codec(Codec::Deflate(DeflateSettings::default()))
864 .block_size(100)
865 .build()
866 }
867
868 fn check_writer(mut writer: Writer<'_, Vec<u8>>, schema: &Schema) -> TestResult {
869 let mut record = Record::new(schema).unwrap();
870 record.put("a", 27i64);
871 record.put("b", "foo");
872
873 let n1 = writer.append_value(record.clone())?;
874 let n2 = writer.append_value(record.clone())?;
875 let n3 = writer.flush()?;
876 let result = writer.into_inner()?;
877
878 assert_eq!(n1 + n2 + n3, result.len());
879
880 let mut data = Vec::new();
881 zig_i64(27, &mut data)?;
882 zig_i64(3, &mut data)?;
883 data.extend(b"foo");
884 data.extend(data.clone());
885 Codec::Deflate(DeflateSettings::default()).compress(&mut data)?;
886
887 assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
889 let last_data_byte = result.len() - 16;
891 assert_eq!(
892 &result[last_data_byte - data.len()..last_data_byte],
893 data.as_slice()
894 );
895
896 Ok(())
897 }
898
899 #[test]
900 fn test_writer_with_codec() -> TestResult {
901 let schema = Schema::parse_str(SCHEMA)?;
902 let writer = make_writer_with_codec(&schema)?;
903 check_writer(writer, &schema)
904 }
905
906 #[test]
907 fn test_writer_with_builder() -> TestResult {
908 let schema = Schema::parse_str(SCHEMA)?;
909 let writer = make_writer_with_builder(&schema)?;
910 check_writer(writer, &schema)
911 }
912
913 #[test]
914 fn test_logical_writer() -> TestResult {
915 const LOGICAL_TYPE_SCHEMA: &str = r#"
916 {
917 "type": "record",
918 "name": "logical_type_test",
919 "fields": [
920 {
921 "name": "a",
922 "type": [
923 "null",
924 {
925 "type": "long",
926 "logicalType": "timestamp-micros"
927 }
928 ]
929 }
930 ]
931 }
932 "#;
933 let codec = Codec::Deflate(DeflateSettings::default());
934 let schema = Schema::parse_str(LOGICAL_TYPE_SCHEMA)?;
935 let mut writer = Writer::builder()
936 .schema(&schema)
937 .codec(codec)
938 .writer(Vec::new())
939 .build()?;
940
941 let mut record1 = Record::new(&schema).unwrap();
942 record1.put(
943 "a",
944 Value::Union(1, Box::new(Value::TimestampMicros(1234_i64))),
945 );
946
947 let mut record2 = Record::new(&schema).unwrap();
948 record2.put("a", Value::Union(0, Box::new(Value::Null)));
949
950 let n1 = writer.append_value(record1)?;
951 let n2 = writer.append_value(record2)?;
952 let n3 = writer.flush()?;
953 let result = writer.into_inner()?;
954
955 assert_eq!(n1 + n2 + n3, result.len());
956
957 let mut data = Vec::new();
958 zig_i64(1, &mut data)?;
960 zig_i64(1234, &mut data)?;
961
962 zig_i64(0, &mut data)?;
964 codec.compress(&mut data)?;
965
966 assert_eq!(&result[..AVRO_OBJECT_HEADER_LEN], AVRO_OBJECT_HEADER);
968 let last_data_byte = result.len() - 16;
970 assert_eq!(
971 &result[last_data_byte - data.len()..last_data_byte],
972 data.as_slice()
973 );
974
975 Ok(())
976 }
977
978 #[test]
979 fn test_avro_3405_writer_add_metadata_success() -> TestResult {
980 let schema = Schema::parse_str(SCHEMA)?;
981 let mut writer = Writer::new(&schema, Vec::new())?;
982
983 writer.add_user_metadata("stringKey".to_string(), String::from("stringValue"))?;
984 writer.add_user_metadata("strKey".to_string(), "strValue")?;
985 writer.add_user_metadata("bytesKey".to_string(), b"bytesValue")?;
986 writer.add_user_metadata("vecKey".to_string(), vec![1, 2, 3])?;
987
988 let mut record = Record::new(&schema).unwrap();
989 record.put("a", 27i64);
990 record.put("b", "foo");
991
992 writer.append_value(record.clone())?;
993 writer.append_value(record.clone())?;
994 writer.flush()?;
995 let result = writer.into_inner()?;
996
997 assert_eq!(result.len(), 244);
998
999 Ok(())
1000 }
1001
1002 #[test]
1003 fn test_avro_3881_metadata_empty_body() -> TestResult {
1004 let schema = Schema::parse_str(SCHEMA)?;
1005 let mut writer = Writer::new(&schema, Vec::new())?;
1006 writer.add_user_metadata("a".to_string(), "b")?;
1007 let result = writer.into_inner()?;
1008
1009 let reader = Reader::builder(&result[..])
1010 .reader_schema(&schema)
1011 .build()?;
1012 let mut expected = HashMap::new();
1013 expected.insert("a".to_string(), vec![b'b']);
1014 assert_eq!(reader.user_metadata(), &expected);
1015 assert_eq!(reader.into_iter().count(), 0);
1016
1017 Ok(())
1018 }
1019
1020 #[test]
1021 fn test_avro_3405_writer_add_metadata_failure() -> TestResult {
1022 let schema = Schema::parse_str(SCHEMA)?;
1023 let mut writer = Writer::new(&schema, Vec::new())?;
1024
1025 let mut record = Record::new(&schema).unwrap();
1026 record.put("a", 27i64);
1027 record.put("b", "foo");
1028 writer.append_value(record.clone())?;
1029
1030 match writer
1031 .add_user_metadata("stringKey".to_string(), String::from("value2"))
1032 .map_err(Error::into_details)
1033 {
1034 Err(e @ Details::FileHeaderAlreadyWritten) => {
1035 assert_eq!(e.to_string(), "The file metadata is already flushed.");
1036 }
1037 Err(e) => panic!("Unexpected error occurred while writing user metadata: {e:?}"),
1038 Ok(_) => panic!("Expected an error that metadata cannot be added after adding data"),
1039 }
1040
1041 Ok(())
1042 }
1043
1044 #[test]
1045 fn test_avro_3405_writer_add_metadata_reserved_prefix_failure() -> TestResult {
1046 let schema = Schema::parse_str(SCHEMA)?;
1047 let mut writer = Writer::new(&schema, Vec::new())?;
1048
1049 let key = "avro.stringKey".to_string();
1050 match writer
1051 .add_user_metadata(key.clone(), "value")
1052 .map_err(Error::into_details)
1053 {
1054 Err(ref e @ Details::InvalidMetadataKey(_)) => {
1055 assert_eq!(
1056 e.to_string(),
1057 format!(
1058 "Metadata keys starting with 'avro.' are reserved for internal usage: {key}."
1059 )
1060 );
1061 }
1062 Err(e) => panic!(
1063 "Unexpected error occurred while writing user metadata with reserved prefix ('avro.'): {e:?}"
1064 ),
1065 Ok(_) => {
1066 panic!("Expected an error that the metadata key cannot be prefixed with 'avro.'")
1067 }
1068 }
1069
1070 Ok(())
1071 }
1072
1073 #[test]
1074 fn test_avro_3405_writer_add_metadata_with_builder_api_success() -> TestResult {
1075 let schema = Schema::parse_str(SCHEMA)?;
1076
1077 let mut user_meta_data: HashMap<String, Value> = HashMap::new();
1078 user_meta_data.insert(
1079 "stringKey".to_string(),
1080 Value::String("stringValue".to_string()),
1081 );
1082 user_meta_data.insert("bytesKey".to_string(), Value::Bytes(b"bytesValue".to_vec()));
1083 user_meta_data.insert("vecKey".to_string(), Value::Bytes(vec![1, 2, 3]));
1084
1085 let writer: Writer<'_, Vec<u8>> = Writer::builder()
1086 .writer(Vec::new())
1087 .schema(&schema)
1088 .user_metadata(user_meta_data.clone())
1089 .build()?;
1090
1091 assert_eq!(writer.user_metadata, user_meta_data);
1092
1093 Ok(())
1094 }
1095
1096 #[test]
1097 fn avro_3894_take_aliases_into_account_when_serializing() -> TestResult {
1098 const SCHEMA: &str = r#"
1099 {
1100 "type": "record",
1101 "name": "Conference",
1102 "fields": [
1103 {"type": "string", "name": "name"},
1104 {"type": ["null", "long"], "name": "date", "aliases" : [ "time2", "time" ]}
1105 ]
1106 }"#;
1107
1108 #[derive(Debug, PartialEq, Eq, Clone, Serialize)]
1109 pub struct Conference {
1110 pub name: String,
1111 pub time: Option<i64>,
1112 }
1113
1114 let conf = Conference {
1115 name: "RustConf".to_string(),
1116 time: Some(1234567890),
1117 };
1118
1119 let schema = Schema::parse_str(SCHEMA)?;
1120 let mut writer = Writer::new(&schema, Vec::new())?;
1121
1122 let bytes = writer.append_ser(conf)?;
1123
1124 assert_eq!(182, bytes);
1125
1126 Ok(())
1127 }
1128
1129 #[test]
1130 fn avro_4014_validation_returns_a_detailed_error() -> TestResult {
1131 const SCHEMA: &str = r#"
1132 {
1133 "type": "record",
1134 "name": "Conference",
1135 "fields": [
1136 {"type": "string", "name": "name"},
1137 {"type": ["null", "long"], "name": "date", "aliases" : [ "time2", "time" ]}
1138 ]
1139 }"#;
1140
1141 #[derive(Debug, PartialEq, Clone, Serialize)]
1142 pub struct Conference {
1143 pub name: String,
1144 pub time: Option<f64>, }
1146
1147 let conf = Conference {
1148 name: "RustConf".to_string(),
1149 time: Some(12345678.90),
1150 };
1151
1152 let schema = Schema::parse_str(SCHEMA)?;
1153 let mut writer = Writer::new(&schema, Vec::new())?;
1154
1155 match writer.append_ser(conf) {
1156 Ok(bytes) => panic!("Expected an error, but got {bytes} bytes written"),
1157 Err(e) => {
1158 assert_eq!(
1159 e.to_string(),
1160 r#"Failed to serialize field 'date' of record RecordSchema { name: Name { name: "Conference", .. }, fields: [RecordField { name: "name", schema: String, .. }, RecordField { name: "date", aliases: ["time2", "time"], schema: Union(UnionSchema { schemas: [Null, Long] }), .. }], .. }: Failed to serialize value of type `f64`: Expected Schema::Double"#
1161 );
1162 }
1163 }
1164 Ok(())
1165 }
1166
1167 #[test]
1168 fn avro_4063_flush_applies_to_inner_writer() -> TestResult {
1169 const SCHEMA: &str = r#"
1170 {
1171 "type": "record",
1172 "name": "ExampleSchema",
1173 "fields": [
1174 {"name": "exampleField", "type": "string"}
1175 ]
1176 }
1177 "#;
1178
1179 #[derive(Clone, Default)]
1180 struct TestBuffer(Rc<RefCell<Vec<u8>>>);
1181
1182 impl TestBuffer {
1183 fn len(&self) -> usize {
1184 self.0.borrow().len()
1185 }
1186 }
1187
1188 impl Write for TestBuffer {
1189 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
1190 self.0.borrow_mut().write(buf)
1191 }
1192
1193 fn flush(&mut self) -> std::io::Result<()> {
1194 Ok(())
1195 }
1196 }
1197
1198 let shared_buffer = TestBuffer::default();
1199
1200 let buffered_writer = std::io::BufWriter::new(shared_buffer.clone());
1201
1202 let schema = Schema::parse_str(SCHEMA)?;
1203
1204 let mut writer = Writer::new(&schema, buffered_writer)?;
1205
1206 let mut record = Record::new(writer.schema()).unwrap();
1207 record.put("exampleField", "value");
1208
1209 writer.append_value(record)?;
1210 writer.flush()?;
1211
1212 assert_eq!(
1213 shared_buffer.len(),
1214 151,
1215 "the test buffer was not fully written to after Writer::flush was called"
1216 );
1217
1218 Ok(())
1219 }
1220
1221 #[test]
1222 fn avro_rs_310_append_unvalidated_value() -> TestResult {
1223 let schema = Schema::String;
1224 let value = Value::Int(1);
1225
1226 let mut writer = Writer::new(&schema, Vec::new())?;
1227 writer.unvalidated_append_value_ref(&value)?;
1228 writer.unvalidated_append_value(value)?;
1229 let buffer = writer.into_inner()?;
1230
1231 assert_eq!(&buffer[buffer.len() - 18..buffer.len() - 16], &[2, 2]);
1233
1234 let mut writer = Writer::new(&schema, Vec::new())?;
1235 let value = Value::Int(1);
1236 let err = writer.append_value_ref(&value).unwrap_err();
1237 assert_eq!(
1238 err.to_string(),
1239 "Value Int(1) does not match schema String: Reason: Unsupported value-schema combination! Value: Int, schema: String"
1240 );
1241 let err = writer.append_value(value).unwrap_err();
1242 assert_eq!(
1243 err.to_string(),
1244 "Value Int(1) does not match schema String: Reason: Unsupported value-schema combination! Value: Int, schema: String"
1245 );
1246
1247 Ok(())
1248 }
1249
1250 #[test]
1251 fn avro_rs_469_reset_writer() -> TestResult {
1252 let schema = Schema::Boolean;
1253 let values = [true, false, true, false];
1254 let mut writer = Writer::new(&schema, Vec::new())?;
1255
1256 for value in values {
1257 writer.append_value(value)?;
1258 }
1259
1260 writer.flush()?;
1261 let first_buffer = writer.get_ref().clone();
1262
1263 writer.reset();
1264 assert_eq!(writer.get_ref().len(), 0);
1265
1266 for value in values {
1267 writer.append_value(value)?;
1268 }
1269
1270 writer.flush()?;
1271 let second_buffer = writer.get_ref().clone();
1272 assert_eq!(first_buffer.len(), second_buffer.len());
1273 let len = first_buffer.len();
1279 let header = len - 16 - 6 - 16;
1280 let data = header + 16;
1281 assert_eq!(
1282 first_buffer[..header],
1283 second_buffer[..header],
1284 "Written header must be the same, excluding sync marker"
1285 );
1286 assert_ne!(
1287 first_buffer[header..data],
1288 second_buffer[header..data],
1289 "Sync markers should be different"
1290 );
1291 assert_eq!(
1292 first_buffer[data..data + 6],
1293 second_buffer[data..data + 6],
1294 "Written data must be the same"
1295 );
1296 assert_ne!(
1297 first_buffer[len - 16..],
1298 second_buffer[len - 16..],
1299 "Sync markers should be different"
1300 );
1301
1302 Ok(())
1303 }
1304}