From ecedb96ae546691d4855794fc6ebe0d368d0400b Mon Sep 17 00:00:00 2001 From: sergeych Date: Mon, 16 Oct 2023 00:52:10 +0100 Subject: [PATCH] partial serde deserialization --- src/bipack_source.rs | 45 +++++++- src/de.rs | 258 +++++++++++++++++++++++++++++++++++++++++++ src/error.rs | 16 +++ src/lib.rs | 1 + src/ser.rs | 116 +++++++++++-------- src/tools.rs | 10 ++ 6 files changed, 393 insertions(+), 53 deletions(-) create mode 100644 src/de.rs diff --git a/src/bipack_source.rs b/src/bipack_source.rs index cf0cae6..08554d2 100644 --- a/src/bipack_source.rs +++ b/src/bipack_source.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::error::Error; use std::fmt::{Display, Formatter}; use std::string::FromUtf8Error; use crate::bipack_source::BipackError::NoDataError; @@ -27,21 +26,21 @@ pub enum BipackError { BadEncoding(FromUtf8Error), } +impl std::error::Error for BipackError {} + impl Display for BipackError { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{:?}", self) } } -impl Error for BipackError {} - - /// Data source compatible with mp_bintools serialization. It supports /// fixed-size integers in right order and varint ans smartint encodings /// separately. There is out of the box implementation for [`Vec`], and /// it is easy to implements your own. /// -/// To implement source for other type, implement just [BipackSource::get_u8] or mayve also +/// To implement source for other type, implement just [BipackSource::get_u8], +/// [BipackSource::eof] or mayve also /// [BipackSource::get_fixed_bytes] for effectiveness. /// /// Unlike the [crate::bipack_sink::BipackSink] the source is returning errors. This is because @@ -50,6 +49,10 @@ impl Error for BipackError {} pub trait BipackSource { fn get_u8(self: &mut Self) -> Result; + /// Tribute to the tradition. End Of File, if true, there is no unread data left + /// in the source. + fn eof(self: &Self) -> bool; + fn get_u16(self: &mut Self) -> Result { Ok(((self.get_u8()? as u16) << 8) + (self.get_u8()? as u16)) } @@ -179,6 +182,38 @@ impl<'x> BipackSource for SliceSource<'x> { Ok(result) } } + + fn eof(self: &Self) -> bool { + self.data.len() >= self.position + } +} + +pub struct VecSource { + data: Vec, + position: usize, +} + +impl VecSource { + pub fn from(src: Vec) -> VecSource { + VecSource { data: src, position: 0 } + } +} + +impl BipackSource for VecSource { + fn get_u8(self: &mut Self) -> Result { + if self.position >= self.data.len() { + Err(NoDataError) + } else { + let result = self.data[self.position]; + self.position += 1; + Ok(result) + } + } + + fn eof(self: &Self) -> bool { + self.data.len() <= self.position + } } + diff --git a/src/de.rs b/src/de.rs new file mode 100644 index 0000000..b9ff00d --- /dev/null +++ b/src/de.rs @@ -0,0 +1,258 @@ +use serde::de::{ + self, DeserializeSeed, MapAccess, SeqAccess, + Visitor, +}; +use serde::{Serialize}; +use serde::Deserialize; + +use crate::bipack_source::{BipackSource, VecSource}; +use crate::error::{Error, Result}; +use crate::ser::to_bytes; +use crate::tools::to_dump; + +pub struct Deserializer { + // This string starts with the input data and characters are truncated off + // the beginning as data is parsed. + input: VecSource, +} + +pub fn from_bytes<'de,T: Deserialize<'de>>(source: Vec) -> Result { + let mut des = Deserializer { input: VecSource::from(source)}; + T::deserialize(&mut des) +} + +impl<'de, 'a> de::Deserializer<'de> for &'a mut Deserializer { + type Error = Error; + + fn deserialize_any(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + Err(Error::NotPossible) + } + + fn deserialize_bool(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_bool(if self.input.get_u8()? == 0 { false } else { true } ) + } + + fn deserialize_i8(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_i8(self.input.get_i8()?) + } + + fn deserialize_i16(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_i16(self.input.get_signed()? as i16) + } + + fn deserialize_i32(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_i32(self.input.get_signed()? as i32) + } + + fn deserialize_i64(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_i64(self.input.get_signed()?) + } + + fn deserialize_u8(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_u8(self.input.get_u8()?) + } + + fn deserialize_u16(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_u16(self.input.get_unsigned()? as u16) + } + + fn deserialize_u32(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_u32(self.input.get_unsigned()? as u32) + } + + fn deserialize_u64(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_u64(self.input.get_unsigned()?) + } + + fn deserialize_f32(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + Err(Error::NotImplemented) + } + + fn deserialize_f64(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + Err(Error::NotImplemented) + } + + fn deserialize_char(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + let ch = self.input.get_str()?; + if ch.len() != 1 { + Err(Error::BadFormat(format!("Char length is {}, should be 1 {:?}", ch.len(), ch))) + } + else { + visitor.visit_char(ch.chars().next().unwrap()) + } + } + + fn deserialize_str(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_string( self.input.get_str()?) + } + + fn deserialize_string(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_string(self.input.get_str()?) + } + + fn deserialize_bytes(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_byte_buf(self.input.get_var_bytes()?) + } + + fn deserialize_byte_buf(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_byte_buf(self.input.get_var_bytes()?) + } + + fn deserialize_option(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + let marker = self.input.get_u8()?; + if marker == 0 { + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_unit(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_unit() + } + + fn deserialize_unit_struct(self, name: &'static str, visitor: V) -> std::result::Result where V: Visitor<'de> { + self.deserialize_unit(visitor) + } + + fn deserialize_newtype_struct(self, name: &'static str, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_newtype_struct(self) + } + + fn deserialize_seq(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + let size = self.input.get_unsigned()? as usize; + visitor.visit_seq(SimpleSeq::new(self, size)) + } + + fn deserialize_tuple(self, len: usize, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_seq(SimpleSeq::new(self, len)) + } + + fn deserialize_tuple_struct(self, name: &'static str, len: usize, visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_seq(SimpleSeq::new(self, len)) + } + + fn deserialize_map(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + let size = self.input.get_unsigned()?; + visitor.visit_map(SimpleMap { de: self, size: size as usize} ) + } + + fn deserialize_struct(self, name: &'static str, fields: &'static [&'static str], visitor: V) -> std::result::Result where V: Visitor<'de> { + visitor.visit_seq(SimpleSeq::new(self, fields.len() )) + } + + fn deserialize_enum(self, name: &'static str, variants: &'static [&'static str], visitor: V) -> std::result::Result where V: Visitor<'de> { + todo!() + } + + fn deserialize_identifier(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + Err(Error::NotPossible) + } + + fn deserialize_ignored_any(self, visitor: V) -> std::result::Result where V: Visitor<'de> { + Err(Error::NotPossible) + } + + fn is_human_readable(&self) -> bool { + false + } +} + +struct SimpleSeq<'a> { + de: &'a mut Deserializer, + size: usize, +} + +impl<'a> SimpleSeq<'a> { + fn new(de: &'a mut Deserializer,size: usize) -> Self { + SimpleSeq { + de, + size: size, + } + } +} + +// `SeqAccess` is provided to the `Visitor` to give it the ability to iterate +// through elements of the sequence. +impl<'de, 'a> SeqAccess<'de> for SimpleSeq<'a> { + type Error = Error; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: DeserializeSeed<'de>, + { + if self.size < 1 { + return Ok(None); + } + self.size -= 1; + seed.deserialize(&mut *self.de).map(Some) + } +} + + +struct SimpleMap<'a> { + de: &'a mut Deserializer, + size: usize, +} + +impl<'de, 'a> MapAccess<'de> for SimpleMap<'a> { + type Error = Error; + + fn next_key_seed(&mut self, seed: K) -> std::result::Result, Self::Error> where K: DeserializeSeed<'de> { + if self.size < 1 { + Ok(None) + } + else { + self.size -= 1; + seed.deserialize(&mut *self.de).map(Some) + } + } + + fn next_value_seed(&mut self, seed: V) -> std::result::Result where V: DeserializeSeed<'de> { + seed.deserialize(&mut *self.de) + } +} + + +#[test] +fn test_ints() -> Result<()> { + // #[derive(Deserialize, PartialEq, Debug)] + // struct Test { + // int: u32, + // seq: Vec, + // } + + let b = vec![7]; + assert_eq!( 7u8, from_bytes(vec![7u8])?); + + + // let j = r#"{"int":1,"seq":["a","b"]}"#; + // let expected = Test { + // int: 1, + // seq: vec!["a".to_owned(), "b".to_owned()], + // }; + // assert_eq!(expected, from_str(j).unwrap()); + Ok(()) +} + +#[test] +fn test_struct_de() -> Result<()> { + #[derive(Serialize, Deserialize, PartialEq, Debug)] + struct Test { + int: u32, + seq: Vec, + } + let expected = Test { + int: 1, + seq: vec!["a".to_owned(), "b".to_owned()], + }; + + let packed = to_bytes(&expected)?; + println!("::{}", to_dump(&packed)); + let unpacked: Test = from_bytes(packed)?; + println!("::{:?}", unpacked); + Ok(()) + + // let j = r#"{"int":1,"seq":["a","b"]}"#; + // assert_eq!(expected, from_str(j).unwrap()); +} \ No newline at end of file diff --git a/src/error.rs b/src/error.rs index bc40077..ca8d4db 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,14 +1,20 @@ use std; use std::fmt::{self, Display}; use serde::{de, ser}; +use crate::bipack_source::BipackError; pub type Result = std::result::Result; #[derive(Debug)] pub enum Error { Message(String), + BadFormat(String), Eof, + ExtraBytes, BadEncoding, + NotSupported, + NotImplemented, + NotPossible } impl Display for Error { @@ -33,3 +39,13 @@ impl de::Error for Error { Error::Message(msg.to_string()) } } + +impl From for Error { + fn from(value: BipackError) -> Self { + match &value { + BipackError::NoDataError => Error::Eof, + BipackError::BadEncoding(_) => Error::BadEncoding, + // other => Error::Message(format!("BiPack error: {}", value)) + } + } +} diff --git a/src/lib.rs b/src/lib.rs index b039f83..510a6d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -120,6 +120,7 @@ pub mod tools; pub mod bipack; mod error; mod ser; +mod de; #[cfg(test)] mod tests { diff --git a/src/ser.rs b/src/ser.rs index c386262..c623160 100644 --- a/src/ser.rs +++ b/src/ser.rs @@ -1,8 +1,11 @@ -use serde::{ser, Serialize}; -use crate::bipack_sink::{BipackSink, IntoU64}; +use std::string::FromUtf8Error; +use serde::{ser, Serialize}; + +use crate::bipack_sink::{BipackSink, IntoU64}; +use crate::bipack_source::{BipackError, BipackSource, SliceSource}; use crate::error::{Error, Result}; -use crate::tools::to_dump; +use crate::tools::{to_dump, to_hex}; pub struct Serializer { // This string starts empty and JSON is appended as values are serialized. @@ -10,10 +13,10 @@ pub struct Serializer { } pub fn to_bytes(value: &T) -> Result> -where T: Serialize, + where T: Serialize, { let mut serializer = Serializer { output: Vec::new() }; - value.serialize(& mut serializer)?; + value.serialize(&mut serializer)?; Ok(serializer.output) } @@ -29,7 +32,7 @@ impl<'a> ser::Serializer for &'a mut Serializer { type SerializeStructVariant = Self; fn serialize_bool(self, v: bool) -> Result<()> { - self.output.put_u8(if v { 1 } else { 0 } ); + self.output.put_u8(if v { 1 } else { 0 }); Ok(()) } @@ -56,11 +59,11 @@ impl<'a> ser::Serializer for &'a mut Serializer { Ok(()) } fn serialize_u16(self, v: u16) -> Result<()> { - self.output.put_u64(v.into()); + self.serialize_u64(v.into_u64())?; Ok(()) } fn serialize_u32(self, v: u32) -> Result<()> { - self.output.put_u64(v.into_u64()); + self.serialize_u64(v.into())?; Ok(()) } fn serialize_u64(self, v: u64) -> Result<()> { @@ -93,7 +96,7 @@ impl<'a> ser::Serializer for &'a mut Serializer { } fn serialize_none(self) -> Result<()> { - self.serialize_u8(0); + self.serialize_u8(0)?; Ok(()) } @@ -101,12 +104,11 @@ impl<'a> ser::Serializer for &'a mut Serializer { where T: ?Sized + Serialize, { - self.serialize_u8(1); + self.serialize_u8(1)?; value.serialize(self) } fn serialize_unit(self) -> Result<()> { - self.output.put_u8(0); Ok(()) } @@ -144,10 +146,11 @@ impl<'a> ser::Serializer for &'a mut Serializer { where T: ?Sized + Serialize, { - self.serialize_str(_name); - self.serialize_u32(_variant_index) + self.serialize_u32(_variant_index)?; + value.serialize(self) } + /// We expect it to use with vairable-length arrays... fn serialize_seq(self, _len: Option) -> Result { self.output.put_unsigned(_len.unwrap_or(0)); Ok(self) @@ -172,7 +175,7 @@ impl<'a> ser::Serializer for &'a mut Serializer { variant: &'static str, _len: usize, ) -> Result { - variant.serialize(&mut *self)?; + _variant_index.serialize(&mut *self)?; Ok(self) } @@ -186,9 +189,10 @@ impl<'a> ser::Serializer for &'a mut Serializer { _name: &'static str, len: usize, ) -> Result { - self.serialize_map(Some(len)) + // self.serialize_map(Some(len)) + Ok(self) } -fn serialize_struct_variant( + fn serialize_struct_variant( self, _name: &'static str, _variant_index: u32, @@ -312,7 +316,6 @@ impl<'a> ser::SerializeStruct for &'a mut Serializer { where T: ?Sized + Serialize, { - key.serialize(&mut **self)?; value.serialize(&mut **self) } @@ -329,7 +332,6 @@ impl<'a> ser::SerializeStructVariant for &'a mut Serializer { where T: ?Sized + Serialize, { - key.serialize(&mut **self)?; value.serialize(&mut **self) } @@ -339,7 +341,7 @@ impl<'a> ser::SerializeStructVariant for &'a mut Serializer { } #[test] -fn test_struct() { +fn test_struct() -> std::result::Result<(), BipackError> { #[derive(Serialize)] struct Test { int: u32, @@ -347,38 +349,56 @@ fn test_struct() { } let test = Test { - int: 1, + int: 17, seq: vec!["a", "b"], }; - // let expected = r#"{"int":1,"seq":["a","b"]}"#; - // assert_eq!(to_string(&test).unwrap(), expected); let x = to_bytes(&test).unwrap(); println!("!:\n{}", to_dump(&x)); + // let y = x.clone(); + // let z = x.clone(); + let mut src = SliceSource::from(&x); + assert_eq!(test.int, src.get_unsigned()? as u32); + assert_eq!(test.seq.len(), src.get_unsigned()? as usize); + assert_eq!(test.seq[0], src.get_str()?); + assert_eq!(test.seq[1], src.get_str()?); + Ok(()) } -// #[test] -// fn test_enum() { -// #[derive(Serialize)] -// enum E { -// Unit, -// Newtype(u32), -// Tuple(u32, u32), -// Struct { a: u32 }, -// } -// -// let u = E::Unit; -// let expected = r#""Unit""#; -// assert_eq!(to_string(&u).unwrap(), expected); -// -// let n = E::Newtype(1); -// let expected = r#"{"Newtype":1}"#; -// assert_eq!(to_string(&n).unwrap(), expected); -// -// let t = E::Tuple(1, 2); -// let expected = r#"{"Tuple":[1,2]}"#; -// assert_eq!(to_string(&t).unwrap(), expected); -// -// let s = E::Struct { a: 1 }; -// let expected = r#"{"Struct":{"a":1}}"#; -// assert_eq!(to_string(&s).unwrap(), expected); -// } \ No newline at end of file +#[test] +fn test_enum() -> std::result::Result<(), FromUtf8Error> { + #[derive(Serialize)] + enum E { + Unit, + Unit2, + Newtype(u32), + Tuple(u32, u32), + Struct { a: u32 }, + } + + let u = E::Unit; + println!("u:{}",to_dump(to_bytes(&u).unwrap().as_slice())); + assert_eq!("00",to_hex(to_bytes(&u).unwrap())?); + let u2 = E::Unit2; + println!("u:{}",to_dump(to_bytes(&u2).unwrap().as_slice())); + let nt = E::Newtype(17); + println!("u:{}",to_dump(to_bytes(&nt).unwrap().as_slice())); + assert_eq!("08 44",to_hex(to_bytes(&nt).unwrap())?); + let t = E::Tuple(7,17); + println!("u:{}",to_dump(to_bytes(&t).unwrap().as_slice())); + assert_eq!("0c 1c 44",to_hex(to_bytes(&t).unwrap())?); + // let expected = r#""Unit""#; + // assert_eq!(to_string(&u).unwrap(), expected); + // + // let n = E::Newtype(1); + // let expected = r#"{"Newtype":1}"#; + // assert_eq!(to_string(&n).unwrap(), expected); + // + // let t = E::Tuple(1, 2); + // let expected = r#"{"Tuple":[1,2]}"#; + // assert_eq!(to_string(&t).unwrap(), expected); + // + // let s = E::Struct { a: 1 }; + // let expected = r#"{"Struct":{"a":1}}"#; + // assert_eq!(to_string(&s).unwrap(), expected); + Ok(()) +} \ No newline at end of file diff --git a/src/tools.rs b/src/tools.rs index 0bedb65..d51ce1d 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -37,6 +37,8 @@ impl StringBuilder { } fn new() -> StringBuilder { StringBuilder(Vec::new()) } + + fn len(self: &Self) -> usize { self.0.len() } } @@ -83,3 +85,11 @@ pub fn to_dump(data: &[u8]) -> String { result.string().unwrap() } +pub fn to_hex>(source: T) -> Result { + let mut result = StringBuilder::new(); + for b in source.as_ref() { + if result.len() != 0 { result.append( " ") } + result.append(format!("{:02x}", b)) + } + result.string() +}