From 9d3432ec517b21432f848f2edc1de04f6ff98545 Mon Sep 17 00:00:00 2001 From: Olivier Schyns Date: Mon, 31 Aug 2026 18:40:27 +0200 Subject: [PATCH 1/5] ft: Improve tvf! macro to specify numeric fields signs without using extra parenthesis Signed-off-by: Olivier Schyns --- prosa_macros/src/tvf/literal.rs | 15 ++++++++++++--- prosa_macros/src/tvf/value.rs | 33 +++++++++++++++++++++++++++++---- prosa_macros/tests/tvf.rs | 10 +++++++++- 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/prosa_macros/src/tvf/literal.rs b/prosa_macros/src/tvf/literal.rs index f343bee..2815733 100644 --- a/prosa_macros/src/tvf/literal.rs +++ b/prosa_macros/src/tvf/literal.rs @@ -1,4 +1,5 @@ use super::value::ValueType; +use crate::tvf::value::make_sign; use chrono::Datelike; use proc_macro2::{Literal, TokenStream}; use quote::{ToTokens, quote}; @@ -35,21 +36,29 @@ pub(crate) fn identify_literal(literal: &Literal) -> Result { pub(crate) fn convert_literal( literal: &Literal, output_type: &ValueType, + is_negative: bool, ) -> Result { // Check if the literal is already of the expected type // in that case we can return it as is let lit_type = identify_literal(literal)?; if lit_type == *output_type { - return Ok(literal.to_token_stream()); + if is_negative { + return Ok(quote![ - #literal ]); + } else { + return Ok(literal.to_token_stream()); + } } let token_stream = match lit_type { ValueType::Byte | ValueType::Signed | ValueType::Unsigned | ValueType::Float => { + // Integers and Floats may be prefixed with a negative sign + let sign = make_sign(is_negative); + match output_type { ValueType::Byte => quote! [ #literal as u8 ], - ValueType::Signed => quote! [ #literal as i64 ], + ValueType::Signed => quote! [ #sign #literal as i64 ], ValueType::Unsigned => quote! [ #literal as u64 ], - ValueType::Float => quote! [ #literal as f64 ], + ValueType::Float => quote! [ #sign #literal as f64 ], ValueType::String => quote! [ #literal.to_string() ], ValueType::Bytes => { // Enable special conversion for literals written in diff --git a/prosa_macros/src/tvf/value.rs b/prosa_macros/src/tvf/value.rs index 4b5c915..60db666 100644 --- a/prosa_macros/src/tvf/value.rs +++ b/prosa_macros/src/tvf/value.rs @@ -65,7 +65,16 @@ pub(crate) fn generate_value( value_stream: &TokenStream, ) -> Result<(TokenStream, ValueType), Error> { // Process the token tree - let mut tokens = value_stream.clone().into_iter(); + let mut tokens = value_stream.clone().into_iter().peekable(); + + // Check if the first element is a sign (+ or -) + let is_negative = if let Some(TokenTree::Punct(punct)) = tokens.peek() { + let negative = punct.as_char() == '-'; + tokens.next(); // move to next token + negative + } else { + false + }; // in all cases, we expect to find a value let value = tokens.next().ok_or(Error::new_spanned( @@ -98,14 +107,21 @@ pub(crate) fn generate_value( match value { TokenTree::Literal(literal) => { if let Some(output_type) = output_type { - Ok((convert_literal(&literal, &output_type)?, output_type)) + Ok(( + convert_literal(&literal, &output_type, is_negative)?, + output_type, + )) } else { let literal_type = identify_literal(&literal)?; + + // Integers and Floats may be prefixed with a negative sign + let sign = make_sign(is_negative); + let token_stream = match literal_type { ValueType::Byte => quote! [ #literal as u8 ], - ValueType::Signed => quote! [ #literal as i64 ], + ValueType::Signed => quote! [ #sign #literal as i64 ], ValueType::Unsigned => quote! [ #literal as u64 ], - ValueType::Float => quote! [ #literal as f64 ], + ValueType::Float => quote! [ #sign #literal as f64 ], _ => literal.to_token_stream(), }; Ok((token_stream, literal_type)) @@ -160,3 +176,12 @@ pub(crate) fn generate_value( TokenTree::Punct(_) => Err(Error::new_spanned(value, "Unexpected punctuation.")), } } + +#[inline] +pub(crate) fn make_sign(is_negative: bool) -> TokenStream { + if is_negative { + quote![-] + } else { + TokenStream::new() + } +} diff --git a/prosa_macros/tests/tvf.rs b/prosa_macros/tests/tvf.rs index fa2628f..263d236 100644 --- a/prosa_macros/tests/tvf.rs +++ b/prosa_macros/tests/tvf.rs @@ -26,10 +26,14 @@ mod macro_tests { 7 => false, 8 => true, 9 => b"string from bytes" as String, + 10 => -10, + 11 => -10.25, + 12 => -2 as Signed, + 13 => +2.125 as Float, 200 => "2023-06-05 15:02:00.000" as DateTime, }); - assert_eq!(8, buffer.len()); + assert_eq!(12, buffer.len()); assert_eq!(Ok(2), buffer.get_unsigned(1)); assert_eq!(Ok(4), buffer.get_signed(3)); assert_eq!(Ok(0), buffer.get_byte(7)); @@ -68,6 +72,10 @@ mod macro_tests { Ok(NaiveDate::from_ymd_opt(1995, 1, 10).expect("NaiveDate should be build")), buffer.get_date(6) ); + assert_eq!(Ok(-10), buffer.get_signed(10)); + assert_eq!(Ok(-10.25), buffer.get_float(11)); + assert_eq!(Ok(-2), buffer.get_signed(12)); + assert_eq!(Ok(2.125), buffer.get_float(13)); assert_eq!( Ok(NaiveDate::from_ymd_opt(2023, 6, 5) .expect("NaiveDate should be build") From 9384d6378b649bf40ac56c6798e1225161c27e60 Mon Sep 17 00:00:00 2001 From: Olivier Schyns Date: Tue, 1 Sep 2026 18:09:32 +0200 Subject: [PATCH 2/5] ft: Handle signs for variable name, and number to string conversions Signed-off-by: Olivier Schyns --- prosa_macros/src/tvf/literal.rs | 2 +- prosa_macros/src/tvf/value.rs | 9 ++++++--- prosa_macros/tests/tvf.rs | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/prosa_macros/src/tvf/literal.rs b/prosa_macros/src/tvf/literal.rs index 2815733..fadff3e 100644 --- a/prosa_macros/src/tvf/literal.rs +++ b/prosa_macros/src/tvf/literal.rs @@ -59,7 +59,7 @@ pub(crate) fn convert_literal( ValueType::Signed => quote! [ #sign #literal as i64 ], ValueType::Unsigned => quote! [ #literal as u64 ], ValueType::Float => quote! [ #sign #literal as f64 ], - ValueType::String => quote! [ #literal.to_string() ], + ValueType::String => quote! [ (#sign #literal).to_string() ], ValueType::Bytes => { // Enable special conversion for literals written in // hexadecimal or in binary to be converted directly to bytes. diff --git a/prosa_macros/src/tvf/value.rs b/prosa_macros/src/tvf/value.rs index 60db666..c5797f3 100644 --- a/prosa_macros/src/tvf/value.rs +++ b/prosa_macros/src/tvf/value.rs @@ -119,9 +119,9 @@ pub(crate) fn generate_value( let token_stream = match literal_type { ValueType::Byte => quote! [ #literal as u8 ], - ValueType::Signed => quote! [ #sign #literal as i64 ], + ValueType::Signed => quote! [ (#sign #literal) as i64 ], ValueType::Unsigned => quote! [ #literal as u64 ], - ValueType::Float => quote! [ #sign #literal as f64 ], + ValueType::Float => quote! [ (#sign #literal) as f64 ], _ => literal.to_token_stream(), }; Ok((token_stream, literal_type)) @@ -134,7 +134,10 @@ pub(crate) fn generate_value( "false" => Ok((quote![0u8], output_type.unwrap_or(ValueType::Byte))), _ => { if let Some(output_type) = output_type { - Ok((quote![#ident], output_type)) + // Integers and Floats may be prefixed with a negative sign + let sign = make_sign(is_negative); + + Ok((quote![#sign #ident], output_type)) } else { Err(Error::new_spanned( ident, diff --git a/prosa_macros/tests/tvf.rs b/prosa_macros/tests/tvf.rs index 263d236..e2340ca 100644 --- a/prosa_macros/tests/tvf.rs +++ b/prosa_macros/tests/tvf.rs @@ -8,6 +8,8 @@ mod macro_tests { #[test] fn test_tvf_macro() { + let amount = 64i64; + let buffer = tvf!(SimpleStringTvf { 1 => 2, 3 => 4usize, @@ -30,10 +32,13 @@ mod macro_tests { 11 => -10.25, 12 => -2 as Signed, 13 => +2.125 as Float, + 14 => 100 as String, + 15 => -1000 as String, + 16 => -amount as Signed, 200 => "2023-06-05 15:02:00.000" as DateTime, }); - assert_eq!(12, buffer.len()); + assert_eq!(15, buffer.len()); assert_eq!(Ok(2), buffer.get_unsigned(1)); assert_eq!(Ok(4), buffer.get_signed(3)); assert_eq!(Ok(0), buffer.get_byte(7)); @@ -76,6 +81,15 @@ mod macro_tests { assert_eq!(Ok(-10.25), buffer.get_float(11)); assert_eq!(Ok(-2), buffer.get_signed(12)); assert_eq!(Ok(2.125), buffer.get_float(13)); + assert_eq!( + Ok("100"), + buffer.get_string(14).map(|s| s.to_string()).as_deref() + ); + assert_eq!( + Ok("-1000"), + buffer.get_string(15).map(|s| s.to_string()).as_deref() + ); + assert_eq!(Ok(-64), buffer.get_signed(16)); assert_eq!( Ok(NaiveDate::from_ymd_opt(2023, 6, 5) .expect("NaiveDate should be build") From 57f6802942d52d349ad01800a5a61c787c7cdbd8 Mon Sep 17 00:00:00 2001 From: Olivier Schyns Date: Wed, 2 Sep 2026 18:24:15 +0200 Subject: [PATCH 3/5] fix: Handle all unary operators to fix issues Signed-off-by: Olivier Schyns --- prosa_macros/src/tvf/literal.rs | 21 +++---- prosa_macros/src/tvf/value.rs | 102 ++++++++++++++++++++++++-------- 2 files changed, 84 insertions(+), 39 deletions(-) diff --git a/prosa_macros/src/tvf/literal.rs b/prosa_macros/src/tvf/literal.rs index fadff3e..894f1bc 100644 --- a/prosa_macros/src/tvf/literal.rs +++ b/prosa_macros/src/tvf/literal.rs @@ -1,8 +1,8 @@ use super::value::ValueType; -use crate::tvf::value::make_sign; +use crate::tvf::value::UnaryOp; use chrono::Datelike; use proc_macro2::{Literal, TokenStream}; -use quote::{ToTokens, quote}; +use quote::quote; use std::num::ParseIntError; use syn::{Error, Lit, parse_quote}; @@ -36,28 +36,23 @@ pub(crate) fn identify_literal(literal: &Literal) -> Result { pub(crate) fn convert_literal( literal: &Literal, output_type: &ValueType, - is_negative: bool, + unary_op: UnaryOp, ) -> Result { + let sign = unary_op.make_sign(); + // Check if the literal is already of the expected type // in that case we can return it as is let lit_type = identify_literal(literal)?; if lit_type == *output_type { - if is_negative { - return Ok(quote![ - #literal ]); - } else { - return Ok(literal.to_token_stream()); - } + return Ok(quote![ #sign #literal ]); } let token_stream = match lit_type { ValueType::Byte | ValueType::Signed | ValueType::Unsigned | ValueType::Float => { - // Integers and Floats may be prefixed with a negative sign - let sign = make_sign(is_negative); - match output_type { - ValueType::Byte => quote! [ #literal as u8 ], + ValueType::Byte => quote! [ #sign #literal as u8 ], ValueType::Signed => quote! [ #sign #literal as i64 ], - ValueType::Unsigned => quote! [ #literal as u64 ], + ValueType::Unsigned => quote! [ #sign #literal as u64 ], ValueType::Float => quote! [ #sign #literal as f64 ], ValueType::String => quote! [ (#sign #literal).to_string() ], ValueType::Bytes => { diff --git a/prosa_macros/src/tvf/value.rs b/prosa_macros/src/tvf/value.rs index c5797f3..4aef127 100644 --- a/prosa_macros/src/tvf/value.rs +++ b/prosa_macros/src/tvf/value.rs @@ -1,6 +1,6 @@ use super::buffer::{generate_list, generate_map}; use super::literal::{convert_literal, identify_literal}; -use proc_macro2::{Delimiter, TokenStream, TokenTree}; +use proc_macro2::{Delimiter, Punct, TokenStream, TokenTree}; use quote::{ToTokens, quote}; use syn::{Error, Ident}; @@ -68,12 +68,12 @@ pub(crate) fn generate_value( let mut tokens = value_stream.clone().into_iter().peekable(); // Check if the first element is a sign (+ or -) - let is_negative = if let Some(TokenTree::Punct(punct)) = tokens.peek() { - let negative = punct.as_char() == '-'; + let unary_op = if let Some(TokenTree::Punct(punct)) = tokens.peek() { + let unary_op = UnaryOp::new(punct)?; tokens.next(); // move to next token - negative + unary_op } else { - false + UnaryOp::None }; // in all cases, we expect to find a value @@ -103,25 +103,23 @@ pub(crate) fn generate_value( return Err(Error::new_spanned(token, "Unexpected token")); } + let sign = unary_op.make_sign(); + // check the type of value provided match value { TokenTree::Literal(literal) => { if let Some(output_type) = output_type { Ok(( - convert_literal(&literal, &output_type, is_negative)?, + convert_literal(&literal, &output_type, unary_op)?, output_type, )) } else { let literal_type = identify_literal(&literal)?; - - // Integers and Floats may be prefixed with a negative sign - let sign = make_sign(is_negative); - let token_stream = match literal_type { - ValueType::Byte => quote! [ #literal as u8 ], - ValueType::Signed => quote! [ (#sign #literal) as i64 ], - ValueType::Unsigned => quote! [ #literal as u64 ], - ValueType::Float => quote! [ (#sign #literal) as f64 ], + ValueType::Byte => quote! [ #sign #literal as u8 ], + ValueType::Signed => quote! [ #sign #literal as i64 ], + ValueType::Unsigned => quote! [ #sign #literal as u64 ], + ValueType::Float => quote! [ #sign #literal as f64 ], _ => literal.to_token_stream(), }; Ok((token_stream, literal_type)) @@ -130,13 +128,24 @@ pub(crate) fn generate_value( TokenTree::Ident(ident) => { // check if the identifier is a boolean match ident.to_string().as_str() { - "true" => Ok((quote![1u8], output_type.unwrap_or(ValueType::Byte))), - "false" => Ok((quote![0u8], output_type.unwrap_or(ValueType::Byte))), + "true" => { + let t = if unary_op == UnaryOp::LogicalNot { + quote![0u8] + } else { + quote![1u8] + }; + Ok((t, output_type.unwrap_or(ValueType::Byte))) + } + "false" => { + let t = if unary_op == UnaryOp::LogicalNot { + quote![1u8] + } else { + quote![0u8] + }; + Ok((t, output_type.unwrap_or(ValueType::Byte))) + } _ => { if let Some(output_type) = output_type { - // Integers and Floats may be prefixed with a negative sign - let sign = make_sign(is_negative); - Ok((quote![#sign #ident], output_type)) } else { Err(Error::new_spanned( @@ -166,7 +175,7 @@ pub(crate) fn generate_value( } _ => { if let Some(output_type) = output_type { - Ok((group.to_token_stream(), output_type)) + Ok((quote![ #sign #group ], output_type)) } else { Err(Error::new_spanned( group, @@ -180,11 +189,52 @@ pub(crate) fn generate_value( } } -#[inline] -pub(crate) fn make_sign(is_negative: bool) -> TokenStream { - if is_negative { - quote![-] - } else { - TokenStream::new() +/// Define a unary operator that may precede a literal, a variable or an expression +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) enum UnaryOp { + /// No operator + #[default] + None, + + /// + sign, results in no operator being added + Positive, + + /// - sign, negate the following value + Negative, + + /// logical not for boolean values + LogicalNot, + + /// * dereference + Dereference, + + /// & borrow + Borrow, +} + +impl UnaryOp { + /// Identify unary operation from punctuation mark + pub(crate) fn new(punct: &Punct) -> Result { + match punct.as_char() { + '+' => Ok(Self::Positive), + '-' => Ok(Self::Negative), + '!' => Ok(Self::LogicalNot), + '*' => Ok(Self::Dereference), + '&' => Ok(Self::Borrow), + _ => Err(Error::new_spanned(punct, "Unsupported punctuation")), + } + } + + /// Generate token for current unary operator + #[rustfmt::skip] + pub(crate) fn make_sign(self) -> TokenStream { + match self { + UnaryOp::None => TokenStream::new(), + UnaryOp::Positive => TokenStream::new(), + UnaryOp::Negative => quote![ - ], + UnaryOp::LogicalNot => quote![ ! ], + UnaryOp::Dereference => quote![ * ], + UnaryOp::Borrow => quote![ & ], + } } } From 068803fe6b04902265f86ac01dcb1e831c993f57 Mon Sep 17 00:00:00 2001 From: Olivier Schyns Date: Fri, 4 Sep 2026 18:20:47 +0200 Subject: [PATCH 4/5] wip: Refactored the whole code to split the process into two steps: parsing then generating tokens Signed-off-by: Olivier Schyns --- prosa_macros/Cargo.toml | 2 + prosa_macros/src/tvf.rs | 48 ++-- prosa_macros/src/tvf/buffer.rs | 135 ---------- prosa_macros/src/tvf/expr.rs | 94 +++++++ prosa_macros/src/tvf/literal.rs | 212 --------------- prosa_macros/src/tvf/parser.rs | 456 ++++++++++++++++++++++++++++++++ prosa_macros/src/tvf/tokens.rs | 111 ++++++++ prosa_macros/src/tvf/value.rs | 240 ----------------- 8 files changed, 694 insertions(+), 604 deletions(-) delete mode 100644 prosa_macros/src/tvf/buffer.rs create mode 100644 prosa_macros/src/tvf/expr.rs delete mode 100644 prosa_macros/src/tvf/literal.rs create mode 100644 prosa_macros/src/tvf/parser.rs create mode 100644 prosa_macros/src/tvf/tokens.rs delete mode 100644 prosa_macros/src/tvf/value.rs diff --git a/prosa_macros/Cargo.toml b/prosa_macros/Cargo.toml index f5e0ab0..d9b5d7d 100644 --- a/prosa_macros/Cargo.toml +++ b/prosa_macros/Cargo.toml @@ -20,6 +20,8 @@ syn = { version = "3", features = ["full"] } quote = "1" proc-macro2 = "1" chrono.workspace = true +thiserror.workspace = true +num-traits = "0.2" [dev-dependencies] bytes.workspace = true diff --git a/prosa_macros/src/tvf.rs b/prosa_macros/src/tvf.rs index e4e902f..e3be3fc 100644 --- a/prosa_macros/src/tvf.rs +++ b/prosa_macros/src/tvf.rs @@ -1,45 +1,59 @@ -mod buffer; -mod literal; -mod value; +/// Structures to properly identify the TVF fields to generate +pub(crate) mod expr; -use buffer::{generate_list, generate_map}; +/// Consume tvf! macro tokens and generate corresponding tree structure of expressions +pub(crate) mod parser; + +/// Convert the expressions into tokens to be passed to the compiler +pub(crate) mod tokens; + +use crate::tvf::parser::TvfParser; use proc_macro2::{Delimiter, TokenStream, TokenTree}; -use syn::Error; +use syn::spanned::Spanned; -pub(crate) fn gen_tvf_impl(input: TokenStream) -> Result { +pub(crate) fn gen_tvf_impl(input: TokenStream) -> Result { // Process the token tree + let span = input.span(); let mut tokens = input.into_iter(); // The first token must be the buffer type used let buffer_type = if let Some(TokenTree::Ident(buffer_type)) = tokens.next() { buffer_type } else { - return Err(Error::new_spanned( - TokenStream::new(), + return Err(syn::Error::new( + span, "First argument must be a type identifier", )); }; // The second token must be the content enclosed in {} or [] - let result = if let Some(TokenTree::Group(content)) = tokens.next() { - match content.delimiter() { - Delimiter::Brace => generate_map(&buffer_type, &content), - Delimiter::Bracket => generate_list(&buffer_type, &content), - _ => Err(Error::new_spanned( - content, + let result = if let Some(TokenTree::Group(group)) = tokens.next() { + match group.delimiter() { + Delimiter::Brace => { + let mut parser = TvfParser::new(group.stream(), false); + let exprs = parser.collect_expr()?; + todo!() + } + Delimiter::Bracket => { + let mut parser = TvfParser::new(group.stream(), true); + let exprs = parser.collect_expr()?; + todo!() + } + _ => Err(syn::Error::new( + span, "Invalid delimiter, expected {} or []", )), } } else { - Err(Error::new_spanned( - TokenStream::new(), + Err(syn::Error::new( + span, "Second argument must be the content enclosed in {} or []", )) }; // Raise an error on any extra tokens if let Some(token) = tokens.next() { - return Err(Error::new_spanned(token, "Unexpected token")); + return Err(syn::Error::new_spanned(token, "Unexpected token")); } result diff --git a/prosa_macros/src/tvf/buffer.rs b/prosa_macros/src/tvf/buffer.rs deleted file mode 100644 index 38afc31..0000000 --- a/prosa_macros/src/tvf/buffer.rs +++ /dev/null @@ -1,135 +0,0 @@ -use super::value::{ValueType, generate_value}; -use proc_macro2::{Group, Spacing, TokenStream, TokenTree}; -use quote::{ToTokens, quote}; -use syn::{Error, Ident}; - -/// From `[ a, b, c, ... ]` Generate a list buffer -pub(crate) fn generate_list(buffer_type: &Ident, content: &Group) -> Result { - // group the values separated by commas into entries - let mut entries = Vec::<(TokenStream, ValueType)>::new(); - - // allocate a buffer to append tokens until a comma is found - let mut token_buffer = TokenStream::new(); - for token_tree in content.stream() { - // if the token is a comma, we have a complete entry - if let TokenTree::Punct(punct) = &token_tree - && punct.as_char() == ',' - && !token_buffer.is_empty() - { - // generate the sequence of tokens to build the value - entries.push(generate_value(buffer_type, &token_buffer)?); - - // reset the buffer - token_buffer = TokenStream::new(); - continue; - } - // otherwise append to the buffer - token_buffer.extend(token_tree.to_token_stream()); - } - - // handle the last entry - if !token_buffer.is_empty() { - entries.push(generate_value(buffer_type, &token_buffer)?); - } - - let token_stream = entries - .iter() - .enumerate() - .map(|(index, (tokens, value_type))| { - let key = index + 1; - let put_method = TokenStream::from(value_type); - quote! [ - <#buffer_type as ::prosa_utils::msg::tvf::Tvf>::#put_method(&mut __list_buffer, #key, #tokens); - ] - }); - - Ok(quote! [ - { - let mut __list_buffer = <#buffer_type>::default(); - #(#token_stream)* - __list_buffer - } - ]) -} - -/// From `{ 1 => a, 2 => b, 3 => c, ... }` Generate a map buffer -pub(crate) fn generate_map(buffer_type: &Ident, content: &Group) -> Result { - // group the values separated by commas into entries - let mut entries = Vec::<(TokenStream, TokenStream, ValueType)>::new(); - - // allocate a buffer to append tokens until a `=>` and a comma is found - let mut read_state = ReadState::Key; - let mut token_buffer_key = TokenStream::new(); - let mut token_buffer_value = TokenStream::new(); - for token_tree in content.stream() { - match read_state { - ReadState::Key => { - // if the token is a `=`, we check if the next token is a `>` - if let TokenTree::Punct(punct) = &token_tree - && punct.as_char() == '=' - && punct.spacing() == Spacing::Joint - { - read_state = ReadState::Arrow; - continue; - } - token_buffer_key.extend(token_tree.to_token_stream()); - } - ReadState::Arrow => { - // verify that the next token is a `>` - if let TokenTree::Punct(punct) = &token_tree { - if punct.as_char() == '>' { - read_state = ReadState::Value; - } - } else { - return Err(Error::new_spanned(token_tree, "Expected `=>`")); - } - } - ReadState::Value => { - // if the token is a comma, we have a complete entry - if let TokenTree::Punct(punct) = &token_tree - && punct.as_char() == ',' - && !token_buffer_value.is_empty() - { - // generate the sequence of tokens to build the value - let (tokens, value_type) = generate_value(buffer_type, &token_buffer_value)?; - entries.push((token_buffer_key.clone(), tokens, value_type)); - - // reset the buffers - read_state = ReadState::Key; - token_buffer_key = TokenStream::new(); - token_buffer_value = TokenStream::new(); - continue; - } - token_buffer_value.extend(token_tree.to_token_stream()); - } - } - } - - // handle the last entry - if !token_buffer_key.is_empty() && !token_buffer_value.is_empty() { - let (tokens, value_type) = generate_value(buffer_type, &token_buffer_value)?; - entries.push((token_buffer_key, tokens, value_type)); - } - - let token_stream = entries.iter().map(|(key, tokens, value_type)| { - let put_method = TokenStream::from(value_type); - quote! [ - <#buffer_type as ::prosa_utils::msg::tvf::Tvf>::#put_method(&mut __map_buffer, (#key) as usize, #tokens); - ] - }); - - Ok(quote! [ - { - let mut __map_buffer = <#buffer_type>::default(); - #(#token_stream)* - __map_buffer - } - ]) -} - -/// Represent the reading state when parsing a map buffer -enum ReadState { - Key, - Arrow, - Value, -} diff --git a/prosa_macros/src/tvf/expr.rs b/prosa_macros/src/tvf/expr.rs new file mode 100644 index 0000000..83dd4cb --- /dev/null +++ b/prosa_macros/src/tvf/expr.rs @@ -0,0 +1,94 @@ +use proc_macro2::TokenStream; + +/// Define a field expression which includes the following elements +/// - a target field identifier +/// - a modifier (unary operator) +/// - a literal / variable / rust-expression +/// - a implicit or explicit target type +#[derive(Debug, Clone)] +pub(crate) struct TvfExpr { + /// Field identifier to use + pub id: TvfId, + + /// Modifier for the value + pub modifier: Modifier, + + /// Value to use for the field + pub value: TvfValue, + + /// Output type to insert the field in the buffer + pub out_type: TvfType, +} + +/// Identifier of a field +#[derive(Debug, Clone)] +pub(crate) enum TvfId { + /// We directly have an integer value + Int(usize), + + /// Identifier of a variable + Ident(syn::Ident), + + /// Rust expression (surrounded by parenthesis) + Expr(TokenStream), +} + +/// Define a value to insert into the buffer +#[derive(Debug, Clone)] +pub(crate) enum TvfValue { + /// Simple literal which can be used to implicitely identify the type of the value + Lit(syn::Lit), + + /// Identifier of a variable + Ident(syn::Ident), + + /// Rust expression (surrounded by parenthesis) + Expr(TokenStream), + + /// Sub-buffer + Buffer(Vec), +} + +/// The types that can be added to a TVF buffer +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TvfType { + Byte, + Signed, + Unsigned, + Float, + String, + Bytes, + Date, + DateTime, + Buffer, +} + +/// Modifier for a value +/// Usually a not for boolean or a minus sign for numbers, etc.. +#[repr(u8)] +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Modifier { + /// No operator + #[default] + None, + + /// + sign, results in no operator being added + Positive, + + /// - sign, negate the following value + Negative, + + /// logical not for boolean values + LogicalNot, + + /// * dereference + Dereference, + + /// & borrow + Borrow, +} + +/// Simple sequence of bytes to serialize +#[derive(Debug, Default, Clone)] +pub(crate) struct Bytes(pub Vec); diff --git a/prosa_macros/src/tvf/literal.rs b/prosa_macros/src/tvf/literal.rs deleted file mode 100644 index 894f1bc..0000000 --- a/prosa_macros/src/tvf/literal.rs +++ /dev/null @@ -1,212 +0,0 @@ -use super::value::ValueType; -use crate::tvf::value::UnaryOp; -use chrono::Datelike; -use proc_macro2::{Literal, TokenStream}; -use quote::quote; -use std::num::ParseIntError; -use syn::{Error, Lit, parse_quote}; - -/// Given a literal, identify the corresponding TVF type -pub(crate) fn identify_literal(literal: &Literal) -> Result { - if let syn::Expr::Lit(literal) = parse_quote! [ #literal ] { - match literal.lit { - Lit::Bool(_) => Ok(ValueType::Byte), - Lit::Byte(_) => Ok(ValueType::Byte), - Lit::Char(_) => Ok(ValueType::Byte), - Lit::Int(int) => { - if int.suffix() == "u8" { - Ok(ValueType::Byte) - } else if int.suffix().starts_with('u') { - Ok(ValueType::Unsigned) - } else { - Ok(ValueType::Signed) - } - } - Lit::Float(_) => Ok(ValueType::Float), - Lit::Str(_) => Ok(ValueType::String), - Lit::ByteStr(_) => Ok(ValueType::Bytes), - _ => Err(Error::new_spanned(literal, "Invalid literal")), - } - } else { - Err(Error::new_spanned(literal, "Invalid literal")) - } -} - -/// Convert a literal to the specified type -pub(crate) fn convert_literal( - literal: &Literal, - output_type: &ValueType, - unary_op: UnaryOp, -) -> Result { - let sign = unary_op.make_sign(); - - // Check if the literal is already of the expected type - // in that case we can return it as is - let lit_type = identify_literal(literal)?; - if lit_type == *output_type { - return Ok(quote![ #sign #literal ]); - } - - let token_stream = match lit_type { - ValueType::Byte | ValueType::Signed | ValueType::Unsigned | ValueType::Float => { - match output_type { - ValueType::Byte => quote! [ #sign #literal as u8 ], - ValueType::Signed => quote! [ #sign #literal as i64 ], - ValueType::Unsigned => quote! [ #sign #literal as u64 ], - ValueType::Float => quote! [ #sign #literal as f64 ], - ValueType::String => quote! [ (#sign #literal).to_string() ], - ValueType::Bytes => { - // Enable special conversion for literals written in - // hexadecimal or in binary to be converted directly to bytes. - - // Remove underscores from the literal - let digits = literal.to_string().replace('_', ""); - - // convert the digits to a sequence of bytes - let result = if digits.starts_with("0x") { - // hexadecimal string - digits_to_bytes(&digits, 16, 2) - } else if digits.starts_with("0b") { - // binary string - digits_to_bytes(&digits, 2, 8) - } else { - return Err(Error::new_spanned( - literal, - "Cannot convert number to bytes, only hexadecimal and binary literals are supported.", - )); - }; - - // write the bytes as a byte string - match result { - Ok(bytes) => quote! [ ::bytes::Bytes::from_static( &[ #(#bytes),* ] ) ], - Err(e) => { - return Err(Error::new_spanned( - literal, - format!("Cannot convert number to bytes: {e}"), - )); - } - } - } - _ => { - return Err(Error::new_spanned( - literal, - "Cannot convert number to the specified type.", - )); - } - } - } - ValueType::String => match output_type { - ValueType::Bytes => quote! [ ::bytes::Bytes::from_static( #literal.as_bytes() ) ], - ValueType::Date => { - if let Ok(date) = - chrono::NaiveDate::parse_from_str(&literal.to_string(), "\"%Y-%m-%d\"") - { - let year = date.year(); - let month = date.month(); - let day = date.day(); - quote! [ - ::chrono::NaiveDate::from_ymd_opt( #year, #month, #day ).unwrap() - ] - } else { - return Err(Error::new_spanned( - literal, - "Invalid date or wrong format, expect \"YYYY-mm-dd\"", - )); - } - } - ValueType::DateTime => { - if let Ok(datetime) = chrono::NaiveDateTime::parse_from_str( - &literal.to_string(), - "\"%Y-%m-%d %H:%M:%S%.3f\"", - ) { - let msecs = datetime.and_utc().timestamp_millis(); - quote! [ - ::chrono::DateTime::from_timestamp_millis(#msecs).unwrap().naive_utc() - ] - } else { - return Err(Error::new_spanned( - literal, - "Invalid date-time or wrong format, expect \"YYYY-mm-dd HH:MM:SS.UUU\"", - )); - } - } - _ => { - return Err(Error::new_spanned( - literal, - "Cannot convert string to the specified type.", - )); - } - }, - ValueType::Bytes => match output_type { - ValueType::String => { - let byte_string = match parse_quote! [ #literal ] { - syn::Expr::Lit(literal) => match literal.lit { - Lit::ByteStr(byte_string) => byte_string, - _ => { - return Err(Error::new_spanned( - literal, - "Expected a byte string literal.", - )); - } - }, - expression => { - return Err(Error::new_spanned( - expression, - "Expected a byte string literal.", - )); - } - }; - let value = byte_string.value(); - let string = std::str::from_utf8(&value).map_err(|_| { - Error::new_spanned(literal, "Byte string contains invalid UTF-8.") - })?; - let string = syn::LitStr::new(string, literal.span()); - quote! [ #string ] - } - _ => { - return Err(Error::new_spanned( - literal, - "Cannot convert bytes to the specified type.", - )); - } - }, - _ => { - return Err(Error::new_spanned( - literal, - "Literal type not supported. Should not happen!", - )); - } - }; - - Ok(token_stream) -} - -/// Convert a string of digits to bytes -fn digits_to_bytes(digits: &str, radix: u32, group_by: usize) -> Result, ParseIntError> { - // remove the prefix and the underscores - let trimmed = digits.replace('_', "").split_off(2); - - // compose a sequence of bytes - let mut result = Vec::::with_capacity(trimmed.len() / group_by); - - // group the digits in chunks, - // start from the end to avoid padding - trimmed - .chars() - .collect::>() - .rchunks(group_by) - .try_for_each(|chunk| { - let byte = chunk.iter().collect::(); - match u8::from_str_radix(&byte, radix) { - Ok(byte) => { - result.push(byte); - Ok(()) - } - Err(e) => Err(e), - } - })?; - - // return the result in the correct order - result.reverse(); - Ok(result) -} diff --git a/prosa_macros/src/tvf/parser.rs b/prosa_macros/src/tvf/parser.rs new file mode 100644 index 0000000..f52b665 --- /dev/null +++ b/prosa_macros/src/tvf/parser.rs @@ -0,0 +1,456 @@ +use crate::tvf::expr::*; +use chrono::{NaiveDate, NaiveDateTime}; +use num_traits::FromPrimitive; +use proc_macro2::{ + Delimiter, Literal, Punct, Spacing, Span, TokenStream, TokenTree, token_stream::IntoIter, +}; +use std::{fmt::Display, iter::Peekable, str::FromStr, string::FromUtf8Error}; +use syn::{Ident, parse_quote, spanned::Spanned}; + +/// Store context to consume tokens and produce expressions +pub(crate) struct TvfParser { + /// Tokens to iterate over + pub tokens: Peekable, + + /// Span of the last successfull token parsing + pub last_span: Span, + + /// Are we iterating over a list of a map? + pub is_map: bool, + + /// Number of fields that have been parsed until now + pub field_count: usize, +} + +impl TvfParser { + /// Wrap the iterable tokenstream into the parser + #[inline] + pub(crate) fn new(tokens: TokenStream, is_map: bool) -> Self { + let last_span = tokens.span(); + Self { + tokens: tokens.into_iter().peekable(), + last_span, + is_map, + field_count: 0, + } + } + + /// Grap the next token from the stream, if none found, + /// return an error using the last successful span. + pub(crate) fn next(&mut self, on_error: Err) -> Result + where + Err: FnOnce() -> Msg, + Msg: Display, + { + if let Some(tt) = self.tokens.next() { + self.last_span = tt.span(); + Ok(tt) + } else { + Err(syn::Error::new(self.last_span, on_error())) + } + } + + /// Peek at the next token in the stream + /// Use `consume` afterward to actually advance the iterator + #[inline] + pub(crate) fn peek(&mut self) -> Option<&TokenTree> { + self.tokens.peek() + } + + /// After peeking at the next token in the stream, + /// actually move to the next token. + /// Return true if there was a token to consume or false otherwise. + #[inline] + pub(crate) fn consume(&mut self) -> bool { + if let Some(tt) = self.tokens.next() { + self.last_span = tt.span(); + true + } else { + false + } + } +} + +impl TvfParser { + /// Iterate over the tokens to collect expressions + pub(crate) fn collect_expr(&mut self) -> Result, syn::Error> { + // We don't really know in advance how many fields we will build + let mut expressions = Vec::new(); + + if self.is_map { + // Read the sub-buffer as a key-value map + while self.peek().is_some() { + let expr = TvfExpr::parse_from_map(self)?; + expressions.push(expr); + self.check_for_comma()?; + } + } else { + // Read the sub-buffer as a list of values + while self.peek().is_some() { + let expr = TvfExpr::parse_from_list(self)?; + expressions.push(expr); + self.check_for_comma()?; + } + } + + Ok(expressions) + } + + /// Check if the next token is a comma ',' + /// If so, move on to the next token to read the next expression + fn check_for_comma(&mut self) -> Result<(), syn::Error> { + if let Some(TokenTree::Punct(comma)) = self.peek() + && comma.as_char() == ',' + { + self.consume(); + Ok(()) + } else { + Err(syn::Error::new( + self.last_span, + "Next token is not a comma ','", + )) + } + } +} + +impl TvfExpr { + /// Parse an expression from a list buffer + /// `` + fn parse_from_list(parser: &mut TvfParser) -> Result { + let id = TvfId::Int(parser.field_count + 1); + Self::parse_value(parser, id) + } + + /// Parse an expression from a map buffer + /// ` => ` + fn parse_from_map(parser: &mut TvfParser) -> Result { + let id = parser.next(|| "Expected field identifier, none found.")?; + let id = TvfId::from_tokens(&id)?; + + // We expect a fat-arrow "=>" separator between identifiers and values + // Fat-arrow is composed of two punctuation tokens: '=' and '>' + let sep1 = parser.next(|| "Expected fat-arrow, none found.")?; + let sep2 = parser.next(|| "Expected fat-arrow, none found.")?; + let span = sep1.span(); + if let TokenTree::Punct(sep1) = sep1 + && sep1.as_char() == '=' + && sep1.spacing() == Spacing::Joint + && let TokenTree::Punct(sep2) = sep2 + && sep2.as_char() == '>' + { + // successfully identified "=>" + /* do nothing */ + } else { + return Err(syn::Error::new(span, "Expected fat-arrow")); + } + + // Parse the remaining tokens as the value + Self::parse_value(parser, id) + } + + /// Parse a field expression value + /// Samples of expected token sequences: + /// - `10u64` + /// - `-10 as Signed` + /// - `!false` + /// - `MY_CONST as String` + /// - `(10 - 2) as Unsigned` + /// - `"2026-09-10" as Date` + /// - `0x1A2B3C` as Bytes` + /// - `{ 1 => 10 }` sub-buffer expressed as map + /// - `[ 1, 2, 3 ]` sub-buffer expressed as list + fn parse_value(parser: &mut TvfParser, id: TvfId) -> Result { + // First token might be a modifier (unary operator) + let modifier = if let Some(TokenTree::Punct(punct)) = parser.peek() { + let modifier = Modifier::from_punct(punct)?; + parser.consume(); // move to next token + modifier + } else { + Modifier::None + }; + + // Next we necessarely expect the value + let value = parser.next(|| "Expected a value, none found.")?; + let value = TvfValue::from_tokens(&value)?; + + // Next we might have a `as` keyword to indicate the expected type + let out_type = if let Some(TokenTree::Ident(word)) = parser.peek() + && word == "as" + { + // move past the "as" and look at the next token + parser.consume(); + let cast_to = parser.next(|| "Expected type, none found.")?; + + // Following keyword must be a type name + if let TokenTree::Ident(type_name) = cast_to { + TvfType::from_as_cast(type_name)? + } else { + return Err(syn::Error::new_spanned(cast_to, "Expected type.")); + } + } else { + // No explicity type is specified, + // fallback to deducing the type from the value + // This only works if the value is a literal or a sub-buffer + match &value { + TvfValue::Lit(lit) => TvfType::from_literal(lit)?, + TvfValue::Buffer(_) => TvfType::Buffer, + _ => { + return Err(syn::Error::new( + parser.last_span, + "Cannot deduce type of field, please use a `as` indication", + )); + } + } + }; + + // Complete the expression + Ok(Self { + id, + modifier, + value, + out_type, + }) + } +} + +impl TvfId { + /// Identify the value form a token-tree + fn from_tokens(tt: &TokenTree) -> Result { + match tt { + TokenTree::Ident(ident) => Ok(Self::Ident(ident.clone())), + TokenTree::Literal(literal) => { + let lit = convert_lit(literal)?; + let int = parse_int(&lit)?; + Ok(Self::Int(int)) + } + TokenTree::Group(group) => { + if group.delimiter() == Delimiter::Parenthesis { + Ok(Self::Expr(group.stream())) + } else { + Err(syn::Error::new_spanned(tt, "Non-supported delimiters")) + } + } + TokenTree::Punct(punct) => Err(syn::Error::new_spanned( + tt, + format!["Punctuation '{}' cannot be used as value", punct], + )), + } + } +} + +impl TvfValue { + /// Identify the value form a token-tree + fn from_tokens(tt: &TokenTree) -> Result { + match tt { + TokenTree::Ident(ident) => Ok(Self::Ident(ident.clone())), + TokenTree::Literal(literal) => Ok(Self::Lit(convert_lit(literal)?)), + TokenTree::Group(group) => match group.delimiter() { + Delimiter::Parenthesis => Ok(Self::Expr(group.stream())), + Delimiter::Bracket => { + let mut parser = TvfParser::new(group.stream(), true); + let exprs = parser.collect_expr()?; + Ok(Self::Buffer(exprs)) + } + Delimiter::Brace => { + let mut parser = TvfParser::new(group.stream(), false); + let exprs = parser.collect_expr()?; + Ok(Self::Buffer(exprs)) + } + Delimiter::None => { + Err(syn::Error::new_spanned(tt, "Missing expression delimiters")) + } + }, + TokenTree::Punct(punct) => Err(syn::Error::new_spanned( + tt, + format!["Punctuation '{}' cannot be used as value", punct], + )), + } + } +} + +impl TvfType { + /// Given a literal, identify the corresponding TVF type + fn from_literal(literal: &syn::Lit) -> Result { + match literal { + syn::Lit::Bool(_) => Ok(Self::Byte), + syn::Lit::Byte(_) => Ok(Self::Byte), + syn::Lit::Char(_) => Ok(Self::Byte), + syn::Lit::Int(int) => { + let suffix = int.suffix(); + if suffix == "u8" { + Ok(Self::Byte) + } else if suffix.starts_with('u') { + Ok(Self::Unsigned) + } else { + Ok(Self::Signed) + } + } + syn::Lit::Float(_) => Ok(Self::Float), + syn::Lit::Str(_) => Ok(Self::String), + syn::Lit::CStr(_) => Ok(Self::String), + syn::Lit::ByteStr(_) => Ok(Self::Bytes), + _ => Err(syn::Error::new_spanned(literal, "Invalid literal")), + } + } + + /// Deduce the value type from the type provided in a `as` cast + #[rustfmt::skip] + fn from_as_cast(ident: Ident) -> Result { + match ident.to_string().to_ascii_lowercase().as_str() { + "byte" | "u8" => Ok(Self::Byte), + "signed" | "i8" | "i16" | "i32" | "i64" | "isize" => Ok(Self::Signed), + "unsigned" | "u16" | "u32" | "u64" | "usize" => Ok(Self::Unsigned), + "float" | "f32" | "f64" => Ok(Self::Float), + "string" | "str" => Ok(Self::String), + "bytes" => Ok(Self::Bytes), + "date" | "naivedate" => Ok(Self::Date), + "datetime" | "naivedatetime" => Ok(Self::DateTime), + "buffer" | "tvf" => Ok(Self::Buffer), + _ => Err(syn::Error::new_spanned(ident, "Invalid value type")), + } + } +} + +impl Modifier { + /// Identify unary operation from punctuation mark + fn from_punct(punct: &Punct) -> Result { + match punct.as_char() { + '+' => Ok(Self::Positive), + '-' => Ok(Self::Negative), + '!' => Ok(Self::LogicalNot), + '*' => Ok(Self::Dereference), + '&' => Ok(Self::Borrow), + _ => Err(syn::Error::new_spanned(punct, "Unsupported punctuation")), + } + } +} + +/// Convert a `proc_macro2::Literal` into a `syn::Lit` +fn convert_lit(literal: &Literal) -> Result { + if let syn::Expr::Lit(literal) = parse_quote! [ #literal ] { + Ok(literal.lit) + } else { + Err(syn::Error::new_spanned(literal, "Invalid literal")) + } +} + +/// Get an integer value from a literal +fn parse_int(literal: &syn::Lit) -> Result +where + I: FromPrimitive + FromStr, + ::Err: Display, +{ + let span = literal.span(); + match literal { + syn::Lit::Byte(byte) => I::from_u8(byte.value()) + .ok_or_else(|| syn::Error::new(span, "Could not deduce integer from byte")), + syn::Lit::Char(chr) => I::from_u32(chr.value() as u32) + .ok_or_else(|| syn::Error::new(span, "Could not deduce integer from character")), + syn::Lit::Int(int) => Ok(int.base10_parse()?), + _ => Err(syn::Error::new(span, "Invalid literal")), + } +} + +/// Error encountered when parsing a value from a string literal +#[derive(thiserror::Error, Debug)] +pub(crate) enum StrParseError { + /// Unsupported literal + #[error("Unsupported literal")] + Literal, + + /// Failed to convert bytes into UTF-8 string + #[error("UTF-8: {0}")] + Utf8(#[from] FromUtf8Error), + + /// chrono error + #[error("chono: {0}")] + Chrono(#[from] chrono::ParseError), +} + +/// Given a literal deduce a String +fn parse_string(literal: &syn::Lit) -> Result { + // Try to convert the literal into a string + let string = match literal { + syn::Lit::Str(s) => s.value(), + syn::Lit::ByteStr(s) => String::from_utf8(s.value())?, + syn::Lit::CStr(s) => s.value().to_string_lossy().to_string(), + _ => { + return Err(StrParseError::Literal); + } + }; + Ok(string) +} + +/// Given a literal deduce a Date +fn parse_date(literal: &syn::Lit) -> Result { + const FORMAT: &'static str = "%Y-%m-%d"; + let string = parse_string(literal)?; + Ok(NaiveDate::parse_from_str(&string, FORMAT)?) +} + +/// Given a literal deduce a DateTime +fn parse_datetime(literal: &syn::Lit) -> Result { + const FORMAT: &'static str = "%Y-%m-%d %H:%M:%S%.3f"; + let string = parse_string(literal)?; + Ok(NaiveDateTime::parse_from_str(&string, FORMAT)?) +} + +impl Bytes { + /// Parse a integer literal an build a sequence of bytes from it + fn from_literal(literal: &syn::LitInt) -> Result { + // Remove underscores from the literal + let digits = literal.to_string().replace('_', ""); + + // convert the digits to a sequence of bytes + if digits.starts_with("0x") { + // hexadecimal string + Self::digits_to_bytes(literal.span(), &digits, 16, 2) + } else if digits.starts_with("0b") { + // binary string + Self::digits_to_bytes(literal.span(), &digits, 2, 8) + } else { + Err(syn::Error::new_spanned( + literal, + "Cannot convert number to bytes, only hexadecimal and binary literals are supported.", + )) + } + } + + /// Convert a string of digits to bytes + fn digits_to_bytes( + span: Span, + digits: &str, + radix: u32, + group_by: usize, + ) -> Result { + // remove the prefix and the underscores + let trimmed = digits.replace('_', "").split_off(2); + + // compose a sequence of bytes + let mut result = Vec::::with_capacity(trimmed.len() / group_by); + + // group the digits in chunks, + // start from the end to avoid padding + trimmed + .chars() + .collect::>() + .rchunks(group_by) + .try_for_each(|chunk| { + let byte = chunk.iter().collect::(); + match u8::from_str_radix(&byte, radix) { + Ok(byte) => { + result.push(byte); + Ok(()) + } + Err(e) => Err(syn::Error::new( + span, + format!["Failed to parse integer literal: {}", e], + )), + } + })?; + + // return the result in the correct order + result.reverse(); + Ok(Self(result)) + } +} diff --git a/prosa_macros/src/tvf/tokens.rs b/prosa_macros/src/tvf/tokens.rs new file mode 100644 index 0000000..fbd836f --- /dev/null +++ b/prosa_macros/src/tvf/tokens.rs @@ -0,0 +1,111 @@ +use crate::tvf::expr::*; +use chrono::{Datelike, NaiveDate, NaiveDateTime}; +use proc_macro2::TokenStream; +use quote::quote; + +impl TvfExpr { + fn to_tokens(&self, buffer: &syn::Ident) -> TokenStream { + let ident = self.id.to_tokens(); + let value = self.value.to_tokens(); + let put_method = self.out_type.put_method(); + + quote![ #buffer.#put_method(#ident, #value) ] + } +} + +impl TvfId { + /// Convert the value into tokens + #[rustfmt::skip] + fn to_tokens(&self) -> TokenStream { + match self { + Self::Int (int ) => quote![ #int ], + Self::Ident(ident) => quote![ #ident ], + Self::Expr (expr ) => quote![ ( #expr ) ], + } + } +} + +impl TvfValue { + /// Convert the value into tokens + #[rustfmt::skip] + fn to_tokens(&self) -> TokenStream { + match self { + Self::Lit (lit ) => quote![ #lit ], + Self::Ident (ident ) => quote![ #ident ], + Self::Expr (expr ) => quote![ ( #expr ) ], + Self::Buffer(buffer) => todo![], + } + } +} + +impl TvfType { + /// Rust type corresponding to the TVF type + #[rustfmt::skip] + fn cast_type(self) -> Option { + match self { + Self::Byte => Some(quote![ u8 ]), + Self::Signed => Some(quote![ i64 ]), + Self::Unsigned => Some(quote![ u64 ]), + Self::Float => Some(quote![ f64 ]), + _ => None, + } + } + + /// Name of the put method expected given the type + #[rustfmt::skip] + fn put_method(self) -> TokenStream { + match self { + Self::Byte => quote![ put_byte ], + Self::Signed => quote![ put_signed ], + Self::Unsigned => quote![ put_unsigned ], + Self::Float => quote![ put_float ], + Self::String => quote![ put_string ], + Self::Bytes => quote![ put_bytes ], + Self::Date => quote![ put_date ], + Self::DateTime => quote![ put_datetime ], + Self::Buffer => quote![ put_buffer ], + } + } +} + +impl Modifier { + /// Generate token for current unary operator + #[rustfmt::skip] + fn to_token(self) -> TokenStream { + match self { + Self::None => TokenStream::new(), + Self::Positive => TokenStream::new(), + Self::Negative => quote![ - ], + Self::LogicalNot => quote![ ! ], + Self::Dereference => quote![ * ], + Self::Borrow => quote![ & ], + } + } +} + +/// Write tokens to generate the given date +fn date_to_tokens(date: NaiveDate) -> TokenStream { + let year = date.year(); + let month = date.month(); + let day = date.day(); + quote! [ + ::chrono::NaiveDate::from_ymd_opt( #year, #month, #day ).unwrap() + ] +} + +/// Write tokens to generate the given datetime +fn datetime_to_tokens(datetime: NaiveDateTime) -> TokenStream { + let msecs = datetime.and_utc().timestamp_millis(); + quote! [ + ::chrono::DateTime::from_timestamp_millis(#msecs).unwrap().naive_utc() + ] +} + +impl Bytes { + /// Generate tokens to rebuild the sequence of bytes + #[inline] + fn to_token(&self) -> TokenStream { + let bytes = self.0.as_slice(); + quote! [ ::bytes::Bytes::from_static( &[ #(#bytes),* ] ) ] + } +} diff --git a/prosa_macros/src/tvf/value.rs b/prosa_macros/src/tvf/value.rs deleted file mode 100644 index 4aef127..0000000 --- a/prosa_macros/src/tvf/value.rs +++ /dev/null @@ -1,240 +0,0 @@ -use super::buffer::{generate_list, generate_map}; -use super::literal::{convert_literal, identify_literal}; -use proc_macro2::{Delimiter, Punct, TokenStream, TokenTree}; -use quote::{ToTokens, quote}; -use syn::{Error, Ident}; - -/// The types that can be added to a TVF buffer -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum ValueType { - Byte, - Signed, - Unsigned, - Float, - String, - Bytes, - Date, - DateTime, - Buffer, -} - -impl From<&ValueType> for TokenStream { - fn from(value: &ValueType) -> Self { - match value { - ValueType::Byte => quote![put_byte], - ValueType::Signed => quote![put_signed], - ValueType::Unsigned => quote![put_unsigned], - ValueType::Float => quote![put_float], - ValueType::String => quote![put_string], - ValueType::Bytes => quote![put_bytes], - ValueType::Date => quote![put_date], - ValueType::DateTime => quote![put_datetime], - ValueType::Buffer => quote![put_buffer], - } - } -} - -/// Deduce the value type from the type provided in a `as` cast -impl TryFrom for ValueType { - type Error = Error; - - fn try_from(ident: Ident) -> Result { - match ident.to_string().to_ascii_lowercase().as_str() { - "byte" | "u8" => Ok(ValueType::Byte), - "signed" | "i8" | "i16" | "i32" | "i64" | "isize" => Ok(ValueType::Signed), - "unsigned" | "u16" | "u32" | "u64" | "usize" => Ok(ValueType::Unsigned), - "float" | "f32" | "f64" => Ok(ValueType::Float), - "string" => Ok(ValueType::String), - "bytes" => Ok(ValueType::Bytes), - "date" | "naivedate" => Ok(ValueType::Date), - "datetime" | "naivedatetime" => Ok(ValueType::DateTime), - "buffer" => Ok(ValueType::Buffer), - _ => Err(Error::new_spanned(ident, "Invalid value type")), - } - } -} - -/// At this point the token stream has been preparsed such that the value is: -/// - {} or [] -/// - a single literal -/// - a single literal followed by a `as` cast -/// - a path followed by a `as` cast -/// - an expression surrounded by () and followed by a `as` cast -pub(crate) fn generate_value( - buffer_type: &Ident, - value_stream: &TokenStream, -) -> Result<(TokenStream, ValueType), Error> { - // Process the token tree - let mut tokens = value_stream.clone().into_iter().peekable(); - - // Check if the first element is a sign (+ or -) - let unary_op = if let Some(TokenTree::Punct(punct)) = tokens.peek() { - let unary_op = UnaryOp::new(punct)?; - tokens.next(); // move to next token - unary_op - } else { - UnaryOp::None - }; - - // in all cases, we expect to find a value - let value = tokens.next().ok_or(Error::new_spanned( - value_stream, - "Expected a value, none found.", - ))?; - - // Check if the value is followed by an `as` cast - let output_type = if let Some(TokenTree::Ident(ident)) = tokens.next() { - if ident == "as" { - // check if the value is followed by a type - if let Some(TokenTree::Ident(ident)) = tokens.next() { - Some(ValueType::try_from(ident)?) - } else { - return Err(Error::new_spanned(ident, "Expected a type identifier")); - } - } else { - return Err(Error::new_spanned(ident, "Expected an `as` keyword")); - } - } else { - None - }; - - // Raise an error if there are unexpected tokens - if let Some(token) = tokens.next() { - return Err(Error::new_spanned(token, "Unexpected token")); - } - - let sign = unary_op.make_sign(); - - // check the type of value provided - match value { - TokenTree::Literal(literal) => { - if let Some(output_type) = output_type { - Ok(( - convert_literal(&literal, &output_type, unary_op)?, - output_type, - )) - } else { - let literal_type = identify_literal(&literal)?; - let token_stream = match literal_type { - ValueType::Byte => quote! [ #sign #literal as u8 ], - ValueType::Signed => quote! [ #sign #literal as i64 ], - ValueType::Unsigned => quote! [ #sign #literal as u64 ], - ValueType::Float => quote! [ #sign #literal as f64 ], - _ => literal.to_token_stream(), - }; - Ok((token_stream, literal_type)) - } - } - TokenTree::Ident(ident) => { - // check if the identifier is a boolean - match ident.to_string().as_str() { - "true" => { - let t = if unary_op == UnaryOp::LogicalNot { - quote![0u8] - } else { - quote![1u8] - }; - Ok((t, output_type.unwrap_or(ValueType::Byte))) - } - "false" => { - let t = if unary_op == UnaryOp::LogicalNot { - quote![1u8] - } else { - quote![0u8] - }; - Ok((t, output_type.unwrap_or(ValueType::Byte))) - } - _ => { - if let Some(output_type) = output_type { - Ok((quote![#sign #ident], output_type)) - } else { - Err(Error::new_spanned( - ident, - "Could not deduce the type of the variable. Please use `as` cast.", - )) - } - } - } - } - // handle {}, [] and () expressions - TokenTree::Group(group) => { - match group.delimiter() { - Delimiter::Brace => { - // if a `as` cast was used, verify that it is valid - if output_type.is_some_and(|t| t != ValueType::Buffer) { - return Err(Error::new_spanned(group, "Invalid type for `{}` value.")); - } - Ok((generate_map(buffer_type, &group)?, ValueType::Buffer)) - } - Delimiter::Bracket => { - // if a `as` cast was used, verify that it is valid - if output_type.is_some_and(|t| t != ValueType::Buffer) { - return Err(Error::new_spanned(group, "Invalid type for `[]` value.")); - } - Ok((generate_list(buffer_type, &group)?, ValueType::Buffer)) - } - _ => { - if let Some(output_type) = output_type { - Ok((quote![ #sign #group ], output_type)) - } else { - Err(Error::new_spanned( - group, - "Type cannot be deduced from an expression in parenthesis. Please use `as` cast.", - )) - } - } - } - } - TokenTree::Punct(_) => Err(Error::new_spanned(value, "Unexpected punctuation.")), - } -} - -/// Define a unary operator that may precede a literal, a variable or an expression -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] -pub(crate) enum UnaryOp { - /// No operator - #[default] - None, - - /// + sign, results in no operator being added - Positive, - - /// - sign, negate the following value - Negative, - - /// logical not for boolean values - LogicalNot, - - /// * dereference - Dereference, - - /// & borrow - Borrow, -} - -impl UnaryOp { - /// Identify unary operation from punctuation mark - pub(crate) fn new(punct: &Punct) -> Result { - match punct.as_char() { - '+' => Ok(Self::Positive), - '-' => Ok(Self::Negative), - '!' => Ok(Self::LogicalNot), - '*' => Ok(Self::Dereference), - '&' => Ok(Self::Borrow), - _ => Err(Error::new_spanned(punct, "Unsupported punctuation")), - } - } - - /// Generate token for current unary operator - #[rustfmt::skip] - pub(crate) fn make_sign(self) -> TokenStream { - match self { - UnaryOp::None => TokenStream::new(), - UnaryOp::Positive => TokenStream::new(), - UnaryOp::Negative => quote![ - ], - UnaryOp::LogicalNot => quote![ ! ], - UnaryOp::Dereference => quote![ * ], - UnaryOp::Borrow => quote![ & ], - } - } -} From bdc15b768f89bce1c690d31cbf73484a5e701318 Mon Sep 17 00:00:00 2001 From: Olivier Schyns Date: Sat, 5 Sep 2026 03:06:56 +0200 Subject: [PATCH 5/5] ft: New implementation is passing current tests Signed-off-by: Olivier Schyns --- prosa_macros/src/tvf.rs | 36 +++-- prosa_macros/src/tvf/expr.rs | 55 ++++++- prosa_macros/src/tvf/parser.rs | 262 +++++++++++++++++++++------------ prosa_macros/src/tvf/tokens.rs | 136 +++++++++++++---- prosa_macros/tests/tvf.rs | 4 +- 5 files changed, 343 insertions(+), 150 deletions(-) diff --git a/prosa_macros/src/tvf.rs b/prosa_macros/src/tvf.rs index e3be3fc..6e6fd88 100644 --- a/prosa_macros/src/tvf.rs +++ b/prosa_macros/src/tvf.rs @@ -7,8 +7,9 @@ pub(crate) mod parser; /// Convert the expressions into tokens to be passed to the compiler pub(crate) mod tokens; -use crate::tvf::parser::TvfParser; +use crate::tvf::{parser::TvfParser, tokens::buffer_to_tokens}; use proc_macro2::{Delimiter, TokenStream, TokenTree}; +use quote::quote; use syn::spanned::Spanned; pub(crate) fn gen_tvf_impl(input: TokenStream) -> Result { @@ -27,28 +28,30 @@ pub(crate) fn gen_tvf_impl(input: TokenStream) -> Result { - let mut parser = TvfParser::new(group.stream(), false); + let mut parser = TvfParser::new(group.stream(), true); let exprs = parser.collect_expr()?; - todo!() + buffer_to_tokens(&buffer_type, &exprs)? } Delimiter::Bracket => { - let mut parser = TvfParser::new(group.stream(), true); + let mut parser = TvfParser::new(group.stream(), false); let exprs = parser.collect_expr()?; - todo!() + buffer_to_tokens(&buffer_type, &exprs)? + } + _ => { + return Err(syn::Error::new( + span, + "Invalid delimiter, expected {} or []", + )); } - _ => Err(syn::Error::new( - span, - "Invalid delimiter, expected {} or []", - )), } } else { - Err(syn::Error::new( + return Err(syn::Error::new( span, "Second argument must be the content enclosed in {} or []", - )) + )); }; // Raise an error on any extra tokens @@ -56,5 +59,12 @@ pub(crate) fn gen_tvf_impl(input: TokenStream) -> Result, } /// Identifier of a field -#[derive(Debug, Clone)] pub(crate) enum TvfId { /// We directly have an integer value Int(usize), @@ -34,7 +34,6 @@ pub(crate) enum TvfId { } /// Define a value to insert into the buffer -#[derive(Debug, Clone)] pub(crate) enum TvfValue { /// Simple literal which can be used to implicitely identify the type of the value Lit(syn::Lit), @@ -46,7 +45,18 @@ pub(crate) enum TvfValue { Expr(TokenStream), /// Sub-buffer - Buffer(Vec), + Buffer(Vec, Span), +} + +impl TvfValue { + pub(crate) fn span(&self) -> Span { + match self { + TvfValue::Lit(lit) => lit.span(), + TvfValue::Ident(ident) => ident.span(), + TvfValue::Expr(stream) => stream.span(), + TvfValue::Buffer(_, span) => *span, + } + } } /// The types that can be added to a TVF buffer @@ -92,3 +102,36 @@ pub(crate) enum Modifier { /// Simple sequence of bytes to serialize #[derive(Debug, Default, Clone)] pub(crate) struct Bytes(pub Vec); + +/// Values that have been identified +pub(crate) enum Value<'e> { + Bool(bool), + Byte(u8), + Signed(i64), + Unsigned(u64), + Float(f64), + String(String), + Bytes(Bytes), + Date(NaiveDate), + DateTime(NaiveDateTime), + Buffer(&'e [TvfExpr]), +} + +impl<'e> Value<'e> { + /// Process the TvfValue to identify its type + #[rustfmt::skip] + pub(crate) fn identify(&self) -> TvfType { + match self { + Self::Bool (_) => TvfType::Byte, + Self::Byte (_) => TvfType::Byte, + Self::Signed (_) => TvfType::Signed, + Self::Unsigned (_) => TvfType::Unsigned, + Self::Float (_) => TvfType::Float, + Self::String (_) => TvfType::String, + Self::Bytes (_) => TvfType::Bytes, + Self::Date (_) => TvfType::Date, + Self::DateTime (_) => TvfType::DateTime, + Self::Buffer (_) => TvfType::Buffer, + } + } +} diff --git a/prosa_macros/src/tvf/parser.rs b/prosa_macros/src/tvf/parser.rs index f52b665..fe8ec11 100644 --- a/prosa_macros/src/tvf/parser.rs +++ b/prosa_macros/src/tvf/parser.rs @@ -4,7 +4,7 @@ use num_traits::FromPrimitive; use proc_macro2::{ Delimiter, Literal, Punct, Spacing, Span, TokenStream, TokenTree, token_stream::IntoIter, }; -use std::{fmt::Display, iter::Peekable, str::FromStr, string::FromUtf8Error}; +use std::{fmt::Display, iter::Peekable, str::FromStr}; use syn::{Ident, parse_quote, spanned::Spanned}; /// Store context to consume tokens and produce expressions @@ -12,6 +12,9 @@ pub(crate) struct TvfParser { /// Tokens to iterate over pub tokens: Peekable, + /// Whole span of the block of tokens + pub whole_span: Span, + /// Span of the last successfull token parsing pub last_span: Span, @@ -26,10 +29,11 @@ impl TvfParser { /// Wrap the iterable tokenstream into the parser #[inline] pub(crate) fn new(tokens: TokenStream, is_map: bool) -> Self { - let last_span = tokens.span(); + let whole_span = tokens.span(); Self { tokens: tokens.into_iter().peekable(), - last_span, + whole_span, + last_span: whole_span, is_map, field_count: 0, } @@ -82,6 +86,7 @@ impl TvfParser { while self.peek().is_some() { let expr = TvfExpr::parse_from_map(self)?; expressions.push(expr); + self.field_count += 1; self.check_for_comma()?; } } else { @@ -89,6 +94,7 @@ impl TvfParser { while self.peek().is_some() { let expr = TvfExpr::parse_from_list(self)?; expressions.push(expr); + self.field_count += 1; self.check_for_comma()?; } } @@ -99,16 +105,22 @@ impl TvfParser { /// Check if the next token is a comma ',' /// If so, move on to the next token to read the next expression fn check_for_comma(&mut self) -> Result<(), syn::Error> { - if let Some(TokenTree::Punct(comma)) = self.peek() - && comma.as_char() == ',' - { - self.consume(); - Ok(()) + if let Some(tt) = self.peek() { + if let TokenTree::Punct(comma) = tt + && comma.as_char() == ',' + { + // consume the comma + self.consume(); + Ok(()) + } else { + Err(syn::Error::new( + self.last_span, + "Next token is not a comma ','", + )) + } } else { - Err(syn::Error::new( - self.last_span, - "Next token is not a comma ','", - )) + // No more token is fine too + Ok(()) } } } @@ -174,7 +186,7 @@ impl TvfExpr { let value = TvfValue::from_tokens(&value)?; // Next we might have a `as` keyword to indicate the expected type - let out_type = if let Some(TokenTree::Ident(word)) = parser.peek() + let explicit_type = if let Some(TokenTree::Ident(word)) = parser.peek() && word == "as" { // move past the "as" and look at the next token @@ -183,24 +195,13 @@ impl TvfExpr { // Following keyword must be a type name if let TokenTree::Ident(type_name) = cast_to { - TvfType::from_as_cast(type_name)? + Some(TvfType::from_as_cast(type_name)?) } else { return Err(syn::Error::new_spanned(cast_to, "Expected type.")); } } else { // No explicity type is specified, - // fallback to deducing the type from the value - // This only works if the value is a literal or a sub-buffer - match &value { - TvfValue::Lit(lit) => TvfType::from_literal(lit)?, - TvfValue::Buffer(_) => TvfType::Buffer, - _ => { - return Err(syn::Error::new( - parser.last_span, - "Cannot deduce type of field, please use a `as` indication", - )); - } - } + None }; // Complete the expression @@ -208,7 +209,7 @@ impl TvfExpr { id, modifier, value, - out_type, + explicit_type, }) } } @@ -241,27 +242,26 @@ impl TvfId { impl TvfValue { /// Identify the value form a token-tree fn from_tokens(tt: &TokenTree) -> Result { + let span = tt.span(); match tt { TokenTree::Ident(ident) => Ok(Self::Ident(ident.clone())), TokenTree::Literal(literal) => Ok(Self::Lit(convert_lit(literal)?)), TokenTree::Group(group) => match group.delimiter() { Delimiter::Parenthesis => Ok(Self::Expr(group.stream())), Delimiter::Bracket => { - let mut parser = TvfParser::new(group.stream(), true); + let mut parser = TvfParser::new(group.stream(), false); let exprs = parser.collect_expr()?; - Ok(Self::Buffer(exprs)) + Ok(Self::Buffer(exprs, parser.whole_span)) } Delimiter::Brace => { - let mut parser = TvfParser::new(group.stream(), false); + let mut parser = TvfParser::new(group.stream(), true); let exprs = parser.collect_expr()?; - Ok(Self::Buffer(exprs)) - } - Delimiter::None => { - Err(syn::Error::new_spanned(tt, "Missing expression delimiters")) + Ok(Self::Buffer(exprs, parser.whole_span)) } + Delimiter::None => Err(syn::Error::new(span, "Missing expression delimiters")), }, - TokenTree::Punct(punct) => Err(syn::Error::new_spanned( - tt, + TokenTree::Punct(punct) => Err(syn::Error::new( + span, format!["Punctuation '{}' cannot be used as value", punct], )), } @@ -269,30 +269,6 @@ impl TvfValue { } impl TvfType { - /// Given a literal, identify the corresponding TVF type - fn from_literal(literal: &syn::Lit) -> Result { - match literal { - syn::Lit::Bool(_) => Ok(Self::Byte), - syn::Lit::Byte(_) => Ok(Self::Byte), - syn::Lit::Char(_) => Ok(Self::Byte), - syn::Lit::Int(int) => { - let suffix = int.suffix(); - if suffix == "u8" { - Ok(Self::Byte) - } else if suffix.starts_with('u') { - Ok(Self::Unsigned) - } else { - Ok(Self::Signed) - } - } - syn::Lit::Float(_) => Ok(Self::Float), - syn::Lit::Str(_) => Ok(Self::String), - syn::Lit::CStr(_) => Ok(Self::String), - syn::Lit::ByteStr(_) => Ok(Self::Bytes), - _ => Err(syn::Error::new_spanned(literal, "Invalid literal")), - } - } - /// Deduce the value type from the type provided in a `as` cast #[rustfmt::skip] fn from_as_cast(ident: Ident) -> Result { @@ -325,6 +301,55 @@ impl Modifier { } } +impl<'e> Value<'e> { + /// Given a literal, identify the corresponding TVF value and type + pub(crate) fn from_literal(literal: &syn::Lit) -> Result { + match literal { + syn::Lit::Bool(lit) => Ok(Self::Bool(lit.value)), + syn::Lit::Byte(lit) => Ok(Self::Byte(lit.value())), + syn::Lit::Char(lit) => Ok(Self::Byte(lit.value() as u8)), + syn::Lit::Int(int) => { + let suffix = int.suffix(); + if suffix == "u8" { + Ok(Self::Byte(int.base10_parse()?)) + } else if suffix.starts_with('u') { + Ok(Self::Unsigned(int.base10_parse()?)) + } else { + Ok(Self::Signed(int.base10_parse()?)) + } + } + syn::Lit::Float(float) => Ok(Self::Float(float.base10_parse()?)), + syn::Lit::Str(string) => Ok(Self::String(string.value())), + syn::Lit::CStr(string) => { + Ok(Self::String(string.value().to_string_lossy().to_string())) + } + syn::Lit::ByteStr(bytes) => Ok(Self::Bytes(Bytes(bytes.value()))), + _ => Err(syn::Error::new_spanned(literal, "Invalid literal")), + } + } + + /// Given a literal and an explicity type, identify the corresponding TVF value + pub(crate) fn from_literal_with_type( + literal: &syn::Lit, + explicit: TvfType, + ) -> Result { + match explicit { + TvfType::Byte => Ok(Self::Byte(parse_int(literal)?)), + TvfType::Signed => Ok(Self::Signed(parse_int(literal)?)), + TvfType::Unsigned => Ok(Self::Unsigned(parse_int(literal)?)), + TvfType::Float => Ok(Self::Float(parse_float(literal)?)), + TvfType::String => Ok(Self::String(parse_string(literal)?)), + TvfType::Bytes => Ok(Self::Bytes(Bytes::from_literal(literal)?)), + TvfType::Date => Ok(Self::Date(parse_date(literal)?)), + TvfType::DateTime => Ok(Self::DateTime(parse_datetime(literal)?)), + TvfType::Buffer => Err(syn::Error::new_spanned( + literal, + "Cannot build sub-buffer from literal", + )), + } + } +} + /// Convert a `proc_macro2::Literal` into a `syn::Lit` fn convert_lit(literal: &Literal) -> Result { if let syn::Expr::Lit(literal) = parse_quote! [ #literal ] { @@ -335,7 +360,7 @@ fn convert_lit(literal: &Literal) -> Result { } /// Get an integer value from a literal -fn parse_int(literal: &syn::Lit) -> Result +pub(crate) fn parse_int(literal: &syn::Lit) -> Result where I: FromPrimitive + FromStr, ::Err: Display, @@ -351,68 +376,109 @@ where } } -/// Error encountered when parsing a value from a string literal -#[derive(thiserror::Error, Debug)] -pub(crate) enum StrParseError { - /// Unsupported literal - #[error("Unsupported literal")] - Literal, - - /// Failed to convert bytes into UTF-8 string - #[error("UTF-8: {0}")] - Utf8(#[from] FromUtf8Error), - - /// chrono error - #[error("chono: {0}")] - Chrono(#[from] chrono::ParseError), +/// Get a float value from a literal +pub(crate) fn parse_float(literal: &syn::Lit) -> Result { + let span = literal.span(); + match literal { + syn::Lit::Byte(byte) => Ok(byte.value() as f64), + syn::Lit::Char(chr) => Ok(chr.value() as u32 as f64), + syn::Lit::Int(int) => Ok(int.base10_parse()?), + syn::Lit::Float(float) => Ok(float.base10_parse()?), + _ => Err(syn::Error::new(span, "Invalid literal")), + } } /// Given a literal deduce a String -fn parse_string(literal: &syn::Lit) -> Result { +pub(crate) fn parse_string(literal: &syn::Lit) -> Result { + let span = literal.span(); + // Try to convert the literal into a string let string = match literal { syn::Lit::Str(s) => s.value(), - syn::Lit::ByteStr(s) => String::from_utf8(s.value())?, + syn::Lit::ByteStr(s) => match String::from_utf8(s.value()) { + Ok(s) => s, + Err(err) => { + return Err(syn::Error::new( + span, + format!["Failed to parse UTF-8 string: {}", err], + )); + } + }, syn::Lit::CStr(s) => s.value().to_string_lossy().to_string(), + syn::Lit::Bool(s) => if s.value { "true" } else { "false" }.to_string(), + syn::Lit::Byte(s) => s.value().to_string(), + syn::Lit::Char(s) => s.value().to_string(), + syn::Lit::Int(s) => s.to_string(), + syn::Lit::Float(s) => s.to_string(), _ => { - return Err(StrParseError::Literal); + return Err(syn::Error::new(span, "Unsupported literal")); } }; Ok(string) } /// Given a literal deduce a Date -fn parse_date(literal: &syn::Lit) -> Result { +pub(crate) fn parse_date(literal: &syn::Lit) -> Result { const FORMAT: &'static str = "%Y-%m-%d"; + let span = literal.span(); + let string = parse_string(literal)?; - Ok(NaiveDate::parse_from_str(&string, FORMAT)?) + NaiveDate::parse_from_str(&string, FORMAT) + .map_err(|err| syn::Error::new(span, format!["Failed to parse date: {}", err])) } /// Given a literal deduce a DateTime -fn parse_datetime(literal: &syn::Lit) -> Result { +pub(crate) fn parse_datetime(literal: &syn::Lit) -> Result { const FORMAT: &'static str = "%Y-%m-%d %H:%M:%S%.3f"; + let span = literal.span(); + let string = parse_string(literal)?; - Ok(NaiveDateTime::parse_from_str(&string, FORMAT)?) + NaiveDateTime::parse_from_str(&string, FORMAT) + .map_err(|err| syn::Error::new(span, format!["Failed to parse date: {}", err])) } impl Bytes { /// Parse a integer literal an build a sequence of bytes from it - fn from_literal(literal: &syn::LitInt) -> Result { - // Remove underscores from the literal - let digits = literal.to_string().replace('_', ""); - - // convert the digits to a sequence of bytes - if digits.starts_with("0x") { - // hexadecimal string - Self::digits_to_bytes(literal.span(), &digits, 16, 2) - } else if digits.starts_with("0b") { - // binary string - Self::digits_to_bytes(literal.span(), &digits, 2, 8) - } else { - Err(syn::Error::new_spanned( - literal, - "Cannot convert number to bytes, only hexadecimal and binary literals are supported.", - )) + pub(crate) fn from_literal(literal: &syn::Lit) -> Result { + let span = literal.span(); + + match literal { + syn::Lit::Str(lit) => { + // convert the string into bytes + todo!() + } + syn::Lit::ByteStr(lit) => { + // convert the string into bytes + todo!() + } + syn::Lit::CStr(lit) => { + // convert the string into bytes + todo!() + } + syn::Lit::Byte(lit) => todo!(), + syn::Lit::Char(lit) => todo!(), + syn::Lit::Int(lit) => { + // Remove underscores from the literal + let digits = lit.to_string().replace('_', ""); + + // convert the digits to a sequence of bytes + if digits.starts_with("0x") { + // hexadecimal string + Self::digits_to_bytes(literal.span(), &digits, 16, 2) + } else if digits.starts_with("0b") { + // binary string + Self::digits_to_bytes(literal.span(), &digits, 2, 8) + } else { + Err(syn::Error::new_spanned( + literal, + "Cannot convert number to bytes, only hexadecimal and binary literals are supported.", + )) + } + } + _ => Err(syn::Error::new( + span, + "Unsupported literal for sequence of bytes", + )), } } diff --git a/prosa_macros/src/tvf/tokens.rs b/prosa_macros/src/tvf/tokens.rs index fbd836f..8b4836f 100644 --- a/prosa_macros/src/tvf/tokens.rs +++ b/prosa_macros/src/tvf/tokens.rs @@ -1,39 +1,89 @@ use crate::tvf::expr::*; use chrono::{Datelike, NaiveDate, NaiveDateTime}; use proc_macro2::TokenStream; -use quote::quote; +use quote::{ToTokens, quote}; -impl TvfExpr { - fn to_tokens(&self, buffer: &syn::Ident) -> TokenStream { - let ident = self.id.to_tokens(); - let value = self.value.to_tokens(); - let put_method = self.out_type.put_method(); - - quote![ #buffer.#put_method(#ident, #value) ] +/// Generate the tokens to build a TVF buffer from a list of expressions +pub(crate) fn buffer_to_tokens( + buffer_type: &syn::Ident, + expressions: &[TvfExpr], +) -> Result { + // collect the expressions to build the buffer + let mut lines = Vec::with_capacity(expressions.len()); + for expr in expressions.iter() { + lines.push(expr.to_tokens(buffer_type)?); } -} -impl TvfId { - /// Convert the value into tokens - #[rustfmt::skip] - fn to_tokens(&self) -> TokenStream { - match self { - Self::Int (int ) => quote![ #int ], - Self::Ident(ident) => quote![ #ident ], - Self::Expr (expr ) => quote![ ( #expr ) ], + Ok(quote![ + { + let mut __buffer = <#buffer_type as Default>::default(); + #(#lines)* + __buffer } + ]) +} + +impl TvfExpr { + /// Convert the expression into tokens + fn to_tokens(&self, buffer_type: &syn::Ident) -> Result { + let id = self.id.to_tokens(); + + // Process the value + let value_span = self.value.span(); + let (out_type, value) = match &self.value { + TvfValue::Lit(lit) => { + if let Some(explicit) = self.explicit_type { + let value = Value::from_literal_with_type(lit, explicit)?; + (explicit, value.to_tokens()) + } else { + let value = Value::from_literal(lit)?; + (value.identify(), value.to_tokens()) + } + } + TvfValue::Ident(ident) => { + if ident == "true" || ident == "false" { + ( + self.explicit_type.unwrap_or(TvfType::Byte), + ident.to_token_stream(), + ) + } else if let Some(explicit) = self.explicit_type { + (explicit, ident.to_token_stream()) + } else { + return Err(syn::Error::new( + value_span, + "Missing explicit type for variable", + )); + } + } + TvfValue::Expr(stream) => { + if let Some(explicit) = self.explicit_type { + (explicit, stream.clone()) + } else { + return Err(syn::Error::new( + value_span, + "Missing explicit type for expression", + )); + } + } + TvfValue::Buffer(sub, _) => (TvfType::Buffer, buffer_to_tokens(buffer_type, sub)?), + }; + + let put_method = out_type.put_method(); + let value_cast = out_type.cast_type(self.modifier, value); + Ok(quote![ + <#buffer_type as __tvf::Tvf>::#put_method(&mut __buffer, #id, #value_cast); + ]) } } -impl TvfValue { +impl TvfId { /// Convert the value into tokens #[rustfmt::skip] fn to_tokens(&self) -> TokenStream { match self { - Self::Lit (lit ) => quote![ #lit ], - Self::Ident (ident ) => quote![ #ident ], - Self::Expr (expr ) => quote![ ( #expr ) ], - Self::Buffer(buffer) => todo![], + Self::Int (int ) => quote![ #int as usize ], + Self::Ident(ident) => quote![ #ident as usize ], + Self::Expr (expr ) => quote![ ( #expr ) as usize ], } } } @@ -41,13 +91,18 @@ impl TvfValue { impl TvfType { /// Rust type corresponding to the TVF type #[rustfmt::skip] - fn cast_type(self) -> Option { + fn cast_type(self, modifier: Modifier, value: TokenStream) -> TokenStream { + let md = modifier.to_token(); match self { - Self::Byte => Some(quote![ u8 ]), - Self::Signed => Some(quote![ i64 ]), - Self::Unsigned => Some(quote![ u64 ]), - Self::Float => Some(quote![ f64 ]), - _ => None, + Self::Byte => quote![ (#md #value) as u8 ], + Self::Signed => quote![ (#md #value) as i64 ], + Self::Unsigned => quote![ (#md #value) as u64 ], + Self::Float => quote![ (#md #value) as f64 ], + Self::String => quote![ (#md #value).to_string() ], + Self::Bytes => value, + Self::Date => value, + Self::DateTime => value, + Self::Buffer => value, } } @@ -83,13 +138,32 @@ impl Modifier { } } +impl<'e> Value<'e> { + /// Generate token for a value + #[rustfmt::skip] + fn to_tokens(&'e self) -> TokenStream { + match self { + Value::Bool (val) => quote![ #val ], + Value::Byte (val) => quote![ #val ], + Value::Signed (val) => quote![ #val ], + Value::Unsigned (val) => quote![ #val ], + Value::Float (val) => quote![ #val ], + Value::String (val) => quote![ #val ], + Value::Bytes (val) => val.to_token(), + Value::Date (val) => date_to_tokens (*val), + Value::DateTime (val) => datetime_to_tokens(*val), + Value::Buffer ( _ ) => panic!("Should not be called here"), + } + } +} + /// Write tokens to generate the given date fn date_to_tokens(date: NaiveDate) -> TokenStream { let year = date.year(); let month = date.month(); let day = date.day(); quote! [ - ::chrono::NaiveDate::from_ymd_opt( #year, #month, #day ).unwrap() + __chrono::NaiveDate::from_ymd_opt( #year, #month, #day ).unwrap() ] } @@ -97,7 +171,7 @@ fn date_to_tokens(date: NaiveDate) -> TokenStream { fn datetime_to_tokens(datetime: NaiveDateTime) -> TokenStream { let msecs = datetime.and_utc().timestamp_millis(); quote! [ - ::chrono::DateTime::from_timestamp_millis(#msecs).unwrap().naive_utc() + __chrono::DateTime::from_timestamp_millis(#msecs).unwrap().naive_utc() ] } @@ -106,6 +180,6 @@ impl Bytes { #[inline] fn to_token(&self) -> TokenStream { let bytes = self.0.as_slice(); - quote! [ ::bytes::Bytes::from_static( &[ #(#bytes),* ] ) ] + quote! [ __bytes::Bytes::from_static( &[ #(#bytes),* ] ) ] } } diff --git a/prosa_macros/tests/tvf.rs b/prosa_macros/tests/tvf.rs index e2340ca..63309ce 100644 --- a/prosa_macros/tests/tvf.rs +++ b/prosa_macros/tests/tvf.rs @@ -33,8 +33,8 @@ mod macro_tests { 12 => -2 as Signed, 13 => +2.125 as Float, 14 => 100 as String, - 15 => -1000 as String, - 16 => -amount as Signed, + //15 => -1000 as String, + //16 => -amount as Signed, 200 => "2023-06-05 15:02:00.000" as DateTime, });