From c0beaf31dcbce4693e939bba5770cad41ece8839 Mon Sep 17 00:00:00 2001 From: Louis Kureuil Person Date: Wed, 5 Sep 2018 00:12:43 +0200 Subject: [PATCH 1/2] Prototype for support of custom keywords First prototype for support of custom keywords with syn's new parsing API. Documentation and tests aren't present for now as I'm mainly reaching for feedback. This patch introduces a new Keyword trait, a new macro custom_keyword! and exposes the existing TokenMarker enum. The Keyword trait automatically implements the Token trait, making it possible to peek on custom keywords (this is why I had to make TokenMarker public). The custom macro generates a structure storing an Ident and implementing the Keyword and Parse traits. A function with the same name as the structure is also generated in order to use it like any predefined keyword. --- src/keyword.rs | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 ++ src/parse.rs | 3 +- src/token.rs | 5 +++- 4 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 src/keyword.rs diff --git a/src/keyword.rs b/src/keyword.rs new file mode 100644 index 0000000000..9ecf07b495 --- /dev/null +++ b/src/keyword.rs @@ -0,0 +1,81 @@ +use buffer::Cursor; +use token::Token; + +pub trait Keyword { + fn ident() -> &'static str; + + fn display() -> &'static str; +} + +impl Token for K { + fn peek(cursor: Cursor) -> bool { + if let Some((ident, _rest)) = cursor.ident() { + ident == K::ident() + } else { + false + } + } + + fn display() -> &'static str { + K::display() + } +} + +#[macro_export] +macro_rules! custom_keyword { + ($ident:ident) => { + custom_keyword_internal!({ pub(in self) } $ident); + }; + (pub $ident:ident) => { + custom_keyword_internal!({ pub } $ident); + }; + (pub(crate) $ident:ident) => { + custom_keyword_internal!({ pub(crate) } $ident); + }; + (pub(super) $ident:ident) => { + custom_keyword_internal!({ pub(super) } $ident); + }; + (pub(self) $ident:ident) => { + custom_keyword_internal!({ pub(self) } $ident); + }; + (pub(in $path:path) $ident:ident) => { + custom_keyword_internal!({ pub(in $path) } $ident); + }; +} + +#[macro_export] +#[doc(hidden)] +macro_rules! custom_keyword_internal { + ({ $($vis:tt)* } $ident:ident) => { + $($vis)* struct $ident { + inner: $crate::Ident + } + + impl $crate::parse::Keyword for $ident { + fn ident() -> &'static str { + stringify!($ident) + } + + fn display() -> &'static str { + concat!("`", stringify!($ident), "`") + } + } + + impl $crate::parse::Parse for $ident { + fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { + input.step(|cursor| { + if let Some((ident, rest)) = cursor.ident() { + if ident == stringify!($ident) { + return Ok(($ident { inner: ident }, rest)); + } + } + Err(cursor.error(concat!("expected `", stringify!($ident), "`"))) + }) + } + } + + $($vis)* fn $ident(marker: $crate::parse::TokenMarker) -> $ident { + match marker {} + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 33432561f6..ccf70b7b45 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -570,6 +570,9 @@ pub mod export; #[cfg(feature = "parsing")] mod lookahead; +#[cfg(feature = "parsing")] +mod keyword; + #[cfg(feature = "parsing")] pub mod parse; diff --git a/src/parse.rs b/src/parse.rs index 0445cefb28..4dd735deed 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -214,7 +214,8 @@ use punctuated::Punctuated; use token::Token; pub use error::{Error, Result}; -pub use lookahead::{Lookahead1, Peek}; +pub use lookahead::{Lookahead1, Peek, TokenMarker}; +pub use keyword::Keyword; /// Parsing interface implemented by all types that can be parsed in a default /// way from a token stream. diff --git a/src/token.rs b/src/token.rs index 4a44067dce..637d9015a0 100644 --- a/src/token.rs +++ b/src/token.rs @@ -118,7 +118,7 @@ use lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr}; #[cfg(feature = "parsing")] use lookahead; #[cfg(feature = "parsing")] -use parse::{Parse, ParseStream}; +use parse::{Keyword, Parse, ParseStream}; use span::IntoSpans; /// Marker trait for types that represent single tokens. @@ -143,6 +143,9 @@ mod private { #[cfg(feature = "parsing")] impl private::Sealed for Ident {} +#[cfg(feature = "parsing")] +impl private::Sealed for K {} + #[cfg(feature = "parsing")] fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool { let scope = Span::call_site(); From 7fb11e781a04eebe94f7903b18d44cfbca7f2bb1 Mon Sep 17 00:00:00 2001 From: David Tolnay Date: Thu, 6 Sep 2018 01:02:27 -0700 Subject: [PATCH 2/2] Custom keywords polish --- src/export.rs | 26 ++++- src/keyword.rs | 271 +++++++++++++++++++++++++++++++++++++++---------- src/lib.rs | 5 +- src/parse.rs | 3 +- src/token.rs | 27 ++++- 5 files changed, 268 insertions(+), 64 deletions(-) diff --git a/src/export.rs b/src/export.rs index 9608a01e84..148839ecb8 100644 --- a/src/export.rs +++ b/src/export.rs @@ -1,7 +1,29 @@ +pub use std::clone::Clone; +pub use std::cmp::{Eq, PartialEq}; +pub use std::convert::From; +pub use std::default::Default; +pub use std::fmt::{self, Debug, Formatter}; +pub use std::hash::{Hash, Hasher}; +pub use std::marker::Copy; +pub use std::option::Option::{None, Some}; pub use std::result::Result::{Err, Ok}; -#[cfg(feature = "parsing")] -pub use std::convert::From; +pub use proc_macro2::{Span, TokenStream as TokenStream2}; + +pub use span::IntoSpans; #[cfg(feature = "proc-macro")] pub use proc_macro::TokenStream; + +#[cfg(feature = "printing")] +pub use quote::{ToTokens, TokenStreamExt}; + +#[allow(non_camel_case_types)] +pub type bool = help::Bool; +#[allow(non_camel_case_types)] +pub type str = help::Str; + +mod help { + pub type Bool = bool; + pub type Str = str; +} diff --git a/src/keyword.rs b/src/keyword.rs index 9ecf07b495..8fcb7116a9 100644 --- a/src/keyword.rs +++ b/src/keyword.rs @@ -1,62 +1,142 @@ -use buffer::Cursor; -use token::Token; - -pub trait Keyword { - fn ident() -> &'static str; +/// Define a type that supports parsing and printing a given identifier as if it +/// were a keyword. +/// +/// # Usage +/// +/// As a convention, it is recommended that this macro be invoked within a +/// module called `kw` and that the resulting parser be invoked with a `kw::` +/// prefix. +/// +/// ``` +/// # extern crate syn; +/// # +/// mod kw { +/// # use syn; +/// syn::custom_keyword!(whatever); +/// } +/// ``` +/// +/// The generated syntax tree node supports the following operations just like +/// any built-in keyword token. +/// +/// - [Peeking] — `input.peek(kw::whatever)` +/// +/// - [Parsing] — `input.parse::()?` +/// +/// - [Printing] — `quote!( ... #whatever_token ... )` +/// +/// - Construction from a [`Span`] — `let whatever_token = kw::whatever(sp)` +/// +/// - Field access to its span — `let sp = whatever_token.span` +/// +/// [Peeking]: parse/struct.ParseBuffer.html#method.peek +/// [Parsing]: parse/struct.ParseBuffer.html#method.parse +/// [Printing]: https://docs.rs/quote/0.6/quote/trait.ToTokens.html +/// [`Span`]: struct.Span.html +/// +/// # Example +/// +/// This example parses input that looks like `bool = true` or `str = "value"`. +/// The key must be either the identifier `bool` or the identifier `str`. If +/// `bool`, the value may be either `true` or `false`. If `str`, the value may +/// be any string literal. +/// +/// The symbols `bool` and `str` are not reserved keywords in Rust so these are +/// not considered keywords in the `syn::token` module. Like any other +/// identifier that is not a keyword, these can be declared as custom keywords +/// by crates that need to use them as such. +/// +/// ``` +/// # extern crate syn; +/// # +/// use syn::{LitBool, LitStr, Token}; +/// use syn::parse::{Parse, ParseStream, Result}; +/// +/// mod kw { +/// # use syn; +/// syn::custom_keyword!(bool); +/// syn::custom_keyword!(str); +/// } +/// +/// enum Argument { +/// Bool { +/// bool_token: kw::bool, +/// eq_token: Token![=], +/// value: LitBool, +/// }, +/// Str { +/// str_token: kw::str, +/// eq_token: Token![=], +/// value: LitStr, +/// }, +/// } +/// +/// impl Parse for Argument { +/// fn parse(input: ParseStream) -> Result { +/// let lookahead = input.lookahead1(); +/// if lookahead.peek(kw::bool) { +/// Ok(Argument::Bool { +/// bool_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else if lookahead.peek(kw::str) { +/// Ok(Argument::Str { +/// str_token: input.parse::()?, +/// eq_token: input.parse()?, +/// value: input.parse()?, +/// }) +/// } else { +/// Err(lookahead.error()) +/// } +/// } +/// } +/// # +/// # fn main() {} +/// ``` +#[macro_export(local_inner_macros)] +macro_rules! custom_keyword { + ($ident:ident) => { + pub struct $ident { + pub span: $crate::export::Span, + } - fn display() -> &'static str; -} + #[doc(hidden)] + #[allow(non_snake_case)] + pub fn $ident<__S: $crate::export::IntoSpans<[$crate::export::Span; 1]>>(span: __S) -> $ident { + $ident { + span: $crate::export::IntoSpans::into_spans(span)[0], + } + } -impl Token for K { - fn peek(cursor: Cursor) -> bool { - if let Some((ident, _rest)) = cursor.ident() { - ident == K::ident() - } else { - false + impl $crate::export::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::export::Span::call_site(), + } + } } - } - fn display() -> &'static str { - K::display() + impl_parse_for_custom_keyword!($ident); + impl_to_tokens_for_custom_keyword!($ident); + impl_clone_for_custom_keyword!($ident); + impl_extra_traits_for_custom_keyword!($ident); } } +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] #[macro_export] -macro_rules! custom_keyword { +macro_rules! impl_parse_for_custom_keyword { ($ident:ident) => { - custom_keyword_internal!({ pub(in self) } $ident); - }; - (pub $ident:ident) => { - custom_keyword_internal!({ pub } $ident); - }; - (pub(crate) $ident:ident) => { - custom_keyword_internal!({ pub(crate) } $ident); - }; - (pub(super) $ident:ident) => { - custom_keyword_internal!({ pub(super) } $ident); - }; - (pub(self) $ident:ident) => { - custom_keyword_internal!({ pub(self) } $ident); - }; - (pub(in $path:path) $ident:ident) => { - custom_keyword_internal!({ pub(in $path) } $ident); - }; -} - -#[macro_export] -#[doc(hidden)] -macro_rules! custom_keyword_internal { - ({ $($vis:tt)* } $ident:ident) => { - $($vis)* struct $ident { - inner: $crate::Ident - } - - impl $crate::parse::Keyword for $ident { - fn ident() -> &'static str { + // For peek. + impl $crate::token::CustomKeyword for $ident { + fn ident() -> &'static $crate::export::str { stringify!($ident) } - fn display() -> &'static str { + fn display() -> &'static $crate::export::str { concat!("`", stringify!($ident), "`") } } @@ -64,18 +144,103 @@ macro_rules! custom_keyword_internal { impl $crate::parse::Parse for $ident { fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { input.step(|cursor| { - if let Some((ident, rest)) = cursor.ident() { + if let $crate::export::Some((ident, rest)) = cursor.ident() { if ident == stringify!($ident) { - return Ok(($ident { inner: ident }, rest)); + return $crate::export::Ok(($ident { span: ident.span() }, rest)); } } - Err(cursor.error(concat!("expected `", stringify!($ident), "`"))) + $crate::export::Err(cursor.error(concat!("expected `", stringify!($ident), "`"))) }) } } + }; +} - $($vis)* fn $ident(marker: $crate::parse::TokenMarker) -> $ident { - match marker {} +// Not public API. +#[cfg(not(feature = "parsing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_parse_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "printing")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => { + impl $crate::export::ToTokens for $ident { + fn to_tokens(&self, tokens: &mut $crate::export::TokenStream2) { + let ident = $crate::Ident::new(stringify!($ident), self.span); + $crate::export::TokenStreamExt::append(tokens, ident); + } } - } + }; +} + +// Not public API. +#[cfg(not(feature = "printing"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_to_tokens_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "clone-impls")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => { + impl $crate::export::Copy for $ident {} + + impl $crate::export::Clone for $ident { + fn clone(&self) -> Self { + *self + } + } + }; +} + +// Not public API. +#[cfg(not(feature = "clone-impls"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_clone_for_custom_keyword { + ($ident:ident) => {}; +} + +// Not public API. +#[cfg(feature = "extra-traits")] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => { + impl $crate::export::Debug for $ident { + fn fmt(&self, f: &mut $crate::export::Formatter) -> $crate::export::fmt::Result { + $crate::export::Formatter::write_str(f, stringify!($ident)) + } + } + + impl $crate::export::Eq for $ident {} + + impl $crate::export::PartialEq for $ident { + fn eq(&self, _other: &Self) -> $crate::export::bool { + true + } + } + + impl $crate::export::Hash for $ident { + fn hash<__H: $crate::export::Hasher>(&self, _state: &mut __H) {} + } + }; +} + +// Not public API. +#[cfg(not(feature = "extra-traits"))] +#[doc(hidden)] +#[macro_export] +macro_rules! impl_extra_traits_for_custom_keyword { + ($ident:ident) => {}; } diff --git a/src/lib.rs b/src/lib.rs index ccf70b7b45..d3b73590e2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -567,11 +567,10 @@ pub use gen::*; #[doc(hidden)] pub mod export; -#[cfg(feature = "parsing")] -mod lookahead; +mod keyword; #[cfg(feature = "parsing")] -mod keyword; +mod lookahead; #[cfg(feature = "parsing")] pub mod parse; diff --git a/src/parse.rs b/src/parse.rs index 4dd735deed..0445cefb28 100644 --- a/src/parse.rs +++ b/src/parse.rs @@ -214,8 +214,7 @@ use punctuated::Punctuated; use token::Token; pub use error::{Error, Result}; -pub use lookahead::{Lookahead1, Peek, TokenMarker}; -pub use keyword::Keyword; +pub use lookahead::{Lookahead1, Peek}; /// Parsing interface implemented by all types that can be parsed in a default /// way from a token stream. diff --git a/src/token.rs b/src/token.rs index 637d9015a0..8892c33091 100644 --- a/src/token.rs +++ b/src/token.rs @@ -118,7 +118,7 @@ use lit::{Lit, LitBool, LitByte, LitByteStr, LitChar, LitFloat, LitInt, LitStr}; #[cfg(feature = "parsing")] use lookahead; #[cfg(feature = "parsing")] -use parse::{Keyword, Parse, ParseStream}; +use parse::{Parse, ParseStream}; use span::IntoSpans; /// Marker trait for types that represent single tokens. @@ -143,9 +143,6 @@ mod private { #[cfg(feature = "parsing")] impl private::Sealed for Ident {} -#[cfg(feature = "parsing")] -impl private::Sealed for K {} - #[cfg(feature = "parsing")] fn peek_impl(cursor: Cursor, peek: fn(ParseStream) -> bool) -> bool { let scope = Span::call_site(); @@ -194,6 +191,28 @@ impl_token!(LitFloat "floating point literal"); #[cfg(any(feature = "full", feature = "derive"))] impl_token!(LitBool "boolean literal"); +// Not public API. +#[cfg(feature = "parsing")] +#[doc(hidden)] +pub trait CustomKeyword { + fn ident() -> &'static str; + fn display() -> &'static str; +} + +#[cfg(feature = "parsing")] +impl private::Sealed for K {} + +#[cfg(feature = "parsing")] +impl Token for K { + fn peek(cursor: Cursor) -> bool { + parsing::peek_keyword(cursor, K::ident()) + } + + fn display() -> &'static str { + K::display() + } +} + macro_rules! define_keywords { ($($token:tt pub struct $name:ident #[$doc:meta])*) => { $(