diff --git a/.clippy.toml b/.clippy.toml new file mode 100644 index 0000000000000000000000000000000000000000..3d30690f120489337cb61685b2f0f0d45af983c1 --- /dev/null +++ b/.clippy.toml @@ -0,0 +1 @@ +msrv = "1.31.0" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d9f630a4987d40c7289bf30c1293be9713b1445..e277c9690145f9fdf645b6cbe4becd39a93f50b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,7 +3,6 @@ name: CI on: push: pull_request: - workflow_dispatch: schedule: [cron: "40 1 * * *"] permissions: @@ -13,31 +12,24 @@ env: RUSTFLAGS: -Dwarnings jobs: - pre_ci: - uses: dtolnay/.github/.github/workflows/pre_ci.yml@master - test: name: Tests - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly with: - components: llvm-tools, rustc-dev + components: rustc-dev - run: cargo test --all-features --release --tests build: name: ${{matrix.name || format('Rust {0}', matrix.rust)}} - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ${{matrix.os || 'ubuntu'}}-latest strategy: fail-fast: false matrix: - rust: [stable, beta, 1.56.0] + rust: [stable, beta, 1.31.0] include: - rust: nightly components: rustc-dev @@ -54,7 +46,7 @@ jobs: target: ${{matrix.target && format('--target={0}', matrix.target)}} timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{matrix.rust}} @@ -80,80 +72,65 @@ jobs: examples: name: Examples - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly - run: cargo check --manifest-path examples/dump-syntax/Cargo.toml - run: cargo check --manifest-path examples/heapsize/example/Cargo.toml - run: cargo check --manifest-path examples/lazy-static/example/Cargo.toml - run: cargo check --manifest-path examples/trace-var/example/Cargo.toml - doc: - name: Documentation - needs: pre_ci - if: needs.pre_ci.outputs.continue + docs: + name: Docs runs-on: ubuntu-latest - timeout-minutes: 45 env: - RUSTDOCFLAGS: -Dwarnings + RUSTDOCFLAGS: --cfg=doc_cfg -Dbroken_intra_doc_links + timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly - - uses: dtolnay/install@cargo-docs-rs - - run: cargo docs-rs - - run: cargo docs-rs --manifest-path json/Cargo.toml - run: cargo test --all-features --doc + - run: cargo doc --all-features codegen: name: Codegen - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@stable - - run: cargo run --manifest-path codegen/Cargo.toml + - run: cd codegen && cargo run - run: git diff --exit-code - minimal: + msrv: name: Minimal versions - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly - - run: cargo generate-lockfile -Z minimal-versions - - run: cargo check --all-features --locked + - run: cargo update -Z minimal-versions + - run: cargo check --all-features fuzz: name: Fuzz - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly - uses: dtolnay/install@cargo-fuzz - - run: cargo fuzz check + - run: cargo fuzz build -O miri: name: Miri - needs: pre_ci - if: needs.pre_ci.outputs.continue runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@miri - - run: cargo miri setup - run: cargo miri test --all-features env: MIRIFLAGS: -Zmiri-strict-provenance @@ -164,12 +141,11 @@ jobs: if: github.event_name != 'pull_request' timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@nightly with: components: clippy,rustc-dev - run: cargo clippy --all-features --tests --benches -- -Dclippy::all -Dclippy::pedantic - - run: cargo clippy --manifest-path codegen/Cargo.toml -- -Dclippy::all -Dclippy::pedantic outdated: name: Outdated @@ -177,7 +153,7 @@ jobs: if: github.event_name != 'pull_request' timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: dtolnay/install@cargo-outdated - run: cargo outdated --workspace --exit-code 1 - run: cargo outdated --manifest-path fuzz/Cargo.toml --exit-code 1 diff --git a/BUILD.gn b/BUILD.gn index 21ce2e47d2b8161614d0f8959048eeddab18f14f..e2796827b200253608b02edeac1b81c837c9143e 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -19,8 +19,8 @@ ohos_cargo_crate("lib") { crate_root = "src/lib.rs" sources = [ "src/lib.rs" ] - edition = "2021" - cargo_pkg_version = "2.0.48" + edition = "2018" + cargo_pkg_version = "1.0.107" cargo_pkg_authors = "David Tolnay " cargo_pkg_name = "syn" cargo_pkg_description = "Parser for Rust source code" diff --git a/Cargo.toml b/Cargo.toml index c17ef6589f6dd77118dc7a3d49b61d54af448951..dd32f92d8f883fea9166becc7ad76e58231258ae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,13 +1,14 @@ [package] name = "syn" -version = "2.0.48" # don't forget to update html_root_url and syn.json +version = "1.0.107" # don't forget to update html_root_url and syn.json authors = ["David Tolnay "] categories = ["development-tools::procedural-macro-helpers", "parser-implementations"] description = "Parser for Rust source code" documentation = "https://docs.rs/syn" -edition = "2021" +edition = "2018" include = [ "/benches/**", + "/build.rs", "/Cargo.toml", "/LICENSE-APACHE", "/LICENSE-MIT", @@ -18,7 +19,7 @@ include = [ keywords = ["macros", "syn"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/syn" -rust-version = "1.56" +rust-version = "1.31" [features] default = ["derive", "parsing", "printing", "clone-impls", "proc-macro"] @@ -35,23 +36,23 @@ proc-macro = ["proc-macro2/proc-macro", "quote/proc-macro"] test = ["syn-test-suite/all-features"] [dependencies] -proc-macro2 = { version = "1.0.75", default-features = false } -quote = { version = "1.0.35", optional = true, default-features = false } -unicode-ident = "1" +proc-macro2 = { version = "1.0.46", default-features = false } +quote = { version = "1.0", optional = true, default-features = false } +unicode-ident = "1.0" [dev-dependencies] -anyhow = "1" -automod = "1" -flate2 = "1" -insta = "1" -rayon = "1" -ref-cast = "1" +anyhow = "1.0" +automod = "1.0" +flate2 = "1.0" +insta = "1.0" +rayon = "1.0" +ref-cast = "1.0" +regex = "1.0" reqwest = { version = "0.11", features = ["blocking"] } -rustversion = "1" syn-test-suite = { version = "0", path = "tests/features" } tar = "0.4.16" -termcolor = "1" -walkdir = "2.3.2" +termcolor = "1.0" +walkdir = "2.1" [lib] doc-scrape-examples = false @@ -68,7 +69,7 @@ required-features = ["full", "parsing"] [package.metadata.docs.rs] all-features = true targets = ["x86_64-unknown-linux-gnu"] -rustdoc-args = ["--cfg", "doc_cfg", "--generate-link-to-definition"] +rustdoc-args = ["--cfg", "doc_cfg"] [package.metadata.playground] features = ["full", "visit", "visit-mut", "fold", "extra-traits"] @@ -84,5 +85,7 @@ members = [ "examples/lazy-static/lazy-static", "examples/trace-var/example", "examples/trace-var/trace-var", + "json", + "tests/crates", "tests/features", ] diff --git a/LICENSE-APACHE b/LICENSE-APACHE index 1b5ec8b78e237b5c3b3d812a7c0a6589d0f7161d..16fe87b06e802f094b3fbb0894b137bca2b16ef1 100644 --- a/LICENSE-APACHE +++ b/LICENSE-APACHE @@ -174,3 +174,28 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.OpenSource b/README.OpenSource index aaf9189801e955b0b8af97ce66f3a5a8983099b4..f51c5bd1e79419b03773dfde086f955c3f218432 100644 --- a/README.OpenSource +++ b/README.OpenSource @@ -3,7 +3,7 @@ "Name": "syn", "License": "Apache License V2.0, MIT", "License File": "LICENSE-APACHE, LICENSE-MIT", - "Version Number": "2.0.48", + "Version Number": "1.0.107", "Owner": "fangting12@huawei.com", "Upstream URL": "https://github.com/dtolnay/syn", "Description": "A Rust library that provides support for parsing Rust code." diff --git a/README.md b/README.md index e8d99abcf42daab06a1f7e517b44b44312d0f386..eeef83dd581e5ea3995f861782ca3afe356cd2e5 100644 --- a/README.md +++ b/README.md @@ -39,14 +39,14 @@ contains some APIs that may be useful more generally. procedural macros enable only what they need, and do not pay in compile time for all the rest. -[`syn::File`]: https://docs.rs/syn/2.0/syn/struct.File.html -[`syn::Item`]: https://docs.rs/syn/2.0/syn/enum.Item.html -[`syn::Expr`]: https://docs.rs/syn/2.0/syn/enum.Expr.html -[`syn::Type`]: https://docs.rs/syn/2.0/syn/enum.Type.html -[`syn::DeriveInput`]: https://docs.rs/syn/2.0/syn/struct.DeriveInput.html -[parser functions]: https://docs.rs/syn/2.0/syn/parse/index.html +[`syn::File`]: https://docs.rs/syn/1.0/syn/struct.File.html +[`syn::Item`]: https://docs.rs/syn/1.0/syn/enum.Item.html +[`syn::Expr`]: https://docs.rs/syn/1.0/syn/enum.Expr.html +[`syn::Type`]: https://docs.rs/syn/1.0/syn/enum.Type.html +[`syn::DeriveInput`]: https://docs.rs/syn/1.0/syn/struct.DeriveInput.html +[parser functions]: https://docs.rs/syn/1.0/syn/parse/index.html -*Version requirement: Syn supports rustc 1.56 and up.* +*Version requirement: Syn supports rustc 1.31 and up.* [*Release notes*](https://github.com/dtolnay/syn/releases) @@ -76,7 +76,7 @@ tokens back to the compiler to compile into the user's crate. ```toml [dependencies] -syn = "2.0" +syn = "1.0" quote = "1.0" [lib] @@ -104,8 +104,9 @@ pub fn my_macro(input: TokenStream) -> TokenStream { ``` The [`heapsize`] example directory shows a complete working implementation of a -derive macro. The example derives a `HeapSize` trait which computes an estimate -of the amount of heap memory owned by a value. +derive macro. It works on any Rust compiler 1.31+. The example derives a +`HeapSize` trait which computes an estimate of the amount of heap memory owned +by a value. [`heapsize`]: examples/heapsize diff --git a/benches/file.rs b/benches/file.rs index b424723966905328ba2ca8f1eb57f3f493089c29..bd4a247df662e2c9161520090617c69b1e71a08a 100644 --- a/benches/file.rs +++ b/benches/file.rs @@ -4,11 +4,8 @@ #![recursion_limit = "1024"] #![allow( clippy::items_after_statements, - clippy::manual_let_else, - clippy::match_like_matches_macro, clippy::missing_panics_doc, - clippy::must_use_candidate, - clippy::uninlined_format_args + clippy::must_use_candidate )] extern crate test; @@ -17,9 +14,10 @@ extern crate test; #[path = "../tests/macros/mod.rs"] mod macros; -#[allow(dead_code)] +#[path = "../tests/common/mod.rs"] +mod common; #[path = "../tests/repo/mod.rs"] -mod repo; +pub mod repo; use proc_macro2::{Span, TokenStream}; use std::fs; diff --git a/benches/rust.rs b/benches/rust.rs index 15536db4c8ca2f7874e85fb8fcf6e1f2cd933db1..e3f8f550abb1a1d47d123e80e68fab29cd225126 100644 --- a/benches/rust.rs +++ b/benches/rust.rs @@ -5,21 +5,14 @@ #![cfg_attr(not(syn_only), feature(rustc_private))] #![recursion_limit = "1024"] -#![allow( - clippy::arc_with_non_send_sync, - clippy::cast_lossless, - clippy::let_underscore_untyped, - clippy::manual_let_else, - clippy::match_like_matches_macro, - clippy::uninlined_format_args, - clippy::unnecessary_wraps -)] +#![allow(clippy::cast_lossless, clippy::unnecessary_wraps)] #[macro_use] #[path = "../tests/macros/mod.rs"] mod macros; -#[allow(dead_code)] +#[path = "../tests/common/mod.rs"] +mod common; #[path = "../tests/repo/mod.rs"] mod repo; @@ -45,7 +38,6 @@ mod syn_parse { #[cfg(not(syn_only))] mod librustc_parse { extern crate rustc_data_structures; - extern crate rustc_driver; extern crate rustc_error_messages; extern crate rustc_errors; extern crate rustc_parse; @@ -54,7 +46,7 @@ mod librustc_parse { use rustc_data_structures::sync::Lrc; use rustc_error_messages::FluentBundle; - use rustc_errors::{emitter::Emitter, translation::Translate, DiagCtxt, Diagnostic}; + use rustc_errors::{emitter::Emitter, translation::Translate, Diagnostic, Handler}; use rustc_session::parse::ParseSess; use rustc_span::source_map::{FilePathMapping, SourceMap}; use rustc_span::{edition::Edition, FileName}; @@ -79,10 +71,10 @@ mod librustc_parse { } rustc_span::create_session_if_not_set_then(Edition::Edition2018, |_| { - let source_map = Lrc::new(SourceMap::new(FilePathMapping::empty())); + let cm = Lrc::new(SourceMap::new(FilePathMapping::empty())); let emitter = Box::new(SilentEmitter); - let handler = DiagCtxt::with_emitter(emitter); - let sess = ParseSess::with_dcx(handler, source_map); + let handler = Handler::with_emitter(false, None, emitter); + let sess = ParseSess::with_span_handler(handler, cm); if let Err(diagnostic) = rustc_parse::parse_crate_from_source_str( FileName::Custom("bench".to_owned()), content.to_owned(), @@ -99,7 +91,7 @@ mod librustc_parse { #[cfg(not(syn_only))] mod read_from_disk { pub fn bench(content: &str) -> Result<(), ()> { - let _ = content; + _ = content; Ok(()) } } @@ -109,13 +101,9 @@ fn exec(mut codepath: impl FnMut(&str) -> Result<(), ()>) -> Duration { let mut success = 0; let mut total = 0; - ["tests/rust/compiler", "tests/rust/library"] - .iter() - .flat_map(|dir| { - walkdir::WalkDir::new(dir) - .into_iter() - .filter_entry(repo::base_dir_filter) - }) + walkdir::WalkDir::new("tests/rust/src") + .into_iter() + .filter_entry(repo::base_dir_filter) .for_each(|entry| { let entry = entry.unwrap(); let path = entry.path(); diff --git a/build.rs b/build.rs index f14ce219dd70ece298012cd70951f7b8fa4c77e9..1a2c077bf0e17b992e375e67374f4b5eb3439751 100644 --- a/build.rs +++ b/build.rs @@ -1,81 +1,51 @@ use std::env; -use std::ffi::OsString; -use std::process::{self, Command, Stdio}; +use std::process::Command; +use std::str; // The rustc-cfg strings below are *not* public API. Please let us know by // opening a GitHub issue if your build environment requires some way to enable // these cfgs other than by executing our build script. fn main() { - println!("cargo:rerun-if-changed=build.rs"); - - // Note: add "/build.rs" to package.include in Cargo.toml if adding any - // conditional compilation within the library. - - if !unstable() { - println!("cargo:rustc-cfg=syn_disable_nightly_tests"); - } -} - -fn unstable() -> bool { - let rustc = cargo_env_var("RUSTC"); - - // Pick up Cargo rustc configuration. - let mut cmd = if let Some(wrapper) = env::var_os("RUSTC_WRAPPER") { - let mut cmd = Command::new(wrapper); - // The wrapper's first argument is supposed to be the path to rustc. - cmd.arg(rustc); - cmd - } else { - Command::new(rustc) + let compiler = match rustc_version() { + Some(compiler) => compiler, + None => return, }; - cmd.stdin(Stdio::null()); - cmd.stdout(Stdio::null()); - cmd.stderr(Stdio::null()); - cmd.arg("-"); - - // Find out whether this is a nightly or dev build. - cmd.env_remove("RUSTC_BOOTSTRAP"); - cmd.arg("-Zcrate-attr=feature(rustc_private)"); - - // Pass `-Zunpretty` to terminate earlier without writing out any "emit" - // files. Use `expanded` to proceed far enough to actually apply crate - // attrs. With `unpretty=normal` or `--print`, not enough compilation - // happens to recognize that the feature attribute is unstable. - cmd.arg("-Zunpretty=expanded"); - - // Set #![no_std] to bypass loading libstd.rlib. This is a 7.5% speedup. - cmd.arg("-Zcrate-attr=no_std"); + if compiler.minor < 36 { + println!("cargo:rustc-cfg=syn_omit_await_from_token_macro"); + } - cmd.arg("--crate-type=lib"); - cmd.arg("--edition=2021"); + if compiler.minor < 39 { + println!("cargo:rustc-cfg=syn_no_const_vec_new"); + } - if let Some(target) = env::var_os("TARGET") { - cmd.arg("--target").arg(target); + if compiler.minor < 40 { + println!("cargo:rustc-cfg=syn_no_non_exhaustive"); } - // If Cargo wants to set RUSTFLAGS, use that. - if let Ok(rustflags) = env::var("CARGO_ENCODED_RUSTFLAGS") { - if !rustflags.is_empty() { - for arg in rustflags.split('\x1f') { - cmd.arg(arg); - } - } + if compiler.minor < 56 { + println!("cargo:rustc-cfg=syn_no_negative_literal_parse"); } - // This rustc invocation should take around 0.03 seconds. - match cmd.status() { - Ok(status) => status.success(), - Err(_) => false, + if !compiler.nightly { + println!("cargo:rustc-cfg=syn_disable_nightly_tests"); } } -fn cargo_env_var(key: &str) -> OsString { - env::var_os(key).unwrap_or_else(|| { - eprintln!( - "Environment variable ${} is not set during execution of build script", - key, - ); - process::exit(1); - }) +struct Compiler { + minor: u32, + nightly: bool, +} + +fn rustc_version() -> Option { + let rustc = env::var_os("RUSTC")?; + let output = Command::new(rustc).arg("--version").output().ok()?; + let version = str::from_utf8(&output.stdout).ok()?; + let mut pieces = version.split('.'); + if pieces.next() != Some("rustc 1") { + return None; + } + let minor = pieces.next()?.parse().ok()?; + let nightly = version.contains("nightly") || version.ends_with("-dev"); + Some(Compiler { minor, nightly }) } diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 0cfb10afb44d32ededbc0bc3d1549a74643d0851..41daf83221e9834cc989febb40a4a2aba7ce90c0 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -2,25 +2,24 @@ name = "syn-internal-codegen" version = "0.0.0" authors = ["David Tolnay ", "Nika Layzell "] -edition = "2021" +edition = "2018" publish = false # this is an internal crate which should never be published [dependencies] -anyhow = "1" +anyhow = "1.0" color-backtrace = "0.4" -indexmap = { version = "2", features = ["serde"] } +indexmap = { version = "1.0", features = ["serde-1"] } inflections = "1.1" -prettyplease = "0.2.3" +prettyplease = "0.1" proc-macro2 = { version = "1.0.20", features = ["span-locations"] } -quote = "1" -semver = { version = "1", features = ["serde"] } -serde = "1.0.88" -serde_derive = "1.0.88" +quote = "1.0" +semver = { version = "1.0", features = ["serde"] } +serde = { version = "1.0.88", features = ["derive"] } serde_json = "1.0.38" -syn = { version = "2", features = ["derive", "full", "parsing", "printing"], default-features = false } syn-codegen = { path = "../json" } -thiserror = "1" +syn = { version = "1.0", features = ["derive", "parsing", "printing", "full"], default-features = false } +thiserror = "1.0" toml = "0.5" [workspace] diff --git a/codegen/src/cfg.rs b/codegen/src/cfg.rs index f11dc8693d1b6c9157ee21d586af5480933ac943..5932860f59ba06b71810dab5a3198674d9fe619a 100644 --- a/codegen/src/cfg.rs +++ b/codegen/src/cfg.rs @@ -2,28 +2,11 @@ use proc_macro2::TokenStream; use quote::quote; use syn_codegen::Features; -pub fn features<'a>( - features: &Features, - overriding_cfg: impl Into>, -) -> TokenStream { +pub fn features(features: &Features) -> TokenStream { let features = &features.any; - let cfg = match features.len() { - 0 => None, - 1 => Some(quote! { cfg(feature = #(#features)*) }), - _ => Some(quote! { cfg(any(#(feature = #features),*)) }), - }; - match (cfg, overriding_cfg.into()) { - (Some(cfg), Some(overriding_cfg)) => quote! { - #[#cfg] - #[cfg_attr(doc_cfg, doc(cfg(feature = #overriding_cfg)))] - }, - (Some(cfg), None) => quote! { - #[#cfg] - #[cfg_attr(doc_cfg, doc(#cfg))] - }, - (None, Some(overriding_cfg)) => quote! { - #[cfg_attr(doc_cfg, doc(cfg(feature = #overriding_cfg)))] - }, - (None, None) => TokenStream::new(), + match features.len() { + 0 => quote!(), + 1 => quote!(#[cfg(feature = #(#features)*)]), + _ => quote!(#[cfg(any(#(feature = #features),*))]), } } diff --git a/codegen/src/clone.rs b/codegen/src/clone.rs index 9db771e7817b628a686da49d3cb0b244fd94b18a..7f6718ad041660dade1777f7cb6bf7d6b59b85da 100644 --- a/codegen/src/clone.rs +++ b/codegen/src/clone.rs @@ -4,14 +4,13 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use syn_codegen::{Data, Definitions, Node, Type}; -const CLONE_SRC: &str = "src/gen/clone.rs"; +const DEBUG_SRC: &str = "../src/gen/clone.rs"; fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let type_name = &node.ident; let ident = Ident::new(type_name, Span::call_site()); match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { let arms = variants.iter().map(|(variant_name, fields)| { let variant = Ident::new(variant_name, Span::call_site()); @@ -41,13 +40,18 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } }); - let nonexhaustive = if node.ident == "Expr" { + let nonexhaustive = if node.exhaustive { + None + } else if node.ident == "Expr" { Some(quote! { - #[cfg(not(feature = "full"))] + #[cfg(any(syn_no_non_exhaustive, not(feature = "full")))] _ => unreachable!(), }) } else { - None + Some(quote! { + #[cfg(syn_no_non_exhaustive)] + _ => unreachable!(), + }) }; quote! { match self { @@ -76,7 +80,7 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features, "clone-impls"); + let cfg_features = cfg::features(&node.features); let copy = node.ident == "AttrStyle" || node.ident == "BinOp" @@ -86,8 +90,10 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { if copy { return quote! { #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] impl Copy for #ident {} #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] impl Clone for #ident { fn clone(&self) -> Self { *self @@ -100,6 +106,7 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { quote! { #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "clone-impls")))] impl Clone for #ident { fn clone(&self) -> Self { #body @@ -115,7 +122,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { } file::write( - CLONE_SRC, + DEBUG_SRC, quote! { #![allow(clippy::clone_on_copy, clippy::expl_impl_clone_on_copy)] diff --git a/codegen/src/debug.rs b/codegen/src/debug.rs index b0f517522e35751e35e42dfd400000bcd330af78..331421e6723577c53247375575512798204ebdec 100644 --- a/codegen/src/debug.rs +++ b/codegen/src/debug.rs @@ -2,57 +2,16 @@ use crate::{cfg, file, lookup}; use anyhow::Result; use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; -use std::collections::BTreeSet as Set; use syn_codegen::{Data, Definitions, Node, Type}; -const DEBUG_SRC: &str = "src/gen/debug.rs"; +const DEBUG_SRC: &str = "../src/gen/debug.rs"; -fn syntax_tree_enum<'a>( - enum_name: &str, - variant_name: &str, - fields: &'a [Type], -) -> Option<&'a str> { - if fields.len() != 1 { - return None; - } - const WHITELIST: &[(&str, &str)] = &[ - ("Meta", "Path"), - ("Pat", "Const"), - ("Pat", "Lit"), - ("Pat", "Macro"), - ("Pat", "Path"), - ("Pat", "Range"), - ("PathArguments", "AngleBracketed"), - ("PathArguments", "Parenthesized"), - ("Stmt", "Local"), - ("TypeParamBound", "Lifetime"), - ("Visibility", "Public"), - ("Visibility", "Restricted"), - ]; - match &fields[0] { - Type::Syn(ty) - if WHITELIST.contains(&(enum_name, variant_name)) - || enum_name.to_owned() + variant_name == *ty => - { - Some(ty) - } - _ => None, - } -} - -fn expand_impl_body( - defs: &Definitions, - node: &Node, - syntax_tree_variants: &Set<&str>, -) -> TokenStream { +fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let type_name = &node.ident; let ident = Ident::new(type_name, Span::call_site()); - let is_syntax_tree_variant = syntax_tree_variants.contains(type_name.as_str()); - let body = match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), + match &node.data { Data::Enum(variants) => { - assert!(!is_syntax_tree_variant); let arms = variants.iter().map(|(variant_name, fields)| { let variant = Ident::new(variant_name, Span::call_site()); if fields.is_empty() { @@ -60,6 +19,9 @@ fn expand_impl_body( #ident::#variant => formatter.write_str(#variant_name), } } else { + let pats = (0..fields.len()) + .map(|i| format_ident!("v{}", i)) + .collect::>(); let mut cfg = None; if node.ident == "Expr" { if let Type::Syn(ty) = &fields[0] { @@ -68,37 +30,30 @@ fn expand_impl_body( } } } - if syntax_tree_enum(type_name, variant_name, fields).is_some() { - quote! { - #cfg - #ident::#variant(v0) => v0.debug(formatter, #variant_name), - } - } else { - let pats = (0..fields.len()) - .map(|i| format_ident!("v{}", i)) - .collect::>(); - quote! { - #cfg - #ident::#variant(#(#pats),*) => { - let mut formatter = formatter.debug_tuple(#variant_name); - #(formatter.field(#pats);)* - formatter.finish() - } + quote! { + #cfg + #ident::#variant(#(#pats),*) => { + let mut formatter = formatter.debug_tuple(#variant_name); + #(formatter.field(#pats);)* + formatter.finish() } } } }); - let nonexhaustive = if node.ident == "Expr" { + let nonexhaustive = if node.exhaustive { + None + } else if node.ident == "Expr" { Some(quote! { - #[cfg(not(feature = "full"))] + #[cfg(any(syn_no_non_exhaustive, not(feature = "full")))] _ => unreachable!(), }) } else { - None + Some(quote! { + #[cfg(syn_no_non_exhaustive)] + _ => unreachable!(), + }) }; - let prefix = format!("{}::", type_name); quote! { - formatter.write_str(#prefix)?; match self { #(#arms)* #nonexhaustive @@ -106,11 +61,6 @@ fn expand_impl_body( } } Data::Struct(fields) => { - let type_name = if is_syntax_tree_variant { - quote!(name) - } else { - quote!(#type_name) - }; let fields = fields.keys().map(|f| { let ident = Ident::new(f, Span::call_site()); quote! { @@ -124,40 +74,24 @@ fn expand_impl_body( } } Data::Private => unreachable!(), - }; - - if is_syntax_tree_variant { - quote! { - impl #ident { - fn debug(&self, formatter: &mut fmt::Formatter, name: &str) -> fmt::Result { - #body - } - } - self.debug(formatter, #type_name) - } - } else { - body } } -fn expand_impl(defs: &Definitions, node: &Node, syntax_tree_variants: &Set<&str>) -> TokenStream { +fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { let manual_debug = node.data == Data::Private || node.ident == "LitBool"; if manual_debug { return TokenStream::new(); } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features, "extra-traits"); - let body = expand_impl_body(defs, node, syntax_tree_variants); - let formatter = match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(_formatter), - _ => quote!(formatter), - }; + let cfg_features = cfg::features(&node.features); + let body = expand_impl_body(defs, node); quote! { #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl Debug for #ident { - fn fmt(&self, #formatter: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { #body } } @@ -165,21 +99,9 @@ fn expand_impl(defs: &Definitions, node: &Node, syntax_tree_variants: &Set<&str> } pub fn generate(defs: &Definitions) -> Result<()> { - let mut syntax_tree_variants = Set::new(); - for node in &defs.types { - if let Data::Enum(variants) = &node.data { - let enum_name = &node.ident; - for (variant_name, fields) in variants { - if let Some(inner) = syntax_tree_enum(enum_name, variant_name, fields) { - syntax_tree_variants.insert(inner); - } - } - } - } - let mut impls = TokenStream::new(); for node in &defs.types { - impls.extend(expand_impl(defs, node, &syntax_tree_variants)); + impls.extend(expand_impl(defs, node)); } file::write( diff --git a/codegen/src/eq.rs b/codegen/src/eq.rs index d709a90af2440720d7e93429a4267f9fd8560ef7..b9f652c0440eaab5cdb1540ad7b0a0a7094f3899 100644 --- a/codegen/src/eq.rs +++ b/codegen/src/eq.rs @@ -4,10 +4,11 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use syn_codegen::{Data, Definitions, Node, Type}; -const EQ_SRC: &str = "src/gen/eq.rs"; +const DEBUG_SRC: &str = "../src/gen/eq.rs"; fn always_eq(field_type: &Type) -> bool { match field_type { + Type::Syn(node) => node == "Reserved", Type::Ext(ty) => ty == "Span", Type::Token(_) | Type::Group(_) => true, Type::Box(inner) => always_eq(inner), @@ -21,7 +22,6 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let ident = Ident::new(type_name, Span::call_site()); match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { let arms = variants.iter().map(|(variant_name, fields)| { let variant = Ident::new(variant_name, Span::call_site()); @@ -72,15 +72,10 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } }); - let fallthrough = if variants.len() == 1 { - None - } else { - Some(quote!(_ => false,)) - }; quote! { match (self, other) { #(#arms)* - #fallthrough + _ => false, } } } @@ -114,10 +109,11 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features, "extra-traits"); + let cfg_features = cfg::features(&node.features); let eq = quote! { #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl Eq for #ident {} }; @@ -127,16 +123,17 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let body = expand_impl_body(defs, node); - let other = match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(_other), - Data::Struct(fields) if fields.values().all(always_eq) => quote!(_other), - _ => quote!(other), + let other = if body.to_string() == "true" { + quote!(_other) + } else { + quote!(other) }; quote! { #eq #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl PartialEq for #ident { fn eq(&self, #other: &Self) -> bool { #body @@ -152,7 +149,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { } file::write( - EQ_SRC, + DEBUG_SRC, quote! { #[cfg(any(feature = "derive", feature = "full"))] use crate::tt::TokenStreamHelper; diff --git a/codegen/src/file.rs b/codegen/src/file.rs index beeb66017d46734563182d25e74e5bbb623b6399..002e303b8f021d2a43938c89bb26329b8a2c46db 100644 --- a/codegen/src/file.rs +++ b/codegen/src/file.rs @@ -1,11 +1,10 @@ -use crate::workspace_path; use anyhow::Result; use proc_macro2::TokenStream; use std::fs; use std::io::Write; use std::path::Path; -pub fn write(relative_to_workspace_root: impl AsRef, content: TokenStream) -> Result<()> { +pub fn write>(path: P, content: TokenStream) -> Result<()> { let mut formatted = Vec::new(); writeln!( formatted, @@ -18,8 +17,7 @@ pub fn write(relative_to_workspace_root: impl AsRef, content: TokenStream) let pretty = prettyplease::unparse(&syntax_tree); write!(formatted, "{}", pretty)?; - let path = workspace_path::get(relative_to_workspace_root); - if path.is_file() && fs::read(&path)? == formatted { + if path.as_ref().is_file() && fs::read(&path)? == formatted { return Ok(()); } diff --git a/codegen/src/fold.rs b/codegen/src/fold.rs index 13a5390a9e72f8ea6b3588476e9e6a4e83b2aaf6..281c70df45c2716594b3e56653066630716deea9 100644 --- a/codegen/src/fold.rs +++ b/codegen/src/fold.rs @@ -5,7 +5,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Features, Node, Type}; -const FOLD_SRC: &str = "src/gen/fold.rs"; +const FOLD_SRC: &str = "../src/gen/fold.rs"; fn simple_visit(item: &str, name: &TokenStream) -> TokenStream { let ident = gen::under_name(item); @@ -62,6 +62,29 @@ fn visit( (#code) }) } + Type::Token(t) => { + let repr = &defs.tokens[t]; + let is_keyword = repr.chars().next().unwrap().is_alphabetic(); + let spans = if is_keyword { + quote!(span) + } else { + quote!(spans) + }; + let ty = if repr == "await" { + quote!(crate::token::Await) + } else { + syn::parse_str(&format!("Token![{}]", repr)).unwrap() + }; + Some(quote! { + #ty(tokens_helper(f, &#name.#spans)) + }) + } + Type::Group(t) => { + let ty = Ident::new(t, Span::call_site()); + Some(quote! { + #ty(tokens_helper(f, &#name.span)) + }) + } Type::Syn(t) => { fn requires_full(features: &Features) -> bool { features.any.contains("full") && features.any.len() == 1 @@ -74,7 +97,7 @@ fn visit( Some(res) } Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), - Type::Ext(_) | Type::Std(_) | Type::Token(_) | Type::Group(_) => None, + Type::Ext(_) | Type::Std(_) => None, } } @@ -128,9 +151,19 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi } } + let nonexhaustive = if s.exhaustive { + None + } else { + Some(quote! { + #[cfg(syn_no_non_exhaustive)] + _ => unreachable!(), + }) + }; + fold_impl.extend(quote! { match node { #fold_variants + #nonexhaustive } }); } @@ -138,17 +171,32 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi let mut fold_fields = TokenStream::new(); for (field, ty) in fields { - let id = Ident::new(field, Span::call_site()); + let id = Ident::new(&field, Span::call_site()); let ref_toks = quote!(node.#id); - let fold = visit(ty, &s.features, defs, &ref_toks).unwrap_or(ref_toks); + if let Type::Syn(ty) = ty { + if ty == "Reserved" { + fold_fields.extend(quote! { + #id: #ref_toks, + }); + continue; + } + } + + let fold = visit(&ty, &s.features, defs, &ref_toks).unwrap_or(ref_toks); fold_fields.extend(quote! { #id: #fold, }); } - if fields.is_empty() { + if !fields.is_empty() { + fold_impl.extend(quote! { + #ty { + #fold_fields + } + }) + } else { if ty == "Ident" { fold_impl.extend(quote! { let mut node = node; @@ -159,12 +207,6 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi fold_impl.extend(quote! { node }); - } else { - fold_impl.extend(quote! { - #ty { - #fold_fields - } - }); } } Data::Private => { @@ -216,14 +258,12 @@ pub fn generate(defs: &Definitions) -> Result<()> { quote! { // Unreachable code is generated sometimes without the full feature. #![allow(unreachable_code, unused_variables)] - #![allow( - clippy::match_wildcard_for_single_variants, - clippy::needless_match, - clippy::needless_pass_by_ref_mut, - )] + #![allow(clippy::match_wildcard_for_single_variants)] #[cfg(any(feature = "full", feature = "derive"))] use crate::gen::helper::fold::*; + #[cfg(any(feature = "full", feature = "derive"))] + use crate::token::{Brace, Bracket, Group, Paren}; use crate::*; use proc_macro2::Span; @@ -234,6 +274,8 @@ pub fn generate(defs: &Definitions) -> Result<()> { /// See the [module documentation] for details. /// /// [module documentation]: self + /// + /// *This trait is available only if Syn is built with the `"fold"` feature.* pub trait Fold { #traits } diff --git a/codegen/src/gen.rs b/codegen/src/gen.rs index 57a9b68f320783e5fd84d194fd91d8c0f8d5ede1..3b2bb70ff3e5b91dc0ddc28bde499cab8f287a5f 100644 --- a/codegen/src/gen.rs +++ b/codegen/src/gen.rs @@ -14,9 +14,9 @@ pub fn traverse( node: fn(&mut TokenStream, &mut TokenStream, &Node, &Definitions), ) -> (TokenStream, TokenStream) { let mut types = defs.types.clone(); - for &terminal in TERMINAL_TYPES { + for terminal in TERMINAL_TYPES { types.push(Node { - ident: terminal.to_owned(), + ident: terminal.to_string(), features: Features::default(), data: Data::Private, exhaustive: true, @@ -27,7 +27,10 @@ pub fn traverse( let mut traits = TokenStream::new(); let mut impls = TokenStream::new(); for s in types { - let features = cfg::features(&s.features, None); + if s.ident == "Reserved" { + continue; + } + let features = cfg::features(&s.features); traits.extend(features.clone()); impls.extend(features); node(&mut traits, &mut impls, &s, defs); diff --git a/codegen/src/hash.rs b/codegen/src/hash.rs index c0868e2f09292ab15a9640f686515eee306c4409..c378e9de0e0de3923cc7e36318c5149e7f630613 100644 --- a/codegen/src/hash.rs +++ b/codegen/src/hash.rs @@ -4,10 +4,11 @@ use proc_macro2::{Ident, Span, TokenStream}; use quote::{format_ident, quote}; use syn_codegen::{Data, Definitions, Node, Type}; -const HASH_SRC: &str = "src/gen/hash.rs"; +const DEBUG_SRC: &str = "../src/gen/hash.rs"; fn skip(field_type: &Type) -> bool { match field_type { + Type::Syn(node) => node == "Reserved", Type::Ext(ty) => ty == "Span", Type::Token(_) | Type::Group(_) => true, Type::Box(inner) => skip(inner), @@ -21,13 +22,12 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { let ident = Ident::new(type_name, Span::call_site()); match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(match *self {}), Data::Enum(variants) => { let arms = variants .iter() .enumerate() .map(|(i, (variant_name, fields))| { - let i = u8::try_from(i).unwrap(); + let i = i as u8; let variant = Ident::new(variant_name, Span::call_site()); if fields.is_empty() { quote! { @@ -76,13 +76,18 @@ fn expand_impl_body(defs: &Definitions, node: &Node) -> TokenStream { } } }); - let nonexhaustive = if node.ident == "Expr" { + let nonexhaustive = if node.exhaustive { + None + } else if node.ident == "Expr" { Some(quote! { - #[cfg(not(feature = "full"))] + #[cfg(any(syn_no_non_exhaustive, not(feature = "full")))] _ => unreachable!(), }) } else { - None + Some(quote! { + #[cfg(syn_no_non_exhaustive)] + _ => unreachable!(), + }) }; quote! { match self { @@ -123,20 +128,20 @@ fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { } let ident = Ident::new(&node.ident, Span::call_site()); - let cfg_features = cfg::features(&node.features, "extra-traits"); + let cfg_features = cfg::features(&node.features); let body = expand_impl_body(defs, node); - - let hasher = match &node.data { - Data::Struct(_) if body.is_empty() => quote!(_state), - Data::Enum(variants) if variants.is_empty() => quote!(_state), - _ => quote!(state), + let state = if body.is_empty() { + quote!(_state) + } else { + quote!(state) }; quote! { #cfg_features + #[cfg_attr(doc_cfg, doc(cfg(feature = "extra-traits")))] impl Hash for #ident { - fn hash(&self, #hasher: &mut H) + fn hash(&self, #state: &mut H) where H: Hasher, { @@ -153,7 +158,7 @@ pub fn generate(defs: &Definitions) -> Result<()> { } file::write( - HASH_SRC, + DEBUG_SRC, quote! { #[cfg(any(feature = "derive", feature = "full"))] use crate::tt::TokenStreamHelper; diff --git a/codegen/src/json.rs b/codegen/src/json.rs index 28ceee8b56d55444909a4a90a59e0d931509e00d..53904bfb1c55f1d7bc87faa3b7a39969a3b0a420 100644 --- a/codegen/src/json.rs +++ b/codegen/src/json.rs @@ -1,6 +1,6 @@ -use crate::workspace_path; use anyhow::Result; use std::fs; +use std::path::Path; use syn_codegen::Definitions; pub fn generate(defs: &Definitions) -> Result<()> { @@ -10,7 +10,8 @@ pub fn generate(defs: &Definitions) -> Result<()> { let check: Definitions = serde_json::from_str(&j)?; assert_eq!(*defs, check); - let json_path = workspace_path::get("syn.json"); + let codegen_root = Path::new(env!("CARGO_MANIFEST_DIR")); + let json_path = codegen_root.join("../syn.json"); fs::write(json_path, j)?; Ok(()) diff --git a/codegen/src/main.rs b/codegen/src/main.rs index 8b8670c5e5740ea22b5b5c9e17b45b2e96cc4f64..022494ef135bb4c7eb39571008bb26953daad2d7 100644 --- a/codegen/src/main.rs +++ b/codegen/src/main.rs @@ -9,14 +9,7 @@ // Finally this crate generates the Visit, VisitMut, and Fold traits in Syn // programmatically from the syntax tree description. -#![allow( - clippy::items_after_statements, - clippy::manual_let_else, - clippy::match_like_matches_macro, - clippy::similar_names, - clippy::too_many_lines, - clippy::uninlined_format_args -)] +#![allow(clippy::needless_pass_by_value)] mod cfg; mod clone; @@ -35,7 +28,6 @@ mod snapshot; mod version; mod visit; mod visit_mut; -mod workspace_path; fn main() -> anyhow::Result<()> { color_backtrace::install(); diff --git a/codegen/src/parse.rs b/codegen/src/parse.rs index f232fedff3730bce75543eab580590053c07ef80..716ba0dc7cbab0f602d5c4a470f3697d84770247 100644 --- a/codegen/src/parse.rs +++ b/codegen/src/parse.rs @@ -1,4 +1,4 @@ -use crate::{version, workspace_path}; +use crate::version; use anyhow::{bail, Result}; use indexmap::IndexMap; use quote::quote; @@ -8,46 +8,35 @@ use std::path::{Path, PathBuf}; use syn::parse::{Error, Parser}; use syn::{ parse_quote, Attribute, Data, DataEnum, DataStruct, DeriveInput, Fields, GenericArgument, - Ident, Item, PathArguments, TypeMacro, TypePath, TypeTuple, UseTree, Visibility, + Ident, Item, PathArguments, TypeMacro, TypePath, TypeTuple, Visibility, }; use syn_codegen as types; use thiserror::Error; -const SYN_CRATE_ROOT: &str = "src/lib.rs"; -const TOKEN_SRC: &str = "src/token.rs"; +const SYN_CRATE_ROOT: &str = "../src/lib.rs"; +const TOKEN_SRC: &str = "../src/token.rs"; const IGNORED_MODS: &[&str] = &["fold", "visit", "visit_mut"]; const EXTRA_TYPES: &[&str] = &["Lifetime"]; -struct Lookup { - items: BTreeMap, - // "+" => "Add" - tokens: BTreeMap, - // "PatLit" => "ExprLit" - aliases: BTreeMap, -} +// NOTE: BTreeMap is used here instead of HashMap to have deterministic output. +type ItemLookup = BTreeMap; +type TokenLookup = BTreeMap; /// Parse the contents of `src` and return a list of AST types. pub fn parse() -> Result { - let tokens = load_token_file(TOKEN_SRC)?; - - let mut lookup = Lookup { - items: BTreeMap::new(), - tokens, - aliases: BTreeMap::new(), - }; + let mut item_lookup = BTreeMap::new(); + load_file(SYN_CRATE_ROOT, &[], &mut item_lookup)?; - load_file(SYN_CRATE_ROOT, &[], &mut lookup)?; + let token_lookup = load_token_file(TOKEN_SRC)?; let version = version::get()?; - let types = lookup - .items + let types = item_lookup .values() - .map(|item| introspect_item(item, &lookup)) + .map(|item| introspect_item(item, &item_lookup, &token_lookup)) .collect(); - let tokens = lookup - .tokens + let tokens = token_lookup .into_iter() .map(|(name, ty)| (ty, name)) .collect(); @@ -65,23 +54,22 @@ pub struct AstItem { features: Vec, } -fn introspect_item(item: &AstItem, lookup: &Lookup) -> types::Node { +fn introspect_item(item: &AstItem, items: &ItemLookup, tokens: &TokenLookup) -> types::Node { let features = introspect_features(&item.features); match &item.ast.data { - Data::Enum(data) => types::Node { + Data::Enum(ref data) => types::Node { ident: item.ast.ident.to_string(), features, - data: types::Data::Enum(introspect_enum(data, lookup)), - exhaustive: !(is_non_exhaustive(&item.ast.attrs) - || data.variants.iter().any(|v| is_doc_hidden(&v.attrs))), + data: types::Data::Enum(introspect_enum(data, items, tokens)), + exhaustive: !data.variants.iter().any(|v| is_doc_hidden(&v.attrs)), }, - Data::Struct(data) => types::Node { + Data::Struct(ref data) => types::Node { ident: item.ast.ident.to_string(), features, data: { if data.fields.iter().all(|f| is_pub(&f.vis)) { - types::Data::Struct(introspect_struct(data, lookup)) + types::Data::Struct(introspect_struct(data, items, tokens)) } else { types::Data::Private } @@ -92,7 +80,7 @@ fn introspect_item(item: &AstItem, lookup: &Lookup) -> types::Node { } } -fn introspect_enum(item: &DataEnum, lookup: &Lookup) -> types::Variants { +fn introspect_enum(item: &DataEnum, items: &ItemLookup, tokens: &TokenLookup) -> types::Variants { item.variants .iter() .filter_map(|variant| { @@ -103,17 +91,17 @@ fn introspect_enum(item: &DataEnum, lookup: &Lookup) -> types::Variants { Fields::Unnamed(fields) => fields .unnamed .iter() - .map(|field| introspect_type(&field.ty, lookup)) + .map(|field| introspect_type(&field.ty, items, tokens)) .collect(), Fields::Unit => vec![], - Fields::Named(_) => panic!("Enum representation not supported"), + _ => panic!("Enum representation not supported"), }; Some((variant.ident.to_string(), fields)) }) .collect() } -fn introspect_struct(item: &DataStruct, lookup: &Lookup) -> types::Fields { +fn introspect_struct(item: &DataStruct, items: &ItemLookup, tokens: &TokenLookup) -> types::Fields { match &item.fields { Fields::Named(fields) => fields .named @@ -121,71 +109,74 @@ fn introspect_struct(item: &DataStruct, lookup: &Lookup) -> types::Fields { .map(|field| { ( field.ident.as_ref().unwrap().to_string(), - introspect_type(&field.ty, lookup), + introspect_type(&field.ty, items, tokens), ) }) .collect(), Fields::Unit => IndexMap::new(), - Fields::Unnamed(_) => panic!("Struct representation not supported"), + _ => panic!("Struct representation not supported"), } } -fn introspect_type(item: &syn::Type, lookup: &Lookup) -> types::Type { +fn introspect_type(item: &syn::Type, items: &ItemLookup, tokens: &TokenLookup) -> types::Type { match item { - syn::Type::Path(TypePath { qself: None, path }) => { + syn::Type::Path(TypePath { + qself: None, + ref path, + }) => { let last = path.segments.last().unwrap(); let string = last.ident.to_string(); match string.as_str() { "Option" => { - let nested = introspect_type(first_arg(&last.arguments), lookup); + let nested = introspect_type(first_arg(&last.arguments), items, tokens); types::Type::Option(Box::new(nested)) } "Punctuated" => { - let nested = introspect_type(first_arg(&last.arguments), lookup); - let types::Type::Token(punct) = - introspect_type(last_arg(&last.arguments), lookup) - else { - panic!() + let nested = introspect_type(first_arg(&last.arguments), items, tokens); + let punct = match introspect_type(last_arg(&last.arguments), items, tokens) { + types::Type::Token(s) => s, + _ => panic!(), }; + types::Type::Punctuated(types::Punctuated { element: Box::new(nested), punct, }) } "Vec" => { - let nested = introspect_type(first_arg(&last.arguments), lookup); + let nested = introspect_type(first_arg(&last.arguments), items, tokens); types::Type::Vec(Box::new(nested)) } "Box" => { - let nested = introspect_type(first_arg(&last.arguments), lookup); + let nested = introspect_type(first_arg(&last.arguments), items, tokens); types::Type::Box(Box::new(nested)) } "Brace" | "Bracket" | "Paren" | "Group" => types::Type::Group(string), "TokenStream" | "Literal" | "Ident" | "Span" => types::Type::Ext(string), "String" | "u32" | "usize" | "bool" => types::Type::Std(string), + "Await" => types::Type::Token("Await".to_string()), _ => { - let mut resolved = &last.ident; - while let Some(alias) = lookup.aliases.get(resolved) { - resolved = alias; - } - if lookup.items.get(resolved).is_some() { - types::Type::Syn(resolved.to_string()) + if items.get(&last.ident).is_some() || last.ident == "Reserved" { + types::Type::Syn(string) } else { - unimplemented!("{}", resolved); + unimplemented!("{}", string); } } } } - syn::Type::Tuple(TypeTuple { elems, .. }) => { - let tys = elems.iter().map(|ty| introspect_type(ty, lookup)).collect(); + syn::Type::Tuple(TypeTuple { ref elems, .. }) => { + let tys = elems + .iter() + .map(|ty| introspect_type(&ty, items, tokens)) + .collect(); types::Type::Tuple(tys) } - syn::Type::Macro(TypeMacro { mac }) + syn::Type::Macro(TypeMacro { ref mac }) if mac.path.segments.last().unwrap().ident == "Token" => { let content = mac.tokens.to_string(); - let ty = lookup.tokens.get(&content).unwrap().to_string(); + let ty = tokens.get(&content).unwrap().to_string(); types::Type::Token(ty) } @@ -197,11 +188,11 @@ fn introspect_features(attrs: &[Attribute]) -> types::Features { let mut ret = types::Features::default(); for attr in attrs { - if !attr.path().is_ident("cfg") { + if !attr.path.is_ident("cfg") { continue; } - let features = attr.parse_args_with(parsing::parse_features).unwrap(); + let features = parsing::parse_features.parse2(attr.tokens.clone()).unwrap(); if ret.any.is_empty() { ret = features; @@ -223,65 +214,60 @@ fn is_pub(vis: &Visibility) -> bool { } } -fn is_non_exhaustive(attrs: &[Attribute]) -> bool { - for attr in attrs { - if attr.path().is_ident("non_exhaustive") { - return true; - } - } - false -} - fn is_doc_hidden(attrs: &[Attribute]) -> bool { for attr in attrs { - if attr.path().is_ident("doc") && attr.parse_args::().is_ok() { - return true; + if attr.path.is_ident("doc") { + if parsing::parse_doc_hidden_attr + .parse2(attr.tokens.clone()) + .is_ok() + { + return true; + } } } false } fn first_arg(params: &PathArguments) -> &syn::Type { - let data = match params { - PathArguments::AngleBracketed(data) => data, + let data = match *params { + PathArguments::AngleBracketed(ref data) => data, _ => panic!("Expected at least 1 type argument here"), }; - match data + match *data .args .first() .expect("Expected at least 1 type argument here") { - GenericArgument::Type(ty) => ty, + GenericArgument::Type(ref ty) => ty, _ => panic!("Expected at least 1 type argument here"), } } fn last_arg(params: &PathArguments) -> &syn::Type { - let data = match params { - PathArguments::AngleBracketed(data) => data, + let data = match *params { + PathArguments::AngleBracketed(ref data) => data, _ => panic!("Expected at least 1 type argument here"), }; - match data + match *data .args .last() .expect("Expected at least 1 type argument here") { - GenericArgument::Type(ty) => ty, + GenericArgument::Type(ref ty) => ty, _ => panic!("Expected at least 1 type argument here"), } } mod parsing { - use super::AstItem; - use proc_macro2::TokenStream; + use super::{AstItem, TokenLookup}; + use proc_macro2::{TokenStream, TokenTree}; use quote::quote; use std::collections::{BTreeMap, BTreeSet}; - use syn::parse::{ParseStream, Result}; + use syn::parse::{ParseStream, Parser, Result}; use syn::{ - braced, bracketed, parenthesized, parse_quote, token, Attribute, Expr, Ident, Lit, LitStr, - Path, Token, + braced, bracketed, parenthesized, parse_quote, token, Attribute, Ident, LitStr, Path, Token, }; use syn_codegen as types; @@ -327,18 +313,32 @@ mod parsing { Ok(res) } - pub fn ast_enum(input: ParseStream) -> Result { - let attrs = input.call(Attribute::parse_outer)?; + fn no_visit(input: ParseStream) -> bool { + if peek_tag(input, "no_visit") { + input.parse::().unwrap(); + input.parse::().unwrap(); + true + } else { + false + } + } + + pub fn ast_enum(input: ParseStream) -> Result> { + input.call(Attribute::parse_outer)?; input.parse::()?; input.parse::()?; let ident: Ident = input.parse()?; + let no_visit = no_visit(input); let rest: TokenStream = input.parse()?; - Ok(AstItem { - ast: syn::parse2(quote! { - #(#attrs)* - pub enum #ident #rest - })?, - features: vec![], + Ok(if no_visit { + None + } else { + Some(AstItem { + ast: syn::parse2(quote! { + pub enum #ident #rest + })?, + features: vec![], + }) }) } @@ -368,7 +368,7 @@ mod parsing { } pub fn ast_enum_of_structs(input: ParseStream) -> Result { - let attrs = input.call(Attribute::parse_outer)?; + input.call(Attribute::parse_outer)?; input.parse::()?; input.parse::()?; let ident: Ident = input.parse()?; @@ -380,18 +380,20 @@ mod parsing { variants.push(content.call(eos_variant)?); } + if let Some(ident) = input.parse::>()? { + assert_eq!(ident, "do_not_generate_to_tokens"); + } + let enum_item = { let variants = variants.iter().map(|v| { let attrs = &v.attrs; let name = &v.name; - if let Some(member) = &v.member { - quote!(#(#attrs)* #name(#member)) - } else { - quote!(#(#attrs)* #name) + match v.member { + Some(ref member) => quote!(#(#attrs)* #name(#member)), + None => quote!(#(#attrs)* #name), } }); parse_quote! { - #(#attrs)* pub enum #ident { #(#variants),* } @@ -403,26 +405,47 @@ mod parsing { }) } - pub mod kw { + mod kw { syn::custom_keyword!(hidden); syn::custom_keyword!(macro_rules); syn::custom_keyword!(Token); } - pub fn parse_token_macro(input: ParseStream) -> Result> { + pub fn parse_token_macro(input: ParseStream) -> Result { + input.parse::()?; + input.parse::]>()?; + + let definition; + braced!(definition in input); + definition.call(Attribute::parse_outer)?; + definition.parse::()?; + definition.parse::()?; + definition.parse::()?; + + let rules; + braced!(rules in definition); + input.parse::()?; + let mut tokens = BTreeMap::new(); - while !input.is_empty() { - let pattern; - bracketed!(pattern in input); - let token = pattern.parse::()?.to_string(); - input.parse::]>()?; - let expansion; - braced!(expansion in input); - input.parse::()?; - expansion.parse::()?; - let path: Path = expansion.parse()?; - let ty = path.segments.last().unwrap().ident.to_string(); - tokens.insert(token, ty.to_string()); + while !rules.is_empty() { + if rules.peek(Token![$]) { + rules.parse::()?; + rules.parse::()?; + rules.parse::()?; + tokens.insert("await".to_owned(), "Await".to_owned()); + } else { + let pattern; + bracketed!(pattern in rules); + let token = pattern.parse::()?.to_string(); + rules.parse::]>()?; + let expansion; + braced!(expansion in rules); + rules.parse::()?; + expansion.parse::()?; + let path: Path = expansion.parse()?; + let ty = path.segments.last().unwrap().ident.to_string(); + tokens.insert(token, ty.to_string()); + } } Ok(tokens) } @@ -440,43 +463,56 @@ mod parsing { pub fn parse_features(input: ParseStream) -> Result { let mut features = BTreeSet::new(); - let i: Ident = input.fork().parse()?; + let level_1; + parenthesized!(level_1 in input); + + let i: Ident = level_1.fork().parse()?; if i == "any" { - input.parse::()?; + level_1.parse::()?; - let nested; - parenthesized!(nested in input); + let level_2; + parenthesized!(level_2 in level_1); - while !nested.is_empty() { - features.insert(parse_feature(&nested)?); + while !level_2.is_empty() { + features.insert(parse_feature(&level_2)?); - if !nested.is_empty() { - nested.parse::()?; + if !level_2.is_empty() { + level_2.parse::()?; } } } else if i == "feature" { - features.insert(parse_feature(input)?); - assert!(input.is_empty()); + features.insert(parse_feature(&level_1)?); + assert!(level_1.is_empty()); } else { panic!("{:?}", i); } + assert!(input.is_empty()); + Ok(types::Features { any: features }) } - pub fn path_attr(attrs: &[Attribute]) -> Result> { + pub fn path_attr(attrs: &[Attribute]) -> Result> { for attr in attrs { - if attr.path().is_ident("path") { - if let Expr::Lit(expr) = &attr.meta.require_name_value()?.value { - if let Lit::Str(lit) = &expr.lit { - return Ok(Some(lit)); - } + if attr.path.is_ident("path") { + fn parser(input: ParseStream) -> Result { + input.parse::()?; + input.parse() } + let filename = parser.parse2(attr.tokens.clone())?; + return Ok(Some(filename)); } } Ok(None) } + + pub fn parse_doc_hidden_attr(input: ParseStream) -> Result<()> { + let content; + parenthesized!(content in input); + content.parse::()?; + Ok(()) + } } fn clone_features(features: &[Attribute]) -> Vec { @@ -487,7 +523,7 @@ fn get_features(attrs: &[Attribute], base: &[Attribute]) -> Vec { let mut ret = clone_features(base); for attr in attrs { - if attr.path().is_ident("cfg") { + if attr.path.is_ident("cfg") { ret.push(parse_quote!(#attr)); } } @@ -504,12 +540,12 @@ struct LoadFileError { error: Error, } -fn load_file( - relative_to_workspace_root: impl AsRef, +fn load_file>( + name: P, features: &[Attribute], - lookup: &mut Lookup, + lookup: &mut ItemLookup, ) -> Result<()> { - let error = match do_load_file(&relative_to_workspace_root, features, lookup).err() { + let error = match do_load_file(&name, features, lookup).err() { None => return Ok(()), Some(error) => error, }; @@ -518,23 +554,23 @@ fn load_file( let span = error.span().start(); bail!(LoadFileError { - path: relative_to_workspace_root.as_ref().to_owned(), + path: name.as_ref().to_owned(), line: span.line, column: span.column + 1, error, }) } -fn do_load_file( - relative_to_workspace_root: impl AsRef, +fn do_load_file>( + name: P, features: &[Attribute], - lookup: &mut Lookup, + lookup: &mut ItemLookup, ) -> Result<()> { - let relative_to_workspace_root = relative_to_workspace_root.as_ref(); - let parent = relative_to_workspace_root.parent().expect("no parent path"); + let name = name.as_ref(); + let parent = name.parent().expect("no parent path"); // Parse the file - let src = fs::read_to_string(workspace_path::get(relative_to_workspace_root))?; + let src = fs::read_to_string(name)?; let file = syn::parse_file(&src)?; // Collect all of the interesting AstItems declared in this file or submodules. @@ -567,10 +603,9 @@ fn do_load_file( // Look up the submodule file, and recursively parse it. // Only handles same-directory .rs file submodules for now. - let filename = if let Some(filename) = parsing::path_attr(&item.attrs)? { - filename.value() - } else { - format!("{}.rs", item.ident) + let filename = match parsing::path_attr(&item.attrs)? { + Some(filename) => filename.value(), + None => format!("{}.rs", item.ident), }; let path = parent.join(filename); load_file(path, &features, lookup)?; @@ -582,24 +617,28 @@ fn do_load_file( // Try to parse the AstItem declaration out of the item. let tts = item.mac.tokens.clone(); - let mut found = if item.mac.path.is_ident("ast_struct") { - parsing::ast_struct.parse2(tts) + let found = if item.mac.path.is_ident("ast_struct") { + Some(parsing::ast_struct.parse2(tts)?) } else if item.mac.path.is_ident("ast_enum") { - parsing::ast_enum.parse2(tts) + parsing::ast_enum.parse2(tts)? } else if item.mac.path.is_ident("ast_enum_of_structs") { - parsing::ast_enum_of_structs.parse2(tts) + Some(parsing::ast_enum_of_structs.parse2(tts)?) } else { continue; - }?; + }; // Record our features on the parsed AstItems. - found.features.extend(clone_features(&features)); - lookup.items.insert(found.ast.ident.clone(), found); + if let Some(mut item) = found { + if item.ast.ident != "Reserved" { + item.features.extend(clone_features(&features)); + lookup.insert(item.ast.ident.clone(), item); + } + } } Item::Struct(item) => { let ident = item.ident; if EXTRA_TYPES.contains(&&ident.to_string()[..]) { - lookup.items.insert( + lookup.insert( ident.clone(), AstItem { ast: DeriveInput { @@ -618,43 +657,20 @@ fn do_load_file( ); } } - Item::Use(item) - if relative_to_workspace_root == Path::new(SYN_CRATE_ROOT) - && matches!(item.vis, Visibility::Public(_)) => - { - load_aliases(item.tree, lookup); - } _ => {} } } Ok(()) } -fn load_aliases(use_tree: UseTree, lookup: &mut Lookup) { - match use_tree { - UseTree::Path(use_tree) => load_aliases(*use_tree.tree, lookup), - UseTree::Rename(use_tree) => { - lookup.aliases.insert(use_tree.rename, use_tree.ident); - } - UseTree::Group(use_tree) => { - for use_tree in use_tree.items { - load_aliases(use_tree, lookup); - } - } - UseTree::Name(_) | UseTree::Glob(_) => {} - } -} - -fn load_token_file( - relative_to_workspace_root: impl AsRef, -) -> Result> { - let path = workspace_path::get(relative_to_workspace_root); - let src = fs::read_to_string(path)?; +fn load_token_file>(name: P) -> Result { + let name = name.as_ref(); + let src = fs::read_to_string(name)?; let file = syn::parse_file(&src)?; for item in file.items { if let Item::Macro(item) = item { match item.ident { - Some(i) if i == "Token" => {} + Some(ref i) if i == "export_token_macro" => {} _ => continue, } let tokens = item.mac.parse_body_with(parsing::parse_token_macro)?; diff --git a/codegen/src/snapshot.rs b/codegen/src/snapshot.rs index 17588dbcc07b1a5452d010cb52666350f1eb986f..46c6a8f1c27a9963a92d35a397f2a2d932938b73 100644 --- a/codegen/src/snapshot.rs +++ b/codegen/src/snapshot.rs @@ -1,4 +1,3 @@ -use crate::operand::{Borrowed, Operand, Owned}; use crate::{file, lookup}; use anyhow::Result; use proc_macro2::{Ident, Span, TokenStream}; @@ -6,7 +5,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Node, Type}; -const TESTS_DEBUG_SRC: &str = "tests/debug/gen.rs"; +const DEBUG_SRC: &str = "../tests/debug/gen.rs"; fn rust_type(ty: &Type) -> TokenStream { match ty { @@ -56,44 +55,44 @@ fn is_printable(ty: &Type) -> bool { Type::Box(ty) => is_printable(ty), Type::Tuple(ty) => ty.iter().any(is_printable), Type::Token(_) | Type::Group(_) => false, - Type::Syn(_) | Type::Std(_) | Type::Punctuated(_) | Type::Option(_) | Type::Vec(_) => true, + Type::Syn(name) => name != "Reserved", + Type::Std(_) | Type::Punctuated(_) | Type::Option(_) | Type::Vec(_) => true, } } -fn format_field(val: &Operand, ty: &Type) -> Option { +fn format_field(val: &TokenStream, ty: &Type) -> Option { if !is_printable(ty) { return None; } let format = match ty { Type::Option(ty) => { - if let Some(format) = format_field(&Borrowed(quote!(_val)), ty) { - let ty = rust_type(ty); - let val = val.ref_tokens(); - quote!({ - #[derive(RefCast)] - #[repr(transparent)] - struct Print(Option<#ty>); - impl Debug for Print { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - match &self.0 { - Some(_val) => { - formatter.write_str("Some(")?; - Debug::fmt(#format, formatter)?; - formatter.write_str(")")?; - Ok(()) - } - None => formatter.write_str("None"), + let inner = quote!(_val); + let format = format_field(&inner, ty).map(|format| { + quote! { + formatter.write_str("(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; + } + }); + let ty = rust_type(ty); + quote!({ + #[derive(RefCast)] + #[repr(transparent)] + struct Print(Option<#ty>); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + match &self.0 { + Some(#inner) => { + formatter.write_str("Some")?; + #format + Ok(()) } + None => formatter.write_str("None"), } } - Print::ref_cast(#val) - }) - } else { - let val = val.tokens(); - quote! { - &super::Option { present: #val.is_some() } } - } + Print::ref_cast(#val) + }) } Type::Tuple(ty) => { let printable: Vec = ty @@ -101,9 +100,8 @@ fn format_field(val: &Operand, ty: &Type) -> Option { .enumerate() .filter_map(|(i, ty)| { let index = Index::from(i); - let val = val.tokens(); - let inner = Owned(quote!(#val.#index)); - format_field(&inner, ty) + let val = quote!(&#val.#index); + format_field(&val, ty) }) .collect(); if printable.len() == 1 { @@ -114,10 +112,7 @@ fn format_field(val: &Operand, ty: &Type) -> Option { } } } - _ => { - let val = val.ref_tokens(); - quote! { Lite(#val) } - } + _ => quote! { Lite(#val) }, }; Some(format) } @@ -126,43 +121,27 @@ fn syntax_tree_enum<'a>(outer: &str, inner: &str, fields: &'a [Type]) -> Option< if fields.len() != 1 { return None; } - const WHITELIST: &[(&str, &str)] = &[ - ("Meta", "Path"), - ("PathArguments", "AngleBracketed"), - ("PathArguments", "Parenthesized"), - ("Stmt", "Local"), - ("TypeParamBound", "Lifetime"), - ("Visibility", "Public"), - ("Visibility", "Restricted"), - ]; + const WHITELIST: &[&str] = &["PathArguments", "Visibility"]; match &fields[0] { - Type::Syn(ty) if WHITELIST.contains(&(outer, inner)) || outer.to_owned() + inner == *ty => { - Some(ty) - } + Type::Syn(ty) if WHITELIST.contains(&outer) || outer.to_owned() + inner == *ty => Some(ty), _ => None, } } -fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) -> TokenStream { +fn expand_impl_body(defs: &Definitions, node: &Node, name: &str) -> TokenStream { let ident = Ident::new(&node.ident, Span::call_site()); match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(unreachable!()), Data::Enum(variants) => { let arms = variants.iter().map(|(v, fields)| { - let path = format!("{}::{}", name, v); let variant = Ident::new(v, Span::call_site()); if fields.is_empty() { quote! { - syn::#ident::#variant => formatter.write_str(#path), + syn::#ident::#variant => formatter.write_str(#v), } } else if let Some(inner) = syntax_tree_enum(name, v, fields) { - let format = expand_impl_body( - defs, - lookup::node(defs, inner), - &path, - &Borrowed(quote!(_val)), - ); + let path = format!("{}::{}", name, v); + let format = expand_impl_body(defs, lookup::node(defs, inner), &path); quote! { syn::#ident::#variant(_val) => { #format @@ -178,7 +157,7 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) }) } else { let ty = &fields[0]; - format_field(&Borrowed(val), ty).map(|format| { + format_field(&val, ty).map(|format| { quote! { formatter.write_str("(")?; Debug::fmt(#format, formatter)?; @@ -188,7 +167,7 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) }; quote! { syn::#ident::#variant(_val) => { - formatter.write_str(#path)?; + formatter.write_str(#v)?; #format Ok(()) } @@ -198,14 +177,14 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) let fields = fields.iter().enumerate().filter_map(|(i, ty)| { let index = format_ident!("_v{}", i); let val = quote!(#index); - let format = format_field(&Borrowed(val), ty)?; + let format = format_field(&val, ty)?; Some(quote! { formatter.field(#format); }) }); quote! { syn::#ident::#variant(#(#pats),*) => { - let mut formatter = formatter.debug_tuple(#path); + let mut formatter = formatter.debug_tuple(#v); #(#fields)* formatter.finish() } @@ -217,9 +196,8 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) } else { Some(quote!(_ => unreachable!())) }; - let val = val.ref_tokens(); quote! { - match #val { + match _val { #(#arms)* #nonexhaustive } @@ -229,65 +207,43 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) let fields = fields.iter().filter_map(|(f, ty)| { let ident = Ident::new(f, Span::call_site()); if let Type::Option(ty) = ty { - Some(if let Some(format) = format_field(&Owned(quote!(self.0)), ty) { - let val = val.tokens(); - let ty = rust_type(ty); + let inner = quote!(_val); + let format = format_field(&inner, ty).map(|format| { quote! { - if let Some(val) = &#val.#ident { - #[derive(RefCast)] - #[repr(transparent)] - struct Print(#ty); - impl Debug for Print { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("Some(")?; - Debug::fmt(#format, formatter)?; - formatter.write_str(")")?; - Ok(()) - } - } - formatter.field(#f, Print::ref_cast(val)); - } + let #inner = &self.0; + formatter.write_str("(")?; + Debug::fmt(#format, formatter)?; + formatter.write_str(")")?; } - } else { - let val = val.tokens(); - quote! { - if #val.#ident.is_some() { - formatter.field(#f, &Present); + }); + let ty = rust_type(ty); + Some(quote! { + if let Some(val) = &_val.#ident { + #[derive(RefCast)] + #[repr(transparent)] + struct Print(#ty); + impl Debug for Print { + fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("Some")?; + #format + Ok(()) + } } + formatter.field(#f, Print::ref_cast(val)); } }) } else { - let val = val.tokens(); - let inner = Owned(quote!(#val.#ident)); - let format = format_field(&inner, ty)?; + let val = quote!(&_val.#ident); + let format = format_field(&val, ty)?; let mut call = quote! { formatter.field(#f, #format); }; - if node.ident == "Block" && f == "stmts" { - // Format regardless of whether is_empty(). - } else if let Type::Vec(_) | Type::Punctuated(_) = ty { + if let Type::Vec(_) | Type::Punctuated(_) = ty { call = quote! { - if !#val.#ident.is_empty() { + if !_val.#ident.is_empty() { #call } }; - } else if let Type::Syn(inner) = ty { - for node in &defs.types { - if node.ident == *inner { - if let Data::Enum(variants) = &node.data { - if variants.get("None").map_or(false, Vec::is_empty) { - let ty = rust_type(ty); - call = quote! { - match #val.#ident { - #ty::None => {} - _ => { #call } - } - }; - } - } - break; - } - } } Some(call) } @@ -300,14 +256,12 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) } Data::Private => { if node.ident == "LitInt" || node.ident == "LitFloat" { - let val = val.ref_tokens(); quote! { - write!(formatter, "{}", #val) + write!(formatter, "{}", _val) } } else { - let val = val.tokens(); quote! { - write!(formatter, "{:?}", #val.value()) + write!(formatter, "{:?}", _val.value()) } } } @@ -315,30 +269,18 @@ fn expand_impl_body(defs: &Definitions, node: &Node, name: &str, val: &Operand) } fn expand_impl(defs: &Definitions, node: &Node) -> TokenStream { - let ident = Ident::new(&node.ident, Span::call_site()); - let body = expand_impl_body(defs, node, &node.ident, &Owned(quote!(self.value))); - let formatter = match &node.data { - Data::Enum(variants) if variants.is_empty() => quote!(_formatter), - _ => quote!(formatter), - }; - - quote! { - impl Debug for Lite { - fn fmt(&self, #formatter: &mut fmt::Formatter) -> fmt::Result { - #body - } - } + if node.ident == "Reserved" { + return TokenStream::new(); } -} -fn expand_token_impl(name: &str, symbol: &str) -> TokenStream { - let ident = Ident::new(name, Span::call_site()); - let repr = format!("Token![{}]", symbol); + let ident = Ident::new(&node.ident, Span::call_site()); + let body = expand_impl_body(defs, node, &node.ident); quote! { - impl Debug for Lite { + impl Debug for Lite { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str(#repr) + let _val = &self.value; + #body } } } @@ -347,22 +289,13 @@ fn expand_token_impl(name: &str, symbol: &str) -> TokenStream { pub fn generate(defs: &Definitions) -> Result<()> { let mut impls = TokenStream::new(); for node in &defs.types { - impls.extend(expand_impl(defs, node)); - } - for (name, symbol) in &defs.tokens { - impls.extend(expand_token_impl(name, symbol)); + impls.extend(expand_impl(&defs, node)); } file::write( - TESTS_DEBUG_SRC, + DEBUG_SRC, quote! { - // False positive: https://github.com/rust-lang/rust/issues/78586#issuecomment-1722680482 - #![allow(repr_transparent_external_private_fields)] - - #![allow(clippy::match_wildcard_for_single_variants)] - - use super::{Lite, Present}; - use ref_cast::RefCast; + use super::{Lite, RefCast}; use std::fmt::{self, Debug, Display}; #impls diff --git a/codegen/src/version.rs b/codegen/src/version.rs index db6a8f8cde6f3b2b7b3828302739088f3a845a5f..3a1f967c7303072f38b9c82ef0abb88bf42fada9 100644 --- a/codegen/src/version.rs +++ b/codegen/src/version.rs @@ -1,11 +1,12 @@ -use crate::workspace_path; use anyhow::Result; use semver::Version; -use serde_derive::Deserialize; +use serde::Deserialize; use std::fs; +use std::path::Path; pub fn get() -> Result { - let syn_cargo_toml = workspace_path::get("Cargo.toml"); + let codegen_root = Path::new(env!("CARGO_MANIFEST_DIR")); + let syn_cargo_toml = codegen_root.join("../Cargo.toml"); let manifest = fs::read_to_string(syn_cargo_toml)?; let parsed: Manifest = toml::from_str(&manifest)?; Ok(parsed.package.version) diff --git a/codegen/src/visit.rs b/codegen/src/visit.rs index 573798a1f5e913ee91e4f04da96dab0db847d0fb..9576d098be8ac800afeb3e62eae66bf85baf856a 100644 --- a/codegen/src/visit.rs +++ b/codegen/src/visit.rs @@ -6,7 +6,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Features, Node, Type}; -const VISIT_SRC: &str = "src/gen/visit.rs"; +const VISIT_SRC: &str = "../src/gen/visit.rs"; fn simple_visit(item: &str, name: &Operand) -> TokenStream { let ident = gen::under_name(item); @@ -51,17 +51,20 @@ fn visit( let name = name.ref_tokens(); Some(quote! { for el in Punctuated::pairs(#name) { - let it = el.value(); + let (it, p) = el.into_tuple(); #val; + if let Some(p) = p { + tokens_helper(v, &p.spans); + } } }) } Type::Option(t) => { let it = Borrowed(quote!(it)); let val = visit(t, features, defs, &it)?; - let name = name.ref_tokens(); + let name = name.owned_tokens(); Some(quote! { - if let Some(it) = #name { + if let Some(it) = &#name { #val; } }) @@ -78,6 +81,25 @@ fn visit( } Some(code) } + Type::Token(t) => { + let name = name.tokens(); + let repr = &defs.tokens[t]; + let is_keyword = repr.chars().next().unwrap().is_alphabetic(); + let spans = if is_keyword { + quote!(span) + } else { + quote!(spans) + }; + Some(quote! { + tokens_helper(v, &#name.#spans); + }) + } + Type::Group(_) => { + let name = name.tokens(); + Some(quote! { + tokens_helper(v, &#name.span); + }) + } Type::Syn(t) => { fn requires_full(features: &Features) -> bool { features.any.contains("full") && features.any.len() == 1 @@ -90,7 +112,7 @@ fn visit( Some(res) } Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), - Type::Ext(_) | Type::Std(_) | Type::Token(_) | Type::Group(_) => None, + Type::Ext(_) | Type::Std(_) => None, } } @@ -102,11 +124,6 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi let mut visit_impl = TokenStream::new(); match &s.data { - Data::Enum(variants) if variants.is_empty() => { - visit_impl.extend(quote! { - match *node {} - }); - } Data::Enum(variants) => { let mut visit_variants = TokenStream::new(); @@ -146,17 +163,33 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi } } + let nonexhaustive = if s.exhaustive { + None + } else { + Some(quote! { + #[cfg(syn_no_non_exhaustive)] + _ => unreachable!(), + }) + }; + visit_impl.extend(quote! { match node { #visit_variants + #nonexhaustive } }); } Data::Struct(fields) => { for (field, ty) in fields { - let id = Ident::new(field, Span::call_site()); + if let Type::Syn(ty) = ty { + if ty == "Reserved" { + continue; + } + } + + let id = Ident::new(&field, Span::call_site()); let ref_toks = Owned(quote!(node.#id)); - let visit_field = visit(ty, &s.features, defs, &ref_toks) + let visit_field = visit(&ty, &s.features, defs, &ref_toks) .unwrap_or_else(|| noop_visit(&ref_toks)); visit_impl.extend(quote! { #visit_field; @@ -201,8 +234,9 @@ pub fn generate(defs: &Definitions) -> Result<()> { VISIT_SRC, quote! { #![allow(unused_variables)] - #![allow(clippy::needless_pass_by_ref_mut)] + #[cfg(any(feature = "full", feature = "derive"))] + use crate::gen::helper::visit::*; #[cfg(any(feature = "full", feature = "derive"))] use crate::punctuated::Punctuated; use crate::*; @@ -219,6 +253,8 @@ pub fn generate(defs: &Definitions) -> Result<()> { /// See the [module documentation] for details. /// /// [module documentation]: self + /// + /// *This trait is available only if Syn is built with the `"visit"` feature.* pub trait Visit<'ast> { #traits } diff --git a/codegen/src/visit_mut.rs b/codegen/src/visit_mut.rs index 5b96c871101c22ea93b7e33c3f55c8e06aed2a8a..294d247f7860b4ab7a5341b654071151416d89eb 100644 --- a/codegen/src/visit_mut.rs +++ b/codegen/src/visit_mut.rs @@ -6,7 +6,7 @@ use quote::{format_ident, quote}; use syn::Index; use syn_codegen::{Data, Definitions, Features, Node, Type}; -const VISIT_MUT_SRC: &str = "src/gen/visit_mut.rs"; +const VISIT_MUT_SRC: &str = "../src/gen/visit_mut.rs"; fn simple_visit(item: &str, name: &Operand) -> TokenStream { let ident = gen::under_name(item); @@ -50,18 +50,21 @@ fn visit( let val = visit(&p.element, features, defs, &operand)?; let name = name.ref_mut_tokens(); Some(quote! { - for mut el in Punctuated::pairs_mut(#name) { - let it = el.value_mut(); + for el in Punctuated::pairs_mut(#name) { + let (it, p) = el.into_tuple(); #val; + if let Some(p) = p { + tokens_helper(v, &mut p.spans); + } } }) } Type::Option(t) => { let it = Borrowed(quote!(it)); let val = visit(t, features, defs, &it)?; - let name = name.ref_mut_tokens(); + let name = name.owned_tokens(); Some(quote! { - if let Some(it) = #name { + if let Some(it) = &mut #name { #val; } }) @@ -78,6 +81,25 @@ fn visit( } Some(code) } + Type::Token(t) => { + let name = name.tokens(); + let repr = &defs.tokens[t]; + let is_keyword = repr.chars().next().unwrap().is_alphabetic(); + let spans = if is_keyword { + quote!(span) + } else { + quote!(spans) + }; + Some(quote! { + tokens_helper(v, &mut #name.#spans); + }) + } + Type::Group(_) => { + let name = name.tokens(); + Some(quote! { + tokens_helper(v, &mut #name.span); + }) + } Type::Syn(t) => { fn requires_full(features: &Features) -> bool { features.any.contains("full") && features.any.len() == 1 @@ -90,7 +112,7 @@ fn visit( Some(res) } Type::Ext(t) if gen::TERMINAL_TYPES.contains(&&t[..]) => Some(simple_visit(t, name)), - Type::Ext(_) | Type::Std(_) | Type::Token(_) | Type::Group(_) => None, + Type::Ext(_) | Type::Std(_) => None, } } @@ -102,11 +124,6 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi let mut visit_mut_impl = TokenStream::new(); match &s.data { - Data::Enum(variants) if variants.is_empty() => { - visit_mut_impl.extend(quote! { - match *node {} - }); - } Data::Enum(variants) => { let mut visit_mut_variants = TokenStream::new(); @@ -146,17 +163,33 @@ fn node(traits: &mut TokenStream, impls: &mut TokenStream, s: &Node, defs: &Defi } } + let nonexhaustive = if s.exhaustive { + None + } else { + Some(quote! { + #[cfg(syn_no_non_exhaustive)] + _ => unreachable!(), + }) + }; + visit_mut_impl.extend(quote! { match node { #visit_mut_variants + #nonexhaustive } }); } Data::Struct(fields) => { for (field, ty) in fields { - let id = Ident::new(field, Span::call_site()); + if let Type::Syn(ty) = ty { + if ty == "Reserved" { + continue; + } + } + + let id = Ident::new(&field, Span::call_site()); let ref_toks = Owned(quote!(node.#id)); - let visit_mut_field = visit(ty, &s.features, defs, &ref_toks) + let visit_mut_field = visit(&ty, &s.features, defs, &ref_toks) .unwrap_or_else(|| noop_visit(&ref_toks)); visit_mut_impl.extend(quote! { #visit_mut_field; @@ -197,8 +230,9 @@ pub fn generate(defs: &Definitions) -> Result<()> { VISIT_MUT_SRC, quote! { #![allow(unused_variables)] - #![allow(clippy::needless_pass_by_ref_mut)] + #[cfg(any(feature = "full", feature = "derive"))] + use crate::gen::helper::visit_mut::*; #[cfg(any(feature = "full", feature = "derive"))] use crate::punctuated::Punctuated; use crate::*; @@ -216,6 +250,8 @@ pub fn generate(defs: &Definitions) -> Result<()> { /// See the [module documentation] for details. /// /// [module documentation]: self + /// + /// *This trait is available only if Syn is built with the `"visit-mut"` feature.* pub trait VisitMut { #traits } diff --git a/codegen/src/workspace_path.rs b/codegen/src/workspace_path.rs deleted file mode 100644 index eb29bce2e8ca80a48c6268aa5b78b7315e5e4fea..0000000000000000000000000000000000000000 --- a/codegen/src/workspace_path.rs +++ /dev/null @@ -1,8 +0,0 @@ -use std::path::{Path, PathBuf}; - -pub fn get(relative_to_workspace_root: impl AsRef) -> PathBuf { - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - assert!(path.pop()); - path.push(relative_to_workspace_root); - path -} diff --git a/dev/Cargo.toml b/dev/Cargo.toml index b61b56189dbdaa8627a7f37c874b5002c3f0d149..79486c122a3d01bf520fd3044973d3f582abaa57 100644 --- a/dev/Cargo.toml +++ b/dev/Cargo.toml @@ -2,7 +2,7 @@ name = "syn-dev" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [lib] @@ -14,9 +14,9 @@ path = "main.rs" name = "syn-dev" [dependencies] -quote = "1" +quote = "1.0" [dependencies.syn] -default-features = false -features = ["extra-traits", "full", "parsing", "proc-macro"] path = ".." +default-features = false +features = ["parsing", "full", "extra-traits", "proc-macro"] diff --git a/examples/dump-syntax/Cargo.toml b/examples/dump-syntax/Cargo.toml index 6239141bb741b92b4713b15726d05171903acad2..2363f829b69c1a79db3b802e0151c25523a31fc2 100644 --- a/examples/dump-syntax/Cargo.toml +++ b/examples/dump-syntax/Cargo.toml @@ -2,14 +2,14 @@ name = "dump-syntax" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [dependencies] colored = "2" -proc-macro2 = { version = "1", features = ["span-locations"] } +proc-macro2 = { version = "1.0", features = ["span-locations"] } [dependencies.syn] -default-features = false -features = ["extra-traits", "full", "parsing"] path = "../.." +default-features = false +features = ["parsing", "full", "extra-traits"] diff --git a/examples/dump-syntax/src/main.rs b/examples/dump-syntax/src/main.rs index 855047a025e7e07e07453c4f099741236e350749..d2b31025ba0da306862d6277a5864ad8f78ad352 100644 --- a/examples/dump-syntax/src/main.rs +++ b/examples/dump-syntax/src/main.rs @@ -13,7 +13,7 @@ //! attrs: [ //! Attribute { //! pound_token: Pound, -//! style: AttrStyle::Inner( +//! style: Inner( //! ... //! } @@ -55,14 +55,14 @@ impl Display for Error { fn main() { if let Err(error) = try_main() { - let _ = writeln!(io::stderr(), "{}", error); + _ = writeln!(io::stderr(), "{}", error); process::exit(1); } } fn try_main() -> Result<(), Error> { let mut args = env::args_os(); - let _ = args.next(); // executable name + _ = args.next(); // executable name let filepath = match (args.next(), args.next()) { (Some(arg), None) => PathBuf::from(arg), @@ -99,7 +99,11 @@ fn render_location( let start = err.span().start(); let mut end = err.span().end(); - let code_line = match start.line.checked_sub(1).and_then(|n| code.lines().nth(n)) { + if start.line == end.line && start.column == end.column { + return render_fallback(formatter, err); + } + + let code_line = match code.lines().nth(start.line - 1) { Some(line) => line, None => return render_fallback(formatter, err), }; @@ -134,10 +138,7 @@ fn render_location( label = start.line.to_string().blue().bold(), code = code_line.trim_end(), offset = " ".repeat(start.column), - underline = "^" - .repeat(end.column.saturating_sub(start.column).max(1)) - .red() - .bold(), + underline = "^".repeat(end.column - start.column).red().bold(), message = err.to_string().red(), ) } diff --git a/examples/heapsize/example/Cargo.toml b/examples/heapsize/example/Cargo.toml index ab2c4266fd102b62d4aab4c5b478555cc721a1f2..85c7699c86718cd95b3d4c3e54ca000df9edcfb8 100644 --- a/examples/heapsize/example/Cargo.toml +++ b/examples/heapsize/example/Cargo.toml @@ -2,7 +2,7 @@ name = "heapsize_example" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [dependencies] diff --git a/examples/heapsize/heapsize/Cargo.toml b/examples/heapsize/heapsize/Cargo.toml index 2b2c31e984c919a8e9719866f6988c2cca2db174..27bb95414c2cb352ec4f061913a2060290f4492d 100644 --- a/examples/heapsize/heapsize/Cargo.toml +++ b/examples/heapsize/heapsize/Cargo.toml @@ -2,7 +2,7 @@ name = "heapsize" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [dependencies] diff --git a/examples/heapsize/heapsize_derive/Cargo.toml b/examples/heapsize/heapsize_derive/Cargo.toml index 06066ccab163834700c0381c0eabc7d13fbcd5a2..f4357b983029765bcda7f76829200d9c49c366a8 100644 --- a/examples/heapsize/heapsize_derive/Cargo.toml +++ b/examples/heapsize/heapsize_derive/Cargo.toml @@ -2,13 +2,13 @@ name = "heapsize_derive" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [lib] proc-macro = true [dependencies] -proc-macro2 = "1" -quote = "1" +proc-macro2 = "1.0" +quote = "1.0" syn = { path = "../../.." } diff --git a/examples/lazy-static/example/Cargo.toml b/examples/lazy-static/example/Cargo.toml index b6e95a2bbc0d0a549040f7800ecd12b1898d4ed1..44d155eb8164ddbc54b04e2b4a8bb3652f489559 100644 --- a/examples/lazy-static/example/Cargo.toml +++ b/examples/lazy-static/example/Cargo.toml @@ -2,7 +2,7 @@ name = "lazy-static-example" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [dependencies] diff --git a/examples/lazy-static/lazy-static/Cargo.toml b/examples/lazy-static/lazy-static/Cargo.toml index be966caa41b13af55be91b4392a7be9961a8de5e..bf65787c0ae15cb58e47c086a082e7298f59b6b3 100644 --- a/examples/lazy-static/lazy-static/Cargo.toml +++ b/examples/lazy-static/lazy-static/Cargo.toml @@ -2,13 +2,13 @@ name = "lazy_static" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [lib] proc-macro = true [dependencies] -proc-macro2 = { version = "1", features = ["nightly"] } -quote = "1" +proc-macro2 = { version = "1.0", features = ["nightly"] } +quote = "1.0" syn = { path = "../../../", features = ["full"] } diff --git a/examples/trace-var/README.md b/examples/trace-var/README.md index 294ed7e4da878fa45a3b924ae0638330f02ec327..b93fae2b2ccfc30368e1525828df45f3fdade816 100644 --- a/examples/trace-var/README.md +++ b/examples/trace-var/README.md @@ -42,7 +42,7 @@ n = 1 The procedural macro uses a syntax tree [`Fold`] to rewrite every `let` statement and assignment expression in the following way: -[`Fold`]: https://docs.rs/syn/2.0/syn/fold/trait.Fold.html +[`Fold`]: https://docs.rs/syn/1.0/syn/fold/trait.Fold.html ```rust // Before diff --git a/examples/trace-var/example/Cargo.toml b/examples/trace-var/example/Cargo.toml index b95e4c351c4d434e34d56e5770d934401b1c4c07..45ab3fd20604cf2e10105e0cde924c4a1b7659e5 100644 --- a/examples/trace-var/example/Cargo.toml +++ b/examples/trace-var/example/Cargo.toml @@ -2,7 +2,7 @@ name = "trace-var-example" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [dependencies] diff --git a/examples/trace-var/trace-var/Cargo.toml b/examples/trace-var/trace-var/Cargo.toml index ffa1da08a8f3c726c20fc953bccdac99a10671ec..72f56e92231447b00cdfa7211ef5d811f1c90783 100644 --- a/examples/trace-var/trace-var/Cargo.toml +++ b/examples/trace-var/trace-var/Cargo.toml @@ -2,13 +2,13 @@ name = "trace-var" version = "0.0.0" authors = ["David Tolnay "] -edition = "2021" +edition = "2018" publish = false [lib] proc-macro = true [dependencies] -proc-macro2 = { version = "1", features = ["nightly"] } -quote = "1" -syn = { path = "../../../", features = ["fold", "full"] } +proc-macro2 = { version = "1.0", features = ["nightly"] } +quote = "1.0" +syn = { path = "../../../", features = ["full", "fold"] } diff --git a/examples/trace-var/trace-var/src/lib.rs b/examples/trace-var/trace-var/src/lib.rs index 0dcaa94bb27a922180cd695a12fbd920dced61d8..6705941693cf1f7cb5139966ac27c66ab85c8a87 100644 --- a/examples/trace-var/trace-var/src/lib.rs +++ b/examples/trace-var/trace-var/src/lib.rs @@ -4,7 +4,7 @@ use std::collections::HashSet as Set; use syn::fold::{self, Fold}; use syn::parse::{Parse, ParseStream, Result}; use syn::punctuated::Punctuated; -use syn::{parse_macro_input, parse_quote, BinOp, Expr, Ident, ItemFn, Local, Pat, Stmt, Token}; +use syn::{parse_macro_input, parse_quote, Expr, Ident, ItemFn, Local, Pat, Stmt, Token}; /// Parses a list of variable names separated by commas. /// @@ -84,7 +84,7 @@ impl Args { /// let VAR = { let VAR = INIT; println!("VAR = {:?}", VAR); VAR }; fn let_and_print(&mut self, local: Local) -> Stmt { let Local { pat, init, .. } = local; - let init = self.fold_expr(*init.unwrap().expr); + let init = self.fold_expr(*init.unwrap().1); let ident = match pat { Pat::Ident(ref p) => &p.ident, _ => unreachable!(), @@ -122,11 +122,11 @@ impl Fold for Args { Expr::Assign(fold::fold_expr_assign(self, e)) } } - Expr::Binary(e) if is_assign_op(e.op) => { + Expr::AssignOp(e) => { if self.should_print_expr(&e.left) { self.assign_and_print(*e.left, &e.op, *e.right) } else { - Expr::Binary(fold::fold_expr_binary(self, e)) + Expr::AssignOp(fold::fold_expr_assign_op(self, e)) } } _ => fold::fold_expr(self, e), @@ -147,22 +147,6 @@ impl Fold for Args { } } -fn is_assign_op(op: BinOp) -> bool { - match op { - BinOp::AddAssign(_) - | BinOp::SubAssign(_) - | BinOp::MulAssign(_) - | BinOp::DivAssign(_) - | BinOp::RemAssign(_) - | BinOp::BitXorAssign(_) - | BinOp::BitAndAssign(_) - | BinOp::BitOrAssign(_) - | BinOp::ShlAssign(_) - | BinOp::ShrAssign(_) => true, - _ => false, - } -} - /// Attribute to print the value of the given variables each time they are /// reassigned. /// diff --git a/fuzz/.gitignore b/fuzz/.gitignore index f83457aece715da65abae687983991a5c762b4d6..188f196098e365ccc8c4934fdd607b012c8cc8a7 100644 --- a/fuzz/.gitignore +++ b/fuzz/.gitignore @@ -1,4 +1,3 @@ artifacts/ corpus/ -coverage/ target/ diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 94578e62104ca68f6d9af044af13e9364c2211c1..ed4b8897457e0252f1284da04757ae5c1e357b99 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -10,12 +10,9 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -proc-macro2 = "1.0.52" +proc-macro2 = "1" syn = { path = "..", default-features = false, features = ["full", "parsing"] } -[features] -span-locations = ["proc-macro2/span-locations"] - [[bin]] name = "create_token_buffer" path = "fuzz_targets/create_token_buffer.rs" diff --git a/fuzz/fuzz_targets/create_token_buffer.rs b/fuzz/fuzz_targets/create_token_buffer.rs index 48540e1b39f9d1e309904f43f00a3e1aad484eca..7b6622a1602fd06e953c990aad0f39347ffd99d4 100644 --- a/fuzz/fuzz_targets/create_token_buffer.rs +++ b/fuzz/fuzz_targets/create_token_buffer.rs @@ -12,7 +12,7 @@ fn immediate_fail(_input: ParseStream) -> syn::Result<()> { fuzz_target!(|data: &[u8]| { if data.len() < 300 { if let Ok(string) = str::from_utf8(data) { - let _ = immediate_fail.parse_str(string); + _ = immediate_fail.parse_str(string); } } }); diff --git a/fuzz/fuzz_targets/parse_file.rs b/fuzz/fuzz_targets/parse_file.rs index b30d694eed7e9800fdbb031b6982e8a7d5ef0786..4fa11dd924648b33108e0b2748f669b1c1d9b789 100644 --- a/fuzz/fuzz_targets/parse_file.rs +++ b/fuzz/fuzz_targets/parse_file.rs @@ -6,7 +6,7 @@ use std::str; fuzz_target!(|data: &[u8]| { if data.len() < 300 { if let Ok(string) = str::from_utf8(data) { - let _ = syn::parse_file(string); + _ = syn::parse_file(string); } } }); diff --git a/json/Cargo.toml b/json/Cargo.toml index e6ff2d544d9935c62339fdc8e1b493b5d3c6abdb..afa160f5e92587a55d063d3f92817b6724566255 100644 --- a/json/Cargo.toml +++ b/json/Cargo.toml @@ -1,29 +1,22 @@ [package] name = "syn-codegen" -version = "0.4.1" # also update html_root_url +version = "0.3.0" # also update html_root_url authors = ["David Tolnay "] categories = ["development-tools::procedural-macro-helpers"] description = "Syntax tree describing Syn's syntax tree" documentation = "https://docs.rs/syn-codegen" -edition = "2021" +edition = "2018" keywords = ["syn"] license = "MIT OR Apache-2.0" repository = "https://github.com/dtolnay/syn" [dependencies] -indexmap = { version = "2", features = ["serde"] } -semver = { version = "1", features = ["serde"] } -serde = "1.0.88" -serde_derive = "1.0.88" +indexmap = { version = "1.0", features = ["serde-1"] } +semver = { version = "1.0", features = ["serde"] } +serde = { version = "1.0.88", features = ["derive"] } [dev-dependencies] -serde_json = "1" - -[lib] -doc-scrape-examples = false +serde_json = "1.0" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] -rustdoc-args = ["--generate-link-to-definition"] - -[workspace] diff --git a/json/src/lib.rs b/json/src/lib.rs index 77f47d4ce9f29ac77ba5e53556148b54eb8c8dbc..be5a02e0aa4aed8b3bec96df8606d5b145f49566 100644 --- a/json/src/lib.rs +++ b/json/src/lib.rs @@ -13,9 +13,9 @@ //! of the [`visit`], [`visit_mut`], and [`fold`] modules can be generated //! programmatically from a description of the syntax tree. //! -//! [`visit`]: https://docs.rs/syn/2.0/syn/visit/index.html -//! [`visit_mut`]: https://docs.rs/syn/2.0/syn/visit_mut/index.html -//! [`fold`]: https://docs.rs/syn/2.0/syn/fold/index.html +//! [`visit`]: https://docs.rs/syn/1.0/syn/visit/index.html +//! [`visit_mut`]: https://docs.rs/syn/1.0/syn/visit_mut/index.html +//! [`fold`]: https://docs.rs/syn/1.0/syn/fold/index.html //! //! To make this type of code as easy as possible to implement in any language, //! every Syn release comes with a machine-readable description of that version @@ -44,12 +44,11 @@ //! } //! ``` -#![doc(html_root_url = "https://docs.rs/syn-codegen/0.4.1")] +#![doc(html_root_url = "https://docs.rs/syn-codegen/0.2.0")] use indexmap::IndexMap; use semver::Version; -use serde::de::{Deserialize, Deserializer}; -use serde_derive::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use std::collections::{BTreeMap, BTreeSet}; /// Top-level content of the syntax tree description. @@ -97,7 +96,7 @@ pub struct Node { /// Content of a syntax tree data structure. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub enum Data { - /// This is an opaque type with no publicly accessible structure. + /// This is an opaque type with no publicy accessible structure. Private, /// This type is a braced struct with named fields. diff --git a/src/attr.rs b/src/attr.rs index b6c4675b763fb4a742889632ee4f4d9aeab693af..bace94f43c8549de54dc429d9fd01fa731b99247 100644 --- a/src/attr.rs +++ b/src/attr.rs @@ -1,15 +1,19 @@ use super::*; +use crate::punctuated::Punctuated; use proc_macro2::TokenStream; use std::iter; use std::slice; #[cfg(feature = "parsing")] -use crate::meta::{self, ParseNestedMeta}; +use crate::parse::{Parse, ParseBuffer, ParseStream, Parser, Result}; #[cfg(feature = "parsing")] -use crate::parse::{Parse, ParseStream, Parser, Result}; +use crate::punctuated::Pair; ast_struct! { - /// An attribute, like `#[repr(transparent)]`. + /// An attribute like `#[repr(transparent)]`. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* /// ///
/// @@ -19,52 +23,27 @@ ast_struct! { /// /// - Outer attributes like `#[repr(transparent)]`. These appear outside or /// in front of the item they describe. - /// /// - Inner attributes like `#![feature(proc_macro)]`. These appear inside /// of the item they describe, usually a module. - /// - /// - Outer one-line doc comments like `/// Example`. - /// - /// - Inner one-line doc comments like `//! Please file an issue`. - /// - /// - Outer documentation blocks `/** Example */`. - /// - /// - Inner documentation blocks `/*! Please file an issue */`. + /// - Outer doc comments like `/// # Example`. + /// - Inner doc comments like `//! Please file an issue`. + /// - Outer block comments `/** # Example */`. + /// - Inner block comments `/*! Please file an issue */`. /// /// The `style` field of type `AttrStyle` distinguishes whether an attribute - /// is outer or inner. - /// - /// Every attribute has a `path` that indicates the intended interpretation - /// of the rest of the attribute's contents. The path and the optional - /// additional contents are represented together in the `meta` field of the - /// attribute in three possible varieties: - /// - /// - Meta::Path — attributes whose information content conveys just a - /// path, for example the `#[test]` attribute. - /// - /// - Meta::List — attributes that carry arbitrary tokens after the - /// path, surrounded by a delimiter (parenthesis, bracket, or brace). For - /// example `#[derive(Copy)]` or `#[precondition(x < 5)]`. - /// - /// - Meta::NameValue — attributes with an `=` sign after the path, - /// followed by a Rust expression. For example `#[path = - /// "sys/windows.rs"]`. - /// - /// All doc comments are represented in the NameValue style with a path of - /// "doc", as this is how they are processed by the compiler and by + /// is outer or inner. Doc comments and block comments are promoted to + /// attributes, as this is how they are processed by the compiler and by /// `macro_rules!` macros. /// - /// ```text - /// #[derive(Copy, Clone)] - /// ~~~~~~Path - /// ^^^^^^^^^^^^^^^^^^^Meta::List + /// The `path` field gives the possibly colon-delimited path against which + /// the attribute is resolved. It is equal to `"doc"` for desugared doc + /// comments. The `tokens` field contains the rest of the attribute body as + /// tokens. /// - /// #[path = "sys/windows.rs"] - /// ~~~~Path - /// ^^^^^^^^^^^^^^^^^^^^^^^Meta::NameValue - /// - /// #[test] - /// ^^^^Meta::Path + /// ```text + /// #[derive(Copy)] #[crate::precondition x < 5] + /// ^^^^^^~~~~~~ ^^^^^^^^^^^^^^^^^^^ ~~~~~ + /// path tokens path tokens /// ``` /// ///
@@ -114,13 +93,18 @@ ast_struct! { /// /// The grammar of attributes in Rust is very flexible, which makes the /// syntax tree not that useful on its own. In particular, arguments of the - /// `Meta::List` variety of attribute are held in an arbitrary `tokens: - /// TokenStream`. Macros are expected to check the `path` of the attribute, - /// decide whether they recognize it, and then parse the remaining tokens - /// according to whatever grammar they wish to require for that kind of - /// attribute. Use [`parse_args()`] to parse those tokens into the expected - /// data structure. - /// + /// attribute are held in an arbitrary `tokens: TokenStream`. Macros are + /// expected to check the `path` of the attribute, decide whether they + /// recognize it, and then parse the remaining tokens according to whatever + /// grammar they wish to require for that kind of attribute. + /// + /// If the attribute you are parsing is expected to conform to the + /// conventional structured form of attribute, use [`parse_meta()`] to + /// obtain that structured representation. If the attribute follows some + /// other grammar of its own, use [`parse_args()`] to parse that into the + /// expected data structure. + /// + /// [`parse_meta()`]: Attribute::parse_meta /// [`parse_args()`]: Attribute::parse_args /// ///


@@ -166,49 +150,65 @@ ast_struct! { pub pound_token: Token![#], pub style: AttrStyle, pub bracket_token: token::Bracket, - pub meta: Meta, + pub path: Path, + pub tokens: TokenStream, } } impl Attribute { - /// Returns the path that identifies the interpretation of this attribute. + /// Parses the content of the attribute, consisting of the path and tokens, + /// as a [`Meta`] if possible. /// - /// For example this would return the `test` in `#[test]`, the `derive` in - /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. - pub fn path(&self) -> &Path { - self.meta.path() + /// *This function is available only if Syn is built with the `"parsing"` + /// feature.* + #[cfg(feature = "parsing")] + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + pub fn parse_meta(&self) -> Result { + fn clone_ident_segment(segment: &PathSegment) -> PathSegment { + PathSegment { + ident: segment.ident.clone(), + arguments: PathArguments::None, + } + } + + let path = Path { + leading_colon: self + .path + .leading_colon + .as_ref() + .map(|colon| Token![::](colon.spans)), + segments: self + .path + .segments + .pairs() + .map(|pair| match pair { + Pair::Punctuated(seg, punct) => { + Pair::Punctuated(clone_ident_segment(seg), Token![::](punct.spans)) + } + Pair::End(seg) => Pair::End(clone_ident_segment(seg)), + }) + .collect(), + }; + + let parser = |input: ParseStream| parsing::parse_meta_after_path(path, input); + parse::Parser::parse2(parser, self.tokens.clone()) } /// Parse the arguments to the attribute as a syntax tree. /// - /// This is similar to pulling out the `TokenStream` from `Meta::List` and - /// doing `syn::parse2::(meta_list.tokens)`, except that using - /// `parse_args` the error message has a more useful span when `tokens` is - /// empty. + /// This is similar to `syn::parse2::(attr.tokens)` except that: /// - /// The surrounding delimiters are *not* included in the input to the - /// parser. + /// - the surrounding delimiters are *not* included in the input to the + /// parser; and + /// - the error message has a more useful span when `tokens` is empty. /// /// ```text /// #[my_attr(value < 5)] /// ^^^^^^^^^ what gets parsed /// ``` /// - /// # Example - /// - /// ``` - /// use syn::{parse_quote, Attribute, Expr}; - /// - /// let attr: Attribute = parse_quote! { - /// #[precondition(value < 5)] - /// }; - /// - /// if attr.path().is_ident("precondition") { - /// let precondition: Expr = attr.parse_args()?; - /// // ... - /// } - /// # anyhow::Ok(()) - /// ``` + /// *This function is available only if Syn is built with the `"parsing"` + /// feature.* #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_args(&self) -> Result { @@ -217,182 +217,22 @@ impl Attribute { /// Parse the arguments to the attribute using the given parser. /// - /// # Example - /// - /// ``` - /// use syn::{parse_quote, Attribute}; - /// - /// let attr: Attribute = parse_quote! { - /// #[inception { #[brrrrrrraaaaawwwwrwrrrmrmrmmrmrmmmmm] }] - /// }; - /// - /// let bwom = attr.parse_args_with(Attribute::parse_outer)?; - /// - /// // Attribute does not have a Parse impl, so we couldn't directly do: - /// // let bwom: Attribute = attr.parse_args()?; - /// # anyhow::Ok(()) - /// ``` + /// *This function is available only if Syn is built with the `"parsing"` + /// feature.* #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_args_with(&self, parser: F) -> Result { - match &self.meta { - Meta::Path(path) => Err(crate::error::new2( - path.segments.first().unwrap().ident.span(), - path.segments.last().unwrap().ident.span(), - format!( - "expected attribute arguments in parentheses: {}[{}(...)]", - parsing::DisplayAttrStyle(&self.style), - parsing::DisplayPath(path), - ), - )), - Meta::NameValue(meta) => Err(Error::new( - meta.eq_token.span, - format_args!( - "expected parentheses: {}[{}(...)]", - parsing::DisplayAttrStyle(&self.style), - parsing::DisplayPath(&meta.path), - ), - )), - Meta::List(meta) => meta.parse_args_with(parser), - } - } - - /// Parse the arguments to the attribute, expecting it to follow the - /// conventional structure used by most of Rust's built-in attributes. - /// - /// The [*Meta Item Attribute Syntax*][syntax] section in the Rust reference - /// explains the convention in more detail. Not all attributes follow this - /// convention, so [`parse_args()`][Self::parse_args] is available if you - /// need to parse arbitrarily goofy attribute syntax. - /// - /// [syntax]: https://doc.rust-lang.org/reference/attributes.html#meta-item-attribute-syntax - /// - /// # Example - /// - /// We'll parse a struct, and then parse some of Rust's `#[repr]` attribute - /// syntax. - /// - /// ``` - /// use syn::{parenthesized, parse_quote, token, ItemStruct, LitInt}; - /// - /// let input: ItemStruct = parse_quote! { - /// #[repr(C, align(4))] - /// pub struct MyStruct(u16, u32); - /// }; - /// - /// let mut repr_c = false; - /// let mut repr_transparent = false; - /// let mut repr_align = None::; - /// let mut repr_packed = None::; - /// for attr in &input.attrs { - /// if attr.path().is_ident("repr") { - /// attr.parse_nested_meta(|meta| { - /// // #[repr(C)] - /// if meta.path.is_ident("C") { - /// repr_c = true; - /// return Ok(()); - /// } - /// - /// // #[repr(transparent)] - /// if meta.path.is_ident("transparent") { - /// repr_transparent = true; - /// return Ok(()); - /// } - /// - /// // #[repr(align(N))] - /// if meta.path.is_ident("align") { - /// let content; - /// parenthesized!(content in meta.input); - /// let lit: LitInt = content.parse()?; - /// let n: usize = lit.base10_parse()?; - /// repr_align = Some(n); - /// return Ok(()); - /// } - /// - /// // #[repr(packed)] or #[repr(packed(N))], omitted N means 1 - /// if meta.path.is_ident("packed") { - /// if meta.input.peek(token::Paren) { - /// let content; - /// parenthesized!(content in meta.input); - /// let lit: LitInt = content.parse()?; - /// let n: usize = lit.base10_parse()?; - /// repr_packed = Some(n); - /// } else { - /// repr_packed = Some(1); - /// } - /// return Ok(()); - /// } - /// - /// Err(meta.error("unrecognized repr")) - /// })?; - /// } - /// } - /// # anyhow::Ok(()) - /// ``` - /// - /// # Alternatives - /// - /// In some cases, for attributes which have nested layers of structured - /// content, the following less flexible approach might be more convenient: - /// - /// ``` - /// # use syn::{parse_quote, ItemStruct}; - /// # - /// # let input: ItemStruct = parse_quote! { - /// # #[repr(C, align(4))] - /// # pub struct MyStruct(u16, u32); - /// # }; - /// # - /// use syn::punctuated::Punctuated; - /// use syn::{parenthesized, token, Error, LitInt, Meta, Token}; - /// - /// let mut repr_c = false; - /// let mut repr_transparent = false; - /// let mut repr_align = None::; - /// let mut repr_packed = None::; - /// for attr in &input.attrs { - /// if attr.path().is_ident("repr") { - /// let nested = attr.parse_args_with(Punctuated::::parse_terminated)?; - /// for meta in nested { - /// match meta { - /// // #[repr(C)] - /// Meta::Path(path) if path.is_ident("C") => { - /// repr_c = true; - /// } - /// - /// // #[repr(align(N))] - /// Meta::List(meta) if meta.path.is_ident("align") => { - /// let lit: LitInt = meta.parse_args()?; - /// let n: usize = lit.base10_parse()?; - /// repr_align = Some(n); - /// } - /// - /// /* ... */ - /// - /// _ => { - /// return Err(Error::new_spanned(meta, "unrecognized repr")); - /// } - /// } - /// } - /// } - /// } - /// # Ok(()) - /// ``` - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn parse_nested_meta( - &self, - logic: impl FnMut(ParseNestedMeta) -> Result<()>, - ) -> Result<()> { - self.parse_args_with(meta::parser(logic)) + let parser = |input: ParseStream| { + let args = enter_args(self, input)?; + parse::parse_stream(parser, &args) + }; + parser.parse2(self.tokens.clone()) } /// Parses zero or more outer attributes from the stream. /// - /// # Example - /// - /// See - /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + /// *This function is available only if Syn is built with the `"parsing"` + /// feature.* #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_outer(input: ParseStream) -> Result> { @@ -405,10 +245,8 @@ impl Attribute { /// Parses zero or more inner attributes from the stream. /// - /// # Example - /// - /// See - /// [*Parsing from tokens to Attribute*](#parsing-from-tokens-to-attribute). + /// *This function is available only if Syn is built with the `"parsing"` + /// feature.* #[cfg(feature = "parsing")] #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_inner(input: ParseStream) -> Result> { @@ -418,10 +256,65 @@ impl Attribute { } } +#[cfg(feature = "parsing")] +fn expected_parentheses(attr: &Attribute) -> String { + let style = match attr.style { + AttrStyle::Outer => "#", + AttrStyle::Inner(_) => "#!", + }; + + let mut path = String::new(); + for segment in &attr.path.segments { + if !path.is_empty() || attr.path.leading_colon.is_some() { + path += "::"; + } + path += &segment.ident.to_string(); + } + + format!("{}[{}(...)]", style, path) +} + +#[cfg(feature = "parsing")] +fn enter_args<'a>(attr: &Attribute, input: ParseStream<'a>) -> Result> { + if input.is_empty() { + let expected = expected_parentheses(attr); + let msg = format!("expected attribute arguments in parentheses: {}", expected); + return Err(crate::error::new2( + attr.pound_token.span, + attr.bracket_token.span, + msg, + )); + } else if input.peek(Token![=]) { + let expected = expected_parentheses(attr); + let msg = format!("expected parentheses: {}", expected); + return Err(input.error(msg)); + }; + + let content; + if input.peek(token::Paren) { + parenthesized!(content in input); + } else if input.peek(token::Bracket) { + bracketed!(content in input); + } else if input.peek(token::Brace) { + braced!(content in input); + } else { + return Err(input.error("unexpected token in attribute arguments")); + } + + if input.is_empty() { + Ok(content) + } else { + Err(input.error("unexpected token in attribute arguments")) + } +} + ast_enum! { /// Distinguishes between attributes that decorate an item and attributes /// that are contained within an item. /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// /// # Outer attributes /// /// - `#[repr(transparent)]` @@ -443,6 +336,9 @@ ast_enum! { ast_enum_of_structs! { /// Content of a compile-time structured attribute. /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// /// ## Path /// /// A meta path is like the `test` in `#[test]`. @@ -475,26 +371,32 @@ ast_enum_of_structs! { ast_struct! { /// A structured list within an attribute, like `derive(Copy, Clone)`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct MetaList { pub path: Path, - pub delimiter: MacroDelimiter, - pub tokens: TokenStream, + pub paren_token: token::Paren, + pub nested: Punctuated, } } ast_struct! { /// A name-value pair within an attribute, like `feature = "nightly"`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct MetaNameValue { pub path: Path, pub eq_token: Token![=], - pub value: Expr, + pub lit: Lit, } } impl Meta { - /// Returns the path that begins this structured meta item. + /// Returns the identifier that begins this structured meta item. /// /// For example this would return the `test` in `#[test]`, the `derive` in /// `#[derive(Copy)]`, and the `path` in `#[path = "sys/windows.rs"]`. @@ -505,84 +407,63 @@ impl Meta { Meta::NameValue(meta) => &meta.path, } } - - /// Error if this is a `Meta::List` or `Meta::NameValue`. - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn require_path_only(&self) -> Result<&Path> { - let error_span = match self { - Meta::Path(path) => return Ok(path), - Meta::List(meta) => meta.delimiter.span().open(), - Meta::NameValue(meta) => meta.eq_token.span, - }; - Err(Error::new(error_span, "unexpected token in attribute")) - } - - /// Error if this is a `Meta::Path` or `Meta::NameValue`. - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn require_list(&self) -> Result<&MetaList> { - match self { - Meta::List(meta) => Ok(meta), - Meta::Path(path) => Err(crate::error::new2( - path.segments.first().unwrap().ident.span(), - path.segments.last().unwrap().ident.span(), - format!( - "expected attribute arguments in parentheses: `{}(...)`", - parsing::DisplayPath(path), - ), - )), - Meta::NameValue(meta) => Err(Error::new(meta.eq_token.span, "expected `(`")), - } - } - - /// Error if this is a `Meta::Path` or `Meta::List`. - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn require_name_value(&self) -> Result<&MetaNameValue> { - match self { - Meta::NameValue(meta) => Ok(meta), - Meta::Path(path) => Err(crate::error::new2( - path.segments.first().unwrap().ident.span(), - path.segments.last().unwrap().ident.span(), - format!( - "expected a value for this attribute: `{} = ...`", - parsing::DisplayPath(path), - ), - )), - Meta::List(meta) => Err(Error::new(meta.delimiter.span().open(), "expected `=`")), - } - } } -impl MetaList { - /// See [`Attribute::parse_args`]. - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn parse_args(&self) -> Result { - self.parse_args_with(T::parse) - } - - /// See [`Attribute::parse_args_with`]. - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn parse_args_with(&self, parser: F) -> Result { - let scope = self.delimiter.span().close(); - crate::parse::parse_scoped(parser, scope, self.tokens.clone()) - } +ast_enum_of_structs! { + /// Element of a compile-time attribute list. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* + #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum NestedMeta { + /// A structured meta item, like the `Copy` in `#[derive(Copy)]` which + /// would be a nested `Meta::Path`. + Meta(Meta), - /// See [`Attribute::parse_nested_meta`]. - #[cfg(feature = "parsing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] - pub fn parse_nested_meta( - &self, - logic: impl FnMut(ParseNestedMeta) -> Result<()>, - ) -> Result<()> { - self.parse_args_with(meta::parser(logic)) + /// A Rust literal, like the `"new_name"` in `#[rename("new_name")]`. + Lit(Lit), } } -pub(crate) trait FilterAttrs<'a> { +/// Conventional argument type associated with an invocation of an attribute +/// macro. +/// +/// For example if we are developing an attribute macro that is intended to be +/// invoked on function items as follows: +/// +/// ``` +/// # const IGNORE: &str = stringify! { +/// #[my_attribute(path = "/v1/refresh")] +/// # }; +/// pub fn refresh() { +/// /* ... */ +/// } +/// ``` +/// +/// The implementation of this macro would want to parse its attribute arguments +/// as type `AttributeArgs`. +/// +/// ``` +/// # extern crate proc_macro; +/// # +/// use proc_macro::TokenStream; +/// use syn::{parse_macro_input, AttributeArgs, ItemFn}; +/// +/// # const IGNORE: &str = stringify! { +/// #[proc_macro_attribute] +/// # }; +/// pub fn my_attribute(args: TokenStream, input: TokenStream) -> TokenStream { +/// let args = parse_macro_input!(args as AttributeArgs); +/// let input = parse_macro_input!(input as ItemFn); +/// +/// /* ... */ +/// # "".parse().unwrap() +/// } +/// ``` +#[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] +pub type AttributeArgs = Vec; + +pub trait FilterAttrs<'a> { type Ret: Iterator; fn outer(self) -> Self::Ret; @@ -614,43 +495,69 @@ impl<'a> FilterAttrs<'a> for &'a [Attribute] { } #[cfg(feature = "parsing")] -pub(crate) mod parsing { +pub mod parsing { use super::*; - use crate::parse::discouraged::Speculative as _; + use crate::ext::IdentExt; use crate::parse::{Parse, ParseStream, Result}; - use std::fmt::{self, Display}; - pub(crate) fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { + pub fn parse_inner(input: ParseStream, attrs: &mut Vec) -> Result<()> { while input.peek(Token![#]) && input.peek2(Token![!]) { attrs.push(input.call(parsing::single_parse_inner)?); } Ok(()) } - pub(crate) fn single_parse_inner(input: ParseStream) -> Result { + pub fn single_parse_inner(input: ParseStream) -> Result { let content; Ok(Attribute { pound_token: input.parse()?, style: AttrStyle::Inner(input.parse()?), bracket_token: bracketed!(content in input), - meta: content.parse()?, + path: content.call(Path::parse_mod_style)?, + tokens: content.parse()?, }) } - pub(crate) fn single_parse_outer(input: ParseStream) -> Result { + pub fn single_parse_outer(input: ParseStream) -> Result { let content; Ok(Attribute { pound_token: input.parse()?, style: AttrStyle::Outer, bracket_token: bracketed!(content in input), - meta: content.parse()?, + path: content.call(Path::parse_mod_style)?, + tokens: content.parse()?, + }) + } + + // Like Path::parse_mod_style but accepts keywords in the path. + fn parse_meta_path(input: ParseStream) -> Result { + Ok(Path { + leading_colon: input.parse()?, + segments: { + let mut segments = Punctuated::new(); + while input.peek(Ident::peek_any) { + let ident = Ident::parse_any(input)?; + segments.push_value(PathSegment::from(ident)); + if !input.peek(Token![::]) { + break; + } + let punct = input.parse()?; + segments.push_punct(punct); + } + if segments.is_empty() { + return Err(input.error("expected path")); + } else if segments.trailing_punct() { + return Err(input.error("expected path segment")); + } + segments + }, }) } #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for Meta { fn parse(input: ParseStream) -> Result { - let path = input.call(Path::parse_mod_style)?; + let path = input.call(parse_meta_path)?; parse_meta_after_path(path, input) } } @@ -658,7 +565,7 @@ pub(crate) mod parsing { #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for MetaList { fn parse(input: ParseStream) -> Result { - let path = input.call(Path::parse_mod_style)?; + let path = input.call(parse_meta_path)?; parse_meta_list_after_path(path, input) } } @@ -666,13 +573,28 @@ pub(crate) mod parsing { #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for MetaNameValue { fn parse(input: ParseStream) -> Result { - let path = input.call(Path::parse_mod_style)?; + let path = input.call(parse_meta_path)?; parse_meta_name_value_after_path(path, input) } } - pub(crate) fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { - if input.peek(token::Paren) || input.peek(token::Bracket) || input.peek(token::Brace) { + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + impl Parse for NestedMeta { + fn parse(input: ParseStream) -> Result { + if input.peek(Lit) && !(input.peek(LitBool) && input.peek2(Token![=])) { + input.parse().map(NestedMeta::Lit) + } else if input.peek(Ident::peek_any) + || input.peek(Token![::]) && input.peek3(Ident::peek_any) + { + input.parse().map(NestedMeta::Meta) + } else { + Err(input.error("expected identifier or literal")) + } + } + } + + pub fn parse_meta_after_path(path: Path, input: ParseStream) -> Result { + if input.peek(token::Paren) { parse_meta_list_after_path(path, input).map(Meta::List) } else if input.peek(Token![=]) { parse_meta_name_value_after_path(path, input).map(Meta::NameValue) @@ -682,60 +604,21 @@ pub(crate) mod parsing { } fn parse_meta_list_after_path(path: Path, input: ParseStream) -> Result { - let (delimiter, tokens) = mac::parse_delimiter(input)?; + let content; Ok(MetaList { path, - delimiter, - tokens, + paren_token: parenthesized!(content in input), + nested: content.parse_terminated(NestedMeta::parse)?, }) } fn parse_meta_name_value_after_path(path: Path, input: ParseStream) -> Result { - let eq_token: Token![=] = input.parse()?; - let ahead = input.fork(); - let lit: Option = ahead.parse()?; - let value = if let (Some(lit), true) = (lit, ahead.is_empty()) { - input.advance_to(&ahead); - Expr::Lit(ExprLit { - attrs: Vec::new(), - lit, - }) - } else if input.peek(Token![#]) && input.peek2(token::Bracket) { - return Err(input.error("unexpected attribute inside of attribute")); - } else { - input.parse()? - }; Ok(MetaNameValue { path, - eq_token, - value, + eq_token: input.parse()?, + lit: input.parse()?, }) } - - pub(super) struct DisplayAttrStyle<'a>(pub &'a AttrStyle); - - impl<'a> Display for DisplayAttrStyle<'a> { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str(match self.0 { - AttrStyle::Outer => "#", - AttrStyle::Inner(_) => "#!", - }) - } - } - - pub(super) struct DisplayPath<'a>(pub &'a Path); - - impl<'a> Display for DisplayPath<'a> { - fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - for (i, segment) in self.0.segments.iter().enumerate() { - if i > 0 || self.0.leading_colon.is_some() { - formatter.write_str("::")?; - } - write!(formatter, "{}", segment.ident)?; - } - Ok(()) - } - } } #[cfg(feature = "printing")] @@ -752,7 +635,8 @@ mod printing { b.to_tokens(tokens); } self.bracket_token.surround(tokens, |tokens| { - self.meta.to_tokens(tokens); + self.path.to_tokens(tokens); + self.tokens.to_tokens(tokens); }); } } @@ -761,7 +645,9 @@ mod printing { impl ToTokens for MetaList { fn to_tokens(&self, tokens: &mut TokenStream) { self.path.to_tokens(tokens); - self.delimiter.surround(tokens, self.tokens.clone()); + self.paren_token.surround(tokens, |tokens| { + self.nested.to_tokens(tokens); + }); } } @@ -770,7 +656,7 @@ mod printing { fn to_tokens(&self, tokens: &mut TokenStream) { self.path.to_tokens(tokens); self.eq_token.to_tokens(tokens); - self.value.to_tokens(tokens); + self.lit.to_tokens(tokens); } } } diff --git a/src/await.rs b/src/await.rs new file mode 100644 index 0000000000000000000000000000000000000000..038c6a5d12cd2364dd8587d55b9b6aad860bf0ab --- /dev/null +++ b/src/await.rs @@ -0,0 +1,2 @@ +// See include!("await.rs") in token.rs. +export_token_macro! {[await]} diff --git a/src/bigint.rs b/src/bigint.rs index 66aaa93725401a441559ecb47ee9efbcda155b53..5397d6beee1d40ae628e7fc8b61f10f922c1d28a 100644 --- a/src/bigint.rs +++ b/src/bigint.rs @@ -1,16 +1,16 @@ use std::ops::{AddAssign, MulAssign}; // For implementing base10_digits() accessor on LitInt. -pub(crate) struct BigInt { +pub struct BigInt { digits: Vec, } impl BigInt { - pub(crate) fn new() -> Self { + pub fn new() -> Self { BigInt { digits: Vec::new() } } - pub(crate) fn to_string(&self) -> String { + pub fn to_string(&self) -> String { let mut repr = String::with_capacity(self.digits.len()); let mut has_nonzero = false; diff --git a/src/buffer.rs b/src/buffer.rs index 86dec46afda0d50d3ed94e62239a73ce26f0d84d..0d5cf30d5794594ccc5bc4216bf411afb02ccd7c 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -1,12 +1,18 @@ //! A stably addressed token buffer supporting efficient traversal based on a //! cheaply copyable cursor. +//! +//! *This module is available only if Syn is built with the `"parsing"` feature.* // This module is heavily commented as it contains most of the unsafe code in // Syn, and caution should be used when editing it. The public-facing interface // is 100% safe but the implementation is fragile internally. +#[cfg(all( + not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), + feature = "proc-macro" +))] +use crate::proc_macro as pm; use crate::Lifetime; -use proc_macro2::extra::DelimSpan; use proc_macro2::{Delimiter, Group, Ident, Literal, Punct, Spacing, Span, TokenStream, TokenTree}; use std::cmp::Ordering; use std::marker::PhantomData; @@ -27,6 +33,8 @@ enum Entry { /// A buffer that can be efficiently traversed multiple times, unlike /// `TokenStream` which requires a deep copy in order to traverse more than /// once. +/// +/// *This type is available only if Syn is built with the `"parsing"` feature.* pub struct TokenBuffer { // NOTE: Do not implement clone on this - while the current design could be // cloned, other designs which could be desirable may not be cloneable. @@ -55,9 +63,14 @@ impl TokenBuffer { /// Creates a `TokenBuffer` containing all the tokens from the input /// `proc_macro::TokenStream`. - #[cfg(feature = "proc-macro")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "proc-macro")))] - pub fn new(stream: proc_macro::TokenStream) -> Self { + /// + /// *This method is available only if Syn is built with both the `"parsing"` and + /// `"proc-macro"` features.* + #[cfg(all( + not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), + feature = "proc-macro" + ))] + pub fn new(stream: pm::TokenStream) -> Self { Self::new2(stream.into()) } @@ -88,6 +101,11 @@ impl TokenBuffer { /// /// An empty `Cursor` can be created directly, or one may create a `TokenBuffer` /// object and get a cursor to its first token with `begin()`. +/// +/// Two cursors are equal if they have the same location in the same input +/// stream, and have the same scope. +/// +/// *This type is available only if Syn is built with the `"parsing"` feature.* pub struct Cursor<'a> { // The current entry which the `Cursor` is pointing at. ptr: *const Entry, @@ -128,11 +146,11 @@ impl<'a> Cursor<'a> { // past it, unless `ptr == scope`, which means that we're at the edge of // our cursor's scope. We should only have `ptr != scope` at the exit // from None-delimited groups entered with `ignore_none`. - while let Entry::End(_) = unsafe { &*ptr } { + while let Entry::End(_) = *ptr { if ptr == scope { break; } - ptr = unsafe { ptr.add(1) }; + ptr = ptr.add(1); } Cursor { @@ -154,7 +172,7 @@ impl<'a> Cursor<'a> { /// If the cursor is looking at an `Entry::Group`, the bumped cursor will /// point at the first token in the group (with the same scope end). unsafe fn bump_ignore_group(self) -> Cursor<'a> { - unsafe { Cursor::create(self.ptr.offset(1), self.scope) } + Cursor::create(self.ptr.offset(1), self.scope) } /// While the cursor is looking at a `None`-delimited group, move it to look @@ -181,7 +199,7 @@ impl<'a> Cursor<'a> { /// If the cursor is pointing at a `Group` with the given delimiter, returns /// a cursor into that group and one pointing to the next `TokenTree`. - pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, DelimSpan, Cursor<'a>)> { + pub fn group(mut self, delim: Delimiter) -> Option<(Cursor<'a>, Span, Cursor<'a>)> { // If we're not trying to enter a none-delimited group, we want to // ignore them. We have to make sure to _not_ ignore them when we want // to enter them, of course. For obvious reasons. @@ -191,40 +209,16 @@ impl<'a> Cursor<'a> { if let Entry::Group(group, end_offset) = self.entry() { if group.delimiter() == delim { - let span = group.delim_span(); let end_of_group = unsafe { self.ptr.add(*end_offset) }; let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; - return Some((inside_of_group, span, after_group)); + return Some((inside_of_group, group.span(), after_group)); } } None } - pub(crate) fn any_group(self) -> Option<(Cursor<'a>, Delimiter, DelimSpan, Cursor<'a>)> { - if let Entry::Group(group, end_offset) = self.entry() { - let delimiter = group.delimiter(); - let span = group.delim_span(); - let end_of_group = unsafe { self.ptr.add(*end_offset) }; - let inside_of_group = unsafe { Cursor::create(self.ptr.add(1), end_of_group) }; - let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; - return Some((inside_of_group, delimiter, span, after_group)); - } - - None - } - - pub(crate) fn any_group_token(self) -> Option<(Group, Cursor<'a>)> { - if let Entry::Group(group, end_offset) = self.entry() { - let end_of_group = unsafe { self.ptr.add(*end_offset) }; - let after_group = unsafe { Cursor::create(end_of_group, self.scope) }; - return Some((group.clone(), after_group)); - } - - None - } - /// If the cursor is pointing at a `Ident`, returns it along with a cursor /// pointing at the next `TokenTree`. pub fn ident(mut self) -> Option<(Ident, Cursor<'a>)> { @@ -319,33 +313,6 @@ impl<'a> Cursor<'a> { } } - /// Returns the `Span` of the token immediately prior to the position of - /// this cursor, or of the current token if there is no previous one. - #[cfg(any(feature = "full", feature = "derive"))] - pub(crate) fn prev_span(mut self) -> Span { - if start_of_buffer(self) < self.ptr { - self.ptr = unsafe { self.ptr.offset(-1) }; - if let Entry::End(_) = self.entry() { - // Locate the matching Group begin token. - let mut depth = 1; - loop { - self.ptr = unsafe { self.ptr.offset(-1) }; - match self.entry() { - Entry::Group(group, _) => { - depth -= 1; - if depth == 0 { - return group.span(); - } - } - Entry::End(_) => depth += 1, - Entry::Literal(_) | Entry::Ident(_) | Entry::Punct(_) => {} - } - } - } - } - self.span() - } - /// Skip over the next token without cloning it. Returns `None` if this /// cursor points to eof. /// @@ -389,7 +356,7 @@ impl<'a> PartialEq for Cursor<'a> { impl<'a> PartialOrd for Cursor<'a> { fn partial_cmp(&self, other: &Self) -> Option { if same_buffer(*self, *other) { - Some(cmp_assuming_same_buffer(*self, *other)) + Some(self.ptr.cmp(&other.ptr)) } else { None } @@ -401,18 +368,17 @@ pub(crate) fn same_scope(a: Cursor, b: Cursor) -> bool { } pub(crate) fn same_buffer(a: Cursor, b: Cursor) -> bool { - start_of_buffer(a) == start_of_buffer(b) -} - -fn start_of_buffer(cursor: Cursor) -> *const Entry { unsafe { - match &*cursor.scope { - Entry::End(offset) => cursor.scope.offset(*offset), + match (&*a.scope, &*b.scope) { + (Entry::End(a_offset), Entry::End(b_offset)) => { + a.scope.offset(*a_offset) == b.scope.offset(*b_offset) + } _ => unreachable!(), } } } +#[cfg(any(feature = "full", feature = "derive"))] pub(crate) fn cmp_assuming_same_buffer(a: Cursor, b: Cursor) -> Ordering { a.ptr.cmp(&b.ptr) } diff --git a/src/custom_keyword.rs b/src/custom_keyword.rs index 6ce23db41ec0c2017aeeb1a9feb8bb96baa745c9..a3ec9d4cb701b33a1eb71aef77f518a4e85a6231 100644 --- a/src/custom_keyword.rs +++ b/src/custom_keyword.rs @@ -96,28 +96,26 @@ macro_rules! custom_keyword { #[doc(hidden)] #[allow(dead_code, non_snake_case)] - pub fn $ident<__S: $crate::__private::IntoSpans<$crate::__private::Span>>( + pub fn $ident<__S: $crate::__private::IntoSpans<[$crate::__private::Span; 1]>>( span: __S, ) -> $ident { $ident { - span: $crate::__private::IntoSpans::into_spans(span), + span: $crate::__private::IntoSpans::into_spans(span)[0], } } - const _: () = { - impl $crate::__private::Default for $ident { - fn default() -> Self { - $ident { - span: $crate::__private::Span::call_site(), - } + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident { + span: $crate::__private::Span::call_site(), } } + } - $crate::impl_parse_for_custom_keyword!($ident); - $crate::impl_to_tokens_for_custom_keyword!($ident); - $crate::impl_clone_for_custom_keyword!($ident); - $crate::impl_extra_traits_for_custom_keyword!($ident); - }; + $crate::impl_parse_for_custom_keyword!($ident); + $crate::impl_to_tokens_for_custom_keyword!($ident); + $crate::impl_clone_for_custom_keyword!($ident); + $crate::impl_extra_traits_for_custom_keyword!($ident); }; } @@ -128,17 +126,17 @@ macro_rules! custom_keyword { macro_rules! impl_parse_for_custom_keyword { ($ident:ident) => { // For peek. - impl $crate::__private::CustomToken for $ident { + impl $crate::token::CustomToken for $ident { fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { if let $crate::__private::Some((ident, _rest)) = cursor.ident() { - ident == $crate::__private::stringify!($ident) + ident == stringify!($ident) } else { false } } fn display() -> &'static $crate::__private::str { - $crate::__private::concat!("`", $crate::__private::stringify!($ident), "`") + concat!("`", stringify!($ident), "`") } } @@ -146,14 +144,14 @@ macro_rules! impl_parse_for_custom_keyword { fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { input.step(|cursor| { if let $crate::__private::Some((ident, rest)) = cursor.ident() { - if ident == $crate::__private::stringify!($ident) { + if ident == stringify!($ident) { return $crate::__private::Ok(($ident { span: ident.span() }, rest)); } } - $crate::__private::Err(cursor.error($crate::__private::concat!( + $crate::__private::Err(cursor.error(concat!( "expected `", - $crate::__private::stringify!($ident), - "`", + stringify!($ident), + "`" ))) }) } @@ -177,7 +175,7 @@ macro_rules! impl_to_tokens_for_custom_keyword { ($ident:ident) => { impl $crate::__private::ToTokens for $ident { fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { - let ident = $crate::Ident::new($crate::__private::stringify!($ident), self.span); + let ident = $crate::Ident::new(stringify!($ident), self.span); $crate::__private::TokenStreamExt::append(tokens, ident); } } @@ -224,14 +222,10 @@ macro_rules! impl_clone_for_custom_keyword { macro_rules! impl_extra_traits_for_custom_keyword { ($ident:ident) => { impl $crate::__private::Debug for $ident { - fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::fmt::Result { $crate::__private::Formatter::write_str( f, - $crate::__private::concat!( - "Keyword [", - $crate::__private::stringify!($ident), - "]", - ), + concat!("Keyword [", stringify!($ident), "]"), ) } } diff --git a/src/custom_punctuation.rs b/src/custom_punctuation.rs index 1b2c768f445859d3570afe508f0ec441c3404cb9..118a8453daabf8105f84f79e431b4b222424c82e 100644 --- a/src/custom_punctuation.rs +++ b/src/custom_punctuation.rs @@ -92,18 +92,16 @@ macro_rules! custom_punctuation { } } - const _: () = { - impl $crate::__private::Default for $ident { - fn default() -> Self { - $ident($crate::__private::Span::call_site()) - } + impl $crate::__private::Default for $ident { + fn default() -> Self { + $ident($crate::__private::Span::call_site()) } + } - $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); - $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); - $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); - $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); - }; + $crate::impl_parse_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_to_tokens_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_clone_for_custom_punctuation!($ident, $($tt)+); + $crate::impl_extra_traits_for_custom_punctuation!($ident, $($tt)+); }; } @@ -113,20 +111,20 @@ macro_rules! custom_punctuation { #[macro_export] macro_rules! impl_parse_for_custom_punctuation { ($ident:ident, $($tt:tt)+) => { - impl $crate::__private::CustomToken for $ident { - fn peek(cursor: $crate::buffer::Cursor) -> $crate::__private::bool { - $crate::__private::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) + impl $crate::token::CustomToken for $ident { + fn peek(cursor: $crate::buffer::Cursor) -> bool { + $crate::token::parsing::peek_punct(cursor, $crate::stringify_punct!($($tt)+)) } fn display() -> &'static $crate::__private::str { - $crate::__private::concat!("`", $crate::stringify_punct!($($tt)+), "`") + concat!("`", $crate::stringify_punct!($($tt)+), "`") } } impl $crate::parse::Parse for $ident { fn parse(input: $crate::parse::ParseStream) -> $crate::parse::Result<$ident> { let spans: $crate::custom_punctuation_repr!($($tt)+) = - $crate::__private::parse_punct(input, $crate::stringify_punct!($($tt)+))?; + $crate::token::parsing::punct(input, $crate::stringify_punct!($($tt)+))?; Ok($ident(spans)) } } @@ -149,7 +147,7 @@ macro_rules! impl_to_tokens_for_custom_punctuation { ($ident:ident, $($tt:tt)+) => { impl $crate::__private::ToTokens for $ident { fn to_tokens(&self, tokens: &mut $crate::__private::TokenStream2) { - $crate::__private::print_punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) + $crate::token::printing::punct($crate::stringify_punct!($($tt)+), &self.spans, tokens) } } }; @@ -195,8 +193,8 @@ macro_rules! impl_clone_for_custom_punctuation { macro_rules! impl_extra_traits_for_custom_punctuation { ($ident:ident, $($tt:tt)+) => { impl $crate::__private::Debug for $ident { - fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::FmtResult { - $crate::__private::Formatter::write_str(f, $crate::__private::stringify!($ident)) + fn fmt(&self, f: &mut $crate::__private::Formatter) -> $crate::__private::fmt::Result { + $crate::__private::Formatter::write_str(f, stringify!($ident)) } } @@ -297,6 +295,6 @@ macro_rules! custom_punctuation_unexpected { #[macro_export] macro_rules! stringify_punct { ($($tt:tt)+) => { - $crate::__private::concat!($($crate::__private::stringify!($tt)),+) + concat!($(stringify!($tt)),+) }; } diff --git a/src/data.rs b/src/data.rs index 134b76bb4132810b0b07d336c1920ca102e74367..3b466618f8abe4210b24f28c980f6cc4fd64f8f2 100644 --- a/src/data.rs +++ b/src/data.rs @@ -3,8 +3,12 @@ use crate::punctuated::Punctuated; ast_struct! { /// An enum variant. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct Variant { + /// Attributes tagged on the variant. pub attrs: Vec, /// Name of the variant. @@ -21,6 +25,9 @@ ast_struct! { ast_enum_of_structs! { /// Data stored within an enum variant or struct. /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// /// # Syntax tree enum /// /// This type is a [syntax tree enum]. @@ -43,6 +50,9 @@ ast_enum_of_structs! { ast_struct! { /// Named fields of a struct or struct variant such as `Point { x: f64, /// y: f64 }`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct FieldsNamed { pub brace_token: token::Brace, @@ -52,6 +62,9 @@ ast_struct! { ast_struct! { /// Unnamed fields of a tuple struct or tuple variant such as `Some(T)`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct FieldsUnnamed { pub paren_token: token::Paren, @@ -134,14 +147,17 @@ impl<'a> IntoIterator for &'a mut Fields { ast_struct! { /// A field of a struct or enum variant. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct Field { + /// Attributes tagged on the field. pub attrs: Vec, + /// Visibility of the field. pub vis: Visibility, - pub mutability: FieldMutability, - /// Name of the field, if any. /// /// Fields of tuple structs have no names. @@ -149,16 +165,82 @@ ast_struct! { pub colon_token: Option, + /// Type of the field. pub ty: Type, } } +ast_enum_of_structs! { + /// The visibility level of an item: inherited or `pub` or + /// `pub(restricted)`. + /// + /// *This type is available only if Syn is built with the `"derive"` or `"full"` + /// feature.* + /// + /// # Syntax tree enum + /// + /// This type is a [syntax tree enum]. + /// + /// [syntax tree enum]: Expr#syntax-tree-enums + #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] + pub enum Visibility { + /// A public visibility level: `pub`. + Public(VisPublic), + + /// A crate-level visibility: `crate`. + Crate(VisCrate), + + /// A visibility level restricted to some path: `pub(self)` or + /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. + Restricted(VisRestricted), + + /// An inherited visibility, which usually means private. + Inherited, + } +} + +ast_struct! { + /// A public visibility level: `pub`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* + #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct VisPublic { + pub pub_token: Token![pub], + } +} + +ast_struct! { + /// A crate-level visibility: `crate`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* + #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct VisCrate { + pub crate_token: Token![crate], + } +} + +ast_struct! { + /// A visibility level restricted to some path: `pub(self)` or + /// `pub(super)` or `pub(crate)` or `pub(in some::module)`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* + #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] + pub struct VisRestricted { + pub pub_token: Token![pub], + pub paren_token: token::Paren, + pub in_token: Option, + pub path: Box, + } +} + #[cfg(feature = "parsing")] -pub(crate) mod parsing { +pub mod parsing { use super::*; - use crate::ext::IdentExt as _; - #[cfg(not(feature = "full"))] - use crate::parse::discouraged::Speculative as _; + use crate::ext::IdentExt; + use crate::parse::discouraged::Speculative; use crate::parse::{Parse, ParseStream, Result}; #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] @@ -176,20 +258,7 @@ pub(crate) mod parsing { }; let discriminant = if input.peek(Token![=]) { let eq_token: Token![=] = input.parse()?; - #[cfg(feature = "full")] let discriminant: Expr = input.parse()?; - #[cfg(not(feature = "full"))] - let discriminant = { - let begin = input.fork(); - let ahead = input.fork(); - let mut discriminant: Result = ahead.parse(); - if discriminant.is_ok() { - input.advance_to(&ahead); - } else if scan_lenient_discriminant(input).is_ok() { - discriminant = Ok(Expr::Verbatim(verbatim::between(&begin, input))); - } - discriminant? - }; Some((eq_token, discriminant)) } else { None @@ -203,86 +272,13 @@ pub(crate) mod parsing { } } - #[cfg(not(feature = "full"))] - pub(crate) fn scan_lenient_discriminant(input: ParseStream) -> Result<()> { - use proc_macro2::Delimiter::{self, Brace, Bracket, Parenthesis}; - - let consume = |delimiter: Delimiter| { - Result::unwrap(input.step(|cursor| match cursor.group(delimiter) { - Some((_inside, _span, rest)) => Ok((true, rest)), - None => Ok((false, *cursor)), - })) - }; - - macro_rules! consume { - [$token:tt] => { - input.parse::>().unwrap().is_some() - }; - } - - let mut initial = true; - let mut depth = 0usize; - loop { - if initial { - if consume![&] { - input.parse::>()?; - } else if consume![if] || consume![match] || consume![while] { - depth += 1; - } else if input.parse::>()?.is_some() - || (consume(Brace) || consume(Bracket) || consume(Parenthesis)) - || (consume![async] || consume![const] || consume![loop] || consume![unsafe]) - && (consume(Brace) || break) - { - initial = false; - } else if consume![let] { - while !consume![=] { - if !((consume![|] || consume![ref] || consume![mut] || consume![@]) - || (consume![!] || input.parse::>()?.is_some()) - || (consume![..=] || consume![..] || consume![&] || consume![_]) - || (consume(Brace) || consume(Bracket) || consume(Parenthesis))) - { - path::parsing::qpath(input, true)?; - } - } - } else if input.parse::>()?.is_some() && !consume![:] { - break; - } else if input.parse::().is_err() { - path::parsing::qpath(input, true)?; - initial = consume![!] || depth == 0 && input.peek(token::Brace); - } - } else if input.is_empty() || input.peek(Token![,]) { - return Ok(()); - } else if depth > 0 && consume(Brace) { - if consume![else] && !consume(Brace) { - initial = consume![if] || break; - } else { - depth -= 1; - } - } else if input.parse::().is_ok() || (consume![..] | consume![=]) { - initial = true; - } else if consume![.] { - if input.parse::>()?.is_none() - && (input.parse::()?.is_named() && consume![::]) - { - AngleBracketedGenericArguments::do_parse(None, input)?; - } - } else if consume![as] { - input.parse::()?; - } else if !(consume(Brace) || consume(Bracket) || consume(Parenthesis)) { - break; - } - } - - Err(input.error("unsupported expression")) - } - #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] impl Parse for FieldsNamed { fn parse(input: ParseStream) -> Result { let content; Ok(FieldsNamed { brace_token: braced!(content in input), - named: content.parse_terminated(Field::parse_named, Token![,])?, + named: content.parse_terminated(Field::parse_named)?, }) } } @@ -293,7 +289,7 @@ pub(crate) mod parsing { let content; Ok(FieldsUnnamed { paren_token: parenthesized!(content in input), - unnamed: content.parse_terminated(Field::parse_unnamed, Token![,])?, + unnamed: content.parse_terminated(Field::parse_unnamed)?, }) } } @@ -302,37 +298,16 @@ pub(crate) mod parsing { /// Parses a named (braced struct) field. #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] pub fn parse_named(input: ParseStream) -> Result { - let attrs = input.call(Attribute::parse_outer)?; - let vis: Visibility = input.parse()?; - - let unnamed_field = cfg!(feature = "full") && input.peek(Token![_]); - let ident = if unnamed_field { - input.call(Ident::parse_any) - } else { - input.parse() - }?; - - let colon_token: Token![:] = input.parse()?; - - let ty: Type = if unnamed_field - && (input.peek(Token![struct]) - || input.peek(Token![union]) && input.peek2(token::Brace)) - { - let begin = input.fork(); - input.call(Ident::parse_any)?; - input.parse::()?; - Type::Verbatim(verbatim::between(&begin, input)) - } else { - input.parse()? - }; - Ok(Field { - attrs, - vis, - mutability: FieldMutability::None, - ident: Some(ident), - colon_token: Some(colon_token), - ty, + attrs: input.call(Attribute::parse_outer)?, + vis: input.parse()?, + ident: Some(if input.peek(Token![_]) { + input.call(Ident::parse_any) + } else { + input.parse() + }?), + colon_token: Some(input.parse()?), + ty: input.parse()?, }) } @@ -342,13 +317,100 @@ pub(crate) mod parsing { Ok(Field { attrs: input.call(Attribute::parse_outer)?, vis: input.parse()?, - mutability: FieldMutability::None, ident: None, colon_token: None, ty: input.parse()?, }) } } + + #[cfg_attr(doc_cfg, doc(cfg(feature = "parsing")))] + impl Parse for Visibility { + fn parse(input: ParseStream) -> Result { + // Recognize an empty None-delimited group, as produced by a $:vis + // matcher that matched no tokens. + if input.peek(token::Group) { + let ahead = input.fork(); + let group = crate::group::parse_group(&ahead)?; + if group.content.is_empty() { + input.advance_to(&ahead); + return Ok(Visibility::Inherited); + } + } + + if input.peek(Token![pub]) { + Self::parse_pub(input) + } else if input.peek(Token![crate]) { + Self::parse_crate(input) + } else { + Ok(Visibility::Inherited) + } + } + } + + impl Visibility { + fn parse_pub(input: ParseStream) -> Result { + let pub_token = input.parse::()?; + + if input.peek(token::Paren) { + let ahead = input.fork(); + + let content; + let paren_token = parenthesized!(content in ahead); + if content.peek(Token![crate]) + || content.peek(Token![self]) + || content.peek(Token![super]) + { + let path = content.call(Ident::parse_any)?; + + // Ensure there are no additional tokens within `content`. + // Without explicitly checking, we may misinterpret a tuple + // field as a restricted visibility, causing a parse error. + // e.g. `pub (crate::A, crate::B)` (Issue #720). + if content.is_empty() { + input.advance_to(&ahead); + return Ok(Visibility::Restricted(VisRestricted { + pub_token, + paren_token, + in_token: None, + path: Box::new(Path::from(path)), + })); + } + } else if content.peek(Token![in]) { + let in_token: Token![in] = content.parse()?; + let path = content.call(Path::parse_mod_style)?; + + input.advance_to(&ahead); + return Ok(Visibility::Restricted(VisRestricted { + pub_token, + paren_token, + in_token: Some(in_token), + path: Box::new(path), + })); + } + } + + Ok(Visibility::Public(VisPublic { pub_token })) + } + + fn parse_crate(input: ParseStream) -> Result { + if input.peek2(Token![::]) { + Ok(Visibility::Inherited) + } else { + Ok(Visibility::Crate(VisCrate { + crate_token: input.parse()?, + })) + } + } + + #[cfg(feature = "full")] + pub(crate) fn is_some(&self) -> bool { + match self { + Visibility::Inherited => false, + _ => true, + } + } + } } #[cfg(feature = "printing")] @@ -401,4 +463,31 @@ mod printing { self.ty.to_tokens(tokens); } } + + #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] + impl ToTokens for VisPublic { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pub_token.to_tokens(tokens); + } + } + + #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] + impl ToTokens for VisCrate { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.crate_token.to_tokens(tokens); + } + } + + #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] + impl ToTokens for VisRestricted { + fn to_tokens(&self, tokens: &mut TokenStream) { + self.pub_token.to_tokens(tokens); + self.paren_token.surround(tokens, |tokens| { + // TODO: If we have a path which is not "self" or "super" or + // "crate", automatically add the "in" token. + self.in_token.to_tokens(tokens); + self.path.to_tokens(tokens); + }); + } + } } diff --git a/src/derive.rs b/src/derive.rs index 25fa4c910b674c62ba78925d042f80d381fd2ae4..af9bb91b7a8d523dedaa8559f1e6340ebdf39935 100644 --- a/src/derive.rs +++ b/src/derive.rs @@ -3,19 +3,32 @@ use crate::punctuated::Punctuated; ast_struct! { /// Data structure sent to a `proc_macro_derive` macro. + /// + /// *This type is available only if Syn is built with the `"derive"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DeriveInput { + /// Attributes tagged on the whole struct or enum. pub attrs: Vec, + + /// Visibility of the struct or enum. pub vis: Visibility, + + /// Name of the struct or enum. pub ident: Ident, + + /// Generics required to complete the definition. pub generics: Generics, + + /// Data within the struct or enum. pub data: Data, } } -ast_enum! { +ast_enum_of_structs! { /// The storage of a struct, enum or union data structure. /// + /// *This type is available only if Syn is built with the `"derive"` feature.* + /// /// # Syntax tree enum /// /// This type is a [syntax tree enum]. @@ -23,14 +36,24 @@ ast_enum! { /// [syntax tree enum]: Expr#syntax-tree-enums #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub enum Data { + /// A struct input to a `proc_macro_derive` macro. Struct(DataStruct), + + /// An enum input to a `proc_macro_derive` macro. Enum(DataEnum), + + /// An untagged union input to a `proc_macro_derive` macro. Union(DataUnion), } + + do_not_generate_to_tokens } ast_struct! { /// A struct input to a `proc_macro_derive` macro. + /// + /// *This type is available only if Syn is built with the `"derive"` + /// feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DataStruct { pub struct_token: Token![struct], @@ -41,6 +64,9 @@ ast_struct! { ast_struct! { /// An enum input to a `proc_macro_derive` macro. + /// + /// *This type is available only if Syn is built with the `"derive"` + /// feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DataEnum { pub enum_token: Token![enum], @@ -51,6 +77,9 @@ ast_struct! { ast_struct! { /// An untagged union input to a `proc_macro_derive` macro. + /// + /// *This type is available only if Syn is built with the `"derive"` + /// feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "derive")))] pub struct DataUnion { pub union_token: Token![union], @@ -59,7 +88,7 @@ ast_struct! { } #[cfg(feature = "parsing")] -pub(crate) mod parsing { +pub mod parsing { use super::*; use crate::parse::{Parse, ParseStream, Result}; @@ -132,7 +161,7 @@ pub(crate) mod parsing { } } - pub(crate) fn data_struct( + pub fn data_struct( input: ParseStream, ) -> Result<(Option, Fields, Option)> { let mut lookahead = input.lookahead1(); @@ -168,7 +197,7 @@ pub(crate) mod parsing { } } - pub(crate) fn data_enum( + pub fn data_enum( input: ParseStream, ) -> Result<( Option, @@ -179,12 +208,12 @@ pub(crate) mod parsing { let content; let brace = braced!(content in input); - let variants = content.parse_terminated(Variant::parse, Token![,])?; + let variants = content.parse_terminated(Variant::parse)?; Ok((where_clause, brace, variants)) } - pub(crate) fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { + pub fn data_union(input: ParseStream) -> Result<(Option, FieldsNamed)> { let where_clause = input.parse()?; let fields = input.parse()?; Ok((where_clause, fields)) diff --git a/src/discouraged.rs b/src/discouraged.rs index fb98d6332c551075a77843c87040b57097a0cd26..a46129b6a159b56b53d21b93368cd53016cd98d3 100644 --- a/src/discouraged.rs +++ b/src/discouraged.rs @@ -1,7 +1,6 @@ //! Extensions to the parsing API with niche applicability. use super::*; -use proc_macro2::extra::DelimSpan; /// Extensions to the `ParseStream` API to support speculative parsing. pub trait Speculative { @@ -193,27 +192,3 @@ impl<'a> Speculative for ParseBuffer<'a> { .set(unsafe { mem::transmute::>(fork.cursor()) }); } } - -/// Extensions to the `ParseStream` API to support manipulating invisible -/// delimiters the same as if they were visible. -pub trait AnyDelimiter { - /// Returns the delimiter, the span of the delimiter token, and the nested - /// contents for further parsing. - fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)>; -} - -impl<'a> AnyDelimiter for ParseBuffer<'a> { - fn parse_any_delimiter(&self) -> Result<(Delimiter, DelimSpan, ParseBuffer)> { - self.step(|cursor| { - if let Some((content, delimiter, span, rest)) = cursor.any_group() { - let scope = crate::buffer::close_span_of_group(*cursor); - let nested = crate::parse::advance_step_cursor(cursor, content); - let unexpected = crate::parse::get_unexpected(self); - let content = crate::parse::new_parse_buffer(scope, nested, unexpected); - Ok(((delimiter, span, content), rest)) - } else { - Err(cursor.error("expected any delimiter")) - } - }) - } -} diff --git a/src/error.rs b/src/error.rs index 71247cde3d35b61ad189a3a8796a348f278b3dbb..e301367d5e4eb43c13649a8700e6edd8cabf4f0f 100644 --- a/src/error.rs +++ b/src/error.rs @@ -7,6 +7,7 @@ use proc_macro2::{ #[cfg(feature = "printing")] use quote::ToTokens; use std::fmt::{self, Debug, Display}; +use std::iter::FromIterator; use std::slice; use std::vec; @@ -33,34 +34,18 @@ pub type Result = std::result::Result; /// # extern crate proc_macro; /// # /// use proc_macro::TokenStream; -/// use syn::parse::{Parse, ParseStream, Result}; -/// use syn::{parse_macro_input, ItemFn}; +/// use syn::{parse_macro_input, AttributeArgs, ItemFn}; /// /// # const IGNORE: &str = stringify! { /// #[proc_macro_attribute] /// # }; /// pub fn my_attr(args: TokenStream, input: TokenStream) -> TokenStream { -/// let args = parse_macro_input!(args as MyAttrArgs); +/// let args = parse_macro_input!(args as AttributeArgs); /// let input = parse_macro_input!(input as ItemFn); /// /// /* ... */ /// # TokenStream::new() /// } -/// -/// struct MyAttrArgs { -/// # _k: [(); { stringify! { -/// ... -/// # }; 0 }] -/// } -/// -/// impl Parse for MyAttrArgs { -/// fn parse(input: ParseStream) -> Result { -/// # stringify! { -/// ... -/// # }; -/// # unimplemented!() -/// } -/// } /// ``` /// /// For errors that arise later than the initial parsing stage, the @@ -104,21 +89,14 @@ pub struct Error { struct ErrorMessage { // Span is implemented as an index into a thread-local interner to keep the // size small. It is not safe to access from a different thread. We want - // errors to be Send and Sync to play nicely with ecosystem crates for error - // handling, so pin the span we're given to its original thread and assume - // it is Span::call_site if accessed from any other thread. - span: ThreadBound, + // errors to be Send and Sync to play nicely with the Failure crate, so pin + // the span we're given to its original thread and assume it is + // Span::call_site if accessed from any other thread. + start_span: ThreadBound, + end_span: ThreadBound, message: String, } -// Cannot use std::ops::Range because that does not implement Copy, -// whereas ThreadBound requires a Copy impl as a way to ensure no Drop impls -// are involved. -struct SpanRange { - start: Span, - end: Span, -} - #[cfg(test)] struct _Test where @@ -161,10 +139,8 @@ impl Error { fn new(span: Span, message: String) -> Error { Error { messages: vec![ErrorMessage { - span: ThreadBound::new(SpanRange { - start: span, - end: span, - }), + start_span: ThreadBound::new(span), + end_span: ThreadBound::new(span), message, }], } @@ -185,7 +161,6 @@ impl Error { /// When in doubt it's recommended to stick to `Error::new` (or /// `ParseStream::error`)! #[cfg(feature = "printing")] - #[cfg_attr(doc_cfg, doc(cfg(feature = "printing")))] pub fn new_spanned(tokens: T, message: U) -> Self { return new_spanned(tokens.into_token_stream(), message.to_string()); @@ -195,7 +170,8 @@ impl Error { let end = iter.last().map_or(start, |t| t.span()); Error { messages: vec![ErrorMessage { - span: ThreadBound::new(SpanRange { start, end }), + start_span: ThreadBound::new(start), + end_span: ThreadBound::new(end), message, }], } @@ -208,7 +184,11 @@ impl Error { /// if called from a different thread than the one on which the `Error` was /// originally created. pub fn span(&self) -> Span { - let SpanRange { start, end } = match self.messages[0].span.get() { + let start = match self.messages[0].start_span.get() { + Some(span) => *span, + None => return Span::call_site(), + }; + let end = match self.messages[0].end_span.get() { Some(span) => *span, None => return Span::call_site(), }; @@ -274,34 +254,15 @@ impl Error { impl ErrorMessage { fn to_compile_error(&self) -> TokenStream { - let (start, end) = match self.span.get() { - Some(range) => (range.start, range.end), - None => (Span::call_site(), Span::call_site()), - }; - - // ::core::compile_error!($message) + let start = self + .start_span + .get() + .cloned() + .unwrap_or_else(Span::call_site); + let end = self.end_span.get().cloned().unwrap_or_else(Span::call_site); + + // compile_error!($message) TokenStream::from_iter(vec![ - TokenTree::Punct({ - let mut punct = Punct::new(':', Spacing::Joint); - punct.set_span(start); - punct - }), - TokenTree::Punct({ - let mut punct = Punct::new(':', Spacing::Alone); - punct.set_span(start); - punct - }), - TokenTree::Ident(Ident::new("core", start)), - TokenTree::Punct({ - let mut punct = Punct::new(':', Spacing::Joint); - punct.set_span(start); - punct - }), - TokenTree::Punct({ - let mut punct = Punct::new(':', Spacing::Alone); - punct.set_span(start); - punct - }), TokenTree::Ident(Ident::new("compile_error", start)), TokenTree::Punct({ let mut punct = Punct::new('!', Spacing::Alone); @@ -324,7 +285,7 @@ impl ErrorMessage { } #[cfg(feature = "parsing")] -pub(crate) fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { +pub fn new_at(scope: Span, cursor: Cursor, message: T) -> Error { if cursor.eof() { Error::new(scope, format!("unexpected end of input, {}", message)) } else { @@ -334,13 +295,14 @@ pub(crate) fn new_at(scope: Span, cursor: Cursor, message: T) -> Err } #[cfg(all(feature = "parsing", any(feature = "full", feature = "derive")))] -pub(crate) fn new2(start: Span, end: Span, message: T) -> Error { +pub fn new2(start: Span, end: Span, message: T) -> Error { return new2(start, end, message.to_string()); fn new2(start: Span, end: Span, message: String) -> Error { Error { messages: vec![ErrorMessage { - span: ThreadBound::new(SpanRange { start, end }), + start_span: ThreadBound::new(start), + end_span: ThreadBound::new(end), message, }], } @@ -385,26 +347,25 @@ impl Clone for Error { impl Clone for ErrorMessage { fn clone(&self) -> Self { + let start = self + .start_span + .get() + .cloned() + .unwrap_or_else(Span::call_site); + let end = self.end_span.get().cloned().unwrap_or_else(Span::call_site); ErrorMessage { - span: self.span, + start_span: ThreadBound::new(start), + end_span: ThreadBound::new(end), message: self.message.clone(), } } } -impl Clone for SpanRange { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for SpanRange {} - impl std::error::Error for Error {} impl From for Error { fn from(err: LexError) -> Self { - Error::new(err.span(), err) + Error::new(err.span(), "lex error") } } diff --git a/src/export.rs b/src/export.rs index b9ea5c747b75a789ae0de31214491527a32f4d06..f478d091ea12090f3f8078d287a23619bbe7e485 100644 --- a/src/export.rs +++ b/src/export.rs @@ -1,73 +1,39 @@ -#[doc(hidden)] pub use std::clone::Clone; -#[doc(hidden)] pub use std::cmp::{Eq, PartialEq}; -#[doc(hidden)] -pub use std::concat; -#[doc(hidden)] pub use std::default::Default; -#[doc(hidden)] -pub use std::fmt::Debug; -#[doc(hidden)] +pub use std::fmt::{self, Debug, Formatter}; pub use std::hash::{Hash, Hasher}; -#[doc(hidden)] pub use std::marker::Copy; -#[doc(hidden)] pub use std::option::Option::{None, Some}; -#[doc(hidden)] pub use std::result::Result::{Err, Ok}; -#[doc(hidden)] -pub use std::stringify; - -#[doc(hidden)] -pub type Formatter<'a> = std::fmt::Formatter<'a>; -#[doc(hidden)] -pub type FmtResult = std::fmt::Result; - -#[doc(hidden)] -pub type bool = std::primitive::bool; -#[doc(hidden)] -pub type str = std::primitive::str; #[cfg(feature = "printing")] -#[doc(hidden)] -pub use quote; +pub extern crate quote; -#[doc(hidden)] -pub type Span = proc_macro2::Span; -#[doc(hidden)] -pub type TokenStream2 = proc_macro2::TokenStream; +pub use proc_macro2::{Span, TokenStream as TokenStream2}; #[cfg(feature = "parsing")] -#[doc(hidden)] pub use crate::group::{parse_braces, parse_brackets, parse_parens}; -#[doc(hidden)] pub use crate::span::IntoSpans; -#[cfg(all(feature = "parsing", feature = "printing"))] -#[doc(hidden)] -pub use crate::parse_quote::parse as parse_quote; - -#[cfg(feature = "parsing")] -#[doc(hidden)] -pub use crate::token::parsing::{peek_punct, punct as parse_punct}; +#[cfg(all( + not(all(target_arch = "wasm32", any(target_os = "unknown", target_os = "wasi"))), + feature = "proc-macro" +))] +pub use proc_macro::TokenStream; #[cfg(feature = "printing")] -#[doc(hidden)] -pub use crate::token::printing::punct as print_punct; - -#[cfg(feature = "parsing")] -#[doc(hidden)] -pub use crate::token::private::CustomToken; +pub use quote::{ToTokens, TokenStreamExt}; -#[cfg(feature = "proc-macro")] -#[doc(hidden)] -pub type TokenStream = proc_macro::TokenStream; +#[allow(non_camel_case_types)] +pub type bool = help::Bool; +#[allow(non_camel_case_types)] +pub type str = help::Str; -#[cfg(feature = "printing")] -#[doc(hidden)] -pub use quote::{ToTokens, TokenStreamExt}; +mod help { + pub type Bool = bool; + pub type Str = str; +} -#[doc(hidden)] pub struct private(pub(crate) ()); diff --git a/src/expr.rs b/src/expr.rs index 7fb0f7b4e2cac0b5d6df0ef0b03b4964400c11f9..93a59b0e20d77e391e1cc4b6619c74d829824998 100644 --- a/src/expr.rs +++ b/src/expr.rs @@ -1,5 +1,7 @@ use super::*; use crate::punctuated::Punctuated; +#[cfg(feature = "full")] +use crate::reserved::Reserved; use proc_macro2::{Span, TokenStream}; #[cfg(feature = "printing")] use quote::IdentFragment; @@ -85,7 +87,7 @@ ast_enum_of_structs! { /// see names getting repeated in your code, like accessing /// `receiver.receiver` or `pat.pat` or `cond.cond`. #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] - #[non_exhaustive] + #[cfg_attr(not(syn_no_non_exhaustive), non_exhaustive)] pub enum Expr { /// A slice literal expression: `[a, b, c, d]`. Array(ExprArray), @@ -93,18 +95,24 @@ ast_enum_of_structs! { /// An assignment expression: `a = compute()`. Assign(ExprAssign), + /// A compound assignment expression: `counter += 1`. + AssignOp(ExprAssignOp), + /// An async block: `async { ... }`. Async(ExprAsync), /// An await expression: `fut.await`. Await(ExprAwait), - /// A binary operation: `a + b`, `a += b`. + /// A binary operation: `a + b`, `a * b`. Binary(ExprBinary), /// A blocked scope: `{ ... }`. Block(ExprBlock), + /// A box expression: `box f`. + Box(ExprBox), + /// A `break`, with an optional label to break and an optional /// expression. Break(ExprBreak), @@ -118,9 +126,6 @@ ast_enum_of_structs! { /// A closure expression: `|a, b| a + b`. Closure(ExprClosure), - /// A const block: `const { ... }`. - Const(ExprConst), - /// A `continue`, with an optional label. Continue(ExprContinue), @@ -148,9 +153,6 @@ ast_enum_of_structs! { /// A square bracketed indexing expression: `vector[2]`. Index(ExprIndex), - /// The inferred value of a const generic argument, denoted `_`. - Infer(ExprInfer), - /// A `let` guard: `let Some(x) = opt`. Let(ExprLet), @@ -205,6 +207,9 @@ ast_enum_of_structs! { /// A tuple expression: `(a, b, c, d)`. Tuple(ExprTuple), + /// A type ascription expression: `foo: f64`. + Type(ExprType), + /// A unary operation: `!x`, `*x`. Unary(ExprUnary), @@ -220,16 +225,17 @@ ast_enum_of_structs! { /// A yield expression: `yield expr`. Yield(ExprYield), + // Not public API. + // // For testing exhaustiveness in downstream code, use the following idiom: // // match expr { - // #![cfg_attr(test, deny(non_exhaustive_omitted_patterns))] - // // Expr::Array(expr) => {...} // Expr::Assign(expr) => {...} // ... // Expr::Yield(expr) => {...} // + // #[cfg_attr(test, deny(non_exhaustive_omitted_patterns))] // _ => { /* some sane fallback */ } // } // @@ -237,11 +243,16 @@ ast_enum_of_structs! { // a variant. You will be notified by a test failure when a variant is // added, so that you can add code to handle it, but your library will // continue to compile and work for downstream users in the interim. + #[cfg(syn_no_non_exhaustive)] + #[doc(hidden)] + __NonExhaustive, } } ast_struct! { /// A slice literal expression: `[a, b, c, d]`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprArray #full { pub attrs: Vec, @@ -252,6 +263,8 @@ ast_struct! { ast_struct! { /// An assignment expression: `a = compute()`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprAssign #full { pub attrs: Vec, @@ -261,8 +274,23 @@ ast_struct! { } } +ast_struct! { + /// A compound assignment expression: `counter += 1`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* + #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] + pub struct ExprAssignOp #full { + pub attrs: Vec, + pub left: Box, + pub op: BinOp, + pub right: Box, + } +} + ast_struct! { /// An async block: `async { ... }`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprAsync #full { pub attrs: Vec, @@ -274,17 +302,22 @@ ast_struct! { ast_struct! { /// An await expression: `fut.await`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprAwait #full { pub attrs: Vec, pub base: Box, pub dot_token: Token![.], - pub await_token: Token![await], + pub await_token: token::Await, } } ast_struct! { - /// A binary operation: `a + b`, `a += b`. + /// A binary operation: `a + b`, `a * b`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprBinary { pub attrs: Vec, @@ -296,6 +329,8 @@ ast_struct! { ast_struct! { /// A blocked scope: `{ ... }`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprBlock #full { pub attrs: Vec, @@ -304,9 +339,23 @@ ast_struct! { } } +ast_struct! { + /// A box expression: `box f`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* + #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] + pub struct ExprBox #full { + pub attrs: Vec, + pub box_token: Token![box], + pub expr: Box, + } +} + ast_struct! { /// A `break`, with an optional label to break and an optional /// expression. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprBreak #full { pub attrs: Vec, @@ -318,6 +367,9 @@ ast_struct! { ast_struct! { /// A function call expression: `invoke(a, b)`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprCall { pub attrs: Vec, @@ -329,6 +381,9 @@ ast_struct! { ast_struct! { /// A cast expression: `foo as f64`. + /// + /// *This type is available only if Syn is built with the `"derive"` or + /// `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprCast { pub attrs: Vec, @@ -340,11 +395,11 @@ ast_struct! { ast_struct! { /// A closure expression: `|a, b| a + b`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprClosure #full { pub attrs: Vec, - pub lifetimes: Option, - pub constness: Option, pub movability: Option, pub asyncness: Option, pub capture: Option, @@ -356,18 +411,10 @@ ast_struct! { } } -ast_struct! { - /// A const block: `const { ... }`. - #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] - pub struct ExprConst #full { - pub attrs: Vec, - pub const_token: Token![const], - pub block: Block, - } -} - ast_struct! { /// A `continue`, with an optional label. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprContinue #full { pub attrs: Vec, @@ -379,6 +426,8 @@ ast_struct! { ast_struct! { /// Access of a named struct field (`obj.k`) or unnamed tuple struct /// field (`obj.0`). + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(any(feature = "full", feature = "derive"))))] pub struct ExprField { pub attrs: Vec, @@ -390,12 +439,14 @@ ast_struct! { ast_struct! { /// A for loop: `for pat in expr { ... }`. + /// + /// *This type is available only if Syn is built with the `"full"` feature.* #[cfg_attr(doc_cfg, doc(cfg(feature = "full")))] pub struct ExprForLoop #full { pub attrs: Vec, pub label: Option