From 91cf0e741186a9fa3bf31b07a65dc89324c10296 Mon Sep 17 00:00:00 2001 From: Matthew Jasper Date: Thu, 30 Jan 2020 20:24:44 +0000 Subject: [PATCH 01/43] Don't requery the param_env of a union Union fields have the ParamEnv of the union. --- src/librustc_typeck/check/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/librustc_typeck/check/mod.rs b/src/librustc_typeck/check/mod.rs index 0a917a1853eb5..8d2cfd9a335d1 100644 --- a/src/librustc_typeck/check/mod.rs +++ b/src/librustc_typeck/check/mod.rs @@ -1545,11 +1545,11 @@ fn check_union_fields(tcx: TyCtxt<'_>, span: Span, item_def_id: DefId) -> bool { if let ty::Adt(def, substs) = item_type.kind { assert!(def.is_union()); let fields = &def.non_enum_variant().fields; + let param_env = tcx.param_env(item_def_id); for field in fields { let field_ty = field.ty(tcx, substs); // We are currently checking the type this field came from, so it must be local. let field_span = tcx.hir().span_if_local(field.did).unwrap(); - let param_env = tcx.param_env(field.did); if field_ty.needs_drop(tcx, param_env) { struct_span_err!( tcx.sess, From 570c1613c1225d5777af5603dcf526da9cf57e19 Mon Sep 17 00:00:00 2001 From: Matthew Jasper Date: Thu, 30 Jan 2020 20:25:39 +0000 Subject: [PATCH 02/43] Remove unnecessary features in rustc_ty --- src/librustc_ty/lib.rs | 2 -- src/librustc_ty/ty.rs | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/librustc_ty/lib.rs b/src/librustc_ty/lib.rs index e5ec98743e0ae..e970faef02f28 100644 --- a/src/librustc_ty/lib.rs +++ b/src/librustc_ty/lib.rs @@ -6,9 +6,7 @@ #![doc(html_root_url = "https://doc.rust-lang.org/nightly/")] #![feature(bool_to_option)] -#![feature(in_band_lifetimes)] #![feature(nll)] -#![cfg_attr(bootstrap, feature(slice_patterns))] #![recursion_limit = "256"] #[macro_use] diff --git a/src/librustc_ty/ty.rs b/src/librustc_ty/ty.rs index 8f882be1a090e..a131b11b07ba3 100644 --- a/src/librustc_ty/ty.rs +++ b/src/librustc_ty/ty.rs @@ -9,7 +9,11 @@ use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE}; use rustc_span::symbol::Symbol; use rustc_span::Span; -fn sized_constraint_for_ty(tcx: TyCtxt<'tcx>, adtdef: &ty::AdtDef, ty: Ty<'tcx>) -> Vec> { +fn sized_constraint_for_ty<'tcx>( + tcx: TyCtxt<'tcx>, + adtdef: &ty::AdtDef, + ty: Ty<'tcx>, +) -> Vec> { use ty::TyKind::*; let result = match ty.kind { From 39733223fc817efba52a4204dd697192bf5da185 Mon Sep 17 00:00:00 2001 From: Matthew Jasper Date: Thu, 30 Jan 2020 20:26:36 +0000 Subject: [PATCH 03/43] Add IS_MANUALLY_DROP to AdtFlags --- src/librustc/ty/mod.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/librustc/ty/mod.rs b/src/librustc/ty/mod.rs index f417b907a3811..2b272d7fe08af 100644 --- a/src/librustc/ty/mod.rs +++ b/src/librustc/ty/mod.rs @@ -1792,19 +1792,22 @@ bitflags! { const IS_STRUCT = 1 << 2; /// Indicates whether the ADT is a struct and has a constructor. const HAS_CTOR = 1 << 3; - /// Indicates whether the type is a `PhantomData`. + /// Indicates whether the type is `PhantomData`. const IS_PHANTOM_DATA = 1 << 4; /// Indicates whether the type has a `#[fundamental]` attribute. const IS_FUNDAMENTAL = 1 << 5; - /// Indicates whether the type is a `Box`. + /// Indicates whether the type is `Box`. const IS_BOX = 1 << 6; + /// Indicates whether the type is `ManuallyDrop`. + const IS_MANUALLY_DROP = 1 << 7; + // FIXME(matthewjasper) replace these with diagnostic items /// Indicates whether the type is an `Arc`. - const IS_ARC = 1 << 7; + const IS_ARC = 1 << 8; /// Indicates whether the type is an `Rc`. - const IS_RC = 1 << 8; + const IS_RC = 1 << 9; /// Indicates whether the variant list of this ADT is `#[non_exhaustive]`. /// (i.e., this flag is never set unless this ADT is an enum). - const IS_VARIANT_LIST_NON_EXHAUSTIVE = 1 << 9; + const IS_VARIANT_LIST_NON_EXHAUSTIVE = 1 << 10; } } @@ -2180,6 +2183,9 @@ impl<'tcx> AdtDef { if Some(did) == tcx.lang_items().owned_box() { flags |= AdtFlags::IS_BOX; } + if Some(did) == tcx.lang_items().manually_drop() { + flags |= AdtFlags::IS_MANUALLY_DROP; + } if Some(did) == tcx.lang_items().arc() { flags |= AdtFlags::IS_ARC; } @@ -2280,6 +2286,12 @@ impl<'tcx> AdtDef { self.flags.contains(AdtFlags::IS_BOX) } + /// Returns `true` if this is ManuallyDrop. + #[inline] + pub fn is_manually_drop(&self) -> bool { + self.flags.contains(AdtFlags::IS_MANUALLY_DROP) + } + /// Returns `true` if this type has a destructor. pub fn has_dtor(&self, tcx: TyCtxt<'tcx>) -> bool { self.destructor(tcx).is_some() From d1965216a34dc2831cf44d2e15ad9d78403d10cc Mon Sep 17 00:00:00 2001 From: Matthew Jasper Date: Thu, 30 Jan 2020 20:28:16 +0000 Subject: [PATCH 04/43] Improve needs_drop query * Handle cycles in `needs_drop` correctly * Normalize types when computing `needs_drop` * Move queries from rustc to rustc_ty --- src/librustc/query/mod.rs | 17 +- src/librustc/traits/misc.rs | 132 -------------- src/librustc/traits/mod.rs | 1 - src/librustc/ty/query/mod.rs | 2 +- src/librustc/ty/query/values.rs | 7 - src/librustc/ty/util.rs | 81 ++++++++- src/librustc_ty/common_traits.rs | 40 ++++ src/librustc_ty/lib.rs | 4 + src/librustc_ty/needs_drop.rs | 171 ++++++++++++++++++ .../ui/type-alias-impl-trait/issue-65918.rs | 2 + 10 files changed, 304 insertions(+), 153 deletions(-) create mode 100644 src/librustc_ty/common_traits.rs create mode 100644 src/librustc_ty/needs_drop.rs diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index 37d5e23535b81..c705956f4540a 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -651,26 +651,27 @@ rustc_queries! { no_force desc { "computing whether `{}` is `Copy`", env.value } } + /// Query backing `TyS::is_sized`. query is_sized_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { no_force desc { "computing whether `{}` is `Sized`", env.value } } + /// Query backing `TyS::is_freeze`. query is_freeze_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { no_force desc { "computing whether `{}` is freeze", env.value } } - - // The cycle error here should be reported as an error by `check_representable`. - // We consider the type as not needing drop in the meanwhile to avoid - // further errors (done in impl Value for NeedsDrop). - // Use `cycle_delay_bug` to delay the cycle error here to be emitted later - // in case we accidentally otherwise don't emit an error. - query needs_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> NeedsDrop { - cycle_delay_bug + /// Query backing `TyS::needs_drop`. + query needs_drop_raw(env: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { no_force desc { "computing whether `{}` needs drop", env.value } } + /// A list of types where the ADT requires drop if and only if any of + /// those types require drop. If the ADT is known to always need drop + /// then `Err(AlwaysRequiresDrop)` is returned. + query adt_drop_tys(_: DefId) -> Result<&'tcx ty::List>, AlwaysRequiresDrop> {} + query layout_raw( env: ty::ParamEnvAnd<'tcx, Ty<'tcx>> ) -> Result<&'tcx ty::layout::LayoutDetails, ty::layout::LayoutError<'tcx>> { diff --git a/src/librustc/traits/misc.rs b/src/librustc/traits/misc.rs index 08c3a77bf3aca..3fd0d12c626aa 100644 --- a/src/librustc/traits/misc.rs +++ b/src/librustc/traits/misc.rs @@ -1,12 +1,9 @@ //! Miscellaneous type-system utilities that are too small to deserve their own modules. -use crate::middle::lang_items; use crate::traits::{self, ObligationCause}; -use crate::ty::util::NeedsDrop; use crate::ty::{self, Ty, TyCtxt, TypeFoldable}; use rustc_hir as hir; -use rustc_span::DUMMY_SP; #[derive(Clone)] pub enum CopyImplementationError<'tcx> { @@ -71,132 +68,3 @@ pub fn can_type_implement_copy( Ok(()) }) } - -fn is_copy_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { - is_item_raw(tcx, query, lang_items::CopyTraitLangItem) -} - -fn is_sized_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { - is_item_raw(tcx, query, lang_items::SizedTraitLangItem) -} - -fn is_freeze_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { - is_item_raw(tcx, query, lang_items::FreezeTraitLangItem) -} - -fn is_item_raw<'tcx>( - tcx: TyCtxt<'tcx>, - query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, - item: lang_items::LangItem, -) -> bool { - let (param_env, ty) = query.into_parts(); - let trait_def_id = tcx.require_lang_item(item, None); - tcx.infer_ctxt().enter(|infcx| { - traits::type_known_to_meet_bound_modulo_regions( - &infcx, - param_env, - ty, - trait_def_id, - DUMMY_SP, - ) - }) -} - -fn needs_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> NeedsDrop { - let (param_env, ty) = query.into_parts(); - - let needs_drop = |ty: Ty<'tcx>| -> bool { tcx.needs_drop_raw(param_env.and(ty)).0 }; - - assert!(!ty.needs_infer()); - - NeedsDrop(match ty.kind { - // Fast-path for primitive types - ty::Infer(ty::FreshIntTy(_)) - | ty::Infer(ty::FreshFloatTy(_)) - | ty::Bool - | ty::Int(_) - | ty::Uint(_) - | ty::Float(_) - | ty::Never - | ty::FnDef(..) - | ty::FnPtr(_) - | ty::Char - | ty::GeneratorWitness(..) - | ty::RawPtr(_) - | ty::Ref(..) - | ty::Str => false, - - // Foreign types can never have destructors - ty::Foreign(..) => false, - - // `ManuallyDrop` doesn't have a destructor regardless of field types. - ty::Adt(def, _) if Some(def.did) == tcx.lang_items().manually_drop() => false, - - // Issue #22536: We first query `is_copy_modulo_regions`. It sees a - // normalized version of the type, and therefore will definitely - // know whether the type implements Copy (and thus needs no - // cleanup/drop/zeroing) ... - _ if ty.is_copy_modulo_regions(tcx, param_env, DUMMY_SP) => false, - - // ... (issue #22536 continued) but as an optimization, still use - // prior logic of asking for the structural "may drop". - - // FIXME(#22815): Note that this is a conservative heuristic; - // it may report that the type "may drop" when actual type does - // not actually have a destructor associated with it. But since - // the type absolutely did not have the `Copy` bound attached - // (see above), it is sound to treat it as having a destructor. - - // User destructors are the only way to have concrete drop types. - ty::Adt(def, _) if def.has_dtor(tcx) => true, - - // Can refer to a type which may drop. - // FIXME(eddyb) check this against a ParamEnv. - ty::Dynamic(..) - | ty::Projection(..) - | ty::Param(_) - | ty::Bound(..) - | ty::Placeholder(..) - | ty::Opaque(..) - | ty::Infer(_) - | ty::Error => true, - - ty::UnnormalizedProjection(..) => bug!("only used with chalk-engine"), - - // Zero-length arrays never contain anything to drop. - ty::Array(_, len) if len.try_eval_usize(tcx, param_env) == Some(0) => false, - - // Structural recursion. - ty::Array(ty, _) | ty::Slice(ty) => needs_drop(ty), - - ty::Closure(def_id, ref substs) => { - substs.as_closure().upvar_tys(def_id, tcx).any(needs_drop) - } - - // Pessimistically assume that all generators will require destructors - // as we don't know if a destructor is a noop or not until after the MIR - // state transformation pass - ty::Generator(..) => true, - - ty::Tuple(..) => ty.tuple_fields().any(needs_drop), - - // unions don't have destructors because of the child types, - // only if they manually implement `Drop` (handled above). - ty::Adt(def, _) if def.is_union() => false, - - ty::Adt(def, substs) => def - .variants - .iter() - .any(|variant| variant.fields.iter().any(|field| needs_drop(field.ty(tcx, substs)))), - }) -} - -pub fn provide(providers: &mut ty::query::Providers<'_>) { - *providers = ty::query::Providers { - is_copy_raw, - is_sized_raw, - is_freeze_raw, - needs_drop_raw, - ..*providers - }; -} diff --git a/src/librustc/traits/mod.rs b/src/librustc/traits/mod.rs index daaba95cf6b13..f6c939c1ce5a2 100644 --- a/src/librustc/traits/mod.rs +++ b/src/librustc/traits/mod.rs @@ -1255,7 +1255,6 @@ impl<'tcx> TraitObligation<'tcx> { } pub fn provide(providers: &mut ty::query::Providers<'_>) { - misc::provide(providers); *providers = ty::query::Providers { is_object_safe: object_safety::is_object_safe_provider, specialization_graph_of: specialize::specialization_graph_provider, diff --git a/src/librustc/ty/query/mod.rs b/src/librustc/ty/query/mod.rs index 973cd81014616..4126b161a821b 100644 --- a/src/librustc/ty/query/mod.rs +++ b/src/librustc/ty/query/mod.rs @@ -33,7 +33,7 @@ use crate::traits::Clauses; use crate::traits::{self, Vtable}; use crate::ty::steal::Steal; use crate::ty::subst::SubstsRef; -use crate::ty::util::NeedsDrop; +use crate::ty::util::AlwaysRequiresDrop; use crate::ty::{self, AdtSizedConstraint, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt}; use crate::util::common::ErrorReported; use rustc_data_structures::fingerprint::Fingerprint; diff --git a/src/librustc/ty/query/values.rs b/src/librustc/ty/query/values.rs index 900a91fe5cf59..b01d15c29b2db 100644 --- a/src/librustc/ty/query/values.rs +++ b/src/librustc/ty/query/values.rs @@ -1,4 +1,3 @@ -use crate::ty::util::NeedsDrop; use crate::ty::{self, AdtSizedConstraint, Ty, TyCtxt}; use rustc_span::symbol::Symbol; @@ -26,12 +25,6 @@ impl<'tcx> Value<'tcx> for ty::SymbolName { } } -impl<'tcx> Value<'tcx> for NeedsDrop { - fn from_cycle_error(_: TyCtxt<'tcx>) -> Self { - NeedsDrop(false) - } -} - impl<'tcx> Value<'tcx> for AdtSizedConstraint<'tcx> { fn from_cycle_error(tcx: TyCtxt<'tcx>) -> Self { AdtSizedConstraint(tcx.intern_type_list(&[tcx.types.err])) diff --git a/src/librustc/ty/util.rs b/src/librustc/ty/util.rs index 4dfff85d53147..eaa7a43b09174 100644 --- a/src/librustc/ty/util.rs +++ b/src/librustc/ty/util.rs @@ -18,6 +18,7 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_macros::HashStable; use rustc_span::Span; +use smallvec::SmallVec; use std::{cmp, fmt}; use syntax::ast; @@ -724,7 +725,23 @@ impl<'tcx> ty::TyS<'tcx> { /// Note that this method is used to check eligible types in unions. #[inline] pub fn needs_drop(&'tcx self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool { - tcx.needs_drop_raw(param_env.and(self)).0 + // Avoid querying in simple cases. + match needs_drop_components(self) { + Err(AlwaysRequiresDrop) => true, + Ok(components) => { + let query_ty = match *components { + [] => return false, + // If we've got a single component, call the query with that + // to increase the chance that we hit the query cache. + [component_ty] => component_ty, + _ => self, + }; + // This doesn't depend on regions, so try to minimize distinct. + // query keys used. + let erased = tcx.normalize_erasing_regions(param_env, query_ty); + tcx.needs_drop_raw(param_env.and(erased)) + } + } } pub fn same_type(a: Ty<'tcx>, b: Ty<'tcx>) -> bool { @@ -923,9 +940,6 @@ impl<'tcx> ty::TyS<'tcx> { } } -#[derive(Clone, HashStable)] -pub struct NeedsDrop(pub bool); - pub enum ExplicitSelf<'tcx> { ByValue, ByReference(ty::Region<'tcx>, hir::Mutability), @@ -974,3 +988,62 @@ impl<'tcx> ExplicitSelf<'tcx> { } } } + +/// Returns a list of types such that the given type needs drop if and only if +/// *any* of the returned types need drop. Returns `Err(AlwaysRequiresDrop)` if +/// this type always needs drop. +pub fn needs_drop_components(ty: Ty<'tcx>) -> Result; 4]>, AlwaysRequiresDrop> { + match ty.kind { + ty::Infer(ty::FreshIntTy(_)) + | ty::Infer(ty::FreshFloatTy(_)) + | ty::Bool + | ty::Int(_) + | ty::Uint(_) + | ty::Float(_) + | ty::Never + | ty::FnDef(..) + | ty::FnPtr(_) + | ty::Char + | ty::GeneratorWitness(..) + | ty::RawPtr(_) + | ty::Ref(..) + | ty::Str => Ok(SmallVec::new()), + + // Foreign types can never have destructors + ty::Foreign(..) => Ok(SmallVec::new()), + + // Pessimistically assume that all generators will require destructors + // as we don't know if a destructor is a noop or not until after the MIR + // state transformation pass + ty::Generator(..) | ty::Dynamic(..) | ty::Error => Err(AlwaysRequiresDrop), + + ty::Slice(ty) => needs_drop_components(ty), + ty::Array(elem_ty, ..) => { + match needs_drop_components(elem_ty) { + Ok(v) if v.is_empty() => Ok(v), + // Arrays of size zero don't need drop, even if their element + // type does. + _ => Ok(smallvec![ty]), + } + } + // If any field needs drop, then the whole tuple does. + ty::Tuple(..) => ty.tuple_fields().try_fold(SmallVec::new(), |mut acc, elem| { + acc.extend(needs_drop_components(elem)?); + Ok(acc) + }), + + // These require checking for `Copy` bounds or `Adt` destructors. + ty::Adt(..) + | ty::Projection(..) + | ty::UnnormalizedProjection(..) + | ty::Param(_) + | ty::Bound(..) + | ty::Placeholder(..) + | ty::Opaque(..) + | ty::Infer(_) + | ty::Closure(..) => Ok(smallvec![ty]), + } +} + +#[derive(Copy, Clone, Debug, HashStable)] +pub struct AlwaysRequiresDrop; diff --git a/src/librustc_ty/common_traits.rs b/src/librustc_ty/common_traits.rs new file mode 100644 index 0000000000000..9fe8a19311fb6 --- /dev/null +++ b/src/librustc_ty/common_traits.rs @@ -0,0 +1,40 @@ +//! Queries for checking whether a type implements one of a few common traits. + +use rustc::middle::lang_items; +use rustc::traits; +use rustc::ty::{self, Ty, TyCtxt}; +use rustc_span::DUMMY_SP; + +fn is_copy_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + is_item_raw(tcx, query, lang_items::CopyTraitLangItem) +} + +fn is_sized_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + is_item_raw(tcx, query, lang_items::SizedTraitLangItem) +} + +fn is_freeze_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + is_item_raw(tcx, query, lang_items::FreezeTraitLangItem) +} + +fn is_item_raw<'tcx>( + tcx: TyCtxt<'tcx>, + query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>, + item: lang_items::LangItem, +) -> bool { + let (param_env, ty) = query.into_parts(); + let trait_def_id = tcx.require_lang_item(item, None); + tcx.infer_ctxt().enter(|infcx| { + traits::type_known_to_meet_bound_modulo_regions( + &infcx, + param_env, + ty, + trait_def_id, + DUMMY_SP, + ) + }) +} + +pub(crate) fn provide(providers: &mut ty::query::Providers<'_>) { + *providers = ty::query::Providers { is_copy_raw, is_sized_raw, is_freeze_raw, ..*providers }; +} diff --git a/src/librustc_ty/lib.rs b/src/librustc_ty/lib.rs index e970faef02f28..7eef19b94e401 100644 --- a/src/librustc_ty/lib.rs +++ b/src/librustc_ty/lib.rs @@ -16,8 +16,12 @@ extern crate log; use rustc::ty::query::Providers; +mod common_traits; +mod needs_drop; mod ty; pub fn provide(providers: &mut Providers<'_>) { + common_traits::provide(providers); + needs_drop::provide(providers); ty::provide(providers); } diff --git a/src/librustc_ty/needs_drop.rs b/src/librustc_ty/needs_drop.rs new file mode 100644 index 0000000000000..1a65acb1f984b --- /dev/null +++ b/src/librustc_ty/needs_drop.rs @@ -0,0 +1,171 @@ +//! Check whether a type has (potentially) non-trivial drop glue. + +use rustc::ty::subst::Subst; +use rustc::ty::util::{needs_drop_components, AlwaysRequiresDrop}; +use rustc::ty::{self, Ty, TyCtxt}; +use rustc_data_structures::fx::FxHashSet; +use rustc_hir::def_id::DefId; +use rustc_span::DUMMY_SP; + +type NeedsDropResult = Result; + +fn needs_drop_raw<'tcx>(tcx: TyCtxt<'tcx>, query: ty::ParamEnvAnd<'tcx, Ty<'tcx>>) -> bool { + let adt_fields = + move |adt_def: &ty::AdtDef| tcx.adt_drop_tys(adt_def.did).map(|tys| tys.iter().copied()); + // If we don't know a type doesn't need drop, say it's a type parameter + // without a `Copy` bound, then we conservatively return that it needs + // drop. + let res = NeedsDropTypes::new(tcx, query.param_env, query.value, adt_fields).next().is_some(); + debug!("needs_drop_raw({:?}) = {:?}", query, res); + res +} + +struct NeedsDropTypes<'tcx, F> { + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + query_ty: Ty<'tcx>, + seen_tys: FxHashSet>, + /// A stack of types left to process. Each round, we pop something from the + /// stack and check if it needs drop. If the result depends on whether some + /// other types need drop we push them onto the stack. + unchecked_tys: Vec<(Ty<'tcx>, usize)>, + recursion_limit: usize, + adt_components: F, +} + +impl<'tcx, F> NeedsDropTypes<'tcx, F> { + fn new( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + ty: Ty<'tcx>, + adt_components: F, + ) -> Self { + let mut seen_tys = FxHashSet::default(); + seen_tys.insert(ty); + let recursion_limit = *tcx.sess.recursion_limit.get(); + Self { + tcx, + param_env, + seen_tys, + query_ty: ty, + unchecked_tys: vec![(ty, 0)], + recursion_limit, + adt_components, + } + } +} + +impl<'tcx, F, I> Iterator for NeedsDropTypes<'tcx, F> +where + F: Fn(&ty::AdtDef) -> NeedsDropResult, + I: Iterator>, +{ + type Item = NeedsDropResult>; + + fn next(&mut self) -> Option>> { + let tcx = self.tcx; + + while let Some((ty, level)) = self.unchecked_tys.pop() { + if level > self.recursion_limit { + // Not having a `Span` isn't great. But there's hopefully some other + // recursion limit error as well. + tcx.sess.span_err( + DUMMY_SP, + &format!("overflow while checking whether `{}` requires drop", self.query_ty), + ); + return Some(Err(AlwaysRequiresDrop)); + } + + let components = match needs_drop_components(ty) { + Err(e) => return Some(Err(e)), + Ok(components) => components, + }; + debug!("needs_drop_components({:?}) = {:?}", ty, components); + + for component in components { + match component.kind { + _ if component.is_copy_modulo_regions(tcx, self.param_env, DUMMY_SP) => (), + + ty::Array(elem_ty, len) => { + // Zero-length arrays never contain anything to drop. + if len.try_eval_usize(tcx, self.param_env) != Some(0) { + if self.seen_tys.insert(elem_ty) { + self.unchecked_tys.push((elem_ty, level + 1)); + } + } + } + + ty::Closure(def_id, substs) => { + for upvar_ty in substs.as_closure().upvar_tys(def_id, tcx) { + if self.seen_tys.insert(upvar_ty) { + self.unchecked_tys.push((upvar_ty, level + 1)); + } + } + } + + // Check for a `Drop` impl and whether this is a union or + // `ManuallyDrop`. If it's a struct or enum without a `Drop` + // impl then check whether the field types need `Drop`. + ty::Adt(adt_def, substs) => { + let tys = match (self.adt_components)(adt_def) { + Err(e) => return Some(Err(e)), + Ok(tys) => tys, + }; + for required_ty in tys { + let subst_ty = tcx.normalize_erasing_regions( + self.param_env, + required_ty.subst(tcx, substs), + ); + if self.seen_tys.insert(subst_ty) { + self.unchecked_tys.push((subst_ty, level + 1)); + } + } + } + ty::Opaque(..) | ty::Projection(..) | ty::Param(_) => { + if ty == component { + // Return the type to the caller so they can decide + // what to do with it. + return Some(Ok(component)); + } else if self.seen_tys.insert(component) { + // Store the type for later. We can't return here + // because we would then lose any other components + // of the type. + self.unchecked_tys.push((component, level + 1)); + } + } + _ => return Some(Err(AlwaysRequiresDrop)), + } + } + } + + return None; + } +} + +fn adt_drop_tys(tcx: TyCtxt<'_>, def_id: DefId) -> Result<&ty::List>, AlwaysRequiresDrop> { + let adt_components = move |adt_def: &ty::AdtDef| { + if adt_def.is_manually_drop() { + debug!("adt_drop_tys: `{:?}` is manually drop", adt_def); + return Ok(Vec::new().into_iter()); + } else if adt_def.destructor(tcx).is_some() { + debug!("adt_drop_tys: `{:?}` implements `Drop`", adt_def); + return Err(AlwaysRequiresDrop); + } else if adt_def.is_union() { + debug!("adt_drop_tys: `{:?}` is a union", adt_def); + return Ok(Vec::new().into_iter()); + } + Ok(adt_def.all_fields().map(|field| tcx.type_of(field.did)).collect::>().into_iter()) + }; + + let adt_ty = tcx.type_of(def_id); + let param_env = tcx.param_env(def_id); + let res: Result, _> = + NeedsDropTypes::new(tcx, param_env, adt_ty, adt_components).collect(); + + debug!("adt_drop_tys(`{}`) = `{:?}`", tcx.def_path_str(def_id), res); + res.map(|components| tcx.intern_type_list(&components)) +} + +pub(crate) fn provide(providers: &mut ty::query::Providers<'_>) { + *providers = ty::query::Providers { needs_drop_raw, adt_drop_tys, ..*providers }; +} diff --git a/src/test/ui/type-alias-impl-trait/issue-65918.rs b/src/test/ui/type-alias-impl-trait/issue-65918.rs index 97efb85ef64c7..4ba778d53ac08 100644 --- a/src/test/ui/type-alias-impl-trait/issue-65918.rs +++ b/src/test/ui/type-alias-impl-trait/issue-65918.rs @@ -1,3 +1,5 @@ +// ignore-test: This now ICEs again. + // build-pass #![feature(type_alias_impl_trait)] From d20646b2d8033f31423b5bda3e56776df115e144 Mon Sep 17 00:00:00 2001 From: Matthew Jasper Date: Fri, 31 Jan 2020 22:22:30 +0000 Subject: [PATCH 05/43] Address review comments * Handle arrays with const-generic lengths * Use closure for repeated code. --- src/librustc/ty/util.rs | 33 ++++++++++++++++++++++----------- src/librustc_ty/needs_drop.rs | 35 ++++++++++++++--------------------- 2 files changed, 36 insertions(+), 32 deletions(-) diff --git a/src/librustc/ty/util.rs b/src/librustc/ty/util.rs index eaa7a43b09174..6191d304719cb 100644 --- a/src/librustc/ty/util.rs +++ b/src/librustc/ty/util.rs @@ -18,6 +18,7 @@ use rustc_hir::def::DefKind; use rustc_hir::def_id::DefId; use rustc_macros::HashStable; use rustc_span::Span; +use rustc_target::abi::TargetDataLayout; use smallvec::SmallVec; use std::{cmp, fmt}; use syntax::ast; @@ -726,7 +727,7 @@ impl<'tcx> ty::TyS<'tcx> { #[inline] pub fn needs_drop(&'tcx self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> bool { // Avoid querying in simple cases. - match needs_drop_components(self) { + match needs_drop_components(self, &tcx.data_layout) { Err(AlwaysRequiresDrop) => true, Ok(components) => { let query_ty = match *components { @@ -736,7 +737,7 @@ impl<'tcx> ty::TyS<'tcx> { [component_ty] => component_ty, _ => self, }; - // This doesn't depend on regions, so try to minimize distinct. + // This doesn't depend on regions, so try to minimize distinct // query keys used. let erased = tcx.normalize_erasing_regions(param_env, query_ty); tcx.needs_drop_raw(param_env.and(erased)) @@ -992,7 +993,10 @@ impl<'tcx> ExplicitSelf<'tcx> { /// Returns a list of types such that the given type needs drop if and only if /// *any* of the returned types need drop. Returns `Err(AlwaysRequiresDrop)` if /// this type always needs drop. -pub fn needs_drop_components(ty: Ty<'tcx>) -> Result; 4]>, AlwaysRequiresDrop> { +pub fn needs_drop_components( + ty: Ty<'tcx>, + target_layout: &TargetDataLayout, +) -> Result; 2]>, AlwaysRequiresDrop> { match ty.kind { ty::Infer(ty::FreshIntTy(_)) | ty::Infer(ty::FreshFloatTy(_)) @@ -1017,18 +1021,25 @@ pub fn needs_drop_components(ty: Ty<'tcx>) -> Result; 4]>, Al // state transformation pass ty::Generator(..) | ty::Dynamic(..) | ty::Error => Err(AlwaysRequiresDrop), - ty::Slice(ty) => needs_drop_components(ty), - ty::Array(elem_ty, ..) => { - match needs_drop_components(elem_ty) { + ty::Slice(ty) => needs_drop_components(ty, target_layout), + ty::Array(elem_ty, size) => { + match needs_drop_components(elem_ty, target_layout) { Ok(v) if v.is_empty() => Ok(v), - // Arrays of size zero don't need drop, even if their element - // type does. - _ => Ok(smallvec![ty]), + res => match size.val.try_to_bits(target_layout.pointer_size) { + // Arrays of size zero don't need drop, even if their element + // type does. + Some(0) => Ok(SmallVec::new()), + Some(_) => res, + // We don't know which of the cases above we are in, so + // return the whole type and let the caller decide what to + // do. + None => Ok(smallvec![ty]), + }, } } // If any field needs drop, then the whole tuple does. - ty::Tuple(..) => ty.tuple_fields().try_fold(SmallVec::new(), |mut acc, elem| { - acc.extend(needs_drop_components(elem)?); + ty::Tuple(..) => ty.tuple_fields().try_fold(SmallVec::new(), move |mut acc, elem| { + acc.extend(needs_drop_components(elem, target_layout)?); Ok(acc) }), diff --git a/src/librustc_ty/needs_drop.rs b/src/librustc_ty/needs_drop.rs index 1a65acb1f984b..c01b3e384aec5 100644 --- a/src/librustc_ty/needs_drop.rs +++ b/src/librustc_ty/needs_drop.rs @@ -76,30 +76,25 @@ where return Some(Err(AlwaysRequiresDrop)); } - let components = match needs_drop_components(ty) { + let components = match needs_drop_components(ty, &tcx.data_layout) { Err(e) => return Some(Err(e)), Ok(components) => components, }; debug!("needs_drop_components({:?}) = {:?}", ty, components); + let queue_type = move |this: &mut Self, component: Ty<'tcx>| { + if this.seen_tys.insert(component) { + this.unchecked_tys.push((component, level + 1)); + } + }; + for component in components { match component.kind { _ if component.is_copy_modulo_regions(tcx, self.param_env, DUMMY_SP) => (), - ty::Array(elem_ty, len) => { - // Zero-length arrays never contain anything to drop. - if len.try_eval_usize(tcx, self.param_env) != Some(0) { - if self.seen_tys.insert(elem_ty) { - self.unchecked_tys.push((elem_ty, level + 1)); - } - } - } - ty::Closure(def_id, substs) => { for upvar_ty in substs.as_closure().upvar_tys(def_id, tcx) { - if self.seen_tys.insert(upvar_ty) { - self.unchecked_tys.push((upvar_ty, level + 1)); - } + queue_type(self, upvar_ty); } } @@ -116,21 +111,19 @@ where self.param_env, required_ty.subst(tcx, substs), ); - if self.seen_tys.insert(subst_ty) { - self.unchecked_tys.push((subst_ty, level + 1)); - } + queue_type(self, subst_ty); } } - ty::Opaque(..) | ty::Projection(..) | ty::Param(_) => { + ty::Array(..) | ty::Opaque(..) | ty::Projection(..) | ty::Param(_) => { if ty == component { - // Return the type to the caller so they can decide - // what to do with it. + // Return the type to the caller: they may be able + // to normalize further than we can. return Some(Ok(component)); - } else if self.seen_tys.insert(component) { + } else { // Store the type for later. We can't return here // because we would then lose any other components // of the type. - self.unchecked_tys.push((component, level + 1)); + queue_type(self, component); } } _ => return Some(Err(AlwaysRequiresDrop)), From 465b86253ce828e215d564fde53adf8742f0e3f6 Mon Sep 17 00:00:00 2001 From: Matthew Jasper Date: Sat, 1 Feb 2020 11:41:58 +0000 Subject: [PATCH 06/43] Use correct `ParamEnv` in `Instance::resolve` --- src/librustc/ty/instance.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/librustc/ty/instance.rs b/src/librustc/ty/instance.rs index 51a18f8eae274..f73e069df36c7 100644 --- a/src/librustc/ty/instance.rs +++ b/src/librustc/ty/instance.rs @@ -285,7 +285,7 @@ impl<'tcx> Instance<'tcx> { _ => { if Some(def_id) == tcx.lang_items().drop_in_place_fn() { let ty = substs.type_at(0); - if ty.needs_drop(tcx, ty::ParamEnv::reveal_all()) { + if ty.needs_drop(tcx, param_env.with_reveal_all()) { debug!(" => nontrivial drop glue"); ty::InstanceDef::DropGlue(def_id, Some(ty)) } else { From 6bf2cc2229768faa8e86e0e8a9f5bd8ebfc817a2 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 5 Feb 2020 09:44:03 +1100 Subject: [PATCH 07/43] Avoid instantiating many `Parser` structs in `generic_extension`. Currently, every iteration of the main loop in `generic_extension` instantiates a `Parser`, which is expensive because `Parser` is a large type. Many of those instantiations are only used immutably, particularly for simple-but-repetitive macros of the sort seen in `html5ever` and PR 68836. This commit initializes a single "base" parser outside the loop, and then uses `Cow` to avoid cloning it except for the mutating iterations. This speeds up `html5ever` runs by up to 15%. --- src/librustc_expand/lib.rs | 1 + src/librustc_expand/mbe/macro_parser.rs | 38 ++++----------- src/librustc_expand/mbe/macro_rules.rs | 62 ++++++++++++++++++------- 3 files changed, 56 insertions(+), 45 deletions(-) diff --git a/src/librustc_expand/lib.rs b/src/librustc_expand/lib.rs index 4fe7c268c4f0b..f119c956ced04 100644 --- a/src/librustc_expand/lib.rs +++ b/src/librustc_expand/lib.rs @@ -1,3 +1,4 @@ +#![feature(cow_is_borrowed)] #![feature(crate_visibility_modifier)] #![feature(decl_macro)] #![feature(proc_macro_diagnostic)] diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs index b14725fd731b1..78f22f3e443b1 100644 --- a/src/librustc_expand/mbe/macro_parser.rs +++ b/src/librustc_expand/mbe/macro_parser.rs @@ -78,13 +78,11 @@ use crate::mbe::{self, TokenTree}; use rustc_ast_pretty::pprust; use rustc_parse::parser::{FollowedByType, Parser, PathStyle}; -use rustc_parse::Directory; use rustc_session::parse::ParseSess; use rustc_span::symbol::{kw, sym, Symbol}; use syntax::ast::{Ident, Name}; use syntax::ptr::P; use syntax::token::{self, DocComment, Nonterminal, Token}; -use syntax::tokenstream::TokenStream; use rustc_errors::{FatalError, PResult}; use rustc_span::Span; @@ -92,6 +90,7 @@ use smallvec::{smallvec, SmallVec}; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sync::Lrc; +use std::borrow::Cow; use std::collections::hash_map::Entry::{Occupied, Vacant}; use std::mem; use std::ops::{Deref, DerefMut}; @@ -613,28 +612,9 @@ fn inner_parse_loop<'root, 'tt>( Success(()) } -/// Use the given sequence of token trees (`ms`) as a matcher. Match the given token stream `tts` -/// against it and return the match. -/// -/// # Parameters -/// -/// - `sess`: The session into which errors are emitted -/// - `tts`: The tokenstream we are matching against the pattern `ms` -/// - `ms`: A sequence of token trees representing a pattern against which we are matching -/// - `directory`: Information about the file locations (needed for the black-box parser) -/// - `recurse_into_modules`: Whether or not to recurse into modules (needed for the black-box -/// parser) -pub(super) fn parse( - sess: &ParseSess, - tts: TokenStream, - ms: &[TokenTree], - directory: Option>, - recurse_into_modules: bool, -) -> NamedParseResult { - // Create a parser that can be used for the "black box" parts. - let mut parser = - Parser::new(sess, tts, directory, recurse_into_modules, true, rustc_parse::MACRO_ARGUMENTS); - +/// Use the given sequence of token trees (`ms`) as a matcher. Match the token +/// stream from the given `parser` against it and return the match. +pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> NamedParseResult { // A queue of possible matcher positions. We initialize it with the matcher position in which // the "dot" is before the first token of the first token tree in `ms`. `inner_parse_loop` then // processes all of these possible matcher positions and produces possible next positions into @@ -659,7 +639,7 @@ pub(super) fn parse( // parsing from the black-box parser done. The result is that `next_items` will contain a // bunch of possible next matcher positions in `next_items`. match inner_parse_loop( - sess, + parser.sess, &mut cur_items, &mut next_items, &mut eof_items, @@ -684,7 +664,7 @@ pub(super) fn parse( if eof_items.len() == 1 { let matches = eof_items[0].matches.iter_mut().map(|dv| Lrc::make_mut(dv).pop().unwrap()); - return nameize(sess, ms, matches); + return nameize(parser.sess, ms, matches); } else if eof_items.len() > 1 { return Error( parser.token.span, @@ -736,13 +716,13 @@ pub(super) fn parse( // If there are no possible next positions AND we aren't waiting for the black-box parser, // then there is a syntax error. else if bb_items.is_empty() && next_items.is_empty() { - return Failure(parser.token.take(), "no rules expected this token in macro call"); + return Failure(parser.token.clone(), "no rules expected this token in macro call"); } // Dump all possible `next_items` into `cur_items` for the next iteration. else if !next_items.is_empty() { // Now process the next token cur_items.extend(next_items.drain(..)); - parser.bump(); + parser.to_mut().bump(); } // Finally, we have the case where we need to call the black-box parser to get some // nonterminal. @@ -754,7 +734,7 @@ pub(super) fn parse( let match_cur = item.match_cur; item.push_match( match_cur, - MatchedNonterminal(Lrc::new(parse_nt(&mut parser, span, ident.name))), + MatchedNonterminal(Lrc::new(parse_nt(parser.to_mut(), span, ident.name))), ); item.idx += 1; item.match_cur += 1; diff --git a/src/librustc_expand/mbe/macro_rules.rs b/src/librustc_expand/mbe/macro_rules.rs index 29d41543fbf8c..9432790e78ced 100644 --- a/src/librustc_expand/mbe/macro_rules.rs +++ b/src/librustc_expand/mbe/macro_rules.rs @@ -1,11 +1,11 @@ -use crate::base::{DummyResult, ExtCtxt, MacResult, TTMacroExpander}; +use crate::base::{DummyResult, ExpansionData, ExtCtxt, MacResult, TTMacroExpander}; use crate::base::{SyntaxExtension, SyntaxExtensionKind}; use crate::expand::{ensure_complete_parse, parse_ast_fragment, AstFragment, AstFragmentKind}; use crate::mbe; use crate::mbe::macro_check; -use crate::mbe::macro_parser::parse; +use crate::mbe::macro_parser::parse_tt; use crate::mbe::macro_parser::{Error, Failure, Success}; -use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq, NamedParseResult}; +use crate::mbe::macro_parser::{MatchedNonterminal, MatchedSeq}; use crate::mbe::transcribe::transcribe; use rustc_ast_pretty::pprust; @@ -166,9 +166,9 @@ impl TTMacroExpander for MacroRulesMacroExpander { } } -fn trace_macros_note(cx: &mut ExtCtxt<'_>, sp: Span, message: String) { +fn trace_macros_note(cx_expansions: &mut FxHashMap>, sp: Span, message: String) { let sp = sp.macro_backtrace().last().map(|trace| trace.call_site).unwrap_or(sp); - cx.expansions.entry(sp).or_default().push(message); + cx_expansions.entry(sp).or_default().push(message); } /// Given `lhses` and `rhses`, this is the new macro we create @@ -184,12 +184,36 @@ fn generic_extension<'cx>( ) -> Box { if cx.trace_macros() { let msg = format!("expanding `{}! {{ {} }}`", name, pprust::tts_to_string(arg.clone())); - trace_macros_note(cx, sp, msg); + trace_macros_note(&mut cx.expansions, sp, msg); } // Which arm's failure should we report? (the one furthest along) let mut best_failure: Option<(Token, &str)> = None; + + // We create a base parser that can be used for the "black box" parts. + // Every iteration needs a fresh copy of that base parser. However, the + // parser is not mutated on many of the iterations, particularly when + // dealing with macros like this: + // + // macro_rules! foo { + // ("a") => (A); + // ("b") => (B); + // ("c") => (C); + // // ... etc. (maybe hundreds more) + // } + // + // as seen in the `html5ever` benchmark. We use a `Cow` so that the base + // parser is only cloned when necessary (upon mutation). Furthermore, we + // reinitialize the `Cow` with the base parser at the start of every + // iteration, so that any mutated parsers are not reused. This is all quite + // hacky, but speeds up the `html5ever` benchmark significantly. (Issue + // 68836 suggests a more comprehensive but more complex change to deal with + // this situation.) + let base_parser = base_parser_from_cx(&cx.current_expansion, &cx.parse_sess, arg.clone()); + for (i, lhs) in lhses.iter().enumerate() { + let mut parser = Cow::Borrowed(&base_parser); + // try each arm's matchers let lhs_tt = match *lhs { mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], @@ -202,7 +226,7 @@ fn generic_extension<'cx>( // are not recorded. On the first `Success(..)`ful matcher, the spans are merged. let mut gated_spans_snaphot = mem::take(&mut *cx.parse_sess.gated_spans.spans.borrow_mut()); - match parse_tt(cx, lhs_tt, arg.clone()) { + match parse_tt(&mut parser, lhs_tt) { Success(named_matches) => { // The matcher was `Success(..)`ful. // Merge the gated spans from parsing the matcher with the pre-existing ones. @@ -232,7 +256,7 @@ fn generic_extension<'cx>( if cx.trace_macros() { let msg = format!("to `{}`", pprust::tts_to_string(tts.clone())); - trace_macros_note(cx, sp, msg); + trace_macros_note(&mut cx.expansions, sp, msg); } let directory = Directory { @@ -269,6 +293,7 @@ fn generic_extension<'cx>( // Restore to the state before snapshotting and maybe try again. mem::swap(&mut gated_spans_snaphot, &mut cx.parse_sess.gated_spans.spans.borrow_mut()); } + drop(base_parser); let (token, label) = best_failure.expect("ran no matchers"); let span = token.span.substitute_dummy(sp); @@ -286,7 +311,9 @@ fn generic_extension<'cx>( mbe::TokenTree::Delimited(_, ref delim) => &delim.tts[..], _ => continue, }; - match parse_tt(cx, lhs_tt, arg.clone()) { + let base_parser = + base_parser_from_cx(&cx.current_expansion, &cx.parse_sess, arg.clone()); + match parse_tt(&mut Cow::Borrowed(&base_parser), lhs_tt) { Success(_) => { if comma_span.is_dummy() { err.note("you might be missing a comma"); @@ -368,7 +395,8 @@ pub fn compile_declarative_macro( ), ]; - let argument_map = match parse(sess, body, &argument_gram, None, true) { + let base_parser = Parser::new(sess, body, None, true, true, rustc_parse::MACRO_ARGUMENTS); + let argument_map = match parse_tt(&mut Cow::Borrowed(&base_parser), &argument_gram) { Success(m) => m, Failure(token, msg) => { let s = parse_failure_msg(&token); @@ -1184,14 +1212,16 @@ fn quoted_tt_to_string(tt: &mbe::TokenTree) -> String { } } -/// Use this token tree as a matcher to parse given tts. -fn parse_tt(cx: &ExtCtxt<'_>, mtch: &[mbe::TokenTree], tts: TokenStream) -> NamedParseResult { - // `None` is because we're not interpolating +fn base_parser_from_cx<'cx>( + current_expansion: &'cx ExpansionData, + sess: &'cx ParseSess, + tts: TokenStream, +) -> Parser<'cx> { let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), - ownership: cx.current_expansion.directory_ownership, + path: Cow::from(current_expansion.module.directory.as_path()), + ownership: current_expansion.directory_ownership, }; - parse(cx.parse_sess(), tts, mtch, Some(directory), true) + Parser::new(sess, tts, Some(directory), true, true, rustc_parse::MACRO_ARGUMENTS) } /// Generates an appropriate parsing failure message. For EOF, this is "unexpected end...". For From f840a955bd449810e75d8320b4c46482d6dbdec1 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 5 Feb 2020 14:33:08 +1100 Subject: [PATCH 08/43] Remove the `Cow` from `Directory`. The previous commit wrapped `Parser` within a `Cow` for the hot macro parsing path. As a result, there's no need for the `Cow` within `Directory`, because it lies within `Parser`. --- src/librustc_expand/mbe/macro_rules.rs | 4 ++-- src/librustc_parse/lib.rs | 9 ++++----- src/librustc_parse/parser/mod.rs | 9 ++++----- src/librustc_parse/parser/module.rs | 6 +++--- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/librustc_expand/mbe/macro_rules.rs b/src/librustc_expand/mbe/macro_rules.rs index 9432790e78ced..9e6edee265c98 100644 --- a/src/librustc_expand/mbe/macro_rules.rs +++ b/src/librustc_expand/mbe/macro_rules.rs @@ -260,7 +260,7 @@ fn generic_extension<'cx>( } let directory = Directory { - path: Cow::from(cx.current_expansion.module.directory.as_path()), + path: cx.current_expansion.module.directory.clone(), ownership: cx.current_expansion.directory_ownership, }; let mut p = Parser::new(cx.parse_sess(), tts, Some(directory), true, false, None); @@ -1218,7 +1218,7 @@ fn base_parser_from_cx<'cx>( tts: TokenStream, ) -> Parser<'cx> { let directory = Directory { - path: Cow::from(current_expansion.module.directory.as_path()), + path: current_expansion.module.directory.clone(), ownership: current_expansion.directory_ownership, }; Parser::new(sess, tts, Some(directory), true, true, rustc_parse::MACRO_ARGUMENTS) diff --git a/src/librustc_parse/lib.rs b/src/librustc_parse/lib.rs index cd674e3c5ebef..4aad2c0f68a29 100644 --- a/src/librustc_parse/lib.rs +++ b/src/librustc_parse/lib.rs @@ -12,8 +12,7 @@ use syntax::ast; use syntax::token::{self, Nonterminal}; use syntax::tokenstream::{self, TokenStream, TokenTree}; -use std::borrow::Cow; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str; use log::info; @@ -29,8 +28,8 @@ pub mod validate_attr; pub mod config; #[derive(Clone)] -pub struct Directory<'a> { - pub path: Cow<'a, Path>, +pub struct Directory { + pub path: PathBuf, pub ownership: DirectoryOwnership, } @@ -274,7 +273,7 @@ pub fn stream_to_parser<'a>( pub fn stream_to_parser_with_base_dir<'a>( sess: &'a ParseSess, stream: TokenStream, - base_dir: Directory<'a>, + base_dir: Directory, ) -> Parser<'a> { Parser::new(sess, stream, Some(base_dir), true, false, None) } diff --git a/src/librustc_parse/parser/mod.rs b/src/librustc_parse/parser/mod.rs index 8c1839da1cb8f..cb95750d984e9 100644 --- a/src/librustc_parse/parser/mod.rs +++ b/src/librustc_parse/parser/mod.rs @@ -29,7 +29,6 @@ use syntax::token::{self, DelimToken, Token, TokenKind}; use syntax::tokenstream::{self, DelimSpan, TokenStream, TokenTree, TreeAndJoint}; use syntax::util::comments::{doc_comment_style, strip_doc_comment_decoration}; -use std::borrow::Cow; use std::path::PathBuf; use std::{cmp, mem, slice}; @@ -114,7 +113,7 @@ pub struct Parser<'a> { prev_token_kind: PrevTokenKind, restrictions: Restrictions, /// Used to determine the path to externally loaded source files. - pub(super) directory: Directory<'a>, + pub(super) directory: Directory, /// `true` to parse sub-modules in other files. // Public for rustfmt usage. pub recurse_into_file_modules: bool, @@ -376,7 +375,7 @@ impl<'a> Parser<'a> { pub fn new( sess: &'a ParseSess, tokens: TokenStream, - directory: Option>, + directory: Option, recurse_into_file_modules: bool, desugar_doc_comments: bool, subparser_name: Option<&'static str>, @@ -390,7 +389,7 @@ impl<'a> Parser<'a> { restrictions: Restrictions::empty(), recurse_into_file_modules, directory: Directory { - path: Cow::from(PathBuf::new()), + path: PathBuf::new(), ownership: DirectoryOwnership::Owned { relative: None }, }, root_module_name: None, @@ -418,7 +417,7 @@ impl<'a> Parser<'a> { &sess.source_map().lookup_char_pos(parser.token.span.lo()).file.unmapped_path { if let Some(directory_path) = path.parent() { - parser.directory.path = Cow::from(directory_path.to_path_buf()); + parser.directory.path = directory_path.to_path_buf(); } } } diff --git a/src/librustc_parse/parser/module.rs b/src/librustc_parse/parser/module.rs index 6ce94d3c6793c..0c8fad03d8690 100644 --- a/src/librustc_parse/parser/module.rs +++ b/src/librustc_parse/parser/module.rs @@ -285,7 +285,7 @@ impl<'a> Parser<'a> { fn push_directory(&mut self, id: Ident, attrs: &[Attribute]) { if let Some(path) = attr::first_attr_value_str_by_name(attrs, sym::path) { - self.directory.path.to_mut().push(&*path.as_str()); + self.directory.path.push(&*path.as_str()); self.directory.ownership = DirectoryOwnership::Owned { relative: None }; } else { // We have to push on the current module name in the case of relative @@ -297,10 +297,10 @@ impl<'a> Parser<'a> { if let DirectoryOwnership::Owned { relative } = &mut self.directory.ownership { if let Some(ident) = relative.take() { // remove the relative offset - self.directory.path.to_mut().push(&*ident.as_str()); + self.directory.path.push(&*ident.as_str()); } } - self.directory.path.to_mut().push(&*id.as_str()); + self.directory.path.push(&*id.as_str()); } } } From 2a13b24d369b8619f0197993cd5dc60f7217ed72 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 5 Feb 2020 15:09:24 +1100 Subject: [PATCH 09/43] Change condition ordering in `parse_tt`. This is a small win, because `Failure` is much more common than `Success`. --- src/librustc_expand/mbe/macro_parser.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/librustc_expand/mbe/macro_parser.rs b/src/librustc_expand/mbe/macro_parser.rs index 78f22f3e443b1..5bf7602ea6e8f 100644 --- a/src/librustc_expand/mbe/macro_parser.rs +++ b/src/librustc_expand/mbe/macro_parser.rs @@ -689,9 +689,14 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na // unnecessary implicit clone later in Rc::make_mut. drop(eof_items); + // If there are no possible next positions AND we aren't waiting for the black-box parser, + // then there is a syntax error. + if bb_items.is_empty() && next_items.is_empty() { + return Failure(parser.token.clone(), "no rules expected this token in macro call"); + } // Another possibility is that we need to call out to parse some rust nonterminal // (black-box) parser. However, if there is not EXACTLY ONE of these, something is wrong. - if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { + else if (!bb_items.is_empty() && !next_items.is_empty()) || bb_items.len() > 1 { let nts = bb_items .iter() .map(|item| match item.top_elts.get_tt(item.idx) { @@ -713,11 +718,6 @@ pub(super) fn parse_tt(parser: &mut Cow<'_, Parser<'_>>, ms: &[TokenTree]) -> Na ), ); } - // If there are no possible next positions AND we aren't waiting for the black-box parser, - // then there is a syntax error. - else if bb_items.is_empty() && next_items.is_empty() { - return Failure(parser.token.clone(), "no rules expected this token in macro call"); - } // Dump all possible `next_items` into `cur_items` for the next iteration. else if !next_items.is_empty() { // Now process the next token From 3998249ae77f3a6e1c6e44de944e9d5928963594 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 22:00:25 -0500 Subject: [PATCH 10/43] remove unnecessary local variable assignment in context manager --- src/bootstrap/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 1935759a5628e..563f12e7bacbd 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -351,7 +351,7 @@ def support_xz(): try: with tempfile.NamedTemporaryFile(delete=False) as temp_file: temp_path = temp_file.name - with tarfile.open(temp_path, "w:xz") as tar: + with tarfile.open(temp_path, "w:xz"): pass return True except tarfile.CompressionError: From 19aaf639459705e2b397f2b8a12f3d73ef0748b2 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 22:04:44 -0500 Subject: [PATCH 11/43] PEP8 format spacing --- src/bootstrap/bootstrap.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 563f12e7bacbd..50e1726240fff 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -80,7 +80,7 @@ def _download(path, url, probably_big, verbose, exception): option = "-s" run(["curl", option, "-y", "30", "-Y", "10", # timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", "30", # timeout if cannot connect within 30 seconds + "--connect-timeout", "30", # timeout if cannot connect within 30 seconds "--retry", "3", "-Sf", "-o", path, url], verbose=verbose, exception=exception) @@ -332,7 +332,6 @@ def __init__(self): self.use_vendored_sources = '' self.verbose = False - def download_stage0(self): """Fetch the build system for Rust, written in Rust @@ -825,7 +824,7 @@ def check_vendored_status(self): if not os.path.exists(vendor_dir): print('error: vendoring required, but vendor directory does not exist.') print(' Run `cargo vendor` without sudo to initialize the ' - 'vendor directory.') + 'vendor directory.') raise Exception("{} not found".format(vendor_dir)) if self.use_vendored_sources: @@ -839,7 +838,7 @@ def check_vendored_status(self): "\n" "[source.vendored-sources]\n" "directory = '{}/vendor'\n" - .format(self.rust_root)) + .format(self.rust_root)) else: if os.path.exists('.cargo'): shutil.rmtree('.cargo') From 6ce8d2b00005143d5f0fb0c9b712ece7709b3ecf Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 22:09:55 -0500 Subject: [PATCH 12/43] PEP8 format spacing --- src/bootstrap/configure.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index 7cfc5385e2104..27bd6d028cd32 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -392,11 +392,12 @@ def set(key, value): def is_number(value): - try: - float(value) - return True - except ValueError: - return False + try: + float(value) + return True + except ValueError: + return False + # Here we walk through the constructed configuration we have from the parsed # command line arguments. We then apply each piece of configuration by From adde3d443d042cd53f7448ce1d6f32e1634fcf28 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 22:30:11 -0500 Subject: [PATCH 13/43] PEP8 format spacing --- src/tools/publish_toolstate.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/tools/publish_toolstate.py b/src/tools/publish_toolstate.py index 61762ae1d9b04..967333c1ace4f 100755 --- a/src/tools/publish_toolstate.py +++ b/src/tools/publish_toolstate.py @@ -1,11 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -## This script publishes the new "current" toolstate in the toolstate repo (not to be -## confused with publishing the test results, which happens in -## `src/ci/docker/x86_64-gnu-tools/checktools.sh`). -## It is set as callback for `src/ci/docker/x86_64-gnu-tools/repo.sh` by the CI scripts -## when a new commit lands on `master` (i.e., after it passed all checks on `auto`). +# This script publishes the new "current" toolstate in the toolstate repo (not to be +# confused with publishing the test results, which happens in +# `src/ci/docker/x86_64-gnu-tools/checktools.sh`). +# It is set as callback for `src/ci/docker/x86_64-gnu-tools/repo.sh` by the CI scripts +# when a new commit lands on `master` (i.e., after it passed all checks on `auto`). from __future__ import print_function @@ -103,6 +103,7 @@ def validate_maintainers(repo, github_token): print("The build will fail due to this.") exit(1) + def read_current_status(current_commit, path): '''Reads build status of `current_commit` from content of `history/*.tsv` ''' @@ -113,14 +114,17 @@ def read_current_status(current_commit, path): return json.loads(status) return {} + def gh_url(): return os.environ['TOOLSTATE_ISSUES_API_URL'] + def maybe_delink(message): if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None: return message.replace("@", "") return message + def issue( tool, status, @@ -164,6 +168,7 @@ def issue( )) response.read() + def update_latest( current_commit, relevant_pr_number, @@ -194,7 +199,7 @@ def update_latest( for status in latest: tool = status['tool'] changed = False - create_issue_for_status = None # set to the status that caused the issue + create_issue_for_status = None # set to the status that caused the issue for os, s in current_status.items(): old = status[os] From 60889d418e37897e272844d15b03fed62c60b92d Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 23:35:27 -0500 Subject: [PATCH 14/43] remove unnecessary semicolons --- src/ci/cpu-usage-over-time.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ci/cpu-usage-over-time.py b/src/ci/cpu-usage-over-time.py index daf21670b3339..78ac060368193 100644 --- a/src/ci/cpu-usage-over-time.py +++ b/src/ci/cpu-usage-over-time.py @@ -148,11 +148,11 @@ def idle_since(self, prev): print('unknown platform', sys.platform) sys.exit(1) -cur_state = State(); +cur_state = State() print("Time,Idle") while True: - time.sleep(1); - next_state = State(); + time.sleep(1) + next_state = State() now = datetime.datetime.utcnow().isoformat() idle = next_state.idle_since(cur_state) print("%s,%s" % (now, idle)) From e30dd86c61c8159b02940b888d876a57d8b07b7e Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 23:47:29 -0500 Subject: [PATCH 15/43] PEP8 format spacing --- src/etc/debugger_pretty_printers_common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/etc/debugger_pretty_printers_common.py b/src/etc/debugger_pretty_printers_common.py index 385ce8efab87b..b3f8f50636bee 100644 --- a/src/etc/debugger_pretty_printers_common.py +++ b/src/etc/debugger_pretty_printers_common.py @@ -212,7 +212,6 @@ def __classify_struct(self): # REGULAR STRUCT return TYPE_KIND_REGULAR_STRUCT - def __classify_union(self): assert self.get_dwarf_type_kind() == DWARF_TYPE_CODE_UNION @@ -233,7 +232,6 @@ def __classify_union(self): else: return TYPE_KIND_REGULAR_UNION - def __conforms_to_field_layout(self, expected_fields): actual_fields = self.get_fields() actual_field_count = len(actual_fields) @@ -363,6 +361,7 @@ def extract_tail_head_ptr_and_cap_from_std_vecdeque(vec_val): assert data_ptr.type.get_dwarf_type_kind() == DWARF_TYPE_CODE_PTR return (tail, head, data_ptr, capacity) + def extract_length_and_ptr_from_slice(slice_val): assert (slice_val.type.get_type_kind() == TYPE_KIND_SLICE or slice_val.type.get_type_kind() == TYPE_KIND_STR_SLICE) @@ -376,8 +375,10 @@ def extract_length_and_ptr_from_slice(slice_val): assert data_ptr.type.get_dwarf_type_kind() == DWARF_TYPE_CODE_PTR return (length, data_ptr) + UNQUALIFIED_TYPE_MARKERS = frozenset(["(", "[", "&", "*"]) + def extract_type_name(qualified_type_name): """Extracts the type name from a fully qualified path""" if qualified_type_name[0] in UNQUALIFIED_TYPE_MARKERS: @@ -393,6 +394,7 @@ def extract_type_name(qualified_type_name): else: return qualified_type_name[index + 2:] + try: compat_str = unicode # Python 2 except NameError: From f38e2701d862379eca06475d196438038ca72564 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Fri, 7 Feb 2020 23:49:40 -0500 Subject: [PATCH 16/43] remove unnecessary sys import --- src/etc/dec2flt_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etc/dec2flt_table.py b/src/etc/dec2flt_table.py index 85395d2ecdfc7..4979882ffeaff 100755 --- a/src/etc/dec2flt_table.py +++ b/src/etc/dec2flt_table.py @@ -14,7 +14,6 @@ even larger, and it's already uncomfortably large (6 KiB). """ from __future__ import print_function -import sys from math import ceil, log from fractions import Fraction from collections import namedtuple @@ -82,6 +81,7 @@ def error(f, e, z): ulp_err = abs_err / Fraction(2) ** z.exp return float(ulp_err) + HEADER = """ //! Tables of approximations of powers of ten. //! DO NOT MODIFY: Generated by `src/etc/dec2flt_table.py` From d36634315a9e07ca4517b9d04c5bd10f218d61d8 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Sat, 8 Feb 2020 00:01:32 -0500 Subject: [PATCH 17/43] PEP8 format spacing --- src/etc/gdb_rust_pretty_printing.py | 43 +++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/src/etc/gdb_rust_pretty_printing.py b/src/etc/gdb_rust_pretty_printing.py index 5da01b96fa5e3..0914c22eb13f0 100755 --- a/src/etc/gdb_rust_pretty_printing.py +++ b/src/etc/gdb_rust_pretty_printing.py @@ -9,7 +9,7 @@ if sys.version_info[0] >= 3: xrange = range -rust_enabled = 'set language rust' in gdb.execute('complete set language ru', to_string = True) +rust_enabled = 'set language rust' in gdb.execute('complete set language ru', to_string=True) # The btree pretty-printers fail in a confusing way unless # https://sourceware.org/bugzilla/show_bug.cgi?id=21763 is fixed. @@ -21,9 +21,10 @@ if int(_match.group(1)) > 8 or (int(_match.group(1)) == 8 and int(_match.group(2)) >= 1): gdb_81 = True -#=============================================================================== +# =============================================================================== # GDB Pretty Printing Module for Rust -#=============================================================================== +# =============================================================================== + class GdbType(rustpp.Type): @@ -133,39 +134,39 @@ def rust_pretty_printer_lookup_function(gdb_val): if type_kind == rustpp.TYPE_KIND_REGULAR_STRUCT: return RustStructPrinter(val, - omit_first_field = False, - omit_type_name = False, - is_tuple_like = False) + omit_first_field=False, + omit_type_name=False, + is_tuple_like=False) if type_kind == rustpp.TYPE_KIND_STRUCT_VARIANT: return RustStructPrinter(val, - omit_first_field = True, - omit_type_name = False, - is_tuple_like = False) + omit_first_field=True, + omit_type_name=False, + is_tuple_like=False) if type_kind == rustpp.TYPE_KIND_STR_SLICE: return RustStringSlicePrinter(val) if type_kind == rustpp.TYPE_KIND_TUPLE: return RustStructPrinter(val, - omit_first_field = False, - omit_type_name = True, - is_tuple_like = True) + omit_first_field=False, + omit_type_name=True, + is_tuple_like=True) if type_kind == rustpp.TYPE_KIND_TUPLE_STRUCT: return RustStructPrinter(val, - omit_first_field = False, - omit_type_name = False, - is_tuple_like = True) + omit_first_field=False, + omit_type_name=False, + is_tuple_like=True) if type_kind == rustpp.TYPE_KIND_CSTYLE_VARIANT: return RustCStyleVariantPrinter(val.get_child_at_index(0)) if type_kind == rustpp.TYPE_KIND_TUPLE_VARIANT: return RustStructPrinter(val, - omit_first_field = True, - omit_type_name = False, - is_tuple_like = True) + omit_first_field=True, + omit_type_name=False, + is_tuple_like=True) if type_kind == rustpp.TYPE_KIND_SINGLETON_ENUM: variant = get_field_at_index(gdb_val, 0) @@ -189,9 +190,9 @@ def rust_pretty_printer_lookup_function(gdb_val): return None -#=------------------------------------------------------------------------------ +# =------------------------------------------------------------------------------ # Pretty Printer Classes -#=------------------------------------------------------------------------------ +# =------------------------------------------------------------------------------ class RustEmptyPrinter(object): def __init__(self, val): self.__val = val @@ -355,6 +356,7 @@ def children_of_node(boxed_node, height, want_values): else: yield keys[i]['value']['value'] + class RustStdBTreeSetPrinter(object): def __init__(self, val): self.__val = val @@ -429,6 +431,7 @@ def to_string(self): def display_hint(self): return "string" + class RustCStyleVariantPrinter(object): def __init__(self, val): assert val.type.get_dwarf_type_kind() == rustpp.DWARF_TYPE_CODE_ENUM From a53f45fe90daf376558d018bc0a474d2a83626f0 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Sat, 8 Feb 2020 00:02:11 -0500 Subject: [PATCH 18/43] PEP8 format spacing, split import statements --- src/etc/generate-deriving-span-tests.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/etc/generate-deriving-span-tests.py b/src/etc/generate-deriving-span-tests.py index afa6bbdae4e9e..c42f942c63cf5 100755 --- a/src/etc/generate-deriving-span-tests.py +++ b/src/etc/generate-deriving-span-tests.py @@ -8,7 +8,8 @@ sample usage: src/etc/generate-deriving-span-tests.py """ -import os, stat +import os +import stat TEST_DIR = os.path.abspath( os.path.join(os.path.dirname(__file__), '../test/ui/derives/')) @@ -56,6 +57,7 @@ ENUM_TUPLE, ENUM_STRUCT, STRUCT_FIELDS, STRUCT_TUPLE = range(4) + def create_test_case(type, trait, super_traits, error_count): string = [ENUM_STRING, ENUM_STRUCT_VARIANT_STRING, STRUCT_STRING, STRUCT_TUPLE_STRING][type] all_traits = ','.join([trait] + super_traits) @@ -63,8 +65,9 @@ def create_test_case(type, trait, super_traits, error_count): error_deriving = '#[derive(%s)]' % super_traits if super_traits else '' errors = '\n'.join('//~%s ERROR' % ('^' * n) for n in range(error_count)) - code = string.format(traits = all_traits, errors = errors) - return TEMPLATE.format(error_deriving=error_deriving, code = code) + code = string.format(traits=all_traits, errors=errors) + return TEMPLATE.format(error_deriving=error_deriving, code=code) + def write_file(name, string): test_file = os.path.join(TEST_DIR, 'derives-span-%s.rs' % name) @@ -86,10 +89,10 @@ def write_file(name, string): traits = { 'Default': (STRUCT, [], 1), - 'FromPrimitive': (0, [], 0), # only works for C-like enums + 'FromPrimitive': (0, [], 0), # only works for C-like enums - 'Decodable': (0, [], 0), # FIXME: quoting gives horrible spans - 'Encodable': (0, [], 0), # FIXME: quoting gives horrible spans + 'Decodable': (0, [], 0), # FIXME: quoting gives horrible spans + 'Encodable': (0, [], 0), # FIXME: quoting gives horrible spans } for (trait, supers, errs) in [('Clone', [], 1), From 8d04b95188fc96236472b7affae73ccfc5547636 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Sat, 8 Feb 2020 00:03:51 -0500 Subject: [PATCH 19/43] remove unnecessary import statement --- src/etc/generate-keyword-tests.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/etc/generate-keyword-tests.py b/src/etc/generate-keyword-tests.py index bc046a8f42d0b..77c3d2758c6dc 100755 --- a/src/etc/generate-keyword-tests.py +++ b/src/etc/generate-keyword-tests.py @@ -11,7 +11,6 @@ import sys import os -import datetime import stat From 85e3661214564010bdb6858d3253c214e686dc04 Mon Sep 17 00:00:00 2001 From: Chris Simpkins Date: Sat, 8 Feb 2020 00:12:25 -0500 Subject: [PATCH 20/43] PEP8 format spacing, remove unnecessary local variable assignment --- src/etc/htmldocck.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/src/etc/htmldocck.py b/src/etc/htmldocck.py index e8be2b9b53710..7789b24b62c83 100644 --- a/src/etc/htmldocck.py +++ b/src/etc/htmldocck.py @@ -131,6 +131,7 @@ except NameError: unichr = chr + class CustomHTMLParser(HTMLParser): """simplified HTML parser. @@ -169,21 +170,25 @@ def close(self): HTMLParser.close(self) return self.__builder.close() + Command = namedtuple('Command', 'negated cmd args lineno context') + class FailedCheck(Exception): pass + class InvalidCheck(Exception): pass + def concat_multi_lines(f): """returns a generator out of the file object, which - removes `\\` then `\n` then a shared prefix with the previous line then optional whitespace; - keeps a line number (starting from 0) of the first line being concatenated.""" - lastline = None # set to the last line when the last line has a backslash + lastline = None # set to the last line when the last line has a backslash firstlineno = None catenated = '' for lineno, line in enumerate(f): @@ -208,6 +213,7 @@ def concat_multi_lines(f): if lastline is not None: print_err(lineno, line, 'Trailing backslash at the end of the file') + LINE_PATTERN = re.compile(r''' (?<=(?!?) (?P[A-Za-z]+(?:-[A-Za-z]+)*) @@ -252,7 +258,7 @@ def flatten(node): def normalize_xpath(path): if path.startswith('//'): - return '.' + path # avoid warnings + return '.' + path # avoid warnings elif path.startswith('.//'): return path else: @@ -316,7 +322,7 @@ def get_dir(self, path): def check_string(data, pat, regexp): if not pat: - return True # special case a presence testing + return True # special case a presence testing elif regexp: return re.search(pat, data, flags=re.UNICODE) is not None else: @@ -353,7 +359,7 @@ def check_tree_text(tree, path, pat, regexp): ret = check_string(value, pat, regexp) if ret: break - except Exception as e: + except Exception: print('Failed to get path "{}"'.format(path)) raise return ret @@ -363,6 +369,7 @@ def get_tree_count(tree, path): path = normalize_xpath(path) return len(tree.findall(path)) + def stderr(*args): if sys.version_info.major < 3: file = codecs.getwriter('utf-8')(sys.stderr) @@ -371,6 +378,7 @@ def stderr(*args): print(*args, file=file) + def print_err(lineno, context, err, message=None): global ERR_COUNT ERR_COUNT += 1 @@ -381,31 +389,33 @@ def print_err(lineno, context, err, message=None): if context: stderr("\t{}".format(context)) + ERR_COUNT = 0 + def check_command(c, cache): try: cerr = "" - if c.cmd == 'has' or c.cmd == 'matches': # string test + if c.cmd == 'has' or c.cmd == 'matches': # string test regexp = (c.cmd == 'matches') - if len(c.args) == 1 and not regexp: # @has = file existence + if len(c.args) == 1 and not regexp: # @has = file existence try: cache.get_file(c.args[0]) ret = True except FailedCheck as err: cerr = str(err) ret = False - elif len(c.args) == 2: # @has/matches = string test + elif len(c.args) == 2: # @has/matches = string test cerr = "`PATTERN` did not match" ret = check_string(cache.get_file(c.args[0]), c.args[1], regexp) - elif len(c.args) == 3: # @has/matches = XML tree test + elif len(c.args) == 3: # @has/matches = XML tree test cerr = "`XPATH PATTERN` did not match" tree = cache.get_tree(c.args[0]) pat, sep, attr = c.args[1].partition('/@') - if sep: # attribute + if sep: # attribute tree = cache.get_tree(c.args[0]) ret = check_tree_attr(tree, pat, attr, c.args[2], regexp) - else: # normalized text + else: # normalized text pat = c.args[1] if pat.endswith('/text()'): pat = pat[:-7] @@ -413,16 +423,16 @@ def check_command(c, cache): else: raise InvalidCheck('Invalid number of @{} arguments'.format(c.cmd)) - elif c.cmd == 'count': # count test - if len(c.args) == 3: # @count = count test + elif c.cmd == 'count': # count test + if len(c.args) == 3: # @count = count test expected = int(c.args[2]) found = get_tree_count(cache.get_tree(c.args[0]), c.args[1]) cerr = "Expected {} occurrences but found {}".format(expected, found) ret = expected == found else: raise InvalidCheck('Invalid number of @{} arguments'.format(c.cmd)) - elif c.cmd == 'has-dir': # has-dir test - if len(c.args) == 1: # @has-dir = has-dir test + elif c.cmd == 'has-dir': # has-dir test + if len(c.args) == 1: # @has-dir = has-dir test try: cache.get_dir(c.args[0]) ret = True @@ -448,11 +458,13 @@ def check_command(c, cache): except InvalidCheck as err: print_err(c.lineno, c.context, str(err)) + def check(target, commands): cache = CachedFiles(target) for c in commands: check_command(c, cache) + if __name__ == '__main__': if len(sys.argv) != 3: stderr('Usage: {}