DX: Test that Transformers are adding only CustomTokens that they def…

…ine and nothing else

.composer-require-checker.json

@@ -1,6 +1,7 @@
 {
     "symbol-whitelist" : [
         "LegacyPHPUnit\\TestCase",
+        "PhpCsFixer\\AccessibleObject\\AccessibleObject",
         "PhpCsFixer\\PhpunitConstraintIsIdenticalString\\Constraint\\IsIdenticalString",
         "PhpCsFixer\\Tests\\Test\\Constraint\\SameStringsConstraint",
         "PhpCsFixer\\Tests\\Test\\IsIdenticalConstraint",

.github/workflows/sca.yml

@@ -114,5 +114,6 @@ jobs:
             grep -v tests/Test/IntegrationCaseFactoryInterface.php |
             grep -v tests/Test/InternalIntegrationCaseFactory.php |
             grep -v tests/Test/IsIdenticalConstraint.php |
+            grep -v tests/Test/TokensWithObservedTransformers.php |
             grep -v tests/TestCase.php \
           && (echo "UNKNOWN FILES DETECTED" && exit 1) || echo "NO UNKNOWN FILES"

composer.json

@@ -71,6 +71,7 @@
             "tests/Test/IntegrationCaseFactoryInterface.php",
             "tests/Test/InternalIntegrationCaseFactory.php",
             "tests/Test/IsIdenticalConstraint.php",
+            "tests/Test/TokensWithObservedTransformers.php",
             "tests/TestCase.php"
         ]
     },

src/Fixer/ControlStructure/NoUnneededControlParenthesesFixer.php

@@ -46,11 +46,12 @@ public function __construct()
     {
         parent::__construct();
 
-        // To be moved back to compile time property declaration when PHP support of PHP CS Fixer will be 7.0+
+        // @TODO: To be moved back to compile time property declaration when PHP support of PHP CS Fixer will be 7.0+
         if (\defined('T_COALESCE')) {
             self::$loops['clone']['forbiddenContents'][] = [T_COALESCE, '??'];
         }
 
+        // @TODO: To be moved back to compile time property declaration when PHP support of PHP CS Fixer will be 7.0+
         if (\defined('T_YIELD_FROM')) {
             self::$loops['yield_from'] = ['lookupTokens' => T_YIELD_FROM, 'neededSuccessors' => [';', ')']];
         }

src/Fixer/ControlStructure/YodaStyleFixer.php

@@ -485,10 +485,12 @@ private function isOfLowerPrecedence(Token $token)
                 T_XOR_EQUAL,    // ^=
             ];
 
+            // @TODO: drop condition when PHP 7.0+ is required
             if (\defined('T_COALESCE')) {
                 $tokens[] = T_COALESCE; // ??
             }
 
+            // @TODO: drop condition when PHP 7.4+ is required
             if (\defined('T_COALESCE_EQUAL')) {
                 $tokens[] = T_COALESCE_EQUAL; // ??=
             }

src/Fixer/LanguageConstruct/SingleSpaceAfterConstructFixer.php

@@ -105,10 +105,12 @@ public function configure(array $configuration = null)
     {
         parent::configure($configuration);
 
+        // @TODO: drop condition when PHP 7.0+ is required
         if (\defined('T_YIELD_FROM')) {
             self::$tokenMap['yield_from'] = T_YIELD_FROM;
         }
 
+        // @TODO: drop condition when PHP 8.0+ is required
         if (\defined('T_MATCH')) {
             self::$tokenMap['match'] = T_MATCH;
         }

src/Fixer/Operator/BinaryOperatorSpacesFixer.php

@@ -457,14 +457,17 @@ private function resolveOperatorsFromConfig()
             }
         }
 
+        // @TODO: drop condition when PHP 7.0+ is required
         if (!\defined('T_SPACESHIP')) {
             unset($operators['<=>']);
         }
 
+        // @TODO: drop condition when PHP 7.0+ is required
         if (!\defined('T_COALESCE')) {
             unset($operators['??']);
         }
 
+        // @TODO: drop condition when PHP 7.4+ is required
         if (!\defined('T_COALESCE_EQUAL')) {
             unset($operators['??=']);
         }

src/Fixer/Operator/NewWithBracesFixer.php

@@ -95,6 +95,7 @@ protected function applyFix(\SplFileInfo $file, Tokens $tokens)
                 [CT::T_BRACE_CLASS_INSTANTIATION_CLOSE],
             ];
 
+            // @TODO: drop condition when PHP 7.0+ is required
             if (\defined('T_SPACESHIP')) {
                 $nextTokenKinds[] = [T_SPACESHIP];
             }

src/Fixer/Whitespace/BlankLineBeforeStatementFixer.php

@@ -71,7 +71,7 @@ public function __construct()
     {
         parent::__construct();
 
-        // To be moved back to compile time property declaration when PHP support of PHP CS Fixer will be 7.0+
+        // @TODO: To be moved back to compile time property declaration when PHP support of PHP CS Fixer will be 7.0+
         if (\defined('T_YIELD_FROM')) {
             self::$tokenMap['yield_from'] = T_YIELD_FROM;
         }

src/Linter/TokenizerLinter.php

@@ -27,7 +27,12 @@ final class TokenizerLinter implements LinterInterface
 {
     public function __construct()
     {
-        if (false === \defined('TOKEN_PARSE') || false === class_exists(\CompileError::class)) {
+        if (
+            // @TODO: drop condition when PHP 7.0+ is required
+            false === \defined('TOKEN_PARSE')
+            // @TODO: drop condition when PHP 7.3+ is required
+            || false === class_exists(\CompileError::class)
+        ) {
             throw new UnavailableLinterException('Cannot use tokenizer as linter.');
         }
     }

src/Tokenizer/AbstractTransformer.php

@@ -43,15 +43,5 @@ public function getPriority()
     /**
      * {@inheritdoc}
      */
-    public function getCustomTokens()
-    {
-        @trigger_error(sprintf('%s is deprecated and will be removed in 3.0.', __METHOD__), E_USER_DEPRECATED);
-
-        return $this->getDeprecatedCustomTokens();
-    }
-
-    /**
-     * @return int[]
-     */
-    abstract protected function getDeprecatedCustomTokens();
+    abstract public function getCustomTokens();
 }

src/Tokenizer/Analyzer/CommentsAnalyzer.php

@@ -85,6 +85,7 @@ public function isBeforeStructuralElement(Tokens $tokens, $index)
         do {
             $nextIndex = $tokens->getNextMeaningfulToken($nextIndex);
 
+            // @TODO: drop condition when PHP 8.0+ is required
             if (\defined('T_ATTRIBUTE')) {
                 while (null !== $nextIndex && $tokens[$nextIndex]->isGivenKind(T_ATTRIBUTE)) {
                     $nextIndex = $tokens->findBlockEnd(Tokens::BLOCK_TYPE_ATTRIBUTE, $nextIndex);

src/Tokenizer/Tokens.php

@@ -276,6 +276,7 @@ public static function getBlockEdgeDefinitions()
             ],
         ];
 
+        // @TODO: drop condition when PHP 8.0+ is required
         if (\defined('T_ATTRIBUTE')) {
             $definitions[self::BLOCK_TYPE_ATTRIBUTE] = [
                 'start' => [T_ATTRIBUTE, '#['],
@@ -1092,7 +1093,7 @@ public function setCode($code)
         // clear memory
         $this->setSize(0);
 
-        $tokens = \defined('TOKEN_PARSE')
+        $tokens = \defined('TOKEN_PARSE') // @TODO: drop condition when PHP 7.0+ is required
             ? token_get_all($code, TOKEN_PARSE)
             : token_get_all($code);
 
@@ -1102,8 +1103,7 @@ public function setCode($code)
             $this[$index] = new Token($token);
         }
 
-        $transformers = Transformers::create();
-        $transformers->transform($this);
+        $this->applyTransformers();
 
         $this->foundTokenKinds = [];
 
@@ -1370,6 +1370,15 @@ public function valid()
         return parent::valid();
     }
 
+    /**
+     * @internal
+     */
+    protected function applyTransformers()
+    {
+        $transformers = Transformers::create();
+        $transformers->transform($this);
+    }
+
     private function warnPhp8SplFixerArrayChange($method)
     {
         if (80000 <= \PHP_VERSION_ID) {

src/Tokenizer/TokensAnalyzer.php

@@ -560,14 +560,17 @@ public function isBinaryOperator($index)
                 CT::T_TYPE_ALTERNATION => true, // |
             ];
 
+            // @TODO: drop condition when PHP 7.0+ is required
             if (\defined('T_SPACESHIP')) {
                 $arrayOperators[T_SPACESHIP] = true; // <=>
             }
 
+            // @TODO: drop condition when PHP 7.0+ is required
             if (\defined('T_COALESCE')) {
                 $arrayOperators[T_COALESCE] = true;  // ??
             }
 
+            // @TODO: drop condition when PHP 7.4+ is required
             if (\defined('T_COALESCE_EQUAL')) {
                 $arrayOperators[T_COALESCE_EQUAL] = true;  // ??=
             }

src/Tokenizer/Transformer/ArrayTypehintTransformer.php

@@ -54,7 +54,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_ARRAY_TYPEHINT];
     }

src/Tokenizer/Transformer/AttributeTransformer.php

@@ -68,7 +68,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [
             CT::T_ATTRIBUTE_CLOSE,

src/Tokenizer/Transformer/BraceClassInstantiationTransformer.php

@@ -81,7 +81,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_BRACE_CLASS_INSTANTIATION_OPEN, CT::T_BRACE_CLASS_INSTANTIATION_CLOSE];
     }

src/Tokenizer/Transformer/ClassConstantTransformer.php

@@ -57,7 +57,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_CLASS_CONSTANT];
     }

src/Tokenizer/Transformer/ConstructorPromotionTransformer.php

@@ -67,7 +67,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [
             CT::T_CONSTRUCTOR_PROPERTY_PROMOTION_PUBLIC,

src/Tokenizer/Transformer/CurlyBraceTransformer.php

@@ -61,7 +61,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [
             CT::T_CURLY_CLOSE,

src/Tokenizer/Transformer/ImportTransformer.php

@@ -60,7 +60,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_CONST_IMPORT, CT::T_FUNCTION_IMPORT];
     }

src/Tokenizer/Transformer/NameQualifiedTransformer.php

@@ -58,7 +58,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [];
     }

src/Tokenizer/Transformer/NamedArgumentTransformer.php

@@ -75,7 +75,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [
             CT::T_NAMED_ARGUMENT_COLON,

src/Tokenizer/Transformer/NamespaceOperatorTransformer.php

@@ -53,7 +53,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_NAMESPACE_OPERATOR];
     }

src/Tokenizer/Transformer/NullableTypeTransformer.php

@@ -75,7 +75,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_NULLABLE_TYPE];
     }

src/Tokenizer/Transformer/ReturnRefTransformer.php

@@ -55,7 +55,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_RETURN_REF];
     }

src/Tokenizer/Transformer/SquareBraceTransformer.php

@@ -70,7 +70,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [
             CT::T_ARRAY_SQUARE_BRACE_OPEN,

src/Tokenizer/Transformer/TypeAlternationTransformer.php

@@ -121,7 +121,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_TYPE_ALTERNATION];
     }

src/Tokenizer/Transformer/TypeColonTransformer.php

@@ -83,7 +83,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_TYPE_COLON];
     }

src/Tokenizer/Transformer/UseTransformer.php

@@ -88,7 +88,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [CT::T_USE_TRAIT, CT::T_USE_LAMBDA];
     }

src/Tokenizer/Transformer/WhitespacyCommentTransformer.php

@@ -64,7 +64,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [];
     }

src/Tokenizer/TransformerInterface.php

@@ -30,8 +30,6 @@ interface TransformerInterface
      * Get tokens created by Transformer.
      *
      * @return int[]
-     *
-     * @deprecated will be removed in 3.0
      */
     public function getCustomTokens();
 

tests/Fixtures/Test/AbstractTransformerTest/FooTransformer.php

@@ -38,7 +38,7 @@ public function process(Tokens $tokens, Token $token, $index)
     /**
      * {@inheritdoc}
      */
-    protected function getDeprecatedCustomTokens()
+    public function getCustomTokens()
     {
         return [];
     }