gherkin (Python): Updating to Python 3 (standard minimum supported version is now 3.7)

.circleci/config.yml

@@ -519,17 +519,6 @@ jobs:
           paths:
             - json-to-messages/python-testdata/acceptance
 
-  gherkin-python-2:
-    executor: docker-circleci-python
-    steps:
-      - attach_workspace:
-          at: '~/cucumber'
-      - run:
-          name: gherkin/python
-          command: |
-            cd gherkin/python
-            make GHERKIN_PYTHON_VERSION=python2
-
   gherkin-python-3:
     executor: docker-circleci-python
     steps:
@@ -766,10 +755,6 @@ workflows:
             - prepare-parallel
             - json-to-messages-ruby-testdata
 
-      - gherkin-python-2:
-          requires:
-            - prepare-parallel
-
       - gherkin-python-3:
           requires:
             - prepare-parallel

.templates/python/default.mk

@@ -27,8 +27,8 @@ publish:
 ifeq ($(IS_TESTDATA),-testdata)
 	# no-op
 else
-	python2 setup.py sdist
-	python2 -m twine upload dist/*
+	python3 setup.py sdist
+	python3 -m twine upload dist/*
 endif
 .PHONY: publish
 

gherkin/CONTRIBUTING.md

@@ -24,7 +24,7 @@ Prerequisites:
   * Maven
 * Node.js or IO.js
 * Ruby
-* Python (both python2 & python3)
+* Python (python3)
 * Go
 * `make`
 * `jq` (>= 1.4 for `--sort-keys` option)

gherkin/python/CONTRIBUTING.md

@@ -21,7 +21,7 @@ Just run `make` from this directory.
 
 ### Using pytest
 
-Just run `pytest` from this directory (you need to `pip install -r requirements.txt` first).
+Just run `pytest` from this directory (you need to `pip install -r test-requirements.txt` first).
 
 Keep in mind that this will only run unit tests. The acceptance tests are only
 run when you build with `make`.

gherkin/python/MANIFEST

@@ -7,8 +7,6 @@ gherkin/__main__.py
 gherkin/ast_builder.py
 gherkin/ast_node.py
 gherkin/count_symbols.py
-gherkin/count_symbols_py2.py
-gherkin/count_symbols_py3_plus.py
 gherkin/dialect.py
 gherkin/errors.py
 gherkin/gherkin-languages.json

gherkin/python/Makefile

@@ -29,10 +29,10 @@ test: gherkin/parser.py
 	# "pytest" is most likely installed in $(HOME)/.local/bin since normal
 	# site-packages most likely is not writeable
 	@echo "$(@): USE PYTHONPATH=$(PYTHONPATH)"
-	PATH=$(PATH):$(HOME)/.local/bin pytest
+	PATH=$(PATH):$(HOME)/.local/bin python3 -m pytest
 
 .pipped:
-	pip install -r requirements.txt
+	pip3 install -r test-requirements.txt
 	touch $@
 
 acceptance/testdata/%.feature.tokens: ../testdata/%.feature ../testdata/%.feature.tokens

gherkin/python/bin/gherkin

@@ -1,5 +1,4 @@
 #!/usr/bin/env sh
-# Use "make GHERKIN_PYTHON_VERSION=python2 ..." to use python2
 if [ -z "$GHERKIN_PYTHON_VERSION" ];
 then
     if [ -x "$(command -v python)" ]
@@ -8,11 +7,8 @@ then
     elif [ -x "$(command -v python3)" ]
     then
         GHERKIN_PYTHON_VERSION=python3
-    elif [ -x "$(command -v python2)" ]
-    then
-        GHERKIN_PYTHON_VERSION=python2
     else
-        echo "Neiter python, python3 or python2 found on PATH, exiting"
+        echo "Neither python or python3 found on PATH, exiting"
         exit 1
     fi
 fi

gherkin/python/bin/gherkin-generate-tokens

@@ -1,5 +1,4 @@
 #!/usr/bin/env sh
-# Use "make GHERKIN_PYTHON_VERSION=python2 ..." to use python2
 if [ -z "$GHERKIN_PYTHON_VERSION" ];
 then
     if [ -x "$(command -v python)" ]
@@ -8,11 +7,8 @@ then
     elif [ -x "$(command -v python3)" ]
     then
         GHERKIN_PYTHON_VERSION=python3
-    elif [ -x "$(command -v python2)" ]
-    then
-        GHERKIN_PYTHON_VERSION=python2
     else
-        echo "Neiter python, python3 or python2 found on PATH, exiting"
+        echo "Neither python or python3 found on PATH, exiting"
         exit 1
     fi
 fi

gherkin/python/bin/gherkin_generate_tokens.py

@@ -1,17 +1,12 @@
-import codecs
 import os
 import sys
-if sys.version_info < (3, 0):
-    import codecs
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 from gherkin.token_scanner import TokenScanner
 from gherkin.token_formatter_builder import TokenFormatterBuilder
 from gherkin.parser import Parser
 
 files = sys.argv[1:]
-if sys.version_info < (3, 0) and os.name != 'nt':  # for Python2 unless on Windows native
-    UTF8Writer = codecs.getwriter('utf8')
-    sys.stdout = UTF8Writer(sys.stdout)
 parser = Parser(TokenFormatterBuilder())
 for file in files:
     scanner = TokenScanner(file)

gherkin/python/default.mk

@@ -27,8 +27,8 @@ publish:
 ifeq ($(IS_TESTDATA),-testdata)
 	# no-op
 else
-	python2 setup.py sdist
-	python2 -m twine upload dist/*
+	python3 setup.py sdist
+	python3 -m twine upload dist/*
 endif
 .PHONY: publish
 

gherkin/python/dev-requirements.txt

@@ -0,0 +1,2 @@
+tox
+coverage >= 4.2

gherkin/python/gherkin-python.razor

@@ -19,14 +19,13 @@
         token.detach
         expected_tokens = ["@Raw(string.Join("\", \"", expectedTokens))"]
         error = UnexpectedEOFException(token, expected_tokens, state_comment) if token.eof() else UnexpectedTokenException(token, expected_tokens, state_comment)
-        if (self.stop_at_first_error):
+        if self.stop_at_first_error:
             raise error
         self.add_error(context, error)
         return @state.Id</text>}
 @helper MatchToken(TokenType tokenType)
 {<text>match_@(tokenType)(context, token)</text>}
 # This file is generated. Do not edit! Edit gherkin-python.razor instead.
-import sys
 from collections import deque
 from .ast_builder import AstBuilder
 from .token_matcher import TokenMatcher
@@ -41,24 +40,22 @@ RULE_TYPE = [
 ]
 
 
-class ParserContext(object):
+class ParserContext:
     def __init__(self, token_scanner, token_matcher, token_queue, errors):
         self.token_scanner = token_scanner
         self.token_matcher = token_matcher
         self.token_queue = token_queue
         self.errors = errors
 
 
-class @(Model.ParserClassName)(object):
+class @(Model.ParserClassName):
     def __init__(self, ast_builder=None):
         self.ast_builder = ast_builder if ast_builder is not None else AstBuilder()
         self.stop_at_first_error = False
 
     def parse(self, token_scanner_or_str, token_matcher=None):
-        if sys.version_info < (3, 0):
-            token_scanner = TokenScanner(token_scanner_or_str) if isinstance(token_scanner_or_str, basestring) else token_scanner_or_str
-        else:
-            token_scanner = TokenScanner(token_scanner_or_str) if isinstance(token_scanner_or_str, str) else token_scanner_or_str
+        token_scanner = TokenScanner(token_scanner_or_str) if isinstance(token_scanner_or_str,
+                                                                         str) else token_scanner_or_str
         self.ast_builder.reset()
         if token_matcher is None:
             token_matcher = TokenMatcher()
@@ -88,7 +85,8 @@ class @(Model.ParserClassName)(object):
     def build(self, context, token):
         self.handle_ast_error(context, token, self.ast_builder.build)
 
-    def add_error(self, context, error):
+    @@staticmethod
+    def add_error(context, error):
         if str(error) not in (str(e) for e in context.errors):
             context.errors.append(error)
             if len(context.errors) > 10:
@@ -103,11 +101,13 @@ class @(Model.ParserClassName)(object):
     def get_result(self):
         return self.ast_builder.get_result()
 
-    def read_token(self, context):
+    @@staticmethod
+    def read_token(context):
         if context.token_queue:
             return context.token_queue.popleft()
         else:
             return context.token_scanner.read()
+
 @foreach(var rule in Model.RuleSet.TokenRules)
 {<text>
     def match_@(rule.Name.Replace("#", ""))(self, context, token):
@@ -116,7 +116,8 @@ class @(Model.ParserClassName)(object):
         @:if token.eof():
         @:    return False
         }
-        return self.handle_external_error(context, False, token, context.token_matcher.match_@(rule.Name.Replace("#", "")))</text>
+        return self.handle_external_error(context, False, token, context.token_matcher.match_@(rule.Name.Replace("#", "")))
+        </text>
 }
 
     def match_token(self, state, token, context):
@@ -129,7 +130,7 @@ class @(Model.ParserClassName)(object):
         if state in state_map:
             return state_map[state](token, context)
         else:
-            raise RuntimeError("Unknown state: " + str(state))
+            raise RuntimeError(f"Unknown state: {str(state)}")
 @foreach(var state in Model.States.Values.Where(s => !s.IsEndState))
 {<text>
     # @Raw(state.Comment)
@@ -148,8 +149,10 @@ class @(Model.ParserClassName)(object):
                 @:return @transition.TargetState
         }
 
-        @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state)</text>
+        @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state)
+        </text>
 }
+
 @foreach(var lookAheadHint in Model.RuleSet.LookAheadHints)
 {
 <text>
@@ -163,16 +166,16 @@ class @(Model.ParserClassName)(object):
             token.detach
             queue.append(token)
 
-            if (@foreach(var tokenType in lookAheadHint.ExpectedTokens) {<text>self.@MatchToken(tokenType) or </text>}False):
+            if @foreach(var tokenType in lookAheadHint.ExpectedTokens) {<text>self.@MatchToken(tokenType) or </text>}False:
                 match = True
                 break
 
             if not (@foreach(var tokenType in lookAheadHint.Skip) {<text>self.@MatchToken(tokenType) or </text>}False):
                 break
 
         context.token_queue.extend(queue)
-
-        return match</text>
+        return match
+        </text>
 }
 
     # private

gherkin/python/gherkin/__main__.py

@@ -1,14 +1,6 @@
 import os
 from optparse import OptionParser
 import sys
-if sys.version_info < (3, 0):
-    string_type = basestring
-    if os.name != 'nt':
-        import codecs
-        UTF8Writer = codecs.getwriter('utf8')
-        sys.stdout = UTF8Writer(sys.stdout)
-else:
-    string_type = str
 
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
 import json