Skip to content

Commit

Permalink
[BEAM-7372] cleanup codes for py2 from apache_beam/transforms (#14544)
Browse files Browse the repository at this point in the history
  • Loading branch information
lazylynx authored Apr 22, 2021
1 parent c4bd6f0 commit 28020ef
Show file tree
Hide file tree
Showing 44 changed files with 130 additions and 422 deletions.
1 change: 0 additions & 1 deletion sdks/python/apache_beam/transforms/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
"""PTransform and descendants."""

# pylint: disable=wildcard-import
from __future__ import absolute_import

from apache_beam.transforms import combiners
from apache_beam.transforms.core import *
Expand Down
9 changes: 1 addition & 8 deletions sdks/python/apache_beam/transforms/combiners.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,12 @@

# pytype: skip-file

from __future__ import absolute_import
from __future__ import division

import copy
import heapq
import operator
import random
import sys
import warnings
from builtins import object
from builtins import zip
from typing import Any
from typing import Dict
from typing import Iterable
Expand All @@ -39,8 +34,6 @@
from typing import TypeVar
from typing import Union

from past.builtins import long

from apache_beam import typehints
from apache_beam.transforms import core
from apache_beam.transforms import cy_combiners
Expand Down Expand Up @@ -96,7 +89,7 @@ def expand(self, pcoll):

# TODO(laolu): This type signature is overly restrictive. This should be
# more general.
@with_input_types(Union[float, int, long])
@with_input_types(Union[float, int])
@with_output_types(float)
class MeanCombineFn(core.CombineFn):
"""CombineFn for computing an arithmetic mean."""
Expand Down
4 changes: 0 additions & 4 deletions sdks/python/apache_beam/transforms/combiners_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,11 @@
"""Unit tests for our libraries of combine PTransforms."""
# pytype: skip-file

from __future__ import absolute_import
from __future__ import division

import itertools
import random
import unittest

import hamcrest as hc
from future.builtins import range
from nose.plugins.attrib import attr

import apache_beam as beam
Expand Down
29 changes: 2 additions & 27 deletions sdks/python/apache_beam/transforms/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,12 @@

# pytype: skip-file

from __future__ import absolute_import

import copy
import inspect
import logging
import random
import types
import typing
from builtins import map
from builtins import object
from builtins import range

from past.builtins import unicode

from apache_beam import coders
from apache_beam import pvalue
Expand Down Expand Up @@ -80,11 +73,6 @@
from apache_beam.transforms.trigger import DefaultTrigger
from apache_beam.transforms.trigger import TriggerFn

try:
import funcsigs # Python 2 only.
except ImportError:
funcsigs = None

__all__ = [
'DoFn',
'CombineFn',
Expand Down Expand Up @@ -385,12 +373,7 @@ def get_function_args_defaults(f):
it doesn't include bound arguments and may follow function wrappers.
"""
signature = get_signature(f)
# Fall back on funcsigs if inspect module doesn't have 'Parameter'; prefer
# inspect.Parameter over funcsigs.Parameter if both are available.
try:
parameter = inspect.Parameter
except AttributeError:
parameter = funcsigs.Parameter
parameter = inspect.Parameter
# TODO(BEAM-5878) support kwonlyargs on Python 3.
_SUPPORTED_ARG_TYPES = [
parameter.POSITIONAL_ONLY, parameter.POSITIONAL_OR_KEYWORD
Expand Down Expand Up @@ -1618,18 +1601,10 @@ def MapTuple(fn, *args, **kwargs): # pylint: disable=invalid-name
beam.MapTuple(lambda a, b, ...: ...)
is equivalent to Python 2
beam.Map(lambda (a, b, ...), ...: ...)
In other words
beam.MapTuple(fn)
is equivalent to
beam.Map(lambda element, ...: fn(\*element, ...))
This can be useful when processing a PCollection of tuples
(e.g. key-value pairs).
Expand Down Expand Up @@ -2904,7 +2879,7 @@ def __init__(self, values, reshuffle=True):
values: An object of values for the PCollection
"""
super(Create, self).__init__()
if isinstance(values, (unicode, str, bytes)):
if isinstance(values, (str, bytes)):
raise TypeError(
'PTransform Create: Refusing to treat string as '
'an iterable. (string=%r)' % values)
Expand Down
7 changes: 0 additions & 7 deletions sdks/python/apache_beam/transforms/create_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,6 @@

# pytype: skip-file

from __future__ import absolute_import
from __future__ import division

from builtins import map
from builtins import next
from builtins import range

from apache_beam.io import iobase
from apache_beam.transforms.core import Create

Expand Down
4 changes: 0 additions & 4 deletions sdks/python/apache_beam/transforms/create_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,8 @@
"""Unit tests for the Create and _CreateSource classes."""
# pytype: skip-file

from __future__ import absolute_import
from __future__ import division

import logging
import unittest
from builtins import range

from apache_beam import Create
from apache_beam.coders import FastPrimitivesCoder
Expand Down
4 changes: 0 additions & 4 deletions sdks/python/apache_beam/transforms/cy_combiners.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,7 @@

# pytype: skip-file

from __future__ import absolute_import
from __future__ import division

import operator
from builtins import object

from apache_beam.transforms import core

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@

# pytype: skip-file

from __future__ import absolute_import

import unittest

from mock import Mock
Expand Down
3 changes: 0 additions & 3 deletions sdks/python/apache_beam/transforms/deduplicate.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@

"""a collection of ptransforms for deduplicating elements."""

from __future__ import absolute_import
from __future__ import division

import typing

from apache_beam import typehints
Expand Down
2 changes: 0 additions & 2 deletions sdks/python/apache_beam/transforms/deduplicate_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@

"""Unit tests for deduplicate transform by using TestStream."""

from __future__ import absolute_import

import unittest

from nose.plugins.attrib import attr
Expand Down
6 changes: 0 additions & 6 deletions sdks/python/apache_beam/transforms/display.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,14 @@

# pytype: skip-file

from __future__ import absolute_import

import calendar
import inspect
import json
from builtins import object
from datetime import datetime
from datetime import timedelta
from typing import TYPE_CHECKING
from typing import List

from past.builtins import unicode

from apache_beam.portability import common_urns
from apache_beam.portability.api import beam_runner_api_pb2

Expand Down Expand Up @@ -231,7 +226,6 @@ class DisplayDataItem(object):
"""
typeDict = {
str: 'STRING',
unicode: 'STRING',
int: 'INTEGER',
float: 'FLOAT',
bool: 'BOOLEAN',
Expand Down
5 changes: 1 addition & 4 deletions sdks/python/apache_beam/transforms/display_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,12 @@

# pytype: skip-file

from __future__ import absolute_import

import unittest
from datetime import datetime

# pylint: disable=ungrouped-imports
import hamcrest as hc
from hamcrest.core.base_matcher import BaseMatcher
from past.builtins import unicode

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
Expand Down Expand Up @@ -167,7 +164,7 @@ def test_unicode_type_display_data(self):
class MyDoFn(beam.DoFn):
def display_data(self):
return {
'unicode_string': unicode('my string'),
'unicode_string': 'my string',
'unicode_literal_string': u'my literal string'
}

Expand Down
2 changes: 0 additions & 2 deletions sdks/python/apache_beam/transforms/dofn_lifecycle_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@

# pytype: skip-file

from __future__ import absolute_import

import unittest

from nose.plugins.attrib import attr
Expand Down
2 changes: 0 additions & 2 deletions sdks/python/apache_beam/transforms/environments.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@

# pytype: skip-file

from __future__ import absolute_import

import json
import logging
import sys
Expand Down
2 changes: 0 additions & 2 deletions sdks/python/apache_beam/transforms/environments_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@

# pytype: skip-file

from __future__ import absolute_import

import logging
import unittest

Expand Down
7 changes: 0 additions & 7 deletions sdks/python/apache_beam/transforms/external.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,6 @@
"""
# pytype: skip-file

from __future__ import absolute_import
from __future__ import print_function

import contextlib
import copy
import functools
Expand Down Expand Up @@ -150,8 +147,6 @@ def _get_named_tuple_instance(self):
class AnnotationBasedPayloadBuilder(SchemaBasedPayloadBuilder):
"""
Build a payload based on an external transform's type annotations.
Supported in python 3 only.
"""
def __init__(self, transform, **values):
"""
Expand All @@ -174,8 +169,6 @@ def _get_named_tuple_instance(self):
class DataclassBasedPayloadBuilder(SchemaBasedPayloadBuilder):
"""
Build a payload based on an external transform that uses dataclasses.
Supported in python 3 only.
"""
def __init__(self, transform):
"""
Expand Down
2 changes: 0 additions & 2 deletions sdks/python/apache_beam/transforms/external_it_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@

# pytype: skip-file

from __future__ import absolute_import

import unittest

from nose.plugins.attrib import attr
Expand Down
5 changes: 1 addition & 4 deletions sdks/python/apache_beam/transforms/external_java.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,13 @@

"""Tests for the Java external transforms."""

from __future__ import absolute_import

import argparse
import logging
import subprocess
import sys

import grpc
from mock import patch
from past.builtins import unicode

import apache_beam as beam
from apache_beam.options.pipeline_options import PipelineOptions
Expand Down Expand Up @@ -139,7 +136,7 @@ def run_pipeline(pipeline_options, expansion_service, wait_until_finish=True):
res = (
p
| beam.Create(list('aaabccxyyzzz'))
| beam.Map(unicode)
| beam.Map(str)
| beam.ExternalTransform(
TEST_FILTER_URN,
ImplicitSchemaPayloadBuilder({'data': u'middle'}),
Expand Down
Loading

0 comments on commit 28020ef

Please sign in to comment.