Merge pull request #227 from censoredplanet/bump

upgrade beam to 2.46
censoredplanet · Mar 24, 2023 · fc6c2a0 · fc6c2a0
2 parents 12b5fea + d530308
commit fc6c2a0
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/pipeline/metadata/beam_metadata.py b/pipeline/metadata/beam_metadata.py
@@ -4,6 +4,8 @@
 
 from typing import Tuple, Dict, List, Iterator, Union, Iterable
 
+import apache_beam as beam
+
 from pipeline.metadata.schema import BigqueryRow, SatelliteRow, PageFetchRow, IpMetadataWithDateKey, IpMetadataWithSourceKey, SatelliteAnswer, SatelliteAnswerWithSourceKey, SatelliteAnswerWithAnyKey, merge_ip_metadata, merge_satellite_answers
 from pipeline.metadata.metrics import ROWS_WITH_METADATA_COUNTER
 
@@ -56,6 +58,9 @@ def make_source_domain_key(row: SatelliteRow) -> SourceDomainKey:
   return (row.source or '', row.domain or '')
 
 
+# We had to remove the beam type hint here
+# So subsequent operations can take either HyperquackRow or SatelliteRow
+@beam.typehints.no_annotations
 def merge_metadata_with_rows(  # pylint: disable=unused-argument
     key: DateIpKey,
     value: Dict[str,

diff --git a/requirements.txt b/requirements.txt
@@ -1,5 +1,5 @@
-apache-beam==2.38.0
-apache-beam[gcp]==2.38.0
+apache-beam==2.46.0
+apache-beam[gcp]==2.46.0
 # Maxmind
 geoip2==4.1.0
 # Imports GCS client needed for beam