[BEAM-12641] Use google-auth instead of oauth2client for GCP auth (#1…

…5004)

Co-authored-by: tvalentyn <[email protected]>
Co-authored-by: Andy Ye <[email protected]>
Co-authored-by: Andy Ye <[email protected]>

ownership/PYTHON_DEPENDENCY_OWNERS.yaml

@@ -63,9 +63,6 @@ deps:
  pymongo:
  owners: yichi
 
- oauth2client:
- owners:
-
  proto-google-cloud-pubsub-v1:
  owners:
 

sdks/python/apache_beam/internal/gcp/auth.py

@@ -23,15 +23,13 @@
 import socket
 import threading
 
-from oauth2client.client import GoogleCredentials
-
-from apache_beam.utils import retry
-
-# Protect against environments where apitools library is not available.
+# google.auth is only available when Beam is installed with the gcp extra.
 try:
- from apitools.base.py.credentials_lib import GceAssertionCredentials
+ import google.auth
+ import google_auth_httplib2
+ _GOOGLE_AUTH_AVAILABLE = True
 except ImportError:
- GceAssertionCredentials = None
+ _GOOGLE_AUTH_AVAILABLE = False
 
 # When we are running in GCE, we can authenticate with VM credentials.
 is_running_in_gce = False
@@ -42,18 +40,6 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-if GceAssertionCredentials is not None:
-
- class _GceAssertionCredentials(GceAssertionCredentials):
- """GceAssertionCredentials with retry wrapper.
-
- For internal use only; no backwards-compatibility guarantees.
- """
- @retry.with_exponential_backoff(
- retry_filter=retry.retry_on_server_errors_and_timeout_filter)
- def _do_refresh_request(self, http_request):
- return super()._do_refresh_request(http_request)
-
 
 def set_running_in_gce(worker_executing_project):
  """For internal use only; no backwards-compatibility guarantees.
@@ -79,12 +65,43 @@ def get_service_credentials():
  Get credentials to access Google services.
 
  Returns:
- A ``oauth2client.client.OAuth2Credentials`` object or None if credentials
+ A ``google.auth.credentials.Credentials`` object or None if credentials
  not found. Returned object is thread-safe.
  """
  return _Credentials.get_service_credentials()
 
 
+if _GOOGLE_AUTH_AVAILABLE:
+
+ class _ApitoolsCredentialsAdapter:
+ """For internal use only; no backwards-compatibility guarantees.
+
+ Adapter allowing use of google-auth credentials with apitools, which
+ normally expects credentials from the oauth2client library. This allows
+ upgrading the auth library used by Beam without simultaneously upgrading
+ all the GCP client libraries (a much larger change).
+ """
+ def __init__(self, google_auth_credentials):
+ self._google_auth_credentials = google_auth_credentials
+
+ def authorize(self, http):
+ """Return an http client authorized with the google-auth credentials.
+
+ Args:
+ http: httplib2.Http, an http object to be used to make the refresh
+ request.
+
+ Returns:
+ google_auth_httplib2.AuthorizedHttp: An authorized http client.
+ """
+ return google_auth_httplib2.AuthorizedHttp(
+ self._google_auth_credentials, http=http)
+
+ def __getattr__(self, attr):
+ """Delegate attribute access to underlying google-auth credentials."""
+ return getattr(self._google_auth_credentials, attr)
+
+
 class _Credentials(object):
  _credentials_lock = threading.Lock()
  _credentials_init = False
@@ -114,29 +131,32 @@ def get_service_credentials(cls):
 
  @staticmethod
  def _get_service_credentials():
- if is_running_in_gce:
- # We are currently running as a GCE taskrunner worker.
- return _GceAssertionCredentials(user_agent='beam-python-sdk/1.0')
- else:
- client_scopes = [
- 'https://www.googleapis.com/auth/bigquery',
- 'https://www.googleapis.com/auth/cloud-platform',
- 'https://www.googleapis.com/auth/devstorage.full_control',
- 'https://www.googleapis.com/auth/userinfo.email',
- 'https://www.googleapis.com/auth/datastore',
- 'https://www.googleapis.com/auth/spanner.admin',
- 'https://www.googleapis.com/auth/spanner.data'
- ]
- try:
- credentials = GoogleCredentials.get_application_default()
- credentials = credentials.create_scoped(client_scopes)
- logging.debug(
- 'Connecting using Google Application Default '
- 'Credentials.')
- return credentials
- except Exception as e:
- _LOGGER.warning(
- 'Unable to find default credentials to use: %s\n'
- 'Connecting anonymously.',
- e)
- return None
+ if not _GOOGLE_AUTH_AVAILABLE:
+ _LOGGER.warning(
+ 'Unable to find default credentials because the google-auth library '
+ 'is not available. Install the gcp extra (apache_beam[gcp]) to use '
+ 'Google default credentials. Connecting anonymously.')
+ return None
+
+ client_scopes = [
+ 'https://www.googleapis.com/auth/bigquery',
+ 'https://www.googleapis.com/auth/cloud-platform',
+ 'https://www.googleapis.com/auth/devstorage.full_control',
+ 'https://www.googleapis.com/auth/userinfo.email',
+ 'https://www.googleapis.com/auth/datastore',
+ 'https://www.googleapis.com/auth/spanner.admin',
+ 'https://www.googleapis.com/auth/spanner.data'
+ ]
+ try:
+ credentials, _ = google.auth.default(scopes=client_scopes) # pylint: disable=c-extension-no-member
+ credentials = _ApitoolsCredentialsAdapter(credentials)
+ logging.debug(
+ 'Connecting using Google Application Default '
+ 'Credentials.')
+ return credentials
+ except Exception as e:
+ _LOGGER.warning(
+ 'Unable to find default credentials to use: %s\n'
+ 'Connecting anonymously.',
+ e)
+ return None

sdks/python/container/py36/base_image_requirements.txt

@@ -36,7 +36,7 @@ clang==5.0
 click==8.0.4
 cloudpickle==2.0.0
 crcmod==1.7
-cryptography==36.0.1
+cryptography==36.0.2
 Cython==0.29.28
 dataclasses==0.8
 deprecation==2.1.0
@@ -47,34 +47,34 @@ execnet==1.9.0
 fastavro==1.4.7
 fasteners==0.17.3
 flatbuffers==1.12
-freezegun==1.2.0
+freezegun==1.2.1
 future==0.18.2
 gast==0.4.0
 google-api-core==1.31.5
-google-api-python-client==2.39.0
+google-api-python-client==2.41.0
 google-apitools==0.5.31
 google-auth==1.35.0
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==0.4.6
-google-cloud-bigquery==2.34.1
-google-cloud-bigquery-storage==2.12.0
+google-cloud-bigquery==2.34.2
+google-cloud-bigquery-storage==2.13.0
 google-cloud-bigtable==1.7.0
 google-cloud-core==1.7.2
 google-cloud-datastore==1.15.3
-google-cloud-dlp==3.6.1
+google-cloud-dlp==3.6.2
 google-cloud-language==1.3.0
 google-cloud-profiler==3.0.7
-google-cloud-pubsub==2.9.0
-google-cloud-pubsublite==1.4.0
+google-cloud-pubsub==2.11.0
+google-cloud-pubsublite==1.4.1
 google-cloud-recommendations-ai==0.2.0
 google-cloud-spanner==1.19.1
 google-cloud-videointelligence==1.16.1
 google-cloud-vision==1.0.0
 google-crc32c==1.3.0
 google-pasta==0.2.0
 google-python-cloud-debugger==2.18
-google-resumable-media==2.3.1
-googleapis-common-protos==1.55.0
+google-resumable-media==2.3.2
+googleapis-common-protos==1.56.0
 greenlet==1.1.2
 grpc-google-iam-v1==0.12.3
 grpcio==1.44.0
@@ -90,12 +90,10 @@ importlib-resources==5.4.0
 joblib==1.1.0
 keras==2.6.0
 Keras-Preprocessing==1.1.2
-libcst==0.4.1
 Markdown==3.3.6
 mmh3==3.0.0
 mock==2.0.0
 more-itertools==8.12.0
-mypy-extensions==0.4.3
 nltk==3.6.7
 nose==1.3.7
 numpy==1.19.5
@@ -128,9 +126,9 @@ pytest-timeout==1.4.2
 pytest-xdist==1.34.0
 python-dateutil==2.8.2
 python-snappy==0.6.1
-pytz==2021.3
+pytz==2022.1
 PyYAML==6.0
-regex==2022.3.2
+regex==2022.3.15
 requests==2.27.1
 requests-mock==1.9.3
 requests-oauthlib==1.3.1
@@ -139,7 +137,7 @@ scikit-learn==0.24.2
 scipy==1.5.4
 six==1.15.0
 soupsieve==2.3.1
-SQLAlchemy==1.4.31
+SQLAlchemy==1.4.32
 tenacity==5.1.5
 tensorboard==2.6.0
 tensorboard-data-server==0.6.1
@@ -151,10 +149,9 @@ testcontainers==3.4.2
 threadpoolctl==3.1.0
 tqdm==4.63.0
 typing-extensions==3.7.4.3
-typing-inspect==0.7.1
 typing-utils==0.1.0
 uritemplate==4.1.1
-urllib3==1.26.8
+urllib3==1.26.9
 wcwidth==0.2.5
 websocket-client==1.3.1
 Werkzeug==2.0.3

sdks/python/container/py37/base_image_requirements.txt

@@ -35,44 +35,44 @@ charset-normalizer==2.0.12
 click==8.0.4
 cloudpickle==2.0.0
 crcmod==1.7
-cryptography==36.0.1
+cryptography==36.0.2
 Cython==0.29.28
 deprecation==2.1.0
 dill==0.3.1.1
 docker==5.0.3
 docopt==0.6.2
 execnet==1.9.0
-fastavro==1.4.9
+fastavro==1.4.10
 fasteners==0.17.3
 flatbuffers==2.0
-freezegun==1.2.0
+freezegun==1.2.1
 future==0.18.2
 gast==0.5.3
 google-api-core==1.31.5
-google-api-python-client==2.39.0
+google-api-python-client==2.41.0
 google-apitools==0.5.31
 google-auth==1.35.0
 google-auth-httplib2==0.1.0
 google-auth-oauthlib==0.4.6
-google-cloud-bigquery==2.34.1
-google-cloud-bigquery-storage==2.12.0
+google-cloud-bigquery==2.34.2
+google-cloud-bigquery-storage==2.13.0
 google-cloud-bigtable==1.7.0
 google-cloud-core==1.7.2
 google-cloud-datastore==1.15.3
-google-cloud-dlp==3.6.1
+google-cloud-dlp==3.6.2
 google-cloud-language==1.3.0
 google-cloud-profiler==3.0.7
-google-cloud-pubsub==2.9.0
-google-cloud-pubsublite==1.4.0
+google-cloud-pubsub==2.11.0
+google-cloud-pubsublite==1.4.1
 google-cloud-recommendations-ai==0.2.0
 google-cloud-spanner==1.19.1
 google-cloud-videointelligence==1.16.1
 google-cloud-vision==1.0.0
 google-crc32c==1.3.0
 google-pasta==0.2.0
 google-python-cloud-debugger==2.18
-google-resumable-media==2.3.1
-googleapis-common-protos==1.55.0
+google-resumable-media==2.3.2
+googleapis-common-protos==1.56.0
 greenlet==1.1.2
 grpc-google-iam-v1==0.12.3
 grpcio==1.44.0
@@ -83,17 +83,15 @@ h5py==3.6.0
 hdfs==2.6.0
 httplib2==0.19.1
 idna==3.3
-importlib-metadata==4.11.2
+importlib-metadata==4.11.3
 joblib==1.1.0
 keras==2.8.0
 Keras-Preprocessing==1.1.2
 libclang==13.0.0
-libcst==0.4.1
 Markdown==3.3.6
 mmh3==3.0.0
 mock==2.0.0
 more-itertools==8.12.0
-mypy-extensions==0.4.3
 nltk==3.7
 nose==1.3.7
 numpy==1.21.5
@@ -128,7 +126,7 @@ python-dateutil==2.8.2
 python-snappy==0.6.1
 pytz==2021.3
 PyYAML==6.0
-regex==2022.3.2
+regex==2022.3.15
 requests==2.27.1
 requests-mock==1.9.3
 requests-oauthlib==1.3.1
@@ -137,7 +135,7 @@ scikit-learn==1.0.2
 scipy==1.7.3
 six==1.16.0
 soupsieve==2.3.1
-SQLAlchemy==1.4.31
+SQLAlchemy==1.4.32
 tenacity==5.1.5
 tensorboard==2.8.0
 tensorboard-data-server==0.6.1
@@ -149,13 +147,12 @@ testcontainers==3.4.2
 tf-estimator-nightly==2.8.0.dev2021122109
 threadpoolctl==3.1.0
 tqdm==4.63.0
-typing-inspect==0.7.1
 typing-utils==0.1.0
 typing_extensions==4.1.1
 uritemplate==4.1.1
-urllib3==1.26.8
+urllib3==1.26.9
 wcwidth==0.2.5
 websocket-client==1.3.1
 Werkzeug==2.0.3
-wrapt==1.13.3
+wrapt==1.14.0
 zipp==3.7.0