Skip to content

Commit 5e89b3d

Browse files
committed
feat(generator): delegate REST transcoding to google-api-core
1 parent a12aeef commit 5e89b3d

146 files changed

Lines changed: 70836 additions & 79 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Subproject commit 3332dec527759859840a3a2ff108c67a54708130

‎packages/gapic-generator/gapic/generator/generator.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def get_response(self, api_schema: api.API, opts: Options) -> CodeGeneratorRespo
120120
for template_name in client_templates:
121121
# Quick check: Skip "private" templates.
122122
filename = template_name.split("/")[-1]
123-
if filename.startswith("_") and filename != "__init__.py.j2":
123+
if filename.startswith("_") and filename not in ("__init__.py.j2", "_compat.py.j2"):
124124
continue
125125

126126
# Append to the output files dictionary.
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
# {% include '_license.j2' %}
2+
3+
from typing import Any, Dict, List, Optional, Tuple
4+
5+
try:
6+
from google.api_core.rest_helpers import (
7+
flatten_query_params,
8+
transcode_request,
9+
)
10+
except ImportError: # pragma: NO COVER
11+
# TODO: Remove these fallbacks when google-api-core >= 2.18.0 is the minimum required version.
12+
import functools
13+
import json
14+
import operator
15+
from google.protobuf import json_format # type: ignore
16+
from google.api_core import path_template # type: ignore
17+
18+
def flatten_query_params(obj, strict=False): # pragma: NO COVER
19+
if obj is not None and not isinstance(obj, dict):
20+
raise TypeError("flatten_query_params must be called with dict object")
21+
return _flatten(obj, key_path=[], strict=strict)
22+
23+
def _flatten(obj, key_path, strict=False): # pragma: NO COVER
24+
if obj is None:
25+
return []
26+
if isinstance(obj, dict):
27+
return _flatten_dict(obj, key_path=key_path, strict=strict)
28+
if isinstance(obj, list):
29+
return _flatten_list(obj, key_path=key_path, strict=strict)
30+
return _flatten_value(obj, key_path=key_path, strict=strict)
31+
32+
def _is_primitive_value(obj): # pragma: NO COVER
33+
if obj is None:
34+
return False
35+
if isinstance(obj, (list, dict)):
36+
raise ValueError("query params may not contain repeated dicts or lists")
37+
return True
38+
39+
def _flatten_value(obj, key_path, strict=False): # pragma: NO COVER
40+
return [(".".join(key_path), _canonicalize(obj, strict=strict))]
41+
42+
def _flatten_dict(obj, key_path, strict=False): # pragma: NO COVER
43+
items = (
44+
_flatten(value, key_path=key_path + [key], strict=strict)
45+
for key, value in obj.items()
46+
)
47+
return functools.reduce(operator.concat, items, [])
48+
49+
def _flatten_list(elems, key_path, strict=False): # pragma: NO COVER
50+
items = (
51+
_flatten_value(elem, key_path=key_path, strict=strict)
52+
for elem in elems
53+
if _is_primitive_value(elem)
54+
)
55+
return functools.reduce(operator.concat, items, [])
56+
57+
def _canonicalize(obj, strict=False): # pragma: NO COVER
58+
if strict:
59+
value = str(obj)
60+
if isinstance(obj, bool):
61+
value = value.lower()
62+
return value
63+
return obj
64+
65+
def transcode_request( # pragma: NO COVER
66+
http_options: List[Dict[str, str]],
67+
request: Any,
68+
required_fields_default_values: Optional[Dict[str, Any]] = None,
69+
rest_numeric_enums: bool = False,
70+
) -> Tuple[Dict[str, Any], Optional[str], Dict[str, Any]]:
71+
pb_request = getattr(request, "_pb", request)
72+
transcoded_request = path_template.transcode(http_options, pb_request)
73+
74+
body_json = None
75+
if transcoded_request.get("body") is not None:
76+
body_json = json_format.MessageToJson(
77+
transcoded_request["body"],
78+
use_integers_for_enums=rest_numeric_enums,
79+
)
80+
81+
query_params_json = {}
82+
if transcoded_request.get("query_params") is not None:
83+
query_params_json = json.loads(json_format.MessageToJson(
84+
transcoded_request["query_params"],
85+
use_integers_for_enums=rest_numeric_enums,
86+
))
87+
88+
if required_fields_default_values:
89+
for k, v in required_fields_default_values.items():
90+
if k not in query_params_json:
91+
query_params_json[k] = v
92+
93+
if rest_numeric_enums:
94+
query_params_json["$alt"] = "json;enum-encoding=int"
95+
96+
return transcoded_request, body_json, query_params_json

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/_shared_macros.j2‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -198,22 +198,23 @@ def _get_http_options():
198198
service: The service.
199199
is_async (bool): Used to determine the code path i.e. whether for sync or async call.
200200
is_request_message_proto_plus_type (bool): Used to determine whether the request message is a proto-plus type. #}
201-
{% macro rest_call_method_common(body_spec, method_name, service, is_async=False, is_request_message_proto_plus_type=False) %}
201+
{% macro rest_call_method_common(body_spec, method_name, service, is_async=False, is_request_message_proto_plus_type=False, rest_numeric_enums=False) %}
202202
{% set service_name = service.name %}
203203
{% set await_prefix = "await " if is_async else "" %}
204204
{% set async_class_prefix = "Async" if is_async else "" %}
205205

206206
http_options = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_http_options()
207-
{# TODO(https://github.com/googleapis/gapic-generator-python/issues/2274): Add debug log before intercepting a request #}
208207
request, metadata = {{ await_prefix }}self._interceptor.pre_{{ method_name|snake_case }}(request, metadata)
209-
transcoded_request = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_transcoded_request(http_options, request)
210-
211-
{% if body_spec %}
212-
body = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_request_body_json(transcoded_request)
213-
{% endif %}{# body_spec #}
214-
215-
# Jsonify the query params
216-
query_params = _Base{{ service_name }}RestTransport._Base{{method_name}}._get_query_params_json(transcoded_request)
208+
transcoded_request, body, query_params = rest_helpers.transcode_request(
209+
http_options,
210+
request,
211+
required_fields_default_values=getattr(
212+
_Base{{ service_name }}RestTransport._Base{{method_name}},
213+
"__REQUIRED_FIELDS_DEFAULT_VALUES",
214+
None,
215+
),
216+
rest_numeric_enums={{ rest_numeric_enums }},
217+
)
217218

218219
if CLIENT_LOGGING_SUPPORTED and _LOGGER.isEnabledFor(logging.DEBUG): # pragma: NO COVER
219220
request_url = "{host}{uri}".format(host=self._host, uri=transcoded_request['uri'])

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/_rest_mixins_base.py.j2‎

Lines changed: 1 addition & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,27 +25,7 @@
2525

2626
{{ shared_macros.http_options_method(api.mixin_http_options["{}".format(name)])|indent(8)}}
2727

28-
@staticmethod
29-
def _get_transcoded_request(http_options, request):
30-
request_kwargs = json_format.MessageToDict(request)
31-
transcoded_request = path_template.transcode(
32-
http_options, **request_kwargs)
33-
return transcoded_request
34-
35-
{% set body_spec = api.mixin_http_options["{}".format(name)][0].body %}
36-
{%- if body_spec %}
37-
38-
@staticmethod
39-
def _get_request_body_json(transcoded_request):
40-
body = json.dumps(transcoded_request['body'])
41-
return body
42-
43-
{%- endif %} {# body_spec #}
44-
45-
@staticmethod
46-
def _get_query_params_json(transcoded_request):
47-
query_params = json.loads(json.dumps(transcoded_request['query_params']))
48-
return query_params
28+
pass
4929

5030
{% endfor %}
5131
{% endif %} {# rest in opts.transport #}

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest.py.j2‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ from google.auth.transport.requests import AuthorizedSession # type: ignore
1010
from google.auth import credentials as ga_credentials # type: ignore
1111
from google.api_core import exceptions as core_exceptions
1212
from google.api_core import retry as retries
13-
from google.api_core import rest_helpers
13+
{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %}
14+
from {{package_path}} import _compat as rest_helpers
1415
from google.api_core import rest_streaming
1516
from google.api_core import gapic_v1
1617
import google.protobuf
@@ -245,7 +246,7 @@ class {{service.name}}RestTransport(_Base{{ service.name }}RestTransport):
245246
{% endif %}
246247
"""
247248

248-
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type)|indent(8) }}
249+
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, False, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }}
249250

250251
{% if not method.void %}
251252
# Return the response

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_asyncio.py.j2‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ from google.iam.v1 import policy_pb2 # type: ignore
3232
from google.cloud.location import locations_pb2 # type: ignore
3333
{% endif %}
3434
from google.api_core import retry_async as retries
35-
from google.api_core import rest_helpers
35+
{% set package_path = api.naming.module_namespace|join('.') + "." + api.naming.versioned_module_name %}
36+
from {{package_path}} import _compat as rest_helpers
3637
from google.api_core import rest_streaming_async # type: ignore
3738
import google.protobuf
3839

@@ -203,7 +204,7 @@ class Async{{service.name}}RestTransport(_Base{{ service.name }}RestTransport):
203204
{% endif %}
204205
"""
205206

206-
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type)|indent(8) }}
207+
{{ shared_macros.rest_call_method_common(body_spec, method.name, service, True, method.input.ident.is_proto_plus_type, opts.rest_numeric_enums)|indent(8) }}
207208

208209
{% if not method.void %}
209210
# Return the response

‎packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/rest_base.py.j2‎

Lines changed: 0 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -120,51 +120,8 @@ class _Base{{ service.name }}RestTransport({{service.name}}Transport):
120120
def _get_unset_required_fields(cls, message_dict):
121121
return {k: v for k, v in cls.__REQUIRED_FIELDS_DEFAULT_VALUES.items() if k not in message_dict}
122122
{% endif %}{# required fields #}
123-
124123
{% set method_http_options = method.http_options %}
125-
126124
{{ shared_macros.http_options_method(method_http_options)|indent(8) }}
127-
128-
@staticmethod
129-
def _get_transcoded_request(http_options, request):
130-
{% if method.input.ident.is_proto_plus_type %}
131-
pb_request = {{method.input.ident}}.pb(request)
132-
{% else %}
133-
pb_request = request
134-
{% endif %}
135-
transcoded_request = path_template.transcode(http_options, pb_request)
136-
return transcoded_request
137-
138-
{% set body_spec = method.http_options[0].body %}
139-
{%- if body_spec %}
140-
141-
@staticmethod
142-
def _get_request_body_json(transcoded_request):
143-
# Jsonify the request body
144-
145-
body = json_format.MessageToJson(
146-
transcoded_request['body'],
147-
use_integers_for_enums={{ opts.rest_numeric_enums }}
148-
)
149-
return body
150-
151-
{%- endif %}{# body_spec #}
152-
153-
@staticmethod
154-
def _get_query_params_json(transcoded_request):
155-
query_params = json.loads(json_format.MessageToJson(
156-
transcoded_request['query_params'],
157-
use_integers_for_enums={{ opts.rest_numeric_enums }},
158-
))
159-
{% if method.input.required_fields %}
160-
query_params.update(_Base{{ service.name }}RestTransport._Base{{method.name}}._get_unset_required_fields(query_params))
161-
{% endif %}{# required fields #}
162-
163-
{% if opts.rest_numeric_enums %}
164-
query_params["$alt"] = "json;enum-encoding=int"
165-
{% endif %}
166-
return query_params
167-
168125
{% endif %}{# method.http_options and not method.client_streaming #}
169126
{% endfor %}
170127

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[run]
2+
branch = True
3+
4+
[report]
5+
show_missing = True
6+
omit =
7+
google/showcase/__init__.py
8+
google/showcase/gapic_version.py
9+
exclude_lines =
10+
# Re-enable the standard pragma
11+
pragma: NO COVER
12+
# Ignore debug-only repr
13+
def __repr__
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# -*- coding: utf-8 -*-
2+
# Copyright 2026 Google LLC
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
[flake8]
17+
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2333):
18+
# Resolve flake8 lint issues
19+
ignore = E203, E231, E266, E501, W503
20+
exclude =
21+
# TODO(https://github.com/googleapis/gapic-generator-python/issues/2333):
22+
# Ensure that generated code passes flake8 lint
23+
**/gapic/**
24+
**/services/**
25+
**/types/**
26+
# Exclude Protobuf gencode
27+
*_pb2.py
28+
29+
# Standard linting exemptions.
30+
**/.nox/**
31+
__pycache__,
32+
.git,
33+
*.pyc,
34+
conf.py

0 commit comments

Comments
 (0)