-
Notifications
You must be signed in to change notification settings - Fork 3.4k
customer opt-in for ascii expansion #48914
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
bc58314
3c0b3b3
adb25de
92c3b51
5af3263
975ed0a
5982c20
52a9b73
e4577c2
587d807
ebf4365
a36fb23
1411090
7c8de9e
f317734
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,3 +1,4 @@ | ||
| apiMdSha256: 1538a79b2c2da38fb83fd37ccea945314bf2f34218acda3d8f15e2f0268f3040 | ||
| parserVersion: 0.3.30 | ||
| pythonVersion: 3.13.14 | ||
| apiMdSha256: 54ea9d4c0c0f08c381ee7e69a7468ed9c08e5fb55c7af0fc1b9f6d0d6f770a86 | ||
| packageVersion: 4.17.0 | ||
| parserVersion: 0.3.31 | ||
| pythonVersion: 3.10.21 |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,6 +21,7 @@ | |
|
|
||
| """Synchronized request in the Azure Cosmos database service. | ||
| """ | ||
| # cspell:ignore surrogatepass | ||
| import copy | ||
| import json | ||
| import time | ||
|
|
@@ -37,6 +38,17 @@ | |
| from ._request_object import RequestObject | ||
| from .documents import _OperationType | ||
|
|
||
| _ITEM_BODY_WRITE_OPERATIONS = frozenset(( | ||
| _OperationType.Create, | ||
| _OperationType.Upsert, | ||
| _OperationType.Replace, | ||
| _OperationType.Patch, | ||
| # Batch membership is necessary but not sufficient: a batch made up only of | ||
| # read and delete operations carries no item body. _should_escape_non_ascii_ | ||
| # in_request_body refines this entry with _batch_contains_item_body below. | ||
| _OperationType.Batch, | ||
|
dibahlfi marked this conversation as resolved.
|
||
| )) | ||
|
|
||
| # cspell:ignore ppaf | ||
| def _is_readable_stream(obj): | ||
| """Checks whether obj is a file-like readable stream. | ||
|
|
@@ -50,26 +62,104 @@ def _is_readable_stream(obj): | |
| return False | ||
|
|
||
|
|
||
| def _request_body_from_data(data): | ||
| """Gets request body from data. | ||
| def _request_body_from_data(data, ensure_ascii=True): | ||
| """Convert supported request data into an HTTP body. | ||
|
|
||
| Dictionaries, lists, and tuples are serialized as compact JSON. Other | ||
| supported body types are returned unchanged. | ||
|
|
||
| When ``ensure_ascii`` is False the serialized body is returned as UTF-8 | ||
| ``bytes``. The escaped default and caller-supplied pre-serialized strings | ||
| keep returning ``str``, preserving existing behavior. | ||
|
|
||
| When `data` is dict and list into unicode string; otherwise return `data` | ||
| without making any change. | ||
| Returning bytes is required for correctness, not just convenience. The sync | ||
| Requests transport forwards the body to ``requests`` unchanged, and the | ||
| supported dependency range still permits urllib3 1.x, whose connection path | ||
| hands ``str`` bodies to ``http.client``. ``http.client`` encodes them as | ||
| Latin-1, so a compact body would be sent with the wrong encoding: text that | ||
| has no Latin-1 representation (any CJK or emoji) raises ``UnicodeEncodeError`` | ||
| before the request is sent, and text that does have one is written in Latin-1 | ||
| rather than UTF-8, so 'é' goes out as the single byte 0xE9 instead of | ||
| 0xC3 0xA9 and the service receives different characters than the caller | ||
| supplied. Handing the transport the encoded bytes removes every subsequent | ||
| re-encoding step. | ||
|
|
||
| :param Union[str, unicode, file-like stream object, dict, list, None] data: | ||
| :returns: the json dump data. | ||
| :rtype: Union[str, unicode, file-like stream object, None] | ||
| :param bool ensure_ascii: Whether non-ASCII characters should be escaped. | ||
| :returns: The serialized or unchanged request body. | ||
| :rtype: Union[str, bytes, file-like stream object, None] | ||
|
|
||
| """ | ||
| if data is None or isinstance(data, str) or _is_readable_stream(data): | ||
| return data | ||
| if isinstance(data, (dict, list, tuple)): | ||
| json_dumped = json.dumps(data, separators=(",", ":")) | ||
|
|
||
| return json_dumped | ||
| if ensure_ascii: | ||
| return json.dumps(data, separators=(",", ":")) | ||
| json_dumped = json.dumps(data, separators=(",", ":"), ensure_ascii=False) | ||
| try: | ||
| # Encode once; callers derive Content-Length directly from these bytes. | ||
| encoded_body = json_dumped.encode("utf-8") | ||
| except UnicodeEncodeError: | ||
| # Rare path for text originating from UTF-16 APIs. Combine adjacent | ||
| # high/low surrogate pairs into their Unicode scalar while preserving | ||
| # unpaired surrogates, then escape only those remaining invalid code | ||
| # units as valid JSON \uXXXX sequences. | ||
| normalized_body = json_dumped.encode( | ||
| "utf-16-le", "surrogatepass" | ||
| ).decode("utf-16-le", "surrogatepass") | ||
| encoded_body = normalized_body.encode("utf-8", "backslashreplace") | ||
| # Send exactly these bytes, so no transport re-encodes them. | ||
| return encoded_body | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🟡 Recommendation — Type contract: update byte-capable retry consumers
This widens SDK-generated item bodies from |
||
| return None | ||
|
|
||
|
|
||
| def _batch_contains_item_body(batch_operations): | ||
| """Return whether a formatted transactional batch contains an item body. | ||
|
|
||
| Read and delete batch operations contain only an item ID. Create, upsert, | ||
| replace, and patch operations contain a resourceBody. | ||
|
|
||
| :param list[dict[str, object]] batch_operations: The formatted batch operations. | ||
| :returns: Whether at least one operation contains an item body. | ||
| :rtype: bool | ||
| """ | ||
| return ( | ||
| isinstance(batch_operations, (list, tuple)) | ||
| and any( | ||
| isinstance(operation, dict) and "resourceBody" in operation | ||
| for operation in batch_operations | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| def _should_escape_non_ascii_in_request_body(client, request_params, request_data): | ||
| """Decide whether a request body must keep non-ASCII characters escaped. | ||
|
|
||
| Compact UTF-8 is only used when the client opted in and the request is one | ||
| of the item write operations the option is scoped to. A transactional batch | ||
| uses compact UTF-8 only when at least one operation contains an item body; | ||
| read/delete-only batches keep the escaped form. Every other request, | ||
| including control-plane bodies and queries, also keeps the escaped form. | ||
|
|
||
| :param object client: the client connection issuing the request. | ||
| :param ~azure.cosmos._request_object.RequestObject request_params: the request parameters. | ||
| :param object request_data: The body data that will be serialized. | ||
| :returns: whether non-ASCII characters should be escaped in the body. | ||
| :rtype: bool | ||
| """ | ||
| # getattr keeps the safe (escaped) default for any caller that supplies a | ||
| # client object without the option, e.g. custom or legacy connections. | ||
| return ( | ||
| not getattr(client, "_enable_compact_utf8_item_writes", False) | ||
| or request_params.resource_type != http_constants.ResourceType.Document | ||
| or request_params.operation_type not in _ITEM_BODY_WRITE_OPERATIONS | ||
| or ( | ||
| request_params.operation_type == _OperationType.Batch | ||
| and not _batch_contains_item_body(request_data) | ||
| ) | ||
| ) | ||
|
|
||
|
|
||
| def _Request(global_endpoint_manager, request_params, connection_policy, pipeline_client, request, **kwargs): # pylint: disable=too-many-statements | ||
| """Makes one http request using the requests module. | ||
|
|
||
|
|
@@ -285,12 +375,21 @@ def SynchronizedRequest( | |
| :return: tuple of (result, headers) | ||
| :rtype: tuple of (dict dict) | ||
| """ | ||
| request.data = _request_body_from_data(request_data) | ||
| if request.data and isinstance(request.data, str): | ||
| request.data = _request_body_from_data( | ||
| request_data, | ||
| ensure_ascii=_should_escape_non_ascii_in_request_body(client, request_params, request_data) | ||
| ) | ||
| if isinstance(request.data, (bytes, bytearray)): | ||
| # Compact UTF-8 bodies reach the transport as their final bytes, so | ||
| # Content-Length cannot drift from the wire body. | ||
| request.headers[http_constants.HttpHeaders.ContentLength] = len(request.data) | ||
| elif request.data and isinstance(request.data, str): | ||
| # Use UTF-8 byte length, not str length (code-point count), so the | ||
| # header matches the bytes the transport actually writes for any | ||
| # non-ASCII payload. | ||
| request.headers[http_constants.HttpHeaders.ContentLength] = len(request.data.encode("utf-8")) | ||
| request.headers[http_constants.HttpHeaders.ContentLength] = len( | ||
| request.data.encode("utf-8") | ||
| ) | ||
| elif request.data is None: | ||
| request.headers[http_constants.HttpHeaders.ContentLength] = 0 | ||
|
|
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.