Skip to content

fix(google-api-core): align HTTP T3 and T4 OpenTelemetry span status and attributes #18868

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
chalmerlowe wants to merge 1 commit into
base: test/showcase-client-timeout-compliance
Choose a base branch
from
Draft

fix(google-api-core): align HTTP T3 and T4 OpenTelemetry span status and attributes #18868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 14 additions & 14 deletions packages/gapic-generator/tests/system/telemetry_requirements_matrix.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,30 @@
Feature ID,Feature Name,Tier,Attempt,Transport,Scenario,Span Count,Span Kind,Span Name,Span Status,rpc.system.name,rpc.method,rpc.response.status_code,http.request.method,http.response.status_code,rpc.grpc.status_code,url.domain,url.template,server.address,server.port,error.type,status.message,resend_count,parent_span_id,Extra / Permitted Attributes
F1.1,Tracing Off (HTTP),T4,N/A,HTTP/REST,Tracing Off,0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,No spans emitted
F1.2,Unary Success (HTTP),T4,0,HTTP/REST,Happy Path,1,CLIENT,POST /v1beta1/echo:echo,OK,NOT SET,NOT SET,NOT SET,POST,200,NOT SET,googleapis.com,/v1beta1/echo:echo,localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,"url.full, http.request.body.size"
F1.3,Server Error (HTTP),T4,0,HTTP/REST,Server Failure,1,CLIENT,POST /v1beta1/echo:echo,ERROR,NOT SET,NOT SET,NOT SET,POST,400,NOT SET,googleapis.com,/v1beta1/echo:echo,localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,"url.full, http.request.body.size"
F1.4,Client Timeout (HTTP),T4,0,HTTP/REST,Client Timeout,1,CLIENT,POST /v1beta1/echo:block,ERROR,NOT SET,NOT SET,NOT SET,POST,NOT SET,NOT SET,googleapis.com,/v1beta1/echo:block,localhost,7469,ReadTimeout,Read timed out,NOT SET,T3.span_id,url.full
F1.5a,Retry Recovery - Attempt 1 (HTTP),T4,0,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,NOT SET,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,url.full
F1.5b,Retry Recovery - Attempt 2 (HTTP),T4,1,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},OK,NOT SET,NOT SET,NOT SET,POST,200,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,url.full
F1.2,Unary Success (HTTP),T4,0,HTTP/REST,Happy Path,1,CLIENT,POST /v1beta1/echo:echo,UNSET,http,NOT SET,NOT SET,POST,200,NOT SET,googleapis.com,/v1beta1/echo:echo,localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,"url.full, http.request.body.size"
F1.3,Server Error (HTTP),T4,0,HTTP/REST,Server Failure,1,CLIENT,POST /v1beta1/echo:echo,ERROR,http,NOT SET,NOT SET,POST,400,NOT SET,googleapis.com,/v1beta1/echo:echo,localhost,7469,400,Simulated unretryable invalid argument error.,NOT SET,T3.span_id,"url.full, http.request.body.size"
F1.4,Client Timeout (HTTP),T4,0,HTTP/REST,Client Timeout,1,CLIENT,POST /v1beta1/echo:block,ERROR,http,NOT SET,NOT SET,POST,NOT SET,NOT SET,googleapis.com,/v1beta1/echo:block,localhost,7469,ReadTimeout,Read timed out,NOT SET,T3.span_id,url.full
F1.5a,Retry Recovery - Attempt 1 (HTTP),T4,0,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,http,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,503,Temporary glitch,NOT SET,T3.span_id,url.full
F1.5b,Retry Recovery - Attempt 2 (HTTP),T4,1,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},UNSET,http,NOT SET,NOT SET,POST,200,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,url.full
F1.6,Tracing Off (gRPC),T4,N/A,gRPC,Tracing Off,0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,No spans emitted
F1.7,Unary Success (gRPC),T4,0,gRPC,Happy Path,1,CLIENT,google.showcase.v1beta1.Echo/Echo,UNSET,grpc,google.showcase.v1beta1.Echo/Echo,OK,NOT SET,NOT SET,0,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,"rpc.system, rpc.service"
F1.8,Server Error (gRPC),T4,0,gRPC,Server Failure,1,CLIENT,google.showcase.v1beta1.Echo/Echo,ERROR,grpc,google.showcase.v1beta1.Echo/Echo,NOT SET,NOT SET,NOT SET,3,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,"rpc.system, rpc.service"
F1.9,Client Timeout (gRPC),T4,0,gRPC,Client Timeout,1,CLIENT,google.showcase.v1beta1.Echo/Block,ERROR,grpc,google.showcase.v1beta1.Echo/Block,NOT SET,NOT SET,NOT SET,4,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,"rpc.system, rpc.service"
F1.10a,Retry Recovery - Attempt 1 (gRPC),T4,0,gRPC,Retry Recovery,2,CLIENT,google.showcase.v1beta1.SequenceService/AttemptSequence,ERROR,grpc,google.showcase.v1beta1.SequenceService/AttemptSequence,NOT SET,NOT SET,NOT SET,14,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,"rpc.system, rpc.service"
F1.10b,Retry Recovery - Attempt 2 (gRPC),T4,1,gRPC,Retry Recovery,2,CLIENT,google.showcase.v1beta1.SequenceService/AttemptSequence,UNSET,grpc,google.showcase.v1beta1.SequenceService/AttemptSequence,OK,NOT SET,NOT SET,0,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,"rpc.system, rpc.service"
F2.1,Tracing Off (HTTP),T3,N/A,HTTP/REST,Tracing Off,0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,No spans emitted
F2.2,Unary Success (HTTP),T3,0,HTTP/REST,Happy Path,1,INTERNAL,google.showcase.v1beta1.Echo/Echo,UNSET,http,google.showcase.v1beta1.Echo/Echo,OK,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,None (Root),Strict ceiling (no unknown attrs)
F2.3,Server Error (HTTP),T3,0,HTTP/REST,Server Failure,1,INTERNAL,google.showcase.v1beta1.Echo/Echo,ERROR,http,google.showcase.v1beta1.Echo/Echo,BAD_REQUEST,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,BAD_REQUEST,Simulated unretryable invalid argument error.,NOT SET,None (Root),"gcp.errors.domain, gcp.errors.metadata.*"
F2.4,Client Timeout (HTTP),T3,0,HTTP/REST,Client Timeout,1,INTERNAL,google.showcase.v1beta1.Echo/Block,ERROR,http,google.showcase.v1beta1.Echo/Block,ReadTimeout,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,ReadTimeout,Read timed out,NOT SET,None (Root),gcp.errors.domain
F2.2,Unary Success (HTTP),T3,0,HTTP/REST,Happy Path,1,INTERNAL,google.showcase.v1beta1.Echo/Echo,UNSET,http,google.showcase.v1beta1.Echo/Echo,OK,NOT SET,200,NOT SET,googleapis.com,/v1beta1/echo:echo,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,None (Root),Strict ceiling (no unknown attrs)
F2.3,Server Error (HTTP),T3,0,HTTP/REST,Server Failure,1,INTERNAL,google.showcase.v1beta1.Echo/Echo,ERROR,http,google.showcase.v1beta1.Echo/Echo,BAD_REQUEST,NOT SET,400,NOT SET,googleapis.com,/v1beta1/echo:echo,NOT SET,NOT SET,BAD_REQUEST,Simulated unretryable invalid argument error.,NOT SET,None (Root),"gcp.errors.domain, gcp.errors.metadata.*"
F2.4,Client Timeout (HTTP),T3,0,HTTP/REST,Client Timeout,1,INTERNAL,google.showcase.v1beta1.Echo/Block,ERROR,http,google.showcase.v1beta1.Echo/Block,ReadTimeout,NOT SET,NOT SET,NOT SET,googleapis.com,/v1beta1/echo:block,NOT SET,NOT SET,ReadTimeout,Read timed out,NOT SET,None (Root),gcp.errors.domain
F2.5,Tracing Off (gRPC),T3,N/A,gRPC,Tracing Off,0,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,N/A,No spans emitted
F2.6,Unary Success (gRPC),T3,0,gRPC,Happy Path,1,INTERNAL,google.showcase.v1beta1.Echo/Echo,UNSET,grpc,google.showcase.v1beta1.Echo/Echo,OK,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,None (Root),Strict ceiling (no unknown attrs)
F2.7,Server Error (gRPC),T3,0,gRPC,Server Failure,1,INTERNAL,google.showcase.v1beta1.Echo/Echo,ERROR,grpc,google.showcase.v1beta1.Echo/Echo,INVALID_ARGUMENT,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,INVALID_ARGUMENT,Simulated unretryable invalid argument error.,NOT SET,None (Root),"gcp.errors.domain, gcp.errors.metadata.*"
F2.8,Client Timeout (gRPC),T3,0,gRPC,Client Timeout,1,INTERNAL,google.showcase.v1beta1.Echo/Block,ERROR,grpc,google.showcase.v1beta1.Echo/Block,DEADLINE_EXCEEDED,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,DEADLINE_EXCEEDED,Deadline Exceeded,NOT SET,None (Root),gcp.errors.domain
F3.1a,Retry Succeeds - Root (HTTP),T3,0,HTTP/REST,Retry Recovery,1,INTERNAL,google.showcase.v1beta1.SequenceService/AttemptSequence,UNSET,http,google.showcase.v1beta1.SequenceService/AttemptSequence,OK,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,None (Root),Parent aggregates OK; T4 children nested under T3
F3.1b,Retry Succeeds - Attempt 1 (HTTP),T4,0,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,NOT SET,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates OK; T4 children nested under T3
F3.1c,Retry Succeeds - Attempt 2 (HTTP),T4,1,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},OK,NOT SET,NOT SET,NOT SET,POST,200,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates OK; T4 children nested under T3
F3.2a,Retries Exhausted - Root (HTTP),T3,0,HTTP/REST,Retries Exhausted,1,INTERNAL,google.showcase.v1beta1.SequenceService/AttemptSequence,ERROR,http,google.showcase.v1beta1.SequenceService/AttemptSequence,UNAVAILABLE,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,UNAVAILABLE,Persistent outage,NOT SET,None (Root),Parent aggregates UNAVAILABLE; all T4 children nested under T3
F3.2b,Retries Exhausted - Attempt 1 (HTTP),T4,0,HTTP/REST,Retries Exhausted,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,NOT SET,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates UNAVAILABLE; all T4 children nested under T3
F3.2c,Retries Exhausted - Attempt 2 (HTTP),T4,1,HTTP/REST,Retries Exhausted,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,NOT SET,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates UNAVAILABLE; all T4 children nested under T3
F3.1a,Retry Succeeds - Root (HTTP),T3,0,HTTP/REST,Retry Recovery,1,INTERNAL,google.showcase.v1beta1.SequenceService/AttemptSequence,UNSET,http,google.showcase.v1beta1.SequenceService/AttemptSequence,OK,NOT SET,200,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,None (Root),Parent aggregates OK; T4 children nested under T3
F3.1b,Retry Succeeds - Attempt 1 (HTTP),T4,0,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,http,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,503,Temporary glitch,NOT SET,T3.span_id,Parent aggregates OK; T4 children nested under T3
F3.1c,Retry Succeeds - Attempt 2 (HTTP),T4,1,HTTP/REST,Retry Recovery,2,CLIENT,POST /v1beta1/{name=sequences/*},UNSET,http,NOT SET,NOT SET,POST,200,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates OK; T4 children nested under T3
F3.2a,Retries Exhausted - Root (HTTP),T3,0,HTTP/REST,Retries Exhausted,1,INTERNAL,google.showcase.v1beta1.SequenceService/AttemptSequence,ERROR,http,google.showcase.v1beta1.SequenceService/AttemptSequence,UNAVAILABLE,NOT SET,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},NOT SET,NOT SET,UNAVAILABLE,Persistent outage,NOT SET,None (Root),Parent aggregates UNAVAILABLE; all T4 children nested under T3
F3.2b,Retries Exhausted - Attempt 1 (HTTP),T4,0,HTTP/REST,Retries Exhausted,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,http,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,503,Persistent outage,NOT SET,T3.span_id,Parent aggregates UNAVAILABLE; all T4 children nested under T3
F3.2c,Retries Exhausted - Attempt 2 (HTTP),T4,1,HTTP/REST,Retries Exhausted,2,CLIENT,POST /v1beta1/{name=sequences/*},ERROR,http,NOT SET,NOT SET,POST,503,NOT SET,googleapis.com,/v1beta1/{name=sequences/*},localhost,7469,503,Persistent outage,NOT SET,T3.span_id,Parent aggregates UNAVAILABLE; all T4 children nested under T3
F3.3a,Retry Succeeds - Root (gRPC),T3,0,gRPC,Retry Recovery,1,INTERNAL,google.showcase.v1beta1.SequenceService/AttemptSequence,UNSET,grpc,google.showcase.v1beta1.SequenceService/AttemptSequence,OK,NOT SET,NOT SET,NOT SET,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,None (Root),Parent aggregates OK; T4 children nested under T3
F3.3b,Retry Succeeds - Attempt 1 (gRPC),T4,0,gRPC,Retry Recovery,2,CLIENT,google.showcase.v1beta1.SequenceService/AttemptSequence,ERROR,grpc,google.showcase.v1beta1.SequenceService/AttemptSequence,NOT SET,NOT SET,NOT SET,14,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates OK; T4 children nested under T3
F3.3c,Retry Succeeds - Attempt 2 (gRPC),T4,1,gRPC,Retry Recovery,2,CLIENT,google.showcase.v1beta1.SequenceService/AttemptSequence,UNSET,grpc,google.showcase.v1beta1.SequenceService/AttemptSequence,OK,NOT SET,NOT SET,0,googleapis.com,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,NOT SET,T3.span_id,Parent aggregates OK; T4 children nested under T3
Expand Down
60 changes: 56 additions & 4 deletions packages/google-api-core/google/api_core/_observability.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,7 @@ def _build_http_span_attributes(
f"{resolved_method} {url_template}" if url_template else resolved_method
)
span_attributes: dict[str, Any] = {
"rpc.system.name": "http",
"http.request.method": resolved_method,
"server.address": server_address or "",
"server.port": server_port or 443,
Expand All @@ -613,6 +614,32 @@ def _build_http_span_attributes(

return span_name, span_attributes, resolved_headers

@staticmethod
def _extract_http_error_message(response: Any) -> str | None:
"""Extract a human-readable error message from an HTTP error response."""
json_fn = getattr(response, "json", None)
if callable(json_fn):
try:
payload = json_fn()
if isinstance(payload, dict):
err_obj = payload.get("error")
if isinstance(err_obj, dict):
msg = err_obj.get("message")
if isinstance(msg, str) and msg:
return msg
elif isinstance(err_obj, str) and err_obj:
return err_obj
except Exception:
pass
for attr in ("text", "reason"):
try:
val = getattr(response, attr, None)
if isinstance(val, str) and val.strip():
return val.strip()
except Exception:
continue
return None

def __init__(
self,
request: Any = None,
Expand All @@ -632,6 +659,7 @@ def __init__(
self._body = body
self._client_options = client_options
self._span: Any = None
self._parent_span: Any = None
self._cm: Any = None

def record_response(self, response: Any) -> None:
Expand All @@ -654,11 +682,18 @@ def record_response(self, response: Any) -> None:
response, "status_code", getattr(response, "status", None)
)
if status_code is not None:
span.set_attribute("http.response.status_code", int(status_code))
if int(status_code) >= 400:
code_int = int(status_code)
span.set_attribute("http.response.status_code", code_int)
if self._parent_span is not None:
self._parent_span.set_attribute(
"http.response.status_code", code_int
)
if code_int >= 400:
span.set_status(Status(StatusCode.ERROR))
else:
span.set_status(Status(StatusCode.OK))
span.set_attribute("error.type", str(code_int))
error_msg = self._extract_http_error_message(response)
if error_msg:
span.set_attribute("status.message", error_msg)

headers = getattr(response, "headers", None)
if headers and "Content-Length" in headers:
Expand Down Expand Up @@ -745,6 +780,23 @@ def __enter__(self) -> "_TraceContext":
)
)

parent_span = trace.get_current_span()
if (
parent_span is not None
and getattr(parent_span, "is_recording", lambda: False)() is True
and hasattr(parent_span, "set_attribute")
):
self._parent_span = parent_span
if self._url_template:
try:
parent_attrs = getattr(parent_span, "attributes", None)
if parent_attrs is None or "url.template" not in parent_attrs:
parent_span.set_attribute(
"url.template", self._url_template
)
except Exception:
pass

self._cm = tracer.start_as_current_span(
span_name,
kind=trace.SpanKind.CLIENT,
Expand Down
Loading
Loading