From 16c17988d1f41262189994dbb73533b7cad1dd26 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 29 Oct 2026 11:59:00 -0700 Subject: [PATCH] openai-v2: enable genai-openai for latest experimental conventions With gen_ai_latest_experimental, enable opentelemetry-instrumentation-genai-openai (now a dependency) instead of doing nothing. Without the flag, keep it disabled: both packages register the 'openai' entry point, so auto-instrumentation would otherwise instrument OpenAI twice. genai-openai brings opentelemetry-util-genai 1.x, which conflicts with the in-repo util-genai, so the package is excluded from the uv workspace and tested via tox only. --- .../.changelog/5261.added | 1 + .../.changelog/5261.changed | 1 + .../README.rst | 7 + .../pyproject.toml | 7 +- .../instrumentation/openai_v2/__init__.py | 159 +++++++++++------- .../tests/conftest.py | 13 +- .../tests/requirements.latest.txt | 4 +- .../tests/requirements.oldest.txt | 13 +- .../tests/test_async_chat_completions.py | 4 +- .../tests/test_async_embeddings.py | 4 +- .../tests/test_chat_completions.py | 4 +- .../tests/test_chat_metrics.py | 4 +- .../tests/test_embeddings.py | 4 +- .../tests/test_genai_openai_coexistence.py | 113 +++++++++++++ .../tests/test_latest_experimental.py | 136 ++++++++++++++- .../tests/test_response_extractors.py | 4 +- .../tests/test_responses.py | 4 +- .../tests/test_utils.py | 5 + pyproject.toml | 5 +- uv.lock | 28 --- 20 files changed, 406 insertions(+), 114 deletions(-) create mode 100644 instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.added create mode 100644 instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.changed create mode 100644 instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_genai_openai_coexistence.py diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.added b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.added new file mode 100644 index 00000000000..531a302faa3 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.added @@ -0,0 +1 @@ +With `OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`, enable `opentelemetry-instrumentation-genai-openai` (new dependency, `~= 1.3.0b0`) and forward `instrument()` arguments to it. Without the flag, keep it disabled so auto-instrumentation does not instrument OpenAI twice. diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.changed b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.changed new file mode 100644 index 00000000000..3f3c2b3d7e7 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.changed @@ -0,0 +1 @@ +Bump minimum `opentelemetry-api` to 1.43 and `opentelemetry-instrumentation`/`opentelemetry-semantic-conventions` to 0.64b0, as required by `opentelemetry-instrumentation-genai-openai`. diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/README.rst b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/README.rst index 0451ebbc389..908d84017e7 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/README.rst +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/README.rst @@ -141,6 +141,13 @@ To enable the latest experimental features, set the environment variable ``OTEL_SEMCONV_STABILITY_OPT_IN`` to ``gen_ai_latest_experimental``. Or, if you use ``OTEL_SEMCONV_STABILITY_OPT_IN`` to enable other features, append ``,gen_ai_latest_experimental`` to its value. +When enabled, this package turns itself off and enables +`opentelemetry-instrumentation-genai-openai `_ +(installed as a dependency) instead. + +Content capturing modes (``span_only``, ``event_only``, ``span_and_event``) and +completion hooks are provided by that package. + Without this setting, OpenAI instrumentation aligns with `Semantic Conventions v1.30.0 `_ and would not capture additional details introduced in later versions. diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/pyproject.toml b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/pyproject.toml index 3de404c25b8..f0c155f153b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/pyproject.toml +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/pyproject.toml @@ -25,9 +25,10 @@ classifiers = [ "Programming Language :: Python :: 3.14", ] dependencies = [ - "opentelemetry-api ~= 1.40", - "opentelemetry-instrumentation ~= 0.61b0", - "opentelemetry-semantic-conventions ~= 0.61b0", + "opentelemetry-api ~= 1.43", + "opentelemetry-instrumentation ~= 0.64b0", + "opentelemetry-semantic-conventions ~= 0.64b0", + "opentelemetry-instrumentation-genai-openai ~= 1.3.0b0", "typing-extensions >= 4.5.0", ] diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py index f0fe836a383..4794c39883b 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py @@ -8,8 +8,9 @@ .. deprecated:: 2.5b0 Use `opentelemetry-instrumentation-genai-openai `_ - instead. This package won't receive new features for `Semantic Conventions v1.30.0 - `_. + instead. This package only supports `Semantic Conventions v1.30.0 + `_ + and won't receive new features. .. _openai: https://pypi.org/project/openai/ @@ -36,42 +37,27 @@ By default, the instrumentation aligns with `Semantic Conventions v1.30.0 `_ -and does not capture prompt or completion content. Behavior is controlled -via environment variables: +and does not capture prompt or completion content. +- ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`` - capture + prompts, completions, tool arguments, and return values on log events. - ``OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`` - opt into the - latest GenAI semantic conventions. Required to access the newer attributes - and the ``span_only`` / ``event_only`` / ``span_and_event`` content modes. - Without this flag, the instrumentation stays on v1.30.0 conventions. -- ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`` - enable capture of - prompts, completions, tool arguments, and return values. Set to ``true`` - on the legacy path, or one of ``span_only``, ``event_only``, - ``span_and_event`` when experimental conventions are enabled. -- ``OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK=upload`` together with - ``OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH=`` - upload - prompts and completions to an ``fsspec``-compatible destination - (local filesystem, ``gs://``, ``s3://``, etc.) and record reference URIs as - ``gen_ai.input.messages.ref`` / ``gen_ai.output.messages.ref`` attributes. - Inline content is not captured unless - ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`` is also set. - -See the `opentelemetry-util-genai README -`_ -for the full list of GenAI configuration variables. - -A custom ``CompletionHook`` implementation can also be passed programmatically:: - - OpenAIInstrumentor().instrument(completion_hook=my_hook) - -When provided, this takes precedence over the hook resolved from -``OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK``. + latest GenAI semantic conventions. This package turns itself off and enables + `opentelemetry-instrumentation-genai-openai + `_ + (installed as a dependency) instead. All ``instrument()`` arguments + (e.g. ``completion_hook``) are forwarded to it. See its documentation for + content capturing modes and completion hooks. API --- """ +from __future__ import annotations + import logging from collections.abc import Collection +from typing import Any from typing_extensions import deprecated from wrapt import wrap_function_wrapper @@ -111,6 +97,17 @@ def _is_latest_experimental_enabled() -> bool: ) +def _get_genai_openai_instrumentor() -> BaseInstrumentor | None: + try: + # pylint: disable-next=import-outside-toplevel,no-name-in-module + from opentelemetry.instrumentation.genai.openai import ( # noqa: PLC0415 + OpenAIInstrumentor as GenAIOpenAIInstrumentor, + ) + except ImportError: + return None + return GenAIOpenAIInstrumentor() + + @deprecated( "opentelemetry-instrumentation-openai-v2 is deprecated. Use the " "opentelemetry-instrumentation-genai-openai package instead. This package " @@ -127,78 +124,120 @@ class OpenAIInstrumentor(BaseInstrumentor): # BaseInstrumentor is a singleton and __init__ re-runs on every # OpenAIInstrumentor() call, so state lives on the class. _legacy_instrumented: bool = False - - def __init__(self): - self._meter = None + _delegate: BaseInstrumentor | None = None + _blocked: BaseInstrumentor | None = None def instrumentation_dependencies(self) -> Collection[str]: return _instruments - def _instrument(self, **kwargs): + def _instrument(self, **kwargs: Any) -> None: """Enable OpenAI instrumentation.""" if _is_latest_experimental_enabled(): - _logger.warning( - "opentelemetry-instrumentation-openai-v2 does not support " - "gen_ai_latest_experimental and is disabled. Use " - "opentelemetry-instrumentation-genai-openai to get the latest " - "GenAI semantic conventions." - ) + self._instrument_latest(**kwargs) return - tracer_provider = kwargs.get("tracer_provider") + self._block_genai_openai() + tracer = get_tracer( __name__, "", - tracer_provider, - schema_url=Schemas.V1_30_0.value, # only used on the legacy path + kwargs.get("tracer_provider"), + schema_url=Schemas.V1_30_0.value, ) - logger_provider = kwargs.get("logger_provider") logger = get_logger( __name__, "", - logger_provider=logger_provider, - schema_url=Schemas.V1_30_0.value, # only used on the legacy path + logger_provider=kwargs.get("logger_provider"), + schema_url=Schemas.V1_30_0.value, ) - meter_provider = kwargs.get("meter_provider") - self._meter = get_meter( + meter = get_meter( __name__, "", - meter_provider, - schema_url=Schemas.V1_30_0.value, # only used on the legacy path + kwargs.get("meter_provider"), + schema_url=Schemas.V1_30_0.value, ) - instruments = Instruments(self._meter) + instruments = Instruments(meter) + capture_content = is_content_enabled() wrap_function_wrapper( "openai.resources.chat.completions", "Completions.create", - chat_completions_create_v_old(tracer, logger, instruments, is_content_enabled()), + chat_completions_create_v_old(tracer, logger, instruments, capture_content), ) - wrap_function_wrapper( "openai.resources.chat.completions", "AsyncCompletions.create", - async_chat_completions_create_v_old(tracer, logger, instruments, is_content_enabled()), + async_chat_completions_create_v_old(tracer, logger, instruments, capture_content), ) - - # Add instrumentation for the embeddings API wrap_function_wrapper( "openai.resources.embeddings", "Embeddings.create", embeddings_create(tracer, instruments, False), ) - wrap_function_wrapper( "openai.resources.embeddings", "AsyncEmbeddings.create", async_embeddings_create(tracer, instruments, False), ) - type(self)._legacy_instrumented = True - def _uninstrument(self, **kwargs): - if not type(self)._legacy_instrumented: + def _instrument_latest(self, **kwargs: Any) -> None: + delegate = _get_genai_openai_instrumentor() + if delegate is None: + _logger.warning( + "opentelemetry-instrumentation-openai-v2 does not support " + "gen_ai_latest_experimental and is disabled. Install " + "opentelemetry-instrumentation-genai-openai to get the latest " + "GenAI semantic conventions." + ) + return + + # Auto-instrumentation may have already enabled it via its own entry point. + if delegate.is_instrumented_by_opentelemetry: + return + + delegate.instrument(**kwargs) + type(self)._delegate = delegate + + def _block_genai_openai(self) -> None: + """Keep opentelemetry-instrumentation-genai-openai off while the legacy path is active. + + Both packages register the ``openai`` entry point and auto-instrumentation loads them + in no particular order, so undo it if it ran first and turn its ``instrument()`` into + a no-op if it runs later. Its ``_is_instrumented_by_opentelemetry`` flag is left + untouched so a stray ``uninstrument()`` can't unwrap our wrappers. + """ + delegate = _get_genai_openai_instrumentor() + if delegate is None: + return + if delegate.is_instrumented_by_opentelemetry: + delegate.uninstrument() + + def _blocked_instrument(**kwargs: Any) -> None: + _logger.warning( + "opentelemetry-instrumentation-genai-openai is disabled because " + "opentelemetry-instrumentation-openai-v2 is active. Set " + "OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental or uninstall " + "opentelemetry-instrumentation-openai-v2 to use it." + ) + + # instance attribute shadows BaseInstrumentor.instrument on the singleton + setattr(delegate, "instrument", _blocked_instrument) + type(self)._blocked = delegate + + def _uninstrument(self, **kwargs: Any) -> None: + cls = type(self) + if cls._delegate is not None: + cls._delegate.uninstrument(**kwargs) + cls._delegate = None + + if cls._blocked is not None: + cls._blocked.__dict__.pop("instrument", None) + cls._blocked = None + + if not cls._legacy_instrumented: return import openai # pylint: disable=import-outside-toplevel # noqa: PLC0415 @@ -207,4 +246,4 @@ def _uninstrument(self, **kwargs): unwrap(openai.resources.chat.completions.AsyncCompletions, "create") unwrap(openai.resources.embeddings.Embeddings, "create") unwrap(openai.resources.embeddings.AsyncEmbeddings, "create") - type(self)._legacy_instrumented = False + cls._legacy_instrumented = False diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py index 11a1b27b10f..382d8b0d6b0 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py @@ -14,6 +14,9 @@ OTEL_SEMCONV_STABILITY_OPT_IN, _OpenTelemetrySemanticConventionStability, ) +from opentelemetry.instrumentation.genai.openai import ( + OpenAIInstrumentor as GenAIOpenAIInstrumentor, +) from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.util.genai.environment_variables import ( @@ -235,9 +238,13 @@ def fixture_reset_instrumentors(monkeypatch): monkeypatch.delenv(OTEL_SEMCONV_STABILITY_OPT_IN, raising=False) _OpenTelemetrySemanticConventionStability._initialized = False yield - instrumentor = OpenAIInstrumentor() - if instrumentor.is_instrumented_by_opentelemetry: - instrumentor.uninstrument() + v2, genai_openai = OpenAIInstrumentor(), GenAIOpenAIInstrumentor() + for instrumentor in (v2, genai_openai): + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + genai_openai.__dict__.pop("instrument", None) + OpenAIInstrumentor._blocked = None + OpenAIInstrumentor._delegate = None _OpenTelemetrySemanticConventionStability._initialized = False diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt index 63de54f8ee6..6cb9aae4d56 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt @@ -51,5 +51,5 @@ wrapt==2.1.2 # test with the latest version of opentelemetry-api, sdk, and semantic conventions -e opentelemetry-instrumentation --e instrumentation-genai/opentelemetry-instrumentation-openai-v2 --e util/opentelemetry-util-genai \ No newline at end of file +opentelemetry-instrumentation-genai-openai~=1.3.0b0 +-e instrumentation-genai/opentelemetry-instrumentation-openai-v2 \ No newline at end of file diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt index 32114829c36..f2d5170c3f7 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt @@ -27,11 +27,12 @@ packaging==24.0 pytest==7.4.4 pytest-vcr==1.0.2 pytest-asyncio==0.21.0 -wrapt==1.16.0 -opentelemetry-exporter-otlp-proto-http~=1.30 -opentelemetry-api==1.40 # when updating, also update in pyproject.toml -opentelemetry-sdk==1.40 # when updating, also update in pyproject.toml -opentelemetry-semantic-conventions==0.61b0 # when updating, also update in pyproject.toml +wrapt==1.17.0 +opentelemetry-exporter-otlp-proto-http==1.43.0 +opentelemetry-api==1.43.0 # when updating, also update in pyproject.toml +opentelemetry-sdk==1.43.0 # when updating, also update in pyproject.toml +opentelemetry-semantic-conventions==0.64b0 # when updating, also update in pyproject.toml +opentelemetry-instrumentation==0.64b0 # when updating, also update in pyproject.toml +opentelemetry-instrumentation-genai-openai==1.3b0 # when updating, also update in pyproject.toml --e util/opentelemetry-util-genai -e instrumentation-genai/opentelemetry-instrumentation-openai-v2 diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py index af5fbc59f36..cbc4a7da604 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py @@ -8,6 +8,9 @@ import pytest from openai import APIConnectionError, AsyncOpenAI, NotFoundError +from opentelemetry.instrumentation.openai_v2 import ( + _is_latest_experimental_enabled as is_experimental_mode, +) from opentelemetry.instrumentation.openai_v2.patch import ( AsyncLegacyChatStreamWrapper, ) @@ -23,7 +26,6 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) -from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_MODEL, diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_embeddings.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_embeddings.py index f7d6f0a2743..02784e94399 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_embeddings.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_embeddings.py @@ -6,6 +6,9 @@ import pytest from openai import AsyncOpenAI, NotFoundError +from opentelemetry.instrumentation.openai_v2 import ( + _is_latest_experimental_enabled as is_experimental_mode, +) from opentelemetry.semconv._incubating.attributes import ( error_attributes as ErrorAttributes, ) @@ -13,7 +16,6 @@ gen_ai_attributes as GenAIAttributes, ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics -from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_EMBEDDING_MODEL, diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py index f25712d3946..cbfdad5e87f 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py @@ -19,6 +19,9 @@ except ImportError: not_given = NOT_GIVEN +from opentelemetry.instrumentation.openai_v2 import ( + _is_latest_experimental_enabled as is_experimental_mode, +) from opentelemetry.semconv._incubating.attributes import ( error_attributes as ErrorAttributes, ) @@ -32,7 +35,6 @@ server_attributes as ServerAttributes, ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics -from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_MODEL, diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py index 8f1d1c72681..cbe30aebe9d 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_metrics.py @@ -4,6 +4,9 @@ import pytest from tests.test_utils import DEFAULT_MODEL, USER_ONLY_PROMPT +from opentelemetry.instrumentation.openai_v2 import ( + _is_latest_experimental_enabled as is_experimental_mode, +) from opentelemetry.semconv._incubating.attributes import ( gen_ai_attributes as GenAIAttributes, ) @@ -14,7 +17,6 @@ server_attributes as ServerAttributes, ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics -from opentelemetry.util.genai.utils import is_experimental_mode _DURATION_BUCKETS = ( 0.01, diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_embeddings.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_embeddings.py index 71da074df69..e30678d750e 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_embeddings.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_embeddings.py @@ -16,6 +16,9 @@ except ImportError: not_given = NOT_GIVEN +from opentelemetry.instrumentation.openai_v2 import ( + _is_latest_experimental_enabled as is_experimental_mode, +) from opentelemetry.semconv._incubating.attributes import ( error_attributes as ErrorAttributes, ) @@ -26,7 +29,6 @@ server_attributes as ServerAttributes, ) from opentelemetry.semconv._incubating.metrics import gen_ai_metrics -from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_EMBEDDING_MODEL, diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_genai_openai_coexistence.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_genai_openai_coexistence.py new file mode 100644 index 00000000000..abd1901d255 --- /dev/null +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_genai_openai_coexistence.py @@ -0,0 +1,113 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import logging +from importlib.metadata import EntryPoint, entry_points + +import pytest + +from opentelemetry.instrumentation import openai_v2 +from opentelemetry.instrumentation.auto_instrumentation import _load +from opentelemetry.instrumentation.distro import DefaultDistro +from opentelemetry.instrumentation.genai.openai import ( + OpenAIInstrumentor as GenAIOpenAIInstrumentor, +) +from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) + +from .test_utils import ( + DEFAULT_MODEL, + USER_ONLY_PROMPT, + V2_SCOPE, + wrapped_by_genai_openai, + wrapped_by_v2, +) + +pytestmark = pytest.mark.usefixtures("reset_instrumentors") + + +def _openai_entry_points() -> dict[str, EntryPoint]: + return {ep.dist.name: ep for ep in entry_points(group="opentelemetry_instrumentor") if ep.name == "openai"} + + +def test_v2_first_blocks_genai_openai(tracer_provider, caplog): + caplog.set_level(logging.WARNING, logger=openai_v2.__name__) + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider) + + genai_openai = GenAIOpenAIInstrumentor() + genai_openai.instrument(tracer_provider=tracer_provider) + + assert not genai_openai.is_instrumented_by_opentelemetry + assert "opentelemetry-instrumentation-genai-openai is disabled" in caplog.text + assert wrapped_by_v2() + + +def test_genai_openai_first_is_uninstrumented(tracer_provider): + genai_openai = GenAIOpenAIInstrumentor() + genai_openai.instrument(tracer_provider=tracer_provider) + assert wrapped_by_genai_openai() + + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider) + + assert not genai_openai.is_instrumented_by_opentelemetry + assert wrapped_by_v2() + + +@pytest.mark.parametrize( + "dist_order", + [ + ("opentelemetry-instrumentation-openai-v2", "opentelemetry-instrumentation-genai-openai"), + ("opentelemetry-instrumentation-genai-openai", "opentelemetry-instrumentation-openai-v2"), + ], + ids=["v2_first", "genai_openai_first"], +) +def test_auto_instrumentation_only_v2_active(monkeypatch, dist_order): + openai_eps = _openai_entry_points() + ordered = [openai_eps[name] for name in dist_order] + + def fake_entry_points(group: str, **_: object) -> list[EntryPoint]: + return ordered if group == "opentelemetry_instrumentor" else [] + + monkeypatch.setattr(_load, "entry_points", fake_entry_points) + + _load._load_instrumentors(DefaultDistro()) + + assert OpenAIInstrumentor().is_instrumented_by_opentelemetry + assert not GenAIOpenAIInstrumentor().is_instrumented_by_opentelemetry + assert wrapped_by_v2() + + +def test_stray_genai_openai_uninstrument_keeps_v2(tracer_provider): + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider) + + GenAIOpenAIInstrumentor().uninstrument() + + assert wrapped_by_v2() + + +def test_v2_uninstrument_lifts_block(tracer_provider): + v2 = OpenAIInstrumentor() + v2.instrument(tracer_provider=tracer_provider) + v2.uninstrument() + + genai_openai = GenAIOpenAIInstrumentor() + genai_openai.instrument(tracer_provider=tracer_provider) + + assert genai_openai.is_instrumented_by_opentelemetry + assert wrapped_by_genai_openai() + + +def test_legacy_chat_completion_uses_v2_scope(span_exporter, openai_client, tracer_provider, vcr): + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider) + + with vcr.use_cassette("test_chat_completion_no_content.yaml"): + openai_client.chat.completions.create(messages=USER_ONLY_PROMPT, model=DEFAULT_MODEL, stream=False) + + (span,) = span_exporter.get_finished_spans() + assert span.instrumentation_scope.name == V2_SCOPE + assert span.attributes[GenAIAttributes.GEN_AI_SYSTEM] == "openai" + assert GenAIAttributes.GEN_AI_PROVIDER_NAME not in span.attributes diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_latest_experimental.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_latest_experimental.py index 39cf203bb19..4ce876be76f 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_latest_experimental.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_latest_experimental.py @@ -4,14 +4,31 @@ from __future__ import annotations import logging +import sys +from typing import Any import pytest from opentelemetry.instrumentation import openai_v2 from opentelemetry.instrumentation._semconv import OTEL_SEMCONV_STABILITY_OPT_IN +from opentelemetry.instrumentation.genai.openai import ( + OpenAIInstrumentor as GenAIOpenAIInstrumentor, +) from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.semconv._incubating.attributes import ( + gen_ai_attributes as GenAIAttributes, +) +from opentelemetry.semconv._incubating.metrics import gen_ai_metrics -from .test_utils import not_wrapped +from .test_utils import ( + DEFAULT_EMBEDDING_MODEL, + DEFAULT_MODEL, + GENAI_OPENAI_SCOPE, + USER_ONLY_PROMPT, + not_wrapped, + wrapped_by_genai_openai, + wrapped_by_v2, +) pytestmark = pytest.mark.usefixtures("latest_experimental") @@ -21,7 +38,36 @@ def fixture_latest_experimental(monkeypatch: pytest.MonkeyPatch, reset_instrumen monkeypatch.setenv(OTEL_SEMCONV_STABILITY_OPT_IN, "gen_ai_latest_experimental") -def test_latest_experimental_disables_instrumentation(tracer_provider, caplog): +@pytest.fixture(name="genai_openai_instrument_calls") +def fixture_genai_openai_instrument_calls( + monkeypatch: pytest.MonkeyPatch, +) -> list[dict[str, Any]]: + calls: list[dict[str, Any]] = [] + original = GenAIOpenAIInstrumentor.instrument + + def instrument(self: GenAIOpenAIInstrumentor, **kwargs: Any) -> Any: + calls.append(kwargs) + return original(self, **kwargs) + + monkeypatch.setattr(GenAIOpenAIInstrumentor, "instrument", instrument) + return calls + + +@pytest.fixture(name="instrument_latest") +def fixture_instrument_latest( + tracer_provider, logger_provider, meter_provider, latest_experimental: None +) -> OpenAIInstrumentor: + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + logger_provider=logger_provider, + meter_provider=meter_provider, + ) + return instrumentor + + +def test_genai_openai_not_installed(tracer_provider, monkeypatch, caplog): + monkeypatch.setitem(sys.modules, GENAI_OPENAI_SCOPE, None) caplog.set_level(logging.WARNING, logger=openai_v2.__name__) instrumentor = OpenAIInstrumentor() @@ -32,3 +78,89 @@ def test_latest_experimental_disables_instrumentation(tracer_provider, caplog): instrumentor.uninstrument() assert not instrumentor.is_instrumented_by_opentelemetry + + +def test_delegates_to_genai_openai(tracer_provider, meter_provider, genai_openai_instrument_calls): + completion_hook = object() + instrumentor = OpenAIInstrumentor() + instrumentor.instrument( + tracer_provider=tracer_provider, + meter_provider=meter_provider, + completion_hook=completion_hook, + ) + + genai_openai = GenAIOpenAIInstrumentor() + assert genai_openai.is_instrumented_by_opentelemetry + assert genai_openai_instrument_calls == [ + { + "tracer_provider": tracer_provider, + "meter_provider": meter_provider, + "completion_hook": completion_hook, + } + ] + assert wrapped_by_genai_openai() + assert not wrapped_by_v2() + + instrumentor.uninstrument() + + assert not genai_openai.is_instrumented_by_opentelemetry + assert not_wrapped() + + +def test_genai_openai_already_instrumented(tracer_provider, genai_openai_instrument_calls): + genai_openai = GenAIOpenAIInstrumentor() + genai_openai.instrument() + + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=tracer_provider) + + assert genai_openai_instrument_calls == [{}] + assert wrapped_by_genai_openai() + + instrumentor.uninstrument() + + assert genai_openai.is_instrumented_by_opentelemetry + assert wrapped_by_genai_openai() + + +def _assert_genai_openai_telemetry(span_exporter, metric_reader, operation_name: str, request_model: str): + (span,) = span_exporter.get_finished_spans() + assert span.instrumentation_scope.name.startswith(GENAI_OPENAI_SCOPE) + assert span.name == f"{operation_name} {request_model}" + assert span.attributes[GenAIAttributes.GEN_AI_OPERATION_NAME] == operation_name + assert span.attributes[GenAIAttributes.GEN_AI_PROVIDER_NAME] == "openai" + assert GenAIAttributes.GEN_AI_SYSTEM not in span.attributes + + metric_names = { + metric.name + for resource_metrics in metric_reader.get_metrics_data().resource_metrics + for scope_metrics in resource_metrics.scope_metrics + if scope_metrics.scope.name.startswith(GENAI_OPENAI_SCOPE) + for metric in scope_metrics.metrics + } + assert gen_ai_metrics.GEN_AI_CLIENT_OPERATION_DURATION in metric_names + + +@pytest.mark.usefixtures("instrument_latest") +def test_chat_completion_smoke(span_exporter, metric_reader, openai_client, vcr): + with vcr.use_cassette("test_chat_completion_no_content.yaml"): + openai_client.chat.completions.create(messages=USER_ONLY_PROMPT, model=DEFAULT_MODEL, stream=False) + + _assert_genai_openai_telemetry(span_exporter, metric_reader, "chat", DEFAULT_MODEL) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("instrument_latest") +async def test_async_chat_completion_smoke(span_exporter, metric_reader, async_openai_client, vcr): + with vcr.use_cassette("test_async_chat_completion_no_content.yaml"): + await async_openai_client.chat.completions.create(messages=USER_ONLY_PROMPT, model=DEFAULT_MODEL, stream=False) + + _assert_genai_openai_telemetry(span_exporter, metric_reader, "chat", DEFAULT_MODEL) + + +@pytest.mark.usefixtures("instrument_latest") +def test_embeddings_smoke(span_exporter, metric_reader, openai_client, vcr): + with vcr.use_cassette("test_embeddings_no_content.yaml"): + openai_client.embeddings.create(model=DEFAULT_EMBEDDING_MODEL, input="This is a test for embeddings") + + _assert_genai_openai_telemetry(span_exporter, metric_reader, "embeddings", DEFAULT_EMBEDDING_MODEL) diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py index 15f095422dc..b96dac3690c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_response_extractors.py @@ -265,8 +265,8 @@ def test_set_invocation_response_attributes_populates_usage_and_metadata( assert invocation.response_id == "resp_123" assert invocation.input_tokens == 11 assert invocation.output_tokens == 7 - assert invocation.cache_read_input_tokens == 3 - assert invocation.cache_creation_input_tokens == 5 + assert invocation.cache_read_input_tokens == 3 # pylint: disable=no-member + assert invocation.cache_creation_input_tokens == 5 # pylint: disable=no-member assert invocation.attributes == { OpenAIAttributes.OPENAI_RESPONSE_SERVICE_TIER: "scale", } diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_responses.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_responses.py index 5fafcb1139b..c8bc2a88817 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_responses.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_responses.py @@ -8,6 +8,9 @@ from openai import APIConnectionError, BadRequestError, NotFoundError, OpenAI from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor +from opentelemetry.instrumentation.openai_v2 import ( + _is_latest_experimental_enabled as is_experimental_mode, +) from opentelemetry.instrumentation.openai_v2.response_wrappers import ( ResponseStreamWrapper, ) @@ -20,7 +23,6 @@ from opentelemetry.semconv._incubating.attributes import ( server_attributes as ServerAttributes, ) -from opentelemetry.util.genai.utils import is_experimental_mode from .test_utils import ( DEFAULT_MODEL, diff --git a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py index 321d66552d2..7d3644f500c 100644 --- a/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py +++ b/instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_utils.py @@ -24,6 +24,7 @@ ) V2_SCOPE = "opentelemetry.instrumentation.openai_v2" +GENAI_OPENAI_SCOPE = "opentelemetry.instrumentation.genai.openai" PATCHED_CLASSES = (Completions, AsyncCompletions, Embeddings, AsyncEmbeddings) @@ -38,6 +39,10 @@ def wrapped_by_v2() -> bool: return all(_wrapper_module(cls) == f"{V2_SCOPE}.patch" for cls in PATCHED_CLASSES) +def wrapped_by_genai_openai() -> bool: + return all((_wrapper_module(cls) or "").startswith(GENAI_OPENAI_SCOPE) for cls in PATCHED_CLASSES) + + def not_wrapped() -> bool: return all(_wrapper_module(cls) is None for cls in PATCHED_CLASSES) diff --git a/pyproject.toml b/pyproject.toml index cac655f02ef..f76d48882c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,6 @@ dependencies = [ "opentelemetry-util-http", "opentelemetry-util-genai", "opentelemetry-instrumentation-vertexai[instruments]", - "opentelemetry-instrumentation-openai-v2[instruments]", ] @@ -137,7 +136,6 @@ opentelemetry-propagator-aws-xray = { workspace = true } opentelemetry-util-http = { workspace = true } opentelemetry-util-genai = { workspace = true } opentelemetry-instrumentation-vertexai = { workspace = true } -opentelemetry-instrumentation-openai-v2 = { workspace = true } # https://docs.astral.sh/uv/reference/settings/#workspace [tool.uv.workspace] @@ -150,6 +148,9 @@ members = [ "propagator/*", "util/*", ] +# Depends on opentelemetry-util-genai 1.x from PyPI (via opentelemetry-instrumentation-genai-openai), +# which conflicts with the in-repo util/opentelemetry-util-genai. Tested via tox only. +exclude = ["instrumentation-genai/opentelemetry-instrumentation-openai-v2"] [tool.ruff] # https://docs.astral.sh/ruff/configuration/ diff --git a/uv.lock b/uv.lock index 3ff4666e214..69264302cd4 100644 --- a/uv.lock +++ b/uv.lock @@ -46,7 +46,6 @@ members = [ "opentelemetry-instrumentation-mysql", "opentelemetry-instrumentation-mysqlclient", "opentelemetry-instrumentation-openai-agents-v2", - "opentelemetry-instrumentation-openai-v2", "opentelemetry-instrumentation-pika", "opentelemetry-instrumentation-psycopg", "opentelemetry-instrumentation-psycopg2", @@ -3386,31 +3385,6 @@ requires-dist = [ ] provides-extras = ["instruments"] -[[package]] -name = "opentelemetry-instrumentation-openai-v2" -source = { editable = "instrumentation-genai/opentelemetry-instrumentation-openai-v2" } -dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, -] - -[package.optional-dependencies] -instruments = [ - { name = "openai" }, -] - -[package.metadata] -requires-dist = [ - { name = "openai", marker = "extra == 'instruments'", specifier = ">=1.26.0,<3" }, - { name = "opentelemetry-api", git = "https://github.com/open-telemetry/opentelemetry-python?subdirectory=opentelemetry-api&branch=main" }, - { name = "opentelemetry-instrumentation", editable = "opentelemetry-instrumentation" }, - { name = "opentelemetry-semantic-conventions", git = "https://github.com/open-telemetry/opentelemetry-python?subdirectory=opentelemetry-semantic-conventions&branch=main" }, - { name = "typing-extensions", specifier = ">=4.5.0" }, -] -provides-extras = ["instruments"] - [[package]] name = "opentelemetry-instrumentation-pika" source = { editable = "instrumentation/opentelemetry-instrumentation-pika" } @@ -4053,7 +4027,6 @@ dependencies = [ { name = "opentelemetry-instrumentation-logging" }, { name = "opentelemetry-instrumentation-mysql", extra = ["instruments"] }, { name = "opentelemetry-instrumentation-mysqlclient", extra = ["instruments"] }, - { name = "opentelemetry-instrumentation-openai-v2", extra = ["instruments"] }, { name = "opentelemetry-instrumentation-pika", extra = ["instruments"] }, { name = "opentelemetry-instrumentation-psycopg", extra = ["instruments"] }, { name = "opentelemetry-instrumentation-psycopg2", extra = ["instruments-any"] }, @@ -4126,7 +4099,6 @@ requires-dist = [ { name = "opentelemetry-instrumentation-logging", editable = "instrumentation/opentelemetry-instrumentation-logging" }, { name = "opentelemetry-instrumentation-mysql", extras = ["instruments"], editable = "instrumentation/opentelemetry-instrumentation-mysql" }, { name = "opentelemetry-instrumentation-mysqlclient", extras = ["instruments"], editable = "instrumentation/opentelemetry-instrumentation-mysqlclient" }, - { name = "opentelemetry-instrumentation-openai-v2", extras = ["instruments"], editable = "instrumentation-genai/opentelemetry-instrumentation-openai-v2" }, { name = "opentelemetry-instrumentation-pika", extras = ["instruments"], editable = "instrumentation/opentelemetry-instrumentation-pika" }, { name = "opentelemetry-instrumentation-psycopg", extras = ["instruments"], editable = "instrumentation/opentelemetry-instrumentation-psycopg" }, { name = "opentelemetry-instrumentation-psycopg2", extras = ["instruments"], editable = "instrumentation/opentelemetry-instrumentation-psycopg2" },