openai-v2: enable opentelemetry-instrumentation-genai-openai for latest experimental conventions

instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.added

@@ -0,0 +1 @@
+With `OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`, enable `opentelemetry-instrumentation-genai-openai` (new dependency, `~= 1.3.0b0`) and forward `instrument()` arguments to it. Without the flag, keep it disabled so auto-instrumentation does not instrument OpenAI twice.

instrumentation-genai/opentelemetry-instrumentation-openai-v2/.changelog/5261.changed

@@ -0,0 +1 @@
+Bump minimum `opentelemetry-api` to 1.43 and `opentelemetry-instrumentation`/`opentelemetry-semantic-conventions` to 0.64b0, as required by `opentelemetry-instrumentation-genai-openai`.

instrumentation-genai/opentelemetry-instrumentation-openai-v2/README.rst

@@ -141,6 +141,13 @@ To enable the latest experimental features, set the environment variable
 ``OTEL_SEMCONV_STABILITY_OPT_IN`` to ``gen_ai_latest_experimental``. Or, if you use
 ``OTEL_SEMCONV_STABILITY_OPT_IN`` to enable other features, append ``,gen_ai_latest_experimental`` to its value.
 
+When enabled, this package turns itself off and enables
+`opentelemetry-instrumentation-genai-openai <https://pypi.org/project/opentelemetry-instrumentation-genai-openai/>`_
+(installed as a dependency) instead.
+
+Content capturing modes (``span_only``, ``event_only``, ``span_and_event``) and
+completion hooks are provided by that package.
+
 Without this setting, OpenAI instrumentation aligns with `Semantic Conventions v1.30.0 <https://github.com/open-telemetry/semantic-conventions/tree/v1.30.0/docs/gen-ai>`_
 and would not capture additional details introduced in later versions.
 

instrumentation-genai/opentelemetry-instrumentation-openai-v2/pyproject.toml

@@ -25,9 +25,10 @@ classifiers = [
   "Programming Language :: Python :: 3.14",
 ]
 dependencies = [
-  "opentelemetry-api ~= 1.40",
-  "opentelemetry-instrumentation ~= 0.61b0",
-  "opentelemetry-semantic-conventions ~= 0.61b0",
+  "opentelemetry-api ~= 1.43",
+  "opentelemetry-instrumentation ~= 0.64b0",
+  "opentelemetry-semantic-conventions ~= 0.64b0",
+  "opentelemetry-instrumentation-genai-openai ~= 1.3.0b0",
   "typing-extensions >= 4.5.0",
 ]
 

instrumentation-genai/opentelemetry-instrumentation-openai-v2/src/opentelemetry/instrumentation/openai_v2/__init__.py

@@ -8,8 +8,9 @@
 .. deprecated:: 2.5b0
     Use `opentelemetry-instrumentation-genai-openai
     <https://pypi.org/project/opentelemetry-instrumentation-genai-openai/>`_
-    instead. This package won't receive new features for `Semantic Conventions v1.30.0
-    <https://github.com/open-telemetry/semantic-conventions/tree/v1.30.0/docs/gen-ai>`_.
+    instead. This package only supports `Semantic Conventions v1.30.0
+    <https://github.com/open-telemetry/semantic-conventions/tree/v1.30.0/docs/gen-ai>`_
+    and won't receive new features.
 
 .. _openai: https://pypi.org/project/openai/
 
@@ -36,42 +37,27 @@
 
 By default, the instrumentation aligns with `Semantic Conventions v1.30.0
 <https://github.com/open-telemetry/semantic-conventions/tree/v1.30.0/docs/gen-ai>`_
-and does not capture prompt or completion content. Behavior is controlled
-via environment variables:
+and does not capture prompt or completion content.
 
+- ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=true`` - capture
+  prompts, completions, tool arguments, and return values on log events.
 - ``OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental`` - opt into the
-  latest GenAI semantic conventions. Required to access the newer attributes
-  and the ``span_only`` / ``event_only`` / ``span_and_event`` content modes.
-  Without this flag, the instrumentation stays on v1.30.0 conventions.
-- ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`` - enable capture of
-  prompts, completions, tool arguments, and return values. Set to ``true``
-  on the legacy path, or one of ``span_only``, ``event_only``,
-  ``span_and_event`` when experimental conventions are enabled.
-- ``OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK=upload`` together with
-  ``OTEL_INSTRUMENTATION_GENAI_UPLOAD_BASE_PATH=<fsspec-uri>`` - upload
-  prompts and completions to an ``fsspec``-compatible destination
-  (local filesystem, ``gs://``, ``s3://``, etc.) and record reference URIs as
-  ``gen_ai.input.messages.ref`` / ``gen_ai.output.messages.ref`` attributes.
-  Inline content is not captured unless
-  ``OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT`` is also set.
-
-See the `opentelemetry-util-genai README
-<https://github.com/open-telemetry/opentelemetry-python-contrib/blob/main/util/opentelemetry-util-genai/README.rst>`_
-for the full list of GenAI configuration variables.
-
-A custom ``CompletionHook`` implementation can also be passed programmatically::
-
-    OpenAIInstrumentor().instrument(completion_hook=my_hook)
-
-When provided, this takes precedence over the hook resolved from
-``OTEL_INSTRUMENTATION_GENAI_COMPLETION_HOOK``.
+  latest GenAI semantic conventions. This package turns itself off and enables
+  `opentelemetry-instrumentation-genai-openai
+  <https://pypi.org/project/opentelemetry-instrumentation-genai-openai/>`_
+  (installed as a dependency) instead. All ``instrument()`` arguments
+  (e.g. ``completion_hook``) are forwarded to it. See its documentation for
+  content capturing modes and completion hooks.
 
 API
 ---
 """
 
+from __future__ import annotations
+
 import logging
 from collections.abc import Collection
+from typing import Any
 
 from typing_extensions import deprecated
 from wrapt import wrap_function_wrapper
@@ -111,6 +97,17 @@ def _is_latest_experimental_enabled() -> bool:
     )
 
 
+def _get_genai_openai_instrumentor() -> BaseInstrumentor | None:
+    try:
+        # pylint: disable-next=import-outside-toplevel,no-name-in-module
+        from opentelemetry.instrumentation.genai.openai import (  # noqa: PLC0415
+            OpenAIInstrumentor as GenAIOpenAIInstrumentor,
+        )
+    except ImportError:
+        return None
+    return GenAIOpenAIInstrumentor()
+
+
 @deprecated(
     "opentelemetry-instrumentation-openai-v2 is deprecated. Use the "
     "opentelemetry-instrumentation-genai-openai package instead. This package "
@@ -127,78 +124,120 @@ class OpenAIInstrumentor(BaseInstrumentor):
     # BaseInstrumentor is a singleton and __init__ re-runs on every
     # OpenAIInstrumentor() call, so state lives on the class.
     _legacy_instrumented: bool = False
-
-    def __init__(self):
-        self._meter = None
+    _delegate: BaseInstrumentor | None = None
+    _blocked: BaseInstrumentor | None = None
 
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
 
-    def _instrument(self, **kwargs):
+    def _instrument(self, **kwargs: Any) -> None:
         """Enable OpenAI instrumentation."""
 
         if _is_latest_experimental_enabled():
-            _logger.warning(
-                "opentelemetry-instrumentation-openai-v2 does not support "
-                "gen_ai_latest_experimental and is disabled. Use "
-                "opentelemetry-instrumentation-genai-openai to get the latest "
-                "GenAI semantic conventions."
-            )
+            self._instrument_latest(**kwargs)
             return
 
-        tracer_provider = kwargs.get("tracer_provider")
+        self._block_genai_openai()
+
         tracer = get_tracer(
             __name__,
             "",
-            tracer_provider,
-            schema_url=Schemas.V1_30_0.value,  # only used on the legacy path
+            kwargs.get("tracer_provider"),
+            schema_url=Schemas.V1_30_0.value,
         )
-        logger_provider = kwargs.get("logger_provider")
         logger = get_logger(
             __name__,
             "",
-            logger_provider=logger_provider,
-            schema_url=Schemas.V1_30_0.value,  # only used on the legacy path
+            logger_provider=kwargs.get("logger_provider"),
+            schema_url=Schemas.V1_30_0.value,
         )
-        meter_provider = kwargs.get("meter_provider")
-        self._meter = get_meter(
+        meter = get_meter(
             __name__,
             "",
-            meter_provider,
-            schema_url=Schemas.V1_30_0.value,  # only used on the legacy path
+            kwargs.get("meter_provider"),
+            schema_url=Schemas.V1_30_0.value,
         )
 
-        instruments = Instruments(self._meter)
+        instruments = Instruments(meter)
+        capture_content = is_content_enabled()
 
         wrap_function_wrapper(
             "openai.resources.chat.completions",
             "Completions.create",
-            chat_completions_create_v_old(tracer, logger, instruments, is_content_enabled()),
+            chat_completions_create_v_old(tracer, logger, instruments, capture_content),
         )
-
         wrap_function_wrapper(
             "openai.resources.chat.completions",
             "AsyncCompletions.create",
-            async_chat_completions_create_v_old(tracer, logger, instruments, is_content_enabled()),
+            async_chat_completions_create_v_old(tracer, logger, instruments, capture_content),
         )
-
-        # Add instrumentation for the embeddings API
         wrap_function_wrapper(
             "openai.resources.embeddings",
             "Embeddings.create",
             embeddings_create(tracer, instruments, False),
         )
-
         wrap_function_wrapper(
             "openai.resources.embeddings",
             "AsyncEmbeddings.create",
             async_embeddings_create(tracer, instruments, False),
         )
-
         type(self)._legacy_instrumented = True
 
-    def _uninstrument(self, **kwargs):
-        if not type(self)._legacy_instrumented:
+    def _instrument_latest(self, **kwargs: Any) -> None:
+        delegate = _get_genai_openai_instrumentor()
+        if delegate is None:
+            _logger.warning(
+                "opentelemetry-instrumentation-openai-v2 does not support "
+                "gen_ai_latest_experimental and is disabled. Install "
+                "opentelemetry-instrumentation-genai-openai to get the latest "
+                "GenAI semantic conventions."
+            )
+            return
+
+        # Auto-instrumentation may have already enabled it via its own entry point.
+        if delegate.is_instrumented_by_opentelemetry:
+            return
+
+        delegate.instrument(**kwargs)
+        type(self)._delegate = delegate
+
+    def _block_genai_openai(self) -> None:
+        """Keep opentelemetry-instrumentation-genai-openai off while the legacy path is active.
+
+        Both packages register the ``openai`` entry point and auto-instrumentation loads them
+        in no particular order, so undo it if it ran first and turn its ``instrument()`` into
+        a no-op if it runs later. Its ``_is_instrumented_by_opentelemetry`` flag is left
+        untouched so a stray ``uninstrument()`` can't unwrap our wrappers.
+        """
+        delegate = _get_genai_openai_instrumentor()
+        if delegate is None:
+            return
+        if delegate.is_instrumented_by_opentelemetry:
+            delegate.uninstrument()
+
+        def _blocked_instrument(**kwargs: Any) -> None:
+            _logger.warning(
+                "opentelemetry-instrumentation-genai-openai is disabled because "
+                "opentelemetry-instrumentation-openai-v2 is active. Set "
+                "OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental or uninstall "
+                "opentelemetry-instrumentation-openai-v2 to use it."
+            )
+
+        # instance attribute shadows BaseInstrumentor.instrument on the singleton
+        setattr(delegate, "instrument", _blocked_instrument)
+        type(self)._blocked = delegate
+
+    def _uninstrument(self, **kwargs: Any) -> None:
+        cls = type(self)
+        if cls._delegate is not None:
+            cls._delegate.uninstrument(**kwargs)
+            cls._delegate = None
+
+        if cls._blocked is not None:
+            cls._blocked.__dict__.pop("instrument", None)
+            cls._blocked = None
+
+        if not cls._legacy_instrumented:
             return
 
         import openai  # pylint: disable=import-outside-toplevel  # noqa: PLC0415
@@ -207,4 +246,4 @@ def _uninstrument(self, **kwargs):
         unwrap(openai.resources.chat.completions.AsyncCompletions, "create")
         unwrap(openai.resources.embeddings.Embeddings, "create")
         unwrap(openai.resources.embeddings.AsyncEmbeddings, "create")
-        type(self)._legacy_instrumented = False
+        cls._legacy_instrumented = False

instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/conftest.py

@@ -14,6 +14,9 @@
     OTEL_SEMCONV_STABILITY_OPT_IN,
     _OpenTelemetrySemanticConventionStability,
 )
+from opentelemetry.instrumentation.genai.openai import (
+    OpenAIInstrumentor as GenAIOpenAIInstrumentor,
+)
 from opentelemetry.instrumentation.openai_v2 import OpenAIInstrumentor
 from opentelemetry.sdk._logs import LoggerProvider
 from opentelemetry.util.genai.environment_variables import (
@@ -235,9 +238,13 @@ def fixture_reset_instrumentors(monkeypatch):
     monkeypatch.delenv(OTEL_SEMCONV_STABILITY_OPT_IN, raising=False)
     _OpenTelemetrySemanticConventionStability._initialized = False
     yield
-    instrumentor = OpenAIInstrumentor()
-    if instrumentor.is_instrumented_by_opentelemetry:
-        instrumentor.uninstrument()
+    v2, genai_openai = OpenAIInstrumentor(), GenAIOpenAIInstrumentor()
+    for instrumentor in (v2, genai_openai):
+        if instrumentor.is_instrumented_by_opentelemetry:
+            instrumentor.uninstrument()
+    genai_openai.__dict__.pop("instrument", None)
+    OpenAIInstrumentor._blocked = None
+    OpenAIInstrumentor._delegate = None
     _OpenTelemetrySemanticConventionStability._initialized = False
 
 

instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.latest.txt

@@ -51,5 +51,5 @@ wrapt==2.1.2
 # test with the latest version of opentelemetry-api, sdk, and semantic conventions
 
 -e opentelemetry-instrumentation
--e instrumentation-genai/opentelemetry-instrumentation-openai-v2
--e util/opentelemetry-util-genai
+opentelemetry-instrumentation-genai-openai~=1.3.0b0
+-e instrumentation-genai/opentelemetry-instrumentation-openai-v2

instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/requirements.oldest.txt

@@ -27,11 +27,12 @@ packaging==24.0
 pytest==7.4.4
 pytest-vcr==1.0.2
 pytest-asyncio==0.21.0
-wrapt==1.16.0
-opentelemetry-exporter-otlp-proto-http~=1.30
-opentelemetry-api==1.40  # when updating, also update in pyproject.toml
-opentelemetry-sdk==1.40  # when updating, also update in pyproject.toml
-opentelemetry-semantic-conventions==0.61b0  # when updating, also update in pyproject.toml
+wrapt==1.17.0
+opentelemetry-exporter-otlp-proto-http==1.43.0
+opentelemetry-api==1.43.0  # when updating, also update in pyproject.toml
+opentelemetry-sdk==1.43.0  # when updating, also update in pyproject.toml
+opentelemetry-semantic-conventions==0.64b0  # when updating, also update in pyproject.toml
+opentelemetry-instrumentation==0.64b0  # when updating, also update in pyproject.toml
+opentelemetry-instrumentation-genai-openai==1.3b0  # when updating, also update in pyproject.toml
 
--e util/opentelemetry-util-genai
 -e instrumentation-genai/opentelemetry-instrumentation-openai-v2

instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_chat_completions.py

@@ -8,6 +8,9 @@
 import pytest
 from openai import APIConnectionError, AsyncOpenAI, NotFoundError
 
+from opentelemetry.instrumentation.openai_v2 import (
+    _is_latest_experimental_enabled as is_experimental_mode,
+)
 from opentelemetry.instrumentation.openai_v2.patch import (
     AsyncLegacyChatStreamWrapper,
 )
@@ -23,7 +26,6 @@
 from opentelemetry.semconv._incubating.attributes import (
     server_attributes as ServerAttributes,
 )
-from opentelemetry.util.genai.utils import is_experimental_mode
 
 from .test_utils import (
     DEFAULT_MODEL,

instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_async_embeddings.py

@@ -6,14 +6,16 @@
 import pytest
 from openai import AsyncOpenAI, NotFoundError
 
+from opentelemetry.instrumentation.openai_v2 import (
+    _is_latest_experimental_enabled as is_experimental_mode,
+)
 from opentelemetry.semconv._incubating.attributes import (
     error_attributes as ErrorAttributes,
 )
 from opentelemetry.semconv._incubating.attributes import (
     gen_ai_attributes as GenAIAttributes,
 )
 from opentelemetry.semconv._incubating.metrics import gen_ai_metrics
-from opentelemetry.util.genai.utils import is_experimental_mode
 
 from .test_utils import (
     DEFAULT_EMBEDDING_MODEL,

instrumentation-genai/opentelemetry-instrumentation-openai-v2/tests/test_chat_completions.py

@@ -19,6 +19,9 @@
 except ImportError:
     not_given = NOT_GIVEN
 
+from opentelemetry.instrumentation.openai_v2 import (
+    _is_latest_experimental_enabled as is_experimental_mode,
+)
 from opentelemetry.semconv._incubating.attributes import (
     error_attributes as ErrorAttributes,
 )
@@ -32,7 +35,6 @@
     server_attributes as ServerAttributes,
 )
 from opentelemetry.semconv._incubating.metrics import gen_ai_metrics
-from opentelemetry.util.genai.utils import is_experimental_mode
 
 from .test_utils import (
     DEFAULT_MODEL,