diff --git a/examples/30_extended/flow_id_tutorial.py b/examples/30_extended/flow_id_tutorial.py new file mode 100644 index 000000000..edb14d003 --- /dev/null +++ b/examples/30_extended/flow_id_tutorial.py @@ -0,0 +1,68 @@ +""" +================== +Obtaining Flow IDs +================== + +This tutorial discusses different ways to obtain the ID of a flow in order to perform further +analysis. +""" + +#################################################################################################### +import sklearn.tree + +import openml + +clf = sklearn.tree.DecisionTreeClassifier() + +#################################################################################################### +# 1. Obtaining a flow given a classifier +# ====================================== +# + +flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish() +flow_id = flow.flow_id +print(flow_id) + +#################################################################################################### +# This piece of code is rather involved. First, it retrieves an +# :class:`~openml.extensions.Extension` which is registered and can handle the given model, +# in our case it is :class:`openml.extensions.sklearn.SklearnExtension`. Second, the extension +# converts the classifier into an instance of :class:`openml.flow.OpenMLFlow`. Third and finally, +# the publish method checks whether the current flow is already present on OpenML. If not, +# it uploads the flow, otherwise, it updates the current instance with all information computed +# by the server (which is obviously also done when uploading/publishing a flow). +# +# To simplify the usage we have created a helper function which automates all these steps: + +flow_id = openml.flows.get_flow_id(model=clf) +print(flow_id) + +#################################################################################################### +# 2. Obtaining a flow given its name +# ================================== +# The schema of a flow is given in XSD (`here +# `_). # noqa E501 +# Only two fields are required, a unique name, and an external version. While it should be pretty +# obvious why we need a name, the need for the additional external version information might not +# be immediately clear. However, this information is very important as it allows to have multiple +# flows with the same name for different versions of a software. This might be necessary if an +# algorithm or implementation introduces, renames or drop hyperparameters over time. + +print(flow.name, flow.external_version) + +#################################################################################################### +# The name and external version are automatically added to a flow when constructing it from a +# model. We can then use them to retrieve the flow id as follows: + +flow_id = openml.flows.flow_exists(name=flow.name, external_version=flow.external_version) +print(flow_id) + +#################################################################################################### +# We can also retrieve all flows for a given name: +flow_ids = openml.flows.get_flow_id(name=flow.name) +print(flow_ids) + +#################################################################################################### +# This also work with the actual model (generalizing the first part of this example): +flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False) +print(flow_ids) diff --git a/openml/flows/__init__.py b/openml/flows/__init__.py index 504c37c1a..3bbf1b21b 100644 --- a/openml/flows/__init__.py +++ b/openml/flows/__init__.py @@ -1,11 +1,12 @@ from .flow import OpenMLFlow -from .functions import get_flow, list_flows, flow_exists, assert_flows_equal +from .functions import get_flow, list_flows, flow_exists, get_flow_id, assert_flows_equal __all__ = [ 'OpenMLFlow', 'get_flow', 'list_flows', + 'get_flow_id', 'flow_exists', 'assert_flows_equal', ] diff --git a/openml/flows/functions.py b/openml/flows/functions.py index ad82ffee7..2aa3df85e 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -5,7 +5,7 @@ import re import xmltodict import pandas as pd -from typing import Union, Dict, Optional +from typing import Any, Union, Dict, Optional, List from ..exceptions import OpenMLCacheException import openml._api_calls @@ -268,6 +268,72 @@ def flow_exists(name: str, external_version: str) -> Union[int, bool]: return False +def get_flow_id( + model: Optional[Any] = None, + name: Optional[str] = None, + exact_version=True, +) -> Union[int, bool, List[int]]: + """Retrieves the flow id for a model or a flow name. + + Provide either a model or a name to this function. Depending on the input, it does + + * ``model`` and ``exact_version == True``: This helper function first queries for the necessary + extension. Second, it uses that extension to convert the model into a flow. Third, it + executes ``flow_exists`` to potentially obtain the flow id the flow is published to the + server. + * ``model`` and ``exact_version == False``: This helper function first queries for the + necessary extension. Second, it uses that extension to convert the model into a flow. Third + it calls ``list_flows`` and filters the returned values based on the flow name. + * ``name``: Ignores ``exact_version`` and calls ``list_flows``, then filters the returned + values based on the flow name. + + Parameters + ---------- + model : object + Any model. Must provide either ``model`` or ``name``. + name : str + Name of the flow. Must provide either ``model`` or ``name``. + exact_version : bool + Whether to return the ``flow_id`` of the exact version or all ``flow_id``s where the name + of the flow matches. This is only taken into account for a model where a version number + is available. + + Returns + ------- + int or bool, List + flow id iff exists, ``False`` otherwise, List if exact_version is ``False`` + """ + if model is None and name is None: + raise ValueError( + 'Need to provide either argument `model` or argument `name`, but both are `None`.' + ) + elif model is not None and name is not None: + raise ValueError( + 'Must provide either argument `model` or argument `name`, but not both.' + ) + + if model is not None: + extension = openml.extensions.get_extension_by_model(model, raise_if_no_extension=True) + if extension is None: + # This should never happen and is only here to please mypy will be gone soon once the + # whole function is removed + raise TypeError(extension) + flow = extension.model_to_flow(model) + flow_name = flow.name + external_version = flow.external_version + else: + flow_name = name + exact_version = False + + if exact_version: + return flow_exists(name=flow_name, external_version=external_version) + else: + flows = list_flows(output_format='dataframe') + assert isinstance(flows, pd.DataFrame) # Make mypy happy + flows = flows.query('name == "{}"'.format(flow_name)) + return flows['id'].to_list() + + def __list_flows( api_call: str, output_format: str = 'dict' diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index 9d16ac586..15dc52f78 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -303,3 +303,23 @@ def test_get_flow_reinstantiate_model_wrong_version(self): # ensure that a new flow was created assert flow.flow_id is None assert "0.19.1" not in flow.dependencies + + def test_get_flow_id(self): + clf = sklearn.tree.DecisionTreeClassifier() + flow = openml.extensions.get_extension_by_model(clf).model_to_flow(clf).publish() + + self.assertEqual(openml.flows.get_flow_id(model=clf, exact_version=True), flow.flow_id) + flow_ids = openml.flows.get_flow_id(model=clf, exact_version=False) + self.assertIn(flow.flow_id, flow_ids) + self.assertGreater(len(flow_ids), 2) + + # Check that the output of get_flow_id is identical if only the name is given, no matter + # whether exact_version is set to True or False. + flow_ids_exact_version_True = openml.flows.get_flow_id(name=flow.name, exact_version=True) + flow_ids_exact_version_False = openml.flows.get_flow_id( + name=flow.name, + exact_version=False, + ) + self.assertEqual(flow_ids_exact_version_True, flow_ids_exact_version_False) + self.assertIn(flow.flow_id, flow_ids_exact_version_True) + self.assertGreater(len(flow_ids_exact_version_True), 2)