From c8c3a6a2caa544472ca4930e08d900759fa04b4c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 02:33:31 -0700 Subject: [PATCH 01/51] Filter down Benchmark Metrics --- src/sagemaker/jumpstart/model.py | 22 +++++++++++++++------- src/sagemaker/jumpstart/utils.py | 14 ++++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 619af2f7a9..ca87346030 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -465,6 +465,7 @@ def benchmark_metrics(self) -> pd.DataFrame: self.config_name, self.instance_type ) keys = list(benchmark_metrics_data.keys()) + # Sort by Config Name and Instance Type column values df = pd.DataFrame(benchmark_metrics_data).sort_values(by=[keys[0], keys[1]]) return df @@ -478,12 +479,19 @@ def list_deployment_configs(self) -> List[Dict[str, Any]]: Returns: List[Dict[str, Any]]: A list of deployment configs. """ - return [ - deployment_config.to_json() - for deployment_config in self._get_deployment_configs( - self.config_name, self.instance_type - ) - ] + configs = [] + for deployment_config in self._get_deployment_configs(self.config_name, self.instance_type): + deployment_config_json = deployment_config.to_json() + + deployment_config_json["BenchmarkMetrics"] = { + deployment_config.deployment_args.instance_type: deployment_config_json.get( + "BenchmarkMetrics" + ).get(deployment_config.deployment_args.instance_type) + } + + configs.append(deployment_config_json) + + return configs def _create_sagemaker_model( self, @@ -925,7 +933,7 @@ def _get_deployment_configs( err = None for config_name, metadata_config in self._metadata_configs.items(): - if err is None or "is not authorized to perform: pricing:GetProducts" not in err: + if metadata_config.benchmark_metrics: err, metadata_config.benchmark_metrics = ( add_instance_rate_stats_to_benchmark_metrics( self.region, metadata_config.benchmark_metrics diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index a8c4bd7c21..c5c3ce8a77 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1055,23 +1055,25 @@ def add_instance_rate_stats_to_benchmark_metrics( for instance_type, benchmark_metric_stats in benchmark_metrics.items(): instance_type = instance_type if instance_type.startswith("ml.") else f"ml.{instance_type}" - if not has_instance_rate_stat(benchmark_metric_stats): + if not has_instance_rate_stat(benchmark_metric_stats) and err_message is None: try: instance_type_rate = get_instance_rate_per_hour( instance_type=instance_type, region=region ) - benchmark_metric_stats.append(JumpStartBenchmarkStat(instance_type_rate)) - final_benchmark_metrics[instance_type] = benchmark_metric_stats + if benchmark_metric_stats: + benchmark_metric_stats.append(JumpStartBenchmarkStat(instance_type_rate)) + else: + benchmark_metric_stats = [JumpStartBenchmarkStat(instance_type_rate)] + final_benchmark_metrics[instance_type] = benchmark_metric_stats except ClientError as e: final_benchmark_metrics[instance_type] = benchmark_metric_stats err_message = e.response["Error"]["Message"] except Exception: # pylint: disable=W0703 final_benchmark_metrics[instance_type] = benchmark_metric_stats - err_message = ( - f"Unable to get instance rate per hour for instance type: {instance_type}." - ) + else: + final_benchmark_metrics[instance_type] = benchmark_metric_stats return err_message, final_benchmark_metrics From 24d2c784691832810547ba2f7c9f536f5f35ab1f Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 03:04:36 -0700 Subject: [PATCH 02/51] Filter down Benchmark Metrics --- src/sagemaker/jumpstart/model.py | 36 +++++++++++--------------------- 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index ca87346030..da6e0355ae 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -360,6 +360,7 @@ def _validate_model_id_and_type(): self.model_package_arn = model_init_kwargs.model_package_arn self.init_kwargs = model_init_kwargs.to_kwargs_dict(False) + self._deployment_config = None self._metadata_configs = get_jumpstart_configs( region=self.region, model_id=self.model_id, @@ -449,10 +450,17 @@ def deployment_config(self) -> Optional[Dict[str, Any]]: Returns: Optional[Dict[str, Any]]: Deployment config. """ - deployment_config = self._retrieve_selected_deployment_config( - self.config_name, self.instance_type - ) - return deployment_config.to_json() if deployment_config is not None else None + if self.config_name is None: + self._deployment_config = None + elif self._deployment_config and self.config_name != self._deployment_config.get( + "DeploymentConfigName" + ): + for config in self.list_deployment_configs(): + if config.get("DeploymentConfigName") == self.config_name: + self._deployment_config = config + break + + return self._deployment_config @property def benchmark_metrics(self) -> pd.DataFrame: @@ -897,26 +905,6 @@ def _get_deployment_configs_benchmarks_data( self._get_deployment_configs(config_name, instance_type) ) - @lru_cache - def _retrieve_selected_deployment_config( - self, config_name: str, instance_type: str - ) -> Optional[DeploymentConfigMetadata]: - """Retrieve the deployment config to apply to `This` model. - - Args: - config_name (str): The name of the deployment config to retrieve. - instance_type (str): The instance type of the deployment config to retrieve. - Returns: - Optional[Dict[str, Any]]: The retrieved deployment config. - """ - if config_name is None: - return None - - for deployment_config in self._get_deployment_configs(config_name, instance_type): - if deployment_config.deployment_config_name == config_name: - return deployment_config - return None - @lru_cache def _get_deployment_configs( self, selected_config_name: str, selected_instance_type: str From f96ec671b9488d470eef18bf43195e4c4d726441 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 10:08:57 -0700 Subject: [PATCH 03/51] Testing NB --- src/sagemaker/jumpstart/model.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index da6e0355ae..e2dfb8dab7 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -452,15 +452,19 @@ def deployment_config(self) -> Optional[Dict[str, Any]]: """ if self.config_name is None: self._deployment_config = None - elif self._deployment_config and self.config_name != self._deployment_config.get( + return None + + if self._deployment_config and self.config_name == self._deployment_config.get( "DeploymentConfigName" ): - for config in self.list_deployment_configs(): - if config.get("DeploymentConfigName") == self.config_name: - self._deployment_config = config - break + return self._deployment_config + + for config in self.list_deployment_configs(): + if config.get("DeploymentConfigName") == self.config_name: + self._deployment_config = config + return self._deployment_config - return self._deployment_config + return None @property def benchmark_metrics(self) -> pd.DataFrame: From 9eb4823cfb4c4c8ab474ad8808981dbbaec5fafb Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 10:24:35 -0700 Subject: [PATCH 04/51] Testing MB --- src/sagemaker/serve/builder/jumpstart_builder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/sagemaker/serve/builder/jumpstart_builder.py b/src/sagemaker/serve/builder/jumpstart_builder.py index ec987dd9fe..868db8592a 100644 --- a/src/sagemaker/serve/builder/jumpstart_builder.py +++ b/src/sagemaker/serve/builder/jumpstart_builder.py @@ -454,14 +454,14 @@ def get_deployment_config(self) -> Optional[Dict[str, Any]]: Optional[Dict[str, Any]]: Deployment config to apply to this model. """ if not hasattr(self, "pysdk_model") or self.pysdk_model is None: - self.pysdk_model = self._create_pre_trained_js_model() + self._build_for_jumpstart() return self.pysdk_model.deployment_config def display_benchmark_metrics(self): """Display Markdown Benchmark Metrics for deployment configs.""" if not hasattr(self, "pysdk_model") or self.pysdk_model is None: - self.pysdk_model = self._create_pre_trained_js_model() + self._build_for_jumpstart() self.pysdk_model.display_benchmark_metrics() @@ -472,18 +472,21 @@ def list_deployment_configs(self) -> List[Dict[str, Any]]: List[Dict[str, Any]]: A list of deployment configs. """ if not hasattr(self, "pysdk_model") or self.pysdk_model is None: - self.pysdk_model = self._create_pre_trained_js_model() + self._build_for_jumpstart() return self.pysdk_model.list_deployment_configs() def _build_for_jumpstart(self): """Placeholder docstring""" + if hasattr(self, "pysdk_model") and self.pysdk_model is not None: + return self.pysdk_model + # we do not pickle for jumpstart. set to none self.secret_key = None self.jumpstart = True - if not hasattr(self, "pysdk_model") or self.pysdk_model is None: - self.pysdk_model = self._create_pre_trained_js_model() + # if not hasattr(self, "pysdk_model") or self.pysdk_model is None: + # self.pysdk_model = self._create_pre_trained_js_model() logger.info( "JumpStart ID %s is packaged with Image URI: %s", self.model, self.pysdk_model.image_uri From 4ca3f8425cdfbbc599ff5b89be05b54e8df57a67 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 10:49:16 -0700 Subject: [PATCH 05/51] Testing --- src/sagemaker/serve/builder/jumpstart_builder.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sagemaker/serve/builder/jumpstart_builder.py b/src/sagemaker/serve/builder/jumpstart_builder.py index 868db8592a..f6a4d165df 100644 --- a/src/sagemaker/serve/builder/jumpstart_builder.py +++ b/src/sagemaker/serve/builder/jumpstart_builder.py @@ -485,8 +485,7 @@ def _build_for_jumpstart(self): self.secret_key = None self.jumpstart = True - # if not hasattr(self, "pysdk_model") or self.pysdk_model is None: - # self.pysdk_model = self._create_pre_trained_js_model() + self.pysdk_model = self._create_pre_trained_js_model() logger.info( "JumpStart ID %s is packaged with Image URI: %s", self.model, self.pysdk_model.image_uri From 7a08be38921ddfeebdb8dc2af40abe9f64a4d746 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 12:37:52 -0700 Subject: [PATCH 06/51] Refactoring --- src/sagemaker/jumpstart/model.py | 29 +++++------------------------ src/sagemaker/jumpstart/types.py | 2 ++ src/sagemaker/jumpstart/utils.py | 32 ++++++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index e2dfb8dab7..11c19d5a4c 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -48,6 +48,7 @@ get_jumpstart_configs, get_metrics_from_deployment_configs, add_instance_rate_stats_to_benchmark_metrics, + deployment_config_response_data, ) from sagemaker.jumpstart.constants import JUMPSTART_LOGGER from sagemaker.jumpstart.enums import JumpStartModelType @@ -360,7 +361,6 @@ def _validate_model_id_and_type(): self.model_package_arn = model_init_kwargs.model_package_arn self.init_kwargs = model_init_kwargs.to_kwargs_dict(False) - self._deployment_config = None self._metadata_configs = get_jumpstart_configs( region=self.region, model_id=self.model_id, @@ -451,19 +451,10 @@ def deployment_config(self) -> Optional[Dict[str, Any]]: Optional[Dict[str, Any]]: Deployment config. """ if self.config_name is None: - self._deployment_config = None return None - - if self._deployment_config and self.config_name == self._deployment_config.get( - "DeploymentConfigName" - ): - return self._deployment_config - for config in self.list_deployment_configs(): if config.get("DeploymentConfigName") == self.config_name: - self._deployment_config = config - return self._deployment_config - + return config return None @property @@ -491,19 +482,9 @@ def list_deployment_configs(self) -> List[Dict[str, Any]]: Returns: List[Dict[str, Any]]: A list of deployment configs. """ - configs = [] - for deployment_config in self._get_deployment_configs(self.config_name, self.instance_type): - deployment_config_json = deployment_config.to_json() - - deployment_config_json["BenchmarkMetrics"] = { - deployment_config.deployment_args.instance_type: deployment_config_json.get( - "BenchmarkMetrics" - ).get(deployment_config.deployment_args.instance_type) - } - - configs.append(deployment_config_json) - - return configs + return deployment_config_response_data( + self._get_deployment_configs(self.config_name, self.instance_type) + ) def _create_sagemaker_model( self, diff --git a/src/sagemaker/jumpstart/types.py b/src/sagemaker/jumpstart/types.py index e0a0f9bea7..6e08ffd777 100644 --- a/src/sagemaker/jumpstart/types.py +++ b/src/sagemaker/jumpstart/types.py @@ -2247,6 +2247,8 @@ def _val_to_json(self, val: Any) -> Any: Any: The converted json value. """ if issubclass(type(val), JumpStartDataHolderType): + if isinstance(val, JumpStartBenchmarkStat): + val.name = val.name.replace("_", " ").title() return val.to_json() if isinstance(val, list): list_obj = [] diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index c5c3ce8a77..54a947c19a 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1129,7 +1129,7 @@ def get_metrics_from_deployment_configs( if outer_index == 0 and inner_index == 0: temp_data = {} for metric in current_instance_type_metrics: - column_name = f"{metric.name.replace('_', ' ').title()} ({metric.unit})" + column_name = f"{metric.name} ({metric.unit})" if metric.name.lower() == "instance rate": data[column_name] = [] else: @@ -1137,7 +1137,35 @@ def get_metrics_from_deployment_configs( data = {**data, **temp_data} for metric in current_instance_type_metrics: - column_name = f"{metric.name.replace('_', ' ').title()} ({metric.unit})" + column_name = f"{metric.name} ({metric.unit})" if column_name in data: data[column_name].append(metric.value) return data + + +def deployment_config_response_data( + deployment_configs: List[DeploymentConfigMetadata], +) -> List[Dict[str, Any]]: + """Deployment config api response data. + + Args: + deployment_configs (List[DeploymentConfigMetadata]): List of deployment configs metadata. + Returns: + List[Dict[str, Any]]: List of deployment config api response data. + """ + configs = [] + if deployment_configs is None: + return configs + + for deployment_config in deployment_configs: + deployment_config_json = deployment_config.to_json() + + deployment_config_json["BenchmarkMetrics"] = { + deployment_config.deployment_args.instance_type: deployment_config_json.get( + "BenchmarkMetrics" + ).get(deployment_config.deployment_args.instance_type) + } + + configs.append(deployment_config_json) + + return configs From b934fb16341e7fbccdc9d478adb4afa4b31bb197 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 14:23:28 -0700 Subject: [PATCH 07/51] Unit tests --- src/sagemaker/jumpstart/utils.py | 17 ++++++---- .../sagemaker/jumpstart/model/test_model.py | 8 ++--- tests/unit/sagemaker/jumpstart/test_utils.py | 32 +++++++------------ tests/unit/sagemaker/jumpstart/utils.py | 14 ++++++-- .../serve/builder/test_js_builder.py | 6 ++++ 5 files changed, 43 insertions(+), 34 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 54a947c19a..abcffb2c2e 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1144,12 +1144,13 @@ def get_metrics_from_deployment_configs( def deployment_config_response_data( - deployment_configs: List[DeploymentConfigMetadata], + deployment_configs: Optional[List[DeploymentConfigMetadata]], ) -> List[Dict[str, Any]]: """Deployment config api response data. Args: - deployment_configs (List[DeploymentConfigMetadata]): List of deployment configs metadata. + deployment_configs (Optional[List[DeploymentConfigMetadata]]): + List of deployment configs metadata. Returns: List[Dict[str, Any]]: List of deployment config api response data. """ @@ -1160,11 +1161,13 @@ def deployment_config_response_data( for deployment_config in deployment_configs: deployment_config_json = deployment_config.to_json() - deployment_config_json["BenchmarkMetrics"] = { - deployment_config.deployment_args.instance_type: deployment_config_json.get( - "BenchmarkMetrics" - ).get(deployment_config.deployment_args.instance_type) - } + benchmark_metrics = deployment_config_json.get("BenchmarkMetrics") + if benchmark_metrics: + deployment_config_json["BenchmarkMetrics"] = { + deployment_config.deployment_args.instance_type: benchmark_metrics.get( + deployment_config.deployment_args.instance_type + ) + } configs.append(deployment_config_json) diff --git a/tests/unit/sagemaker/jumpstart/model/test_model.py b/tests/unit/sagemaker/jumpstart/model/test_model.py index cd11d950d5..c89743312d 100644 --- a/tests/unit/sagemaker/jumpstart/model/test_model.py +++ b/tests/unit/sagemaker/jumpstart/model/test_model.py @@ -1733,7 +1733,7 @@ def test_model_list_deployment_configs( mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs(model_id) mock_verify_model_region_and_return_specs.side_effect = ( - lambda *args, **kwargs: get_base_spec_with_prototype_configs() + lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks() ) mock_add_instance_rate_stats_to_benchmark_metrics.side_effect = lambda region, metrics: ( None, @@ -1750,7 +1750,7 @@ def test_model_list_deployment_configs( configs = model.list_deployment_configs() - self.assertEqual(configs, get_base_deployment_configs()) + self.assertEqual(configs, get_base_deployment_configs(True)) @mock.patch("sagemaker.jumpstart.utils.verify_model_region_and_return_specs") @mock.patch("sagemaker.jumpstart.accessors.JumpStartModelsAccessor._get_manifest") @@ -1803,7 +1803,7 @@ def test_model_retrieve_deployment_config( model_id, _ = "pytorch-eqa-bert-base-cased", "*" mock_verify_model_region_and_return_specs.side_effect = ( - lambda *args, **kwargs: get_base_spec_with_prototype_configs_with_missing_benchmarks() + lambda *args, **kwargs: get_base_spec_with_prototype_configs() ) mock_add_instance_rate_stats_to_benchmark_metrics.side_effect = lambda region, metrics: ( None, @@ -1815,7 +1815,7 @@ def test_model_retrieve_deployment_config( ) mock_model_deploy.return_value = default_predictor - expected = get_base_deployment_configs(True)[0] + expected = get_base_deployment_configs()[0] config_name = expected.get("DeploymentConfigName") instance_type = expected.get("InstanceType") mock_get_init_kwargs.side_effect = lambda *args, **kwargs: get_mock_init_kwargs( diff --git a/tests/unit/sagemaker/jumpstart/test_utils.py b/tests/unit/sagemaker/jumpstart/test_utils.py index f7458a29e9..6a2a0a2727 100644 --- a/tests/unit/sagemaker/jumpstart/test_utils.py +++ b/tests/unit/sagemaker/jumpstart/test_utils.py @@ -52,6 +52,7 @@ get_special_model_spec, get_prototype_manifest, get_base_deployment_configs_metadata, + get_base_deployment_configs, ) from mock import MagicMock @@ -1831,26 +1832,6 @@ def test_add_instance_rate_stats_to_benchmark_metrics_client_ex( assert len(out[key]) == 1 -@patch("sagemaker.jumpstart.utils.get_instance_rate_per_hour") -def test_add_instance_rate_stats_to_benchmark_metrics_ex( - mock_get_instance_rate_per_hour, -): - mock_get_instance_rate_per_hour.side_effect = Exception() - - err, out = utils.add_instance_rate_stats_to_benchmark_metrics( - "us-west-2", - { - "ml.p2.xlarge": [ - JumpStartBenchmarkStat({"name": "Latency", "value": "100", "unit": "Tokens/S"}) - ], - }, - ) - - assert err == "Unable to get instance rate per hour for instance type: ml.p2.xlarge." - for key in out: - assert len(out[key]) == 1 - - @pytest.mark.parametrize( "stats, expected", [ @@ -1864,3 +1845,14 @@ def test_add_instance_rate_stats_to_benchmark_metrics_ex( ) def test_has_instance_rate_stat(stats, expected): assert utils.has_instance_rate_stat(stats) is expected + + +@pytest.mark.parametrize( + "data, expected", + [(None, []), ([], []), (get_base_deployment_configs_metadata(), get_base_deployment_configs())], +) +def test_deployment_config_response_data(data, expected): + out = utils.deployment_config_response_data(data) + + print(out) + assert out == expected diff --git a/tests/unit/sagemaker/jumpstart/utils.py b/tests/unit/sagemaker/jumpstart/utils.py index e8a93dff6c..b90da25061 100644 --- a/tests/unit/sagemaker/jumpstart/utils.py +++ b/tests/unit/sagemaker/jumpstart/utils.py @@ -388,9 +388,17 @@ def get_base_deployment_configs_metadata( def get_base_deployment_configs( omit_benchmark_metrics: bool = False, ) -> List[Dict[str, Any]]: - return [ - config.to_json() for config in get_base_deployment_configs_metadata(omit_benchmark_metrics) - ] + configs = [] + for config in get_base_deployment_configs_metadata(omit_benchmark_metrics): + config_json = config.to_json() + if config_json["BenchmarkMetrics"]: + config_json["BenchmarkMetrics"] = { + config.deployment_args.instance_type: config_json["BenchmarkMetrics"].get( + config.deployment_args.instance_type + ) + } + configs.append(config_json) + return configs def append_instance_stat_metrics( diff --git a/tests/unit/sagemaker/serve/builder/test_js_builder.py b/tests/unit/sagemaker/serve/builder/test_js_builder.py index 56b01cd9e3..4ec96e88e3 100644 --- a/tests/unit/sagemaker/serve/builder/test_js_builder.py +++ b/tests/unit/sagemaker/serve/builder/test_js_builder.py @@ -866,6 +866,12 @@ def test_display_benchmark_metrics_initial( model="facebook/galactica-mock-model-id", schema_builder=mock_schema_builder, ) + + mock_pre_trained_model.return_value.image_uri = mock_tgi_image_uri + mock_pre_trained_model.return_value.list_deployment_configs.side_effect = ( + lambda: DEPLOYMENT_CONFIGS + ) + builder.display_benchmark_metrics() mock_pre_trained_model.return_value.display_benchmark_metrics.assert_called_once() From bb1bf4dc8319144fd57a2fc2e3c26371d91af31b Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 15:11:07 -0700 Subject: [PATCH 08/51] Display instance type first, and instance rate last --- src/sagemaker/jumpstart/model.py | 2 +- src/sagemaker/jumpstart/utils.py | 31 +++++++++++++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 11c19d5a4c..4c2b1b26c0 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -483,7 +483,7 @@ def list_deployment_configs(self) -> List[Dict[str, Any]]: List[Dict[str, Any]]: A list of deployment configs. """ return deployment_config_response_data( - self._get_deployment_configs(self.config_name, self.instance_type) + self.config_name, self._get_deployment_configs(self.config_name, self.instance_type) ) def _create_sagemaker_model( diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index abcffb2c2e..03351da1c1 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1105,7 +1105,7 @@ def get_metrics_from_deployment_configs( Args: deployment_configs (List[DeploymentConfigMetadata]): List of deployment configs metadata. """ - data = {"Config Name": [], "Instance Type": []} + data = {"Instance Type": [], "Config Name": []} for outer_index, deployment_config in enumerate(deployment_configs): if deployment_config.deployment_args is None: @@ -1131,9 +1131,9 @@ def get_metrics_from_deployment_configs( for metric in current_instance_type_metrics: column_name = f"{metric.name} ({metric.unit})" if metric.name.lower() == "instance rate": - data[column_name] = [] - else: temp_data[column_name] = [] + else: + data[column_name] = [] data = {**data, **temp_data} for metric in current_instance_type_metrics: @@ -1144,11 +1144,13 @@ def get_metrics_from_deployment_configs( def deployment_config_response_data( + config_name: str, deployment_configs: Optional[List[DeploymentConfigMetadata]], ) -> List[Dict[str, Any]]: """Deployment config api response data. Args: + config_name (str): Name of deployment config. deployment_configs (Optional[List[DeploymentConfigMetadata]]): List of deployment configs metadata. Returns: @@ -1159,16 +1161,17 @@ def deployment_config_response_data( return configs for deployment_config in deployment_configs: - deployment_config_json = deployment_config.to_json() - - benchmark_metrics = deployment_config_json.get("BenchmarkMetrics") - if benchmark_metrics: - deployment_config_json["BenchmarkMetrics"] = { - deployment_config.deployment_args.instance_type: benchmark_metrics.get( - deployment_config.deployment_args.instance_type - ) - } - - configs.append(deployment_config_json) + if config_name == deployment_config.deployment_config_name: + deployment_config_json = deployment_config.to_json() + benchmark_metrics = deployment_config_json.get("BenchmarkMetrics") + if benchmark_metrics: + deployment_config_json["BenchmarkMetrics"] = { + deployment_config.deployment_args.instance_type: benchmark_metrics.get( + deployment_config.deployment_args.instance_type + ) + } + + configs.append(deployment_config_json) + break return configs From bd5f67d593372bca748bfc68129deb25eb1b690c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 16:03:33 -0700 Subject: [PATCH 09/51] Display unbalanced metrics --- src/sagemaker/jumpstart/model.py | 15 ++++--- src/sagemaker/jumpstart/utils.py | 70 ++++++++++++++++++++------------ 2 files changed, 50 insertions(+), 35 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 4c2b1b26c0..c016756355 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -483,7 +483,7 @@ def list_deployment_configs(self) -> List[Dict[str, Any]]: List[Dict[str, Any]]: A list of deployment configs. """ return deployment_config_response_data( - self.config_name, self._get_deployment_configs(self.config_name, self.instance_type) + self._get_deployment_configs(self.config_name, self.instance_type) ) def _create_sagemaker_model( @@ -906,19 +906,18 @@ def _get_deployment_configs( err = None for config_name, metadata_config in self._metadata_configs.items(): - if metadata_config.benchmark_metrics: - err, metadata_config.benchmark_metrics = ( - add_instance_rate_stats_to_benchmark_metrics( - self.region, metadata_config.benchmark_metrics - ) - ) - resolved_config = metadata_config.resolved_config if selected_config_name == config_name: instance_type_to_use = selected_instance_type else: instance_type_to_use = resolved_config.get("default_inference_instance_type") + err, metadata_config.benchmark_metrics = ( + add_instance_rate_stats_to_benchmark_metrics( + self.region, instance_type_to_use, metadata_config.benchmark_metrics + ) + ) + init_kwargs = get_init_kwargs( model_id=self.model_id, instance_type=instance_type_to_use, diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 03351da1c1..d00e34ce9b 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1034,23 +1034,23 @@ def get_jumpstart_configs( def add_instance_rate_stats_to_benchmark_metrics( region: str, + default_instance_type: str, benchmark_metrics: Optional[Dict[str, List[JumpStartBenchmarkStat]]], ) -> Optional[Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]]: """Adds instance types metric stats to the given benchmark_metrics dict. Args: region (str): AWS region. + default_instance_type (str): Instance type. benchmark_metrics (Dict[str, List[JumpStartBenchmarkStat]]): Returns: Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]: Contains Error message and metrics dict. """ - if benchmark_metrics is None: - return None + benchmark_metrics = {default_instance_type: None} final_benchmark_metrics = {} - err_message = None for instance_type, benchmark_metric_stats in benchmark_metrics.items(): instance_type = instance_type if instance_type.startswith("ml.") else f"ml.{instance_type}" @@ -1106,8 +1106,9 @@ def get_metrics_from_deployment_configs( deployment_configs (List[DeploymentConfigMetadata]): List of deployment configs metadata. """ data = {"Instance Type": [], "Config Name": []} + instance_rate_data = {} - for outer_index, deployment_config in enumerate(deployment_configs): + for deployment_config in deployment_configs: if deployment_config.deployment_args is None: continue @@ -1126,25 +1127,42 @@ def get_metrics_from_deployment_configs( ) data["Instance Type"].append(instance_type_to_display) - if outer_index == 0 and inner_index == 0: - temp_data = {} - for metric in current_instance_type_metrics: - column_name = f"{metric.name} ({metric.unit})" - if metric.name.lower() == "instance rate": - temp_data[column_name] = [] - else: - data[column_name] = [] - data = {**data, **temp_data} + # if outer_index == 0 and inner_index == 0: + # temp_data = {} + # for metric in current_instance_type_metrics: + # column_name = f"{metric.name} ({metric.unit})" + # if metric.name.lower() == "instance rate": + # temp_data[column_name] = [] + # else: + # data[column_name] = [] + # data = {**data, **temp_data} + # + # for metric in current_instance_type_metrics: + # column_name = f"{metric.name} ({metric.unit})" + # if column_name in data: + # data[column_name].append(metric.value) for metric in current_instance_type_metrics: column_name = f"{metric.name} ({metric.unit})" - if column_name in data: + + if metric.name.lower() == "instance rate": + if column_name in instance_rate_data: + instance_rate_data[metric.name].append(metric.value) + else: + instance_rate_data[metric.name] = [metric.value] + else: + for _ in range(inner_index): + if column_name in data: + data[column_name].append(" - ") + else: + data[column_name] = [" - "] data[column_name].append(metric.value) + + data = {**data, **instance_rate_data} return data def deployment_config_response_data( - config_name: str, deployment_configs: Optional[List[DeploymentConfigMetadata]], ) -> List[Dict[str, Any]]: """Deployment config api response data. @@ -1161,17 +1179,15 @@ def deployment_config_response_data( return configs for deployment_config in deployment_configs: - if config_name == deployment_config.deployment_config_name: - deployment_config_json = deployment_config.to_json() - benchmark_metrics = deployment_config_json.get("BenchmarkMetrics") - if benchmark_metrics: - deployment_config_json["BenchmarkMetrics"] = { - deployment_config.deployment_args.instance_type: benchmark_metrics.get( - deployment_config.deployment_args.instance_type - ) - } - - configs.append(deployment_config_json) - break + deployment_config_json = deployment_config.to_json() + benchmark_metrics = deployment_config_json.get("BenchmarkMetrics") + if benchmark_metrics: + deployment_config_json["BenchmarkMetrics"] = { + deployment_config.deployment_args.instance_type: benchmark_metrics.get( + deployment_config.deployment_args.instance_type + ) + } + + configs.append(deployment_config_json) return configs From 9934b52d55ae803c74320bb1eff0c6626de6613c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 16:15:57 -0700 Subject: [PATCH 10/51] Testing with NB --- src/sagemaker/jumpstart/utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index d00e34ce9b..eb895564c8 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1151,11 +1151,10 @@ def get_metrics_from_deployment_configs( else: instance_rate_data[metric.name] = [metric.value] else: + if column_name not in data: + data[column_name] = [] for _ in range(inner_index): - if column_name in data: - data[column_name].append(" - ") - else: - data[column_name] = [" - "] + data[column_name].append(" - ") data[column_name].append(metric.value) data = {**data, **instance_rate_data} From 51d593208451b99f2d852fe86d4dcfb7ba0b35a8 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 16:24:58 -0700 Subject: [PATCH 11/51] Testing with NB --- src/sagemaker/jumpstart/model.py | 8 ++++---- src/sagemaker/jumpstart/utils.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index c016756355..b862630c6a 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -912,11 +912,11 @@ def _get_deployment_configs( else: instance_type_to_use = resolved_config.get("default_inference_instance_type") - err, metadata_config.benchmark_metrics = ( - add_instance_rate_stats_to_benchmark_metrics( - self.region, instance_type_to_use, metadata_config.benchmark_metrics - ) + err, metadata_config.benchmark_metrics = ( + add_instance_rate_stats_to_benchmark_metrics( + self.region, instance_type_to_use, metadata_config.benchmark_metrics ) + ) init_kwargs = get_init_kwargs( model_id=self.model_id, diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index eb895564c8..7daf41597d 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1042,7 +1042,7 @@ def add_instance_rate_stats_to_benchmark_metrics( Args: region (str): AWS region. default_instance_type (str): Instance type. - benchmark_metrics (Dict[str, List[JumpStartBenchmarkStat]]): + benchmark_metrics (Optional[Dict[str, List[JumpStartBenchmarkStat]]]): Returns: Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]: Contains Error message and metrics dict. From ed80588b9b49f70a6010efbc6d6081cd12b6c347 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 16:35:42 -0700 Subject: [PATCH 12/51] Debug --- src/sagemaker/jumpstart/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 7daf41597d..06eaae66e8 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1050,8 +1050,8 @@ def add_instance_rate_stats_to_benchmark_metrics( if benchmark_metrics is None: benchmark_metrics = {default_instance_type: None} - final_benchmark_metrics = {} err_message = None + final_benchmark_metrics = {} for instance_type, benchmark_metric_stats in benchmark_metrics.items(): instance_type = instance_type if instance_type.startswith("ml.") else f"ml.{instance_type}" @@ -1144,7 +1144,6 @@ def get_metrics_from_deployment_configs( for metric in current_instance_type_metrics: column_name = f"{metric.name} ({metric.unit})" - if metric.name.lower() == "instance rate": if column_name in instance_rate_data: instance_rate_data[metric.name].append(metric.value) @@ -1158,6 +1157,8 @@ def get_metrics_from_deployment_configs( data[column_name].append(metric.value) data = {**data, **instance_rate_data} + print("******************************") + print(data) return data From 907ae6552132f0eaf55664d9c7c61acd6e46115c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 16:52:53 -0700 Subject: [PATCH 13/51] Debug --- src/sagemaker/jumpstart/utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 06eaae66e8..3cf75f547c 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1143,14 +1143,23 @@ def get_metrics_from_deployment_configs( # data[column_name].append(metric.value) for metric in current_instance_type_metrics: + print(deployment_config.deployment_config_name) + print(current_instance_type) + column_name = f"{metric.name} ({metric.unit})" + print(f"column_name: {column_name}") + if metric.name.lower() == "instance rate": + print("Instance rate metric") if column_name in instance_rate_data: - instance_rate_data[metric.name].append(metric.value) + print("append instance_rate_data[column_name]") + instance_rate_data[column_name].append(metric.value) else: - instance_rate_data[metric.name] = [metric.value] + print("instance_rate_data[column_name] = [metric.value]") + instance_rate_data[column_name] = [metric.value] else: if column_name not in data: + print(f"Column {column_name} not found in data") data[column_name] = [] for _ in range(inner_index): data[column_name].append(" - ") From 0492f635b60522da964617d41e54acd7daf816a5 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 17:03:45 -0700 Subject: [PATCH 14/51] Testing with NB --- src/sagemaker/jumpstart/utils.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 3cf75f547c..37d4f866bf 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1143,25 +1143,17 @@ def get_metrics_from_deployment_configs( # data[column_name].append(metric.value) for metric in current_instance_type_metrics: - print(deployment_config.deployment_config_name) - print(current_instance_type) - column_name = f"{metric.name} ({metric.unit})" - print(f"column_name: {column_name}") if metric.name.lower() == "instance rate": - print("Instance rate metric") if column_name in instance_rate_data: - print("append instance_rate_data[column_name]") instance_rate_data[column_name].append(metric.value) else: - print("instance_rate_data[column_name] = [metric.value]") instance_rate_data[column_name] = [metric.value] else: if column_name not in data: - print(f"Column {column_name} not found in data") data[column_name] = [] - for _ in range(inner_index): + for _ in range(len(data[column_name]), inner_index): data[column_name].append(" - ") data[column_name].append(metric.value) From ef0c12fb22c2da84e62bcbd23a4792a0e1cb1e50 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 17:07:58 -0700 Subject: [PATCH 15/51] Testing with NB --- src/sagemaker/jumpstart/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 37d4f866bf..5f088a086d 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1153,7 +1153,9 @@ def get_metrics_from_deployment_configs( else: if column_name not in data: data[column_name] = [] - for _ in range(len(data[column_name]), inner_index): + for _ in range(len(data[column_name]), inner_index + 1): + print(f"len(data[column_name]): {len(data[column_name])}") + print(f"inner_index: {inner_index}") data[column_name].append(" - ") data[column_name].append(metric.value) From db219992621a7e25340b5d9219cf3b8263f42363 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 17:12:39 -0700 Subject: [PATCH 16/51] Testing with NB --- src/sagemaker/jumpstart/model.py | 2 +- src/sagemaker/jumpstart/utils.py | 19 ------------------- 2 files changed, 1 insertion(+), 20 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index b862630c6a..1f8e5744cb 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -469,7 +469,7 @@ def benchmark_metrics(self) -> pd.DataFrame: ) keys = list(benchmark_metrics_data.keys()) # Sort by Config Name and Instance Type column values - df = pd.DataFrame(benchmark_metrics_data).sort_values(by=[keys[0], keys[1]]) + df = pd.DataFrame(benchmark_metrics_data).sort_values(by=[keys[1], keys[0]]) return df def display_benchmark_metrics(self) -> None: diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 5f088a086d..1a1a0539c7 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1127,21 +1127,6 @@ def get_metrics_from_deployment_configs( ) data["Instance Type"].append(instance_type_to_display) - # if outer_index == 0 and inner_index == 0: - # temp_data = {} - # for metric in current_instance_type_metrics: - # column_name = f"{metric.name} ({metric.unit})" - # if metric.name.lower() == "instance rate": - # temp_data[column_name] = [] - # else: - # data[column_name] = [] - # data = {**data, **temp_data} - # - # for metric in current_instance_type_metrics: - # column_name = f"{metric.name} ({metric.unit})" - # if column_name in data: - # data[column_name].append(metric.value) - for metric in current_instance_type_metrics: column_name = f"{metric.name} ({metric.unit})" @@ -1154,14 +1139,10 @@ def get_metrics_from_deployment_configs( if column_name not in data: data[column_name] = [] for _ in range(len(data[column_name]), inner_index + 1): - print(f"len(data[column_name]): {len(data[column_name])}") - print(f"inner_index: {inner_index}") data[column_name].append(" - ") data[column_name].append(metric.value) data = {**data, **instance_rate_data} - print("******************************") - print(data) return data From 19bc9f3c7faa0c5c51dd51106f6ae5a65a84721c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 17:35:20 -0700 Subject: [PATCH 17/51] Refactoring --- src/sagemaker/jumpstart/model.py | 9 +++++---- src/sagemaker/jumpstart/utils.py | 5 +---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 1f8e5744cb..af962857bf 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -912,11 +912,12 @@ def _get_deployment_configs( else: instance_type_to_use = resolved_config.get("default_inference_instance_type") - err, metadata_config.benchmark_metrics = ( - add_instance_rate_stats_to_benchmark_metrics( - self.region, instance_type_to_use, metadata_config.benchmark_metrics + if metadata_config.benchmark_metrics: + err, metadata_config.benchmark_metrics = ( + add_instance_rate_stats_to_benchmark_metrics( + self.region, metadata_config.benchmark_metrics + ) ) - ) init_kwargs = get_init_kwargs( model_id=self.model_id, diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 1a1a0539c7..155a95449b 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1034,21 +1034,19 @@ def get_jumpstart_configs( def add_instance_rate_stats_to_benchmark_metrics( region: str, - default_instance_type: str, benchmark_metrics: Optional[Dict[str, List[JumpStartBenchmarkStat]]], ) -> Optional[Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]]: """Adds instance types metric stats to the given benchmark_metrics dict. Args: region (str): AWS region. - default_instance_type (str): Instance type. benchmark_metrics (Optional[Dict[str, List[JumpStartBenchmarkStat]]]): Returns: Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]: Contains Error message and metrics dict. """ if benchmark_metrics is None: - benchmark_metrics = {default_instance_type: None} + return None err_message = None final_benchmark_metrics = {} @@ -1152,7 +1150,6 @@ def deployment_config_response_data( """Deployment config api response data. Args: - config_name (str): Name of deployment config. deployment_configs (Optional[List[DeploymentConfigMetadata]]): List of deployment configs metadata. Returns: From 61d1f9dadbbaba3c7c654b38328d63c54ea703a2 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 17:40:29 -0700 Subject: [PATCH 18/51] Refactoring --- src/sagemaker/jumpstart/utils.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 155a95449b..6bbb45e0a7 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1107,11 +1107,8 @@ def get_metrics_from_deployment_configs( instance_rate_data = {} for deployment_config in deployment_configs: - if deployment_config.deployment_args is None: - continue - benchmark_metrics = deployment_config.benchmark_metrics - if benchmark_metrics is None: + if deployment_config.deployment_args is None or benchmark_metrics is None: continue for inner_index, current_instance_type in enumerate(benchmark_metrics): @@ -1136,7 +1133,7 @@ def get_metrics_from_deployment_configs( else: if column_name not in data: data[column_name] = [] - for _ in range(len(data[column_name]), inner_index + 1): + for _ in range(len(data[column_name]), inner_index): data[column_name].append(" - ") data[column_name].append(metric.value) From 2ab3a9c52e314f985fbfb0e091a23c7538568fcb Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Mon, 29 Apr 2024 18:28:26 -0700 Subject: [PATCH 19/51] Refactoring --- src/sagemaker/jumpstart/model.py | 4 +++- src/sagemaker/jumpstart/utils.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index af962857bf..7d0e8aafd9 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -887,7 +887,9 @@ def _get_deployment_configs_benchmarks_data( Dict[str, List[str]]: Deployment config benchmark data. """ return get_metrics_from_deployment_configs( - self._get_deployment_configs(config_name, instance_type) + config_name, + instance_type, + self._get_deployment_configs(config_name, instance_type), ) @lru_cache diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 6bbb45e0a7..373b73632f 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1096,11 +1096,15 @@ def has_instance_rate_stat(benchmark_metric_stats: Optional[List[JumpStartBenchm def get_metrics_from_deployment_configs( + default_config_name: str, + default_instance_type: str, deployment_configs: List[DeploymentConfigMetadata], ) -> Dict[str, List[str]]: """Extracts benchmark metrics from deployment configs metadata. Args: + default_config_name (str): The name of the default deployment config. + default_instance_type (str): The name of the default instance type. deployment_configs (List[DeploymentConfigMetadata]): List of deployment configs metadata. """ data = {"Instance Type": [], "Config Name": []} @@ -1117,7 +1121,8 @@ def get_metrics_from_deployment_configs( data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( f"{current_instance_type} (Default)" - if current_instance_type == deployment_config.deployment_args.default_instance_type + if current_instance_type == default_instance_type + and default_config_name == deployment_config.deployment_config_name else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From 055cf4df3059572f8a0a17f081e75626f494d9e7 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 11:01:41 -0700 Subject: [PATCH 20/51] Unit tests --- src/sagemaker/jumpstart/model.py | 8 ++-- src/sagemaker/jumpstart/utils.py | 50 ++++++++++---------- tests/unit/sagemaker/jumpstart/test_utils.py | 2 +- 3 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 7d0e8aafd9..8a174eb6e3 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -472,9 +472,9 @@ def benchmark_metrics(self) -> pd.DataFrame: df = pd.DataFrame(benchmark_metrics_data).sort_values(by=[keys[1], keys[0]]) return df - def display_benchmark_metrics(self) -> None: + def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" - print(self.benchmark_metrics.to_markdown(index=False)) + print(self.benchmark_metrics.to_markdown(index=False), *args, **kwargs) def list_deployment_configs(self) -> List[Dict[str, Any]]: """List deployment configs for ``This`` model. @@ -903,7 +903,7 @@ def _get_deployment_configs( selected_instance_type (str): The selected instance type. """ deployment_configs = [] - if self._metadata_configs is None: + if not self._metadata_configs: return deployment_configs err = None @@ -940,7 +940,7 @@ def _get_deployment_configs( ) deployment_configs.append(deployment_config_metadata) - if err is not None and "is not authorized to perform: pricing:GetProducts" in err: + if err and "is not authorized to perform: pricing:GetProducts" in err: error_message = "Instance rate metrics will be omitted. Reason: %s" JUMPSTART_LOGGER.warning(error_message, err) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 373b73632f..8c5b5e18ad 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1042,10 +1042,10 @@ def add_instance_rate_stats_to_benchmark_metrics( region (str): AWS region. benchmark_metrics (Optional[Dict[str, List[JumpStartBenchmarkStat]]]): Returns: - Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]: + Optional[Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]]: Contains Error message and metrics dict. """ - if benchmark_metrics is None: + if not benchmark_metrics: return None err_message = None @@ -1053,16 +1053,15 @@ def add_instance_rate_stats_to_benchmark_metrics( for instance_type, benchmark_metric_stats in benchmark_metrics.items(): instance_type = instance_type if instance_type.startswith("ml.") else f"ml.{instance_type}" - if not has_instance_rate_stat(benchmark_metric_stats) and err_message is None: + if not has_instance_rate_stat(benchmark_metric_stats) and not err_message: try: instance_type_rate = get_instance_rate_per_hour( instance_type=instance_type, region=region ) - if benchmark_metric_stats: - benchmark_metric_stats.append(JumpStartBenchmarkStat(instance_type_rate)) - else: - benchmark_metric_stats = [JumpStartBenchmarkStat(instance_type_rate)] + if not benchmark_metric_stats: + benchmark_metric_stats = [] + benchmark_metric_stats.append(JumpStartBenchmarkStat(instance_type_rate)) final_benchmark_metrics[instance_type] = benchmark_metric_stats except ClientError as e: @@ -1087,32 +1086,35 @@ def has_instance_rate_stat(benchmark_metric_stats: Optional[List[JumpStartBenchm """ if benchmark_metric_stats is None: return False - for benchmark_metric_stat in benchmark_metric_stats: if benchmark_metric_stat.name.lower() == "instance rate": return True - return False def get_metrics_from_deployment_configs( - default_config_name: str, - default_instance_type: str, - deployment_configs: List[DeploymentConfigMetadata], + default_config_name: Optional[str], + default_instance_type: Optional[str], + deployment_configs: Optional[List[DeploymentConfigMetadata]], ) -> Dict[str, List[str]]: """Extracts benchmark metrics from deployment configs metadata. Args: - default_config_name (str): The name of the default deployment config. - default_instance_type (str): The name of the default instance type. - deployment_configs (List[DeploymentConfigMetadata]): List of deployment configs metadata. + default_config_name (Optional[str]): The name of the default deployment config. + default_instance_type (Optional[str]): The name of the default instance type. + deployment_configs (Optional[List[DeploymentConfigMetadata]]): + List of deployment configs metadata. + Returns: + Dict[str, List[str]]: Deployment configs bench metrics dict. """ + if not deployment_configs: + return {} + data = {"Instance Type": [], "Config Name": []} instance_rate_data = {} - for deployment_config in deployment_configs: benchmark_metrics = deployment_config.benchmark_metrics - if deployment_config.deployment_args is None or benchmark_metrics is None: + if not deployment_config.deployment_args or not benchmark_metrics: continue for inner_index, current_instance_type in enumerate(benchmark_metrics): @@ -1122,7 +1124,7 @@ def get_metrics_from_deployment_configs( instance_type_to_display = ( f"{current_instance_type} (Default)" if current_instance_type == default_instance_type - and default_config_name == deployment_config.deployment_config_name + and deployment_config.deployment_config_name == default_config_name else current_instance_type ) data["Instance Type"].append(instance_type_to_display) @@ -1131,10 +1133,9 @@ def get_metrics_from_deployment_configs( column_name = f"{metric.name} ({metric.unit})" if metric.name.lower() == "instance rate": - if column_name in instance_rate_data: - instance_rate_data[column_name].append(metric.value) - else: - instance_rate_data[column_name] = [metric.value] + if column_name not in instance_rate_data: + instance_rate_data[column_name] = [] + instance_rate_data[column_name].append(metric.value) else: if column_name not in data: data[column_name] = [] @@ -1158,13 +1159,13 @@ def deployment_config_response_data( List[Dict[str, Any]]: List of deployment config api response data. """ configs = [] - if deployment_configs is None: + if not deployment_configs: return configs for deployment_config in deployment_configs: deployment_config_json = deployment_config.to_json() benchmark_metrics = deployment_config_json.get("BenchmarkMetrics") - if benchmark_metrics: + if benchmark_metrics and deployment_config.deployment_args: deployment_config_json["BenchmarkMetrics"] = { deployment_config.deployment_args.instance_type: benchmark_metrics.get( deployment_config.deployment_args.instance_type @@ -1172,5 +1173,4 @@ def deployment_config_response_data( } configs.append(deployment_config_json) - return configs diff --git a/tests/unit/sagemaker/jumpstart/test_utils.py b/tests/unit/sagemaker/jumpstart/test_utils.py index 6a2a0a2727..7e4fa7b14a 100644 --- a/tests/unit/sagemaker/jumpstart/test_utils.py +++ b/tests/unit/sagemaker/jumpstart/test_utils.py @@ -1770,7 +1770,7 @@ def test_extract_metrics_from_deployment_configs(): configs[0].benchmark_metrics = None configs[2].deployment_args = None - data = utils.get_metrics_from_deployment_configs(configs) + data = utils.get_metrics_from_deployment_configs("neuron-inference", "ml.g5.xlarge", configs) for key in data: assert len(data[key]) == (len(configs) - 2) From 55479225e3f8223f5f4f4fdf830a3c28a389032a Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 11:20:01 -0700 Subject: [PATCH 21/51] Custom lru --- src/sagemaker/jumpstart/model.py | 6 ++-- src/sagemaker/jumpstart/utils.py | 47 ++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 8a174eb6e3..a24b33324d 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -from functools import lru_cache from typing import Dict, List, Optional, Any, Union import pandas as pd from botocore.exceptions import ClientError @@ -49,6 +48,7 @@ get_metrics_from_deployment_configs, add_instance_rate_stats_to_benchmark_metrics, deployment_config_response_data, + _deployment_config_lru_cache, ) from sagemaker.jumpstart.constants import JUMPSTART_LOGGER from sagemaker.jumpstart.enums import JumpStartModelType @@ -874,7 +874,7 @@ def register_deploy_wrapper(*args, **kwargs): return model_package - @lru_cache + @_deployment_config_lru_cache def _get_deployment_configs_benchmarks_data( self, config_name: str, instance_type: str ) -> Dict[str, Any]: @@ -892,7 +892,7 @@ def _get_deployment_configs_benchmarks_data( self._get_deployment_configs(config_name, instance_type), ) - @lru_cache + @_deployment_config_lru_cache def _get_deployment_configs( self, selected_config_name: str, selected_instance_type: str ) -> List[DeploymentConfigMetadata]: diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 8c5b5e18ad..a9f3f0d054 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -14,6 +14,7 @@ from __future__ import absolute_import import logging import os +from functools import lru_cache, wraps from typing import Any, Dict, List, Set, Optional, Tuple, Union from urllib.parse import urlparse import boto3 @@ -1174,3 +1175,49 @@ def deployment_config_response_data( configs.append(deployment_config_json) return configs + + +def _deployment_config_lru_cache(_func=None, *, maxsize: int = 128, typed: bool = False): + """LRU cache for deployment configs.""" + + def has_instance_rate_metric(config: DeploymentConfigMetadata) -> bool: + """Determines whether a benchmark metric stats contains instance rate metric stat.""" + if config.benchmark_metrics is None: + return False + for benchmark_metric_stats in config.benchmark_metrics.values(): + if not has_instance_rate_stat(benchmark_metric_stats): + return False + return True + + def wrapper_cache(f): + f = lru_cache(maxsize=maxsize, typed=typed)(f) + + @wraps(f) + def wrapped_f(*args, **kwargs): + res = f(*args, **kwargs) + + if f.cache_info().hits == 1: + print("******* Not From Cache ***********") + if isinstance(res, list): + for item in res: + if isinstance( + item, DeploymentConfigMetadata + ) and not has_instance_rate_metric(item): + f.cache_clear() + break + elif isinstance(res, dict): + keys = list(res.keys()) + if len(keys) > 3 and "Instance Rate" not in keys[2]: + f.cache_clear() + else: + print("******* From Cache ***********") + + return res + + wrapped_f.cache_info = f.cache_info + wrapped_f.cache_clear = f.cache_clear + return wrapped_f + + if _func is None: + return wrapper_cache + return wrapper_cache(_func) From 3a71233a212f8abc224ebca74127ddaa123e1988 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 11:45:16 -0700 Subject: [PATCH 22/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index a9f3f0d054..34add16f5e 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1198,6 +1198,10 @@ def wrapped_f(*args, **kwargs): if f.cache_info().hits == 1: print("******* Not From Cache ***********") + print(res) + print() + print(f.cache_info().misses) + print() if isinstance(res, list): for item in res: if isinstance( From 0697473335616f17572787d3367dc76624dbc034 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 11:51:35 -0700 Subject: [PATCH 23/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 34add16f5e..c10a4f9cd9 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1200,7 +1200,7 @@ def wrapped_f(*args, **kwargs): print("******* Not From Cache ***********") print(res) print() - print(f.cache_info().misses) + print(f.cache_info()) print() if isinstance(res, list): for item in res: From 4358dad8ff313f2f04873494e24e24576c8af04d Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 11:54:59 -0700 Subject: [PATCH 24/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index c10a4f9cd9..35f36c110e 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1215,6 +1215,9 @@ def wrapped_f(*args, **kwargs): f.cache_clear() else: print("******* From Cache ***********") + print(res) + print() + print(f.cache_info()) return res From 7f715e46b0b9f6ba7db1ef661af91a0f4880c083 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 11:58:06 -0700 Subject: [PATCH 25/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 35f36c110e..6a14e7fb44 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1196,7 +1196,7 @@ def wrapper_cache(f): def wrapped_f(*args, **kwargs): res = f(*args, **kwargs) - if f.cache_info().hits == 1: + if f.cache_info().hits == 0 and f.cache_info().misses == 1: print("******* Not From Cache ***********") print(res) print() From 7c72895ed1c4920621263fab75f904d40587c234 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 12:08:20 -0700 Subject: [PATCH 26/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 6a14e7fb44..3f11ed20c2 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1211,7 +1211,10 @@ def wrapped_f(*args, **kwargs): break elif isinstance(res, dict): keys = list(res.keys()) - if len(keys) > 3 and "Instance Rate" not in keys[2]: + if "Instance Rate" not in keys[-1]: + f.cache_clear() + elif len(res[keys[1]]) > len(res[keys[-1]]): + del res[keys[-1]] f.cache_clear() else: print("******* From Cache ***********") From 707a23eaacc11015663e8dc3dccee7c1c9cbf118 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 12:15:59 -0700 Subject: [PATCH 27/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 3f11ed20c2..347fb7f9f2 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1086,7 +1086,7 @@ def has_instance_rate_stat(benchmark_metric_stats: Optional[List[JumpStartBenchm bool: Whether the benchmark metric stats contains instance rate metric stat. """ if benchmark_metric_stats is None: - return False + return True for benchmark_metric_stat in benchmark_metric_stats: if benchmark_metric_stat.name.lower() == "instance rate": return True @@ -1183,7 +1183,7 @@ def _deployment_config_lru_cache(_func=None, *, maxsize: int = 128, typed: bool def has_instance_rate_metric(config: DeploymentConfigMetadata) -> bool: """Determines whether a benchmark metric stats contains instance rate metric stat.""" if config.benchmark_metrics is None: - return False + return True for benchmark_metric_stats in config.benchmark_metrics.values(): if not has_instance_rate_stat(benchmark_metric_stats): return False From 071e9c498771b8f9dc3491ce69f716051bf3a3a5 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 12:22:09 -0700 Subject: [PATCH 28/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 347fb7f9f2..1758b38f95 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1213,9 +1213,9 @@ def wrapped_f(*args, **kwargs): keys = list(res.keys()) if "Instance Rate" not in keys[-1]: f.cache_clear() - elif len(res[keys[1]]) > len(res[keys[-1]]): - del res[keys[-1]] - f.cache_clear() + # elif len(res[keys[1]]) > len(res[keys[-1]]): + # del res[keys[-1]] + # f.cache_clear() else: print("******* From Cache ***********") print(res) From d122b06955656e52caf3db6988af855303d7e95b Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 12:30:25 -0700 Subject: [PATCH 29/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 1758b38f95..8fd8bfa582 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1197,11 +1197,6 @@ def wrapped_f(*args, **kwargs): res = f(*args, **kwargs) if f.cache_info().hits == 0 and f.cache_info().misses == 1: - print("******* Not From Cache ***********") - print(res) - print() - print(f.cache_info()) - print() if isinstance(res, list): for item in res: if isinstance( @@ -1213,14 +1208,9 @@ def wrapped_f(*args, **kwargs): keys = list(res.keys()) if "Instance Rate" not in keys[-1]: f.cache_clear() - # elif len(res[keys[1]]) > len(res[keys[-1]]): - # del res[keys[-1]] - # f.cache_clear() - else: - print("******* From Cache ***********") - print(res) - print() - print(f.cache_info()) + elif len(res[keys[1]]) > len(res[keys[-1]]): + del res[keys[-1]] + f.cache_clear() return res From 01568fde70cb1a86971d131863796f449e5be520 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 12:57:27 -0700 Subject: [PATCH 30/51] Custom lru --- src/sagemaker/jumpstart/utils.py | 12 ++++++++++-- tests/unit/sagemaker/jumpstart/test_utils.py | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 8fd8bfa582..9502b05411 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1181,7 +1181,13 @@ def _deployment_config_lru_cache(_func=None, *, maxsize: int = 128, typed: bool """LRU cache for deployment configs.""" def has_instance_rate_metric(config: DeploymentConfigMetadata) -> bool: - """Determines whether a benchmark metric stats contains instance rate metric stat.""" + """Determines whether metadata config contains instance rate metric stat. + + Args: + config (DeploymentConfigMetadata): Metadata config metadata. + Returns: + bool: Whether the metadata config contains instance rate metric stat. + """ if config.benchmark_metrics is None: return True for benchmark_metric_stats in config.benchmark_metrics.values(): @@ -1196,6 +1202,9 @@ def wrapper_cache(f): def wrapped_f(*args, **kwargs): res = f(*args, **kwargs) + # Clear cache on first call if + # - The output does not contain Instant rate metrics + # as this is caused by missing policy. if f.cache_info().hits == 0 and f.cache_info().misses == 1: if isinstance(res, list): for item in res: @@ -1211,7 +1220,6 @@ def wrapped_f(*args, **kwargs): elif len(res[keys[1]]) > len(res[keys[-1]]): del res[keys[-1]] f.cache_clear() - return res wrapped_f.cache_info = f.cache_info diff --git a/tests/unit/sagemaker/jumpstart/test_utils.py b/tests/unit/sagemaker/jumpstart/test_utils.py index 7e4fa7b14a..b722473a78 100644 --- a/tests/unit/sagemaker/jumpstart/test_utils.py +++ b/tests/unit/sagemaker/jumpstart/test_utils.py @@ -1835,7 +1835,7 @@ def test_add_instance_rate_stats_to_benchmark_metrics_client_ex( @pytest.mark.parametrize( "stats, expected", [ - (None, False), + (None, True), ( [JumpStartBenchmarkStat({"name": "Instance Rate", "unit": "USD/Hrs", "value": "3.76"})], True, From 10430ea5b56eb52b3bca23e0171bcc57f74bace2 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 15:42:52 -0700 Subject: [PATCH 31/51] Refactoring --- src/sagemaker/jumpstart/model.py | 31 +++++++++----------- src/sagemaker/jumpstart/utils.py | 8 ++--- tests/unit/sagemaker/jumpstart/test_utils.py | 11 +++++-- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index a24b33324d..9b2b0123e6 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -14,6 +14,7 @@ from __future__ import absolute_import +import copy from typing import Dict, List, Optional, Any, Union import pandas as pd from botocore.exceptions import ClientError @@ -361,6 +362,8 @@ def _validate_model_id_and_type(): self.model_package_arn = model_init_kwargs.model_package_arn self.init_kwargs = model_init_kwargs.to_kwargs_dict(False) + self._default_config_name = copy.deepcopy(self.config_name) + self._default_instance_type = copy.deepcopy(self.instance_type) self._metadata_configs = get_jumpstart_configs( region=self.region, model_id=self.model_id, @@ -464,13 +467,12 @@ def benchmark_metrics(self) -> pd.DataFrame: Returns: Benchmark Metrics: Pandas DataFrame object. """ - benchmark_metrics_data = self._get_deployment_configs_benchmarks_data( - self.config_name, self.instance_type - ) - keys = list(benchmark_metrics_data.keys()) + benchmark_metrics_data = self._get_deployment_configs_benchmarks_data() # Sort by Config Name and Instance Type column values - df = pd.DataFrame(benchmark_metrics_data).sort_values(by=[keys[1], keys[0]]) - return df + df = pd.DataFrame(benchmark_metrics_data) + default_mask = df.apply(lambda raw: any("Default" in str(val) for val in raw), axis=1) + sorted_df = pd.concat([df[default_mask], df[~default_mask]]) + return sorted_df def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" @@ -875,21 +877,16 @@ def register_deploy_wrapper(*args, **kwargs): return model_package @_deployment_config_lru_cache - def _get_deployment_configs_benchmarks_data( - self, config_name: str, instance_type: str - ) -> Dict[str, Any]: + def _get_deployment_configs_benchmarks_data(self) -> Dict[str, Any]: """Deployment configs benchmark metrics. - Args: - config_name (str): Name of selected deployment config. - instance_type (str): The selected Instance type. Returns: Dict[str, List[str]]: Deployment config benchmark data. """ return get_metrics_from_deployment_configs( - config_name, - instance_type, - self._get_deployment_configs(config_name, instance_type), + self._default_config_name, + self._default_instance_type, + self._get_deployment_configs(self._default_config_name, self._default_instance_type), ) @_deployment_config_lru_cache @@ -940,9 +937,9 @@ def _get_deployment_configs( ) deployment_configs.append(deployment_config_metadata) - if err and "is not authorized to perform: pricing:GetProducts" in err: + if err and err["Code"] == "AccessDeniedException": error_message = "Instance rate metrics will be omitted. Reason: %s" - JUMPSTART_LOGGER.warning(error_message, err) + JUMPSTART_LOGGER.warning(error_message, err["Message"]) return deployment_configs diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 9502b05411..88a8423cb0 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1036,15 +1036,15 @@ def get_jumpstart_configs( def add_instance_rate_stats_to_benchmark_metrics( region: str, benchmark_metrics: Optional[Dict[str, List[JumpStartBenchmarkStat]]], -) -> Optional[Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]]: +) -> Optional[Tuple[Dict[str, str], Dict[str, List[JumpStartBenchmarkStat]]]]: """Adds instance types metric stats to the given benchmark_metrics dict. Args: region (str): AWS region. benchmark_metrics (Optional[Dict[str, List[JumpStartBenchmarkStat]]]): Returns: - Optional[Tuple[str, Dict[str, List[JumpStartBenchmarkStat]]]]: - Contains Error message and metrics dict. + Optional[Tuple[Dict[str, str], Dict[str, List[JumpStartBenchmarkStat]]]]: + Contains Error and metrics. """ if not benchmark_metrics: return None @@ -1067,7 +1067,7 @@ def add_instance_rate_stats_to_benchmark_metrics( final_benchmark_metrics[instance_type] = benchmark_metric_stats except ClientError as e: final_benchmark_metrics[instance_type] = benchmark_metric_stats - err_message = e.response["Error"]["Message"] + err_message = e.response["Error"] except Exception: # pylint: disable=W0703 final_benchmark_metrics[instance_type] = benchmark_metric_stats else: diff --git a/tests/unit/sagemaker/jumpstart/test_utils.py b/tests/unit/sagemaker/jumpstart/test_utils.py index b722473a78..59c1bf0792 100644 --- a/tests/unit/sagemaker/jumpstart/test_utils.py +++ b/tests/unit/sagemaker/jumpstart/test_utils.py @@ -1815,7 +1815,13 @@ def test_add_instance_rate_stats_to_benchmark_metrics_client_ex( mock_get_instance_rate_per_hour, ): mock_get_instance_rate_per_hour.side_effect = ClientError( - {"Error": {"Message": "is not authorized to perform: pricing:GetProducts"}}, "GetProducts" + { + "Error": { + "Message": "is not authorized to perform: pricing:GetProducts", + "Code": "AccessDenied", + }, + }, + "GetProducts", ) err, out = utils.add_instance_rate_stats_to_benchmark_metrics( @@ -1827,7 +1833,8 @@ def test_add_instance_rate_stats_to_benchmark_metrics_client_ex( }, ) - assert err == "is not authorized to perform: pricing:GetProducts" + assert err["Message"] == "is not authorized to perform: pricing:GetProducts" + assert err["Code"] == "AccessDenied" for key in out: assert len(out[key]) == 1 From 296cc799eb0811a4473dfcc5c4ff260028f2043a Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 16:17:33 -0700 Subject: [PATCH 32/51] Debug --- src/sagemaker/jumpstart/model.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 9b2b0123e6..0bb7842b74 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -906,6 +906,11 @@ def _get_deployment_configs( err = None for config_name, metadata_config in self._metadata_configs.items(): resolved_config = metadata_config.resolved_config + + print("*************************") + print(resolved_config) + print("*************************\n") + if selected_config_name == config_name: instance_type_to_use = selected_instance_type else: From 73118f6137244f54f99189b05811f4f2013084b1 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:10:13 -0700 Subject: [PATCH 33/51] Config ranking --- src/sagemaker/jumpstart/model.py | 5 ----- src/sagemaker/jumpstart/utils.py | 7 +++---- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 0bb7842b74..9b2b0123e6 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -906,11 +906,6 @@ def _get_deployment_configs( err = None for config_name, metadata_config in self._metadata_configs.items(): resolved_config = metadata_config.resolved_config - - print("*************************") - print(resolved_config) - print("*************************\n") - if selected_config_name == config_name: instance_type_to_use = selected_instance_type else: diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 88a8423cb0..88cde29dc1 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1024,7 +1024,7 @@ def get_jumpstart_configs( raise ValueError(f"Unknown script scope: {scope}.") if not config_names: - config_names = metadata_configs.configs.keys() if metadata_configs else [] + config_names = metadata_configs.config_rankings.get("rankings", []) if metadata_configs else [] return ( {config_name: metadata_configs.configs[config_name] for config_name in config_names} @@ -1113,7 +1113,7 @@ def get_metrics_from_deployment_configs( data = {"Instance Type": [], "Config Name": []} instance_rate_data = {} - for deployment_config in deployment_configs: + for index, deployment_config in enumerate(deployment_configs): benchmark_metrics = deployment_config.benchmark_metrics if not deployment_config.deployment_args or not benchmark_metrics: continue @@ -1124,8 +1124,7 @@ def get_metrics_from_deployment_configs( data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( f"{current_instance_type} (Default)" - if current_instance_type == default_instance_type - and deployment_config.deployment_config_name == default_config_name + if index == 0 else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From a5698712c14f4c629c7d4b1f35c3801a39b414c2 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:14:27 -0700 Subject: [PATCH 34/51] Debug --- src/sagemaker/jumpstart/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 88cde29dc1..16f4a2be48 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1023,6 +1023,10 @@ def get_jumpstart_configs( else: raise ValueError(f"Unknown script scope: {scope}.") + print("******************************") + print(metadata_configs.config_rankings) + print("******************************") + if not config_names: config_names = metadata_configs.config_rankings.get("rankings", []) if metadata_configs else [] From c2cfff293faea16f939abebc8960ee4a961efe4d Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:16:12 -0700 Subject: [PATCH 35/51] Debug --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 16f4a2be48..9d7531430e 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1028,7 +1028,7 @@ def get_jumpstart_configs( print("******************************") if not config_names: - config_names = metadata_configs.config_rankings.get("rankings", []) if metadata_configs else [] + config_names = metadata_configs.configs.keys() if metadata_configs else [] return ( {config_name: metadata_configs.configs[config_name] for config_name in config_names} From 360033f6bcbda2c7f7780865fabe490193e04a4d Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:18:50 -0700 Subject: [PATCH 36/51] Debug --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 9d7531430e..5822c30575 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1024,7 +1024,7 @@ def get_jumpstart_configs( raise ValueError(f"Unknown script scope: {scope}.") print("******************************") - print(metadata_configs.config_rankings) + print(metadata_configs.config_rankings.get("overall").rankings) print("******************************") if not config_names: From ce2b50b9faf3cbf159f7f03a5876af6b9a2be0dc Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:21:17 -0700 Subject: [PATCH 37/51] Debug --- src/sagemaker/jumpstart/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 5822c30575..de644c1eaf 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1023,12 +1023,12 @@ def get_jumpstart_configs( else: raise ValueError(f"Unknown script scope: {scope}.") - print("******************************") - print(metadata_configs.config_rankings.get("overall").rankings) - print("******************************") + # print("******************************") + # print(metadata_configs.config_rankings.get("overall").rankings) + # print("******************************") if not config_names: - config_names = metadata_configs.configs.keys() if metadata_configs else [] + config_names = metadata_configs.config_rankings.get("overall").rankings if metadata_configs else [] return ( {config_name: metadata_configs.configs[config_name] for config_name in config_names} From 8eab467dfe9f545e6fe2208cdf159ea8d3327221 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:25:42 -0700 Subject: [PATCH 38/51] Debug --- src/sagemaker/jumpstart/utils.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index de644c1eaf..e25ac7b0ed 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1023,10 +1023,6 @@ def get_jumpstart_configs( else: raise ValueError(f"Unknown script scope: {scope}.") - # print("******************************") - # print(metadata_configs.config_rankings.get("overall").rankings) - # print("******************************") - if not config_names: config_names = metadata_configs.config_rankings.get("overall").rankings if metadata_configs else [] @@ -1128,7 +1124,7 @@ def get_metrics_from_deployment_configs( data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( f"{current_instance_type} (Default)" - if index == 0 + if index == 0 and current_instance_type == deployment_config.deployment_args.default_instance_type else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From e92339202f25f3c4fa597167de057f961b0d03c1 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 17:50:15 -0700 Subject: [PATCH 39/51] Ranking --- src/sagemaker/jumpstart/model.py | 20 +++++--------------- src/sagemaker/jumpstart/utils.py | 11 +++++------ tests/unit/sagemaker/jumpstart/test_utils.py | 2 +- tests/unit/sagemaker/jumpstart/utils.py | 3 ++- 4 files changed, 13 insertions(+), 23 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 9b2b0123e6..fca3ebf772 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -14,7 +14,6 @@ from __future__ import absolute_import -import copy from typing import Dict, List, Optional, Any, Union import pandas as pd from botocore.exceptions import ClientError @@ -362,8 +361,6 @@ def _validate_model_id_and_type(): self.model_package_arn = model_init_kwargs.model_package_arn self.init_kwargs = model_init_kwargs.to_kwargs_dict(False) - self._default_config_name = copy.deepcopy(self.config_name) - self._default_instance_type = copy.deepcopy(self.instance_type) self._metadata_configs = get_jumpstart_configs( region=self.region, model_id=self.model_id, @@ -467,12 +464,7 @@ def benchmark_metrics(self) -> pd.DataFrame: Returns: Benchmark Metrics: Pandas DataFrame object. """ - benchmark_metrics_data = self._get_deployment_configs_benchmarks_data() - # Sort by Config Name and Instance Type column values - df = pd.DataFrame(benchmark_metrics_data) - default_mask = df.apply(lambda raw: any("Default" in str(val) for val in raw), axis=1) - sorted_df = pd.concat([df[default_mask], df[~default_mask]]) - return sorted_df + return pd.DataFrame(self._get_deployment_configs_benchmarks_data()) def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" @@ -884,20 +876,18 @@ def _get_deployment_configs_benchmarks_data(self) -> Dict[str, Any]: Dict[str, List[str]]: Deployment config benchmark data. """ return get_metrics_from_deployment_configs( - self._default_config_name, - self._default_instance_type, - self._get_deployment_configs(self._default_config_name, self._default_instance_type), + self._get_deployment_configs(None, None), ) @_deployment_config_lru_cache def _get_deployment_configs( - self, selected_config_name: str, selected_instance_type: str + self, selected_config_name: Optional[str], selected_instance_type: Optional[str] ) -> List[DeploymentConfigMetadata]: """Retrieve deployment configs metadata. Args: - selected_config_name (str): The name of the selected deployment config. - selected_instance_type (str): The selected instance type. + selected_config_name (Optional[str]): The name of the selected deployment config. + selected_instance_type (Optional[str]): The selected instance type. """ deployment_configs = [] if not self._metadata_configs: diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index e25ac7b0ed..351c914bdd 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1024,7 +1024,9 @@ def get_jumpstart_configs( raise ValueError(f"Unknown script scope: {scope}.") if not config_names: - config_names = metadata_configs.config_rankings.get("overall").rankings if metadata_configs else [] + config_names = ( + metadata_configs.config_rankings.get("overall").rankings if metadata_configs else [] + ) return ( {config_name: metadata_configs.configs[config_name] for config_name in config_names} @@ -1094,15 +1096,11 @@ def has_instance_rate_stat(benchmark_metric_stats: Optional[List[JumpStartBenchm def get_metrics_from_deployment_configs( - default_config_name: Optional[str], - default_instance_type: Optional[str], deployment_configs: Optional[List[DeploymentConfigMetadata]], ) -> Dict[str, List[str]]: """Extracts benchmark metrics from deployment configs metadata. Args: - default_config_name (Optional[str]): The name of the default deployment config. - default_instance_type (Optional[str]): The name of the default instance type. deployment_configs (Optional[List[DeploymentConfigMetadata]]): List of deployment configs metadata. Returns: @@ -1124,7 +1122,8 @@ def get_metrics_from_deployment_configs( data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( f"{current_instance_type} (Default)" - if index == 0 and current_instance_type == deployment_config.deployment_args.default_instance_type + if index == 0 + and current_instance_type == deployment_config.deployment_args.default_instance_type else current_instance_type ) data["Instance Type"].append(instance_type_to_display) diff --git a/tests/unit/sagemaker/jumpstart/test_utils.py b/tests/unit/sagemaker/jumpstart/test_utils.py index 59c1bf0792..4c584f5046 100644 --- a/tests/unit/sagemaker/jumpstart/test_utils.py +++ b/tests/unit/sagemaker/jumpstart/test_utils.py @@ -1770,7 +1770,7 @@ def test_extract_metrics_from_deployment_configs(): configs[0].benchmark_metrics = None configs[2].deployment_args = None - data = utils.get_metrics_from_deployment_configs("neuron-inference", "ml.g5.xlarge", configs) + data = utils.get_metrics_from_deployment_configs(configs) for key in data: assert len(data[key]) == (len(configs) - 2) diff --git a/tests/unit/sagemaker/jumpstart/utils.py b/tests/unit/sagemaker/jumpstart/utils.py index b90da25061..63b964e16e 100644 --- a/tests/unit/sagemaker/jumpstart/utils.py +++ b/tests/unit/sagemaker/jumpstart/utils.py @@ -358,7 +358,8 @@ def get_base_deployment_configs_metadata( else get_base_spec_with_prototype_configs() ) configs = [] - for config_name, jumpstart_config in specs.inference_configs.configs.items(): + for config_name in specs.inference_configs.config_rankings.get("overall").rankings: + jumpstart_config = specs.inference_configs.configs.get(config_name) benchmark_metrics = jumpstart_config.benchmark_metrics if benchmark_metrics: From 42f94342ac1efb1dd8d6947087005204c9bc133c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 18:01:25 -0700 Subject: [PATCH 40/51] Ranking-Debug --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 351c914bdd..88b0e70b84 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1123,7 +1123,7 @@ def get_metrics_from_deployment_configs( instance_type_to_display = ( f"{current_instance_type} (Default)" if index == 0 - and current_instance_type == deployment_config.deployment_args.default_instance_type + # and current_instance_type == deployment_config.deployment_args.default_instance_type else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From a2762532a630829da0b15ebfe75468ab42b07351 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 22:26:05 -0700 Subject: [PATCH 41/51] Ranking-Debug --- src/sagemaker/jumpstart/model.py | 2 +- src/sagemaker/jumpstart/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index fca3ebf772..220e35c5fe 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -464,7 +464,7 @@ def benchmark_metrics(self) -> pd.DataFrame: Returns: Benchmark Metrics: Pandas DataFrame object. """ - return pd.DataFrame(self._get_deployment_configs_benchmarks_data()) + return pd.DataFrame(self._get_deployment_configs_benchmarks_data()).df.apply(lambda raw: any('Default' in str(val) for val in raw), axis=1) def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 88b0e70b84..351c914bdd 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1123,7 +1123,7 @@ def get_metrics_from_deployment_configs( instance_type_to_display = ( f"{current_instance_type} (Default)" if index == 0 - # and current_instance_type == deployment_config.deployment_args.default_instance_type + and current_instance_type == deployment_config.deployment_args.default_instance_type else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From a6bd6608aa40c96fe5c931bb4e2579b4d638993c Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 22:27:48 -0700 Subject: [PATCH 42/51] Ranking-Debug --- src/sagemaker/jumpstart/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 220e35c5fe..7fe5b24e82 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -464,7 +464,7 @@ def benchmark_metrics(self) -> pd.DataFrame: Returns: Benchmark Metrics: Pandas DataFrame object. """ - return pd.DataFrame(self._get_deployment_configs_benchmarks_data()).df.apply(lambda raw: any('Default' in str(val) for val in raw), axis=1) + return pd.DataFrame(self._get_deployment_configs_benchmarks_data()).apply(lambda raw: any('Default' in str(val) for val in raw), axis=1) def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" From 64211009f24cee82a1bea1335c5496f1697dde09 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 22:47:10 -0700 Subject: [PATCH 43/51] Ranking-Debug --- src/sagemaker/jumpstart/model.py | 2 +- src/sagemaker/jumpstart/utils.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 7fe5b24e82..fca3ebf772 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -464,7 +464,7 @@ def benchmark_metrics(self) -> pd.DataFrame: Returns: Benchmark Metrics: Pandas DataFrame object. """ - return pd.DataFrame(self._get_deployment_configs_benchmarks_data()).apply(lambda raw: any('Default' in str(val) for val in raw), axis=1) + return pd.DataFrame(self._get_deployment_configs_benchmarks_data()) def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 351c914bdd..46c4f7edad 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1116,14 +1116,22 @@ def get_metrics_from_deployment_configs( if not deployment_config.deployment_args or not benchmark_metrics: continue - for inner_index, current_instance_type in enumerate(benchmark_metrics): + ranking_benchmark_metrics = {} + if index == 0: + ranking_benchmark_metrics[deployment_config.deployment_args.default_instance_type] = benchmark_metrics.get(deployment_config.deployment_args.default_instance_type) + del benchmark_metrics[deployment_config.deployment_args.default_instance_type] + ranking_benchmark_metrics = {**ranking_benchmark_metrics, **benchmark_metrics} + else: + ranking_benchmark_metrics = benchmark_metrics + + for inner_index, current_instance_type in enumerate(ranking_benchmark_metrics): current_instance_type_metrics = benchmark_metrics[current_instance_type] data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( f"{current_instance_type} (Default)" if index == 0 - and current_instance_type == deployment_config.deployment_args.default_instance_type + # and current_instance_type == deployment_config.deployment_args.default_instance_type else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From a4c58bed9c821b2fa2930d689c833480a63c4975 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 22:49:13 -0700 Subject: [PATCH 44/51] Ranking-Debug --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 46c4f7edad..094a391279 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1125,7 +1125,7 @@ def get_metrics_from_deployment_configs( ranking_benchmark_metrics = benchmark_metrics for inner_index, current_instance_type in enumerate(ranking_benchmark_metrics): - current_instance_type_metrics = benchmark_metrics[current_instance_type] + current_instance_type_metrics = ranking_benchmark_metrics[current_instance_type] data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( From 9c991442fc89353bdf4ed7e4c8df7e51e1928125 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 22:50:36 -0700 Subject: [PATCH 45/51] Ranking-Debug --- src/sagemaker/jumpstart/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 094a391279..4060915eea 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1131,7 +1131,7 @@ def get_metrics_from_deployment_configs( instance_type_to_display = ( f"{current_instance_type} (Default)" if index == 0 - # and current_instance_type == deployment_config.deployment_args.default_instance_type + and current_instance_type == deployment_config.deployment_args.default_instance_type else current_instance_type ) data["Instance Type"].append(instance_type_to_display) From 8e1cb87d58db97b8b94d36915ca36eff4b573ec5 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Wed, 1 May 2024 23:01:30 -0700 Subject: [PATCH 46/51] Debug --- src/sagemaker/jumpstart/utils.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 4060915eea..dec49ef6a9 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -12,6 +12,8 @@ # language governing permissions and limitations under the License. """This module contains utilities related to SageMaker JumpStart.""" from __future__ import absolute_import + +import copy import logging import os from functools import lru_cache, wraps @@ -1116,16 +1118,19 @@ def get_metrics_from_deployment_configs( if not deployment_config.deployment_args or not benchmark_metrics: continue - ranking_benchmark_metrics = {} + copy_benchmark_metrics = copy.deepcopy(benchmark_metrics) + benchmark_metrics = {} if index == 0: - ranking_benchmark_metrics[deployment_config.deployment_args.default_instance_type] = benchmark_metrics.get(deployment_config.deployment_args.default_instance_type) - del benchmark_metrics[deployment_config.deployment_args.default_instance_type] - ranking_benchmark_metrics = {**ranking_benchmark_metrics, **benchmark_metrics} + benchmark_metrics[deployment_config.deployment_args.default_instance_type] = ( + copy_benchmark_metrics.get(deployment_config.deployment_args.default_instance_type) + ) + del copy_benchmark_metrics[deployment_config.deployment_args.default_instance_type] + benchmark_metrics = {**benchmark_metrics, **copy_benchmark_metrics} else: - ranking_benchmark_metrics = benchmark_metrics + benchmark_metrics = copy_benchmark_metrics - for inner_index, current_instance_type in enumerate(ranking_benchmark_metrics): - current_instance_type_metrics = ranking_benchmark_metrics[current_instance_type] + for inner_index, current_instance_type in enumerate(benchmark_metrics): + current_instance_type_metrics = benchmark_metrics[current_instance_type] data["Config Name"].append(deployment_config.deployment_config_name) instance_type_to_display = ( From 3aa1f160401451782779e2c09772103d18b0f2a8 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Thu, 2 May 2024 08:49:53 -0700 Subject: [PATCH 47/51] Debug --- src/sagemaker/jumpstart/utils.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index dec49ef6a9..7d17631eac 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1118,16 +1118,11 @@ def get_metrics_from_deployment_configs( if not deployment_config.deployment_args or not benchmark_metrics: continue - copy_benchmark_metrics = copy.deepcopy(benchmark_metrics) - benchmark_metrics = {} - if index == 0: - benchmark_metrics[deployment_config.deployment_args.default_instance_type] = ( - copy_benchmark_metrics.get(deployment_config.deployment_args.default_instance_type) - ) - del copy_benchmark_metrics[deployment_config.deployment_args.default_instance_type] - benchmark_metrics = {**benchmark_metrics, **copy_benchmark_metrics} - else: - benchmark_metrics = copy_benchmark_metrics + default_instance_type = deployment_config.deployment_args.default_instance_type + benchmark_metrics = { + default_instance_type: benchmark_metrics.pop(default_instance_type), + **benchmark_metrics, + } for inner_index, current_instance_type in enumerate(benchmark_metrics): current_instance_type_metrics = benchmark_metrics[current_instance_type] From a9d4a7e1f34cddf09efcd4e09c9e1e89efcda4f8 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Thu, 2 May 2024 08:54:08 -0700 Subject: [PATCH 48/51] Debug --- src/sagemaker/jumpstart/utils.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 7d17631eac..7a199e5902 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1119,10 +1119,11 @@ def get_metrics_from_deployment_configs( continue default_instance_type = deployment_config.deployment_args.default_instance_type - benchmark_metrics = { - default_instance_type: benchmark_metrics.pop(default_instance_type), - **benchmark_metrics, - } + if default_instance_type in benchmark_metrics: + benchmark_metrics = { + default_instance_type: benchmark_metrics.pop(default_instance_type), + **benchmark_metrics, + } for inner_index, current_instance_type in enumerate(benchmark_metrics): current_instance_type_metrics = benchmark_metrics[current_instance_type] From 477e272da61069a7b6a82f00d3545f2ab5e29a41 Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Thu, 2 May 2024 09:04:39 -0700 Subject: [PATCH 49/51] Debug --- src/sagemaker/jumpstart/model.py | 5 ++++- src/sagemaker/jumpstart/utils.py | 7 ------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index fca3ebf772..5bdbb8fd1d 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -464,7 +464,10 @@ def benchmark_metrics(self) -> pd.DataFrame: Returns: Benchmark Metrics: Pandas DataFrame object. """ - return pd.DataFrame(self._get_deployment_configs_benchmarks_data()) + df = pd.DataFrame(self._get_deployment_configs_benchmarks_data()) + default_mask = df.apply(lambda raw: any('Default' in str(val) for val in raw), axis=1) + sorted_df = pd.concat([df[default_mask], df[~default_mask]]) + return sorted_df def display_benchmark_metrics(self, *args, **kwargs) -> None: """Display deployment configs benchmark metrics.""" diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 7a199e5902..888194212e 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -1118,13 +1118,6 @@ def get_metrics_from_deployment_configs( if not deployment_config.deployment_args or not benchmark_metrics: continue - default_instance_type = deployment_config.deployment_args.default_instance_type - if default_instance_type in benchmark_metrics: - benchmark_metrics = { - default_instance_type: benchmark_metrics.pop(default_instance_type), - **benchmark_metrics, - } - for inner_index, current_instance_type in enumerate(benchmark_metrics): current_instance_type_metrics = benchmark_metrics[current_instance_type] From 2c363af2f5e13cbd63e59566499f8ea122efa4fe Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Thu, 2 May 2024 09:13:36 -0700 Subject: [PATCH 50/51] Refactoring --- src/sagemaker/jumpstart/model.py | 2 +- src/sagemaker/jumpstart/utils.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sagemaker/jumpstart/model.py b/src/sagemaker/jumpstart/model.py index 5bdbb8fd1d..6f263d9a7e 100644 --- a/src/sagemaker/jumpstart/model.py +++ b/src/sagemaker/jumpstart/model.py @@ -465,7 +465,7 @@ def benchmark_metrics(self) -> pd.DataFrame: Benchmark Metrics: Pandas DataFrame object. """ df = pd.DataFrame(self._get_deployment_configs_benchmarks_data()) - default_mask = df.apply(lambda raw: any('Default' in str(val) for val in raw), axis=1) + default_mask = df.apply(lambda row: any("Default" in str(val) for val in row), axis=1) sorted_df = pd.concat([df[default_mask], df[~default_mask]]) return sorted_df diff --git a/src/sagemaker/jumpstart/utils.py b/src/sagemaker/jumpstart/utils.py index 888194212e..0cf46815fa 100644 --- a/src/sagemaker/jumpstart/utils.py +++ b/src/sagemaker/jumpstart/utils.py @@ -13,7 +13,6 @@ """This module contains utilities related to SageMaker JumpStart.""" from __future__ import absolute_import -import copy import logging import os from functools import lru_cache, wraps From 7b8969ea21eb82facfad218fc961bf3c914a34cb Mon Sep 17 00:00:00 2001 From: Jonathan Makunga Date: Thu, 2 May 2024 09:47:23 -0700 Subject: [PATCH 51/51] Contact JumpStart team to fix flaky test. test_list_jumpstart_models_script_filter --- tests/unit/sagemaker/jumpstart/test_notebook_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/sagemaker/jumpstart/test_notebook_utils.py b/tests/unit/sagemaker/jumpstart/test_notebook_utils.py index c00d271ef1..c7a68c0307 100644 --- a/tests/unit/sagemaker/jumpstart/test_notebook_utils.py +++ b/tests/unit/sagemaker/jumpstart/test_notebook_utils.py @@ -228,7 +228,7 @@ def test_list_jumpstart_models_simple_case( patched_get_model_specs.assert_not_called() @pytest.mark.skipif( - datetime.datetime.now() < datetime.datetime(year=2024, month=5, day=1), + datetime.datetime.now() < datetime.datetime(year=2024, month=7, day=1), reason="Contact JumpStart team to fix flaky test.", ) @patch("sagemaker.jumpstart.accessors.JumpStartModelsAccessor._get_manifest")