diff --git a/README.md b/README.md index fe2daad..68a28c6 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,32 @@ - [Open source code for setup, install, and execution of software, with complete developer documentation](https://intel-retail.github.io/documentation/performance-tools/benchmark.html) - [Developer focused website to enable developers to engage and build our partner community](https://www.intel.com/content/www/us/en/developer/articles/reference-implementation/automated-self-checkout.html) +## Benchmark Target Device + +Benchmark scripts support explicit target-device selection via `--target_device` or `TARGET_DEVICE`. + +Accepted values: + +- `CPU` +- `GPU` +- `GPU.` (for example: `GPU.0`, `GPU.1`, `GPU.2`) +- `NPU` + +Examples: + +```bash +# Existing workflow (still supported) +make benchmark TARGET_DEVICE=GPU + +# Explicit GPU device selection +make benchmark TARGET_DEVICE=GPU.1 + +# Direct script usage +python benchmark-scripts/benchmark.py \ + --compose_file ./docker/docker-compose.yaml \ + --target_device GPU.1 +``` + ## Disclaimer GStreamer is an open source framework licensed under LGPL. See https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language=c. You are solely responsible for determining if your use of Gstreamer requires any additional licenses. Intel is not responsible for obtaining any such licenses, nor liable for any licensing fees due, in connection with your use of Gstreamer diff --git a/benchmark-scripts/Makefile b/benchmark-scripts/Makefile index bfca254..3725c68 100644 --- a/benchmark-scripts/Makefile +++ b/benchmark-scripts/Makefile @@ -14,7 +14,7 @@ plot: init-packages python3 usage_graph_plot.py --dir $(ROOT_DIRECTORY)/ python-test: - python -m coverage run -m unittest benchmark_test.py stream_density_test.py + python -m coverage run -m unittest benchmark_test.py stream_density_test.py device_validation_test.py python-integration: python -m coverage run -m unittest benchmark_integration.py diff --git a/benchmark-scripts/benchmark.py b/benchmark-scripts/benchmark.py index 2ede4a7..52568cb 100644 --- a/benchmark-scripts/benchmark.py +++ b/benchmark-scripts/benchmark.py @@ -13,6 +13,7 @@ import csv import json import stream_density +from device_validation import validate_target_device, resolve_target_device_default def parse_args(print=False): @@ -76,8 +77,13 @@ def parse_args(print=False): help='initial time in seconds before ' + 'starting metric data collection') # TODO: change target_device to an env variable in docker compose - parser.add_argument('--target_device', default='CPU', - help='desired running platform [cpu|core|xeon|dgpu.x]') + try: + default_target_device = resolve_target_device_default('CPU') + except argparse.ArgumentTypeError as exc: + parser.error(str(exc)) + parser.add_argument('--target_device', default=default_target_device, + type=validate_target_device, + help='desired running platform [CPU|GPU|GPU.|NPU]') parser.add_argument('--compose_file', default=None, action='append', help='path to docker compose files. ' + 'can be used multiple times') diff --git a/benchmark-scripts/benchmark_order_accuracy.py b/benchmark-scripts/benchmark_order_accuracy.py index e8d5e2e..b6ab804 100644 --- a/benchmark-scripts/benchmark_order_accuracy.py +++ b/benchmark-scripts/benchmark_order_accuracy.py @@ -25,6 +25,7 @@ import traceback from pathlib import Path from typing import List, Dict, Optional +from device_validation import validate_target_device, resolve_target_device_default # Import from performance-tools benchmark scripts sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) @@ -686,12 +687,16 @@ def parse_args(): help='Directory for results output' ) + try: + default_target_device = resolve_target_device_default('GPU') + except argparse.ArgumentTypeError as exc: + parser.error(str(exc)) + parser.add_argument( '--target_device', - type=str, - default='GPU', - choices=['CPU', 'GPU', 'NPU'], - help='Target inference device' + type=validate_target_device, + default=default_target_device, + help='Target inference device [CPU|GPU|GPU.|NPU]' ) parser.add_argument( diff --git a/benchmark-scripts/benchmark_test.py b/benchmark-scripts/benchmark_test.py index 581bca9..b05f63d 100644 --- a/benchmark-scripts/benchmark_test.py +++ b/benchmark-scripts/benchmark_test.py @@ -55,6 +55,5 @@ def test_docker_compose_containers_fail(self): mock_popen.communicate.assert_called_once_with() mock_returncode.assert_called() - if __name__ == '__main__': unittest.main() diff --git a/benchmark-scripts/device_validation.py b/benchmark-scripts/device_validation.py new file mode 100644 index 0000000..2bb6edb --- /dev/null +++ b/benchmark-scripts/device_validation.py @@ -0,0 +1,49 @@ +""" +Utilities for validating benchmark target device arguments. +""" + +import argparse +import os +import re + + +_GPU_INDEX_PATTERN = re.compile(r"^GPU\.(\d+)$", re.IGNORECASE) + + +def validate_target_device(value: str) -> str: + """Validate and normalize target device values. + + Accepted values: + - CPU + - GPU + - GPU. where index is a non-negative integer + - NPU + """ + normalized = value.strip() + upper_value = normalized.upper() + + if upper_value in {"CPU", "GPU", "NPU"}: + return upper_value + + gpu_match = _GPU_INDEX_PATTERN.fullmatch(normalized) + if gpu_match: + return f"GPU.{gpu_match.group(1)}" + + raise argparse.ArgumentTypeError( + "invalid target device '%s'. Expected one of: CPU, GPU, NPU, GPU." + % value + ) + + +def resolve_target_device_default(default_value: str, + env_var_name: str = "TARGET_DEVICE") -> str: + """Resolve default target device using env var and validate/normalize it. + + Precedence: + 1) explicit CLI value (handled by argparse separately) + 2) environment variable + 3) hard-coded default + """ + env_value = os.getenv(env_var_name) + candidate = env_value if env_value and env_value.strip() else default_value + return validate_target_device(candidate) diff --git a/benchmark-scripts/device_validation_test.py b/benchmark-scripts/device_validation_test.py new file mode 100644 index 0000000..d2f8319 --- /dev/null +++ b/benchmark-scripts/device_validation_test.py @@ -0,0 +1,56 @@ +""" +Unit tests for benchmark target device validation. +""" + +import argparse +import os +import unittest +from unittest import mock + +from device_validation import validate_target_device, resolve_target_device_default + + +class TestDeviceValidation(unittest.TestCase): + + def test_valid_target_devices(self): + test_cases = { + 'CPU': 'CPU', + 'GPU': 'GPU', + 'GPU.0': 'GPU.0', + 'GPU.1': 'GPU.1', + 'GPU.2': 'GPU.2', + 'NPU': 'NPU', + } + + for user_value, expected in test_cases.items(): + with self.subTest(user_value=user_value): + self.assertEqual(validate_target_device(user_value), expected) + + def test_invalid_target_devices(self): + invalid_values = ['GPU.', 'GPU.A', 'GPU.-1', 'GPU.abc'] + + for user_value in invalid_values: + with self.subTest(user_value=user_value): + with self.assertRaises(argparse.ArgumentTypeError): + validate_target_device(user_value) + + def test_env_target_device_valid(self): + with mock.patch.dict(os.environ, {'TARGET_DEVICE': 'gpu.2'}, clear=False): + self.assertEqual(resolve_target_device_default('CPU'), 'GPU.2') + + def test_env_target_device_invalid(self): + with mock.patch.dict(os.environ, {'TARGET_DEVICE': 'GPU.-1'}, clear=False): + with self.assertRaises(argparse.ArgumentTypeError): + resolve_target_device_default('CPU') + + def test_env_target_device_fallback_when_missing(self): + with mock.patch.dict(os.environ, {}, clear=True): + self.assertEqual(resolve_target_device_default('CPU'), 'CPU') + + def test_env_target_device_fallback_when_empty(self): + with mock.patch.dict(os.environ, {'TARGET_DEVICE': ' '}, clear=False): + self.assertEqual(resolve_target_device_default('GPU'), 'GPU') + + +if __name__ == '__main__': + unittest.main()