Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,32 @@
- [Open source code for setup, install, and execution of software, with complete developer documentation](https://intel-retail.github.io/documentation/performance-tools/benchmark.html)
- [Developer focused website to enable developers to engage and build our partner community](https://www.intel.com/content/www/us/en/developer/articles/reference-implementation/automated-self-checkout.html)

## Benchmark Target Device

Benchmark scripts support explicit target-device selection via `--target_device` or `TARGET_DEVICE`.

Accepted values:

- `CPU`
- `GPU`
- `GPU.<index>` (for example: `GPU.0`, `GPU.1`, `GPU.2`)
- `NPU`

Examples:

```bash
# Existing workflow (still supported)
make benchmark TARGET_DEVICE=GPU

# Explicit GPU device selection
make benchmark TARGET_DEVICE=GPU.1

# Direct script usage
python benchmark-scripts/benchmark.py \
--compose_file ./docker/docker-compose.yaml \
--target_device GPU.1
```

## Disclaimer

GStreamer is an open source framework licensed under LGPL. See https://gstreamer.freedesktop.org/documentation/frequently-asked-questions/licensing.html?gi-language=c. You are solely responsible for determining if your use of Gstreamer requires any additional licenses. Intel is not responsible for obtaining any such licenses, nor liable for any licensing fees due, in connection with your use of Gstreamer
Expand Down
2 changes: 1 addition & 1 deletion benchmark-scripts/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ plot: init-packages
python3 usage_graph_plot.py --dir $(ROOT_DIRECTORY)/

python-test:
python -m coverage run -m unittest benchmark_test.py stream_density_test.py
python -m coverage run -m unittest benchmark_test.py stream_density_test.py device_validation_test.py

python-integration:
python -m coverage run -m unittest benchmark_integration.py
Expand Down
10 changes: 8 additions & 2 deletions benchmark-scripts/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import csv
import json
import stream_density
from device_validation import validate_target_device, resolve_target_device_default


def parse_args(print=False):
Expand Down Expand Up @@ -76,8 +77,13 @@ def parse_args(print=False):
help='initial time in seconds before ' +
'starting metric data collection')
# TODO: change target_device to an env variable in docker compose
parser.add_argument('--target_device', default='CPU',
help='desired running platform [cpu|core|xeon|dgpu.x]')
try:
default_target_device = resolve_target_device_default('CPU')
except argparse.ArgumentTypeError as exc:
parser.error(str(exc))
parser.add_argument('--target_device', default=default_target_device,
type=validate_target_device,
help='desired running platform [CPU|GPU|GPU.<index>|NPU]')
Comment thread
TanmayeeSharvani22 marked this conversation as resolved.
parser.add_argument('--compose_file', default=None, action='append',
help='path to docker compose files. ' +
'can be used multiple times')
Expand Down
13 changes: 9 additions & 4 deletions benchmark-scripts/benchmark_order_accuracy.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import traceback
from pathlib import Path
from typing import List, Dict, Optional
from device_validation import validate_target_device, resolve_target_device_default

# Import from performance-tools benchmark scripts
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
Expand Down Expand Up @@ -686,12 +687,16 @@ def parse_args():
help='Directory for results output'
)

try:
default_target_device = resolve_target_device_default('GPU')
except argparse.ArgumentTypeError as exc:
parser.error(str(exc))

parser.add_argument(
'--target_device',
type=str,
default='GPU',
choices=['CPU', 'GPU', 'NPU'],
help='Target inference device'
type=validate_target_device,
default=default_target_device,
help='Target inference device [CPU|GPU|GPU.<index>|NPU]'
)

parser.add_argument(
Expand Down
1 change: 0 additions & 1 deletion benchmark-scripts/benchmark_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,5 @@ def test_docker_compose_containers_fail(self):
mock_popen.communicate.assert_called_once_with()
mock_returncode.assert_called()


if __name__ == '__main__':
unittest.main()
49 changes: 49 additions & 0 deletions benchmark-scripts/device_validation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""
Utilities for validating benchmark target device arguments.
"""

import argparse
import os
import re


_GPU_INDEX_PATTERN = re.compile(r"^GPU\.(\d+)$", re.IGNORECASE)


def validate_target_device(value: str) -> str:
"""Validate and normalize target device values.

Accepted values:
- CPU
- GPU
- GPU.<index> where index is a non-negative integer
- NPU
"""
normalized = value.strip()
upper_value = normalized.upper()

if upper_value in {"CPU", "GPU", "NPU"}:
return upper_value

gpu_match = _GPU_INDEX_PATTERN.fullmatch(normalized)
if gpu_match:
return f"GPU.{gpu_match.group(1)}"

raise argparse.ArgumentTypeError(
"invalid target device '%s'. Expected one of: CPU, GPU, NPU, GPU.<index>"
% value
)


def resolve_target_device_default(default_value: str,
env_var_name: str = "TARGET_DEVICE") -> str:
"""Resolve default target device using env var and validate/normalize it.

Precedence:
1) explicit CLI value (handled by argparse separately)
2) environment variable
3) hard-coded default
"""
env_value = os.getenv(env_var_name)
candidate = env_value if env_value and env_value.strip() else default_value
return validate_target_device(candidate)
56 changes: 56 additions & 0 deletions benchmark-scripts/device_validation_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
"""
Unit tests for benchmark target device validation.
"""

import argparse
import os
import unittest
from unittest import mock

from device_validation import validate_target_device, resolve_target_device_default


class TestDeviceValidation(unittest.TestCase):

def test_valid_target_devices(self):
test_cases = {
'CPU': 'CPU',
'GPU': 'GPU',
'GPU.0': 'GPU.0',
'GPU.1': 'GPU.1',
'GPU.2': 'GPU.2',
'NPU': 'NPU',
}

for user_value, expected in test_cases.items():
with self.subTest(user_value=user_value):
self.assertEqual(validate_target_device(user_value), expected)

def test_invalid_target_devices(self):
invalid_values = ['GPU.', 'GPU.A', 'GPU.-1', 'GPU.abc']

for user_value in invalid_values:
with self.subTest(user_value=user_value):
with self.assertRaises(argparse.ArgumentTypeError):
validate_target_device(user_value)

def test_env_target_device_valid(self):
with mock.patch.dict(os.environ, {'TARGET_DEVICE': 'gpu.2'}, clear=False):
self.assertEqual(resolve_target_device_default('CPU'), 'GPU.2')

def test_env_target_device_invalid(self):
with mock.patch.dict(os.environ, {'TARGET_DEVICE': 'GPU.-1'}, clear=False):
with self.assertRaises(argparse.ArgumentTypeError):
resolve_target_device_default('CPU')

def test_env_target_device_fallback_when_missing(self):
with mock.patch.dict(os.environ, {}, clear=True):
self.assertEqual(resolve_target_device_default('CPU'), 'CPU')

def test_env_target_device_fallback_when_empty(self):
with mock.patch.dict(os.environ, {'TARGET_DEVICE': ' '}, clear=False):
self.assertEqual(resolve_target_device_default('GPU'), 'GPU')


if __name__ == '__main__':
unittest.main()
Loading