diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a01660c..8d7ff7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -218,12 +218,12 @@ jobs: - name: Run tests (Windows) if: runner.os == 'Windows' run: | - pytest tests/ -v --tb=short -x --ignore=tests/test_examples_advanced.py --ignore=tests/test_examples_modelica.py --ignore=tests/test_examples_perfectgaz.py --ignore=tests/test_examples_runner.py --ignore=tests/test_examples_telemac.py --ignore=tests/test_interrupt_handling.py --ignore=tests/test_ssh_availability_check.py --ignore=tests/test_ssh_directory_uniqueness.py --ignore=tests/test_ssh_localhost.py --ignore=tests/test_ssh_many_cases.py --ignore=tests/test_ssh_perfectgaz.py --ignore=tests/test_funz_integration.py --ignore=tests/test_funz_protocol.py + pytest tests/ -v --tb=short -x --ignore=tests/test_examples_advanced.py --ignore=tests/test_examples_modelica.py --ignore=tests/test_examples_perfectgaz.py --ignore=tests/test_examples_runner.py --ignore=tests/test_examples_telemac.py --ignore=tests/test_interrupt_handling.py --ignore=tests/test_ssh_availability_check.py --ignore=tests/test_ssh_directory_uniqueness.py --ignore=tests/test_ssh_localhost.py --ignore=tests/test_ssh_many_cases.py --ignore=tests/test_ssh_perfectgaz.py --ignore=tests/test_static_files_ssh.py --ignore=tests/test_funz_integration.py --ignore=tests/test_funz_protocol.py - name: Run tests (macOS) if: runner.os == 'macOS' run: | - pytest tests/ -v --tb=short -x --ignore=tests/test_examples_advanced.py --ignore=tests/test_examples_modelica.py --ignore=tests/test_examples_perfectgaz.py --ignore=tests/test_examples_runner.py --ignore=tests/test_examples_telemac.py --ignore=tests/test_ssh_availability_check.py --ignore=tests/test_ssh_directory_uniqueness.py --ignore=tests/test_ssh_localhost.py --ignore=tests/test_ssh_many_cases.py --ignore=tests/test_ssh_perfectgaz.py --ignore=tests/test_funz_integration.py --ignore=tests/test_funz_protocol.py + pytest tests/ -v --tb=short -x --ignore=tests/test_examples_advanced.py --ignore=tests/test_examples_modelica.py --ignore=tests/test_examples_perfectgaz.py --ignore=tests/test_examples_runner.py --ignore=tests/test_examples_telemac.py --ignore=tests/test_ssh_availability_check.py --ignore=tests/test_ssh_directory_uniqueness.py --ignore=tests/test_ssh_localhost.py --ignore=tests/test_ssh_many_cases.py --ignore=tests/test_ssh_perfectgaz.py --ignore=tests/test_static_files_ssh.py --ignore=tests/test_funz_integration.py --ignore=tests/test_funz_protocol.py - name: Run tests (Linux) if: runner.os == 'Linux' diff --git a/.github/workflows/slurm-localhost.yml b/.github/workflows/slurm-localhost.yml index abc653c..ec60fbd 100644 --- a/.github/workflows/slurm-localhost.yml +++ b/.github/workflows/slurm-localhost.yml @@ -423,6 +423,57 @@ jobs: print("\\n✓ Multiple partition test passed!") PYTHON + - name: Test SLURM calculator - static_files + run: | + echo "Testing FZ static_files with local SLURM..." + SCRIPT_PATH="$HOME/fz_test/slurm_calc.sh" + + python3 << PYTHON + import tempfile + from pathlib import Path + from fz import fzr + + print("=" * 60) + print("Test 4: static_files with local SLURM") + print("=" * 60) + + script_path = "$SCRIPT_PATH" + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + assets_dir = root / "assets" + assets_dir.mkdir() + weather = assets_dir / "weather.csv" + weather.write_text("weather-slurm-local\\n") + + study_dir = root / "study" + study_dir.mkdir() + input_file = study_dir / "input.txt" + input_file.write_text("x = \${x}\\n") + + model = { + "delim": "{}", + "output": {"weather": "cat weather.csv"}, + } + + import os + os.chdir(study_dir) + calculator_uri = f"slurm://:debug/bash {script_path}" + results = fzr( + str(input_file), {"x": 3}, model, + calculators=calculator_uri, results_dir="results", + input_static=["../assets/weather.csv"], + ) + + if hasattr(results, "to_dict"): + row = results.to_dict("records")[0] + else: + row = {k: (v[0] if isinstance(v, list) else v) for k, v in results.items()} + + assert row["weather"].strip() == "weather-slurm-local", row + print("\\n✓ static_files with local SLURM test passed!") + PYTHON + - name: Run SLURM error reporting integration tests run: | python -m pytest tests/test_error_reporting.py::TestSlurmIntegrationErrorReporting -v -s --tb=long diff --git a/.github/workflows/ssh-localhost.yml b/.github/workflows/ssh-localhost.yml index 88bf1d5..9f50500 100644 --- a/.github/workflows/ssh-localhost.yml +++ b/.github/workflows/ssh-localhost.yml @@ -78,6 +78,11 @@ jobs: pip install paramiko python -m pytest tests/test_error_reporting.py::TestSSHIntegrationErrorReporting -v -s --tb=long + - name: Run static_files over SSH test + run: | + pip install paramiko + python -m pytest tests/test_static_files_ssh.py -v -s --tb=long + - name: Test summary if: always() run: | diff --git a/NEWS.md b/NEWS.md index 2135cc1..3200286 100644 --- a/NEWS.md +++ b/NEWS.md @@ -2,6 +2,41 @@ ## Unreleased +### Shared static files across cases (`input_static`) + +- `fzr()`/`fzc()`/`fzi()`/`fzd()` gain an `input_static` parameter (CLI + `--input_static`, repeatable or an inline JSON list): files identical + across every case (e.g. a shared weather CSV or a large reference + dataset) that are never templated/substituted, never re-hashed per case, + and (for relative paths) not duplicated on disk per case. This is a + function argument, not a model field — the model itself doesn't need to + know about it. + - **Absolute path** entries are assumed already present at that same path + on the calculator side too (shared/mounted storage); fz never copies, + symlinks, or transfers them - only hashes them (once per `fzr()`/`fzd()` + call), so `cache://` still reacts if the shared file's content changes. + - **Relative path** entries are resolved against the cwd `fzr()`/`fzd()` + was called from, identified by basename, and symlinked into every case's + result/temp directory (falling back to a real copy if the platform + doesn't allow symlinks, e.g. Windows without developer mode/admin). + Explicitly transferred to `ssh://`, `slurm://` (remote), and `funz://` + calculators, since they live outside `input_path` and wouldn't otherwise + be found by the normal per-case file transfer. + - `fzi()` never scans them for `$variables`; `.fz_hash` always includes + them (once, memoized) so cache matching stays correct. + - `fzd()` passes `input_static` through unchanged to each iteration's + internal `fzr()` call. + - See `doc/core-functions.md` ("fzr" → `input_static`) for the full write-up. + - `fzr()` now logs a one-time warning (per file, not per case) when an + `input_path` file has no variables and is at least + `FZ_STATIC_CANDIDATE_MIN_SIZE` bytes (default 1 MiB), suggesting it be + passed via `input_static` instead; set `FZ_STATIC_CANDIDATE_MIN_SIZE=0` + to disable. + - New `tests/test_static_files.py` (8 tests, `sh://`), + `tests/test_static_files_ssh.py` (real SFTP transfer over `ssh://` to + localhost, wired into `ssh-localhost.yml`), and + `tests/test_static_files_warning.py` (4 tests for the new warning). + ### Configurable case directory naming (`case_naming`), thread-safe signal handling - `fzr()`/CLI `fzr`/`fz run` gain a `case_naming` parameter (`--case_naming`, diff --git a/README.md b/README.md index 9c88808..05c9792 100644 --- a/README.md +++ b/README.md @@ -1083,6 +1083,13 @@ print(results) `info.txt` if the manifest is missing or incomplete). Defaults to the `FZ_CASE_NAMING` env var, or `"path"`. +- `input_static`: Files identical across every case (a shared weather CSV, a large + reference dataset) that are never templated and never duplicated per case — see + `doc/core-functions.md` ("fzr" → `input_static`) for the full write-up. If a large + variable-free file is left in `input_path` instead, `fzr()` logs a one-time warning + suggesting `input_static` (threshold: `FZ_STATIC_CANDIDATE_MIN_SIZE`, default 1 MiB, + `0` disables it). + **Returns**: pandas DataFrame with all results ### fzd - Run Design of Experiments @@ -2467,6 +2474,11 @@ export FZ_RUN_TIMEOUT=3600 # (short content hash, avoids filesystem filename length limits with many # variables), or "index" (case_) export FZ_CASE_NAMING=path + +# Minimum size (bytes) for a variable-free input_path file to trigger a +# one-time warning suggesting input_static instead (default: 1048576 = 1 MiB; +# 0 disables the warning) +export FZ_STATIC_CANDIDATE_MIN_SIZE=1048576 ``` ### Shell Path Configuration (FZ_SHELL_PATH) diff --git a/doc/INDEX.md b/doc/INDEX.md index 66c54b1..06afc92 100644 --- a/doc/INDEX.md +++ b/doc/INDEX.md @@ -218,12 +218,14 @@ Quick reference index for finding specific topics in the FZ context documentatio | Use caching | calculators.md → "Cache Calculator" | | Debug my calculation | quick-examples.md → "Troubleshooting Examples" | | Avoid filename length limits with many variables | core-functions.md → "fzr" → `case_naming` | +| Share a large/static file across all cases without duplicating it | core-functions.md → "fzr" → `input_static` | ## Configuration & Advanced Topics | Topic | File | Section | |-------|------|---------| | Case directory naming (`case_naming`, `FZ_CASE_NAMING`) | core-functions.md | "fzr" | +| Shared static files across cases (`input_static`) | core-functions.md | "fzr" | | FZ_SHELL_PATH overview | shell-path.md | "Overview" | | Shell path setup | shell-path.md | "Usage" | | Windows path configuration | shell-path.md | "Common Configurations" → "Windows with MSYS2" | @@ -271,3 +273,4 @@ Quick keyword search: - **Performance**: parallel-and-caching.md → "Performance Optimization" - **case_naming / FZ_CASE_NAMING**: core-functions.md → "fzr" - **cases.csv manifest**: core-functions.md → "fzo" → "Automatic Variable Extraction" +- **input_static**: core-functions.md → "fzr" → `input_static` diff --git a/doc/core-functions.md b/doc/core-functions.md index 4b6ac5a..b8b85c4 100644 --- a/doc/core-functions.md +++ b/doc/core-functions.md @@ -104,12 +104,14 @@ fzl --format json > config.json ```python import fz -variables = fz.fzi(input_path, model) +variables = fz.fzi(input_path, model, input_static=None) ``` **Parameters**: - `input_path` (str): Path to input file or directory - `model` (dict or str): Model definition or alias +- `input_static` (list of str, optional): Files identical across every case (see `fzr`'s + `input_static` below); never scanned for variables, since they're never templated **Returns**: Dictionary with variable names as keys (values are None) @@ -173,7 +175,7 @@ print(variables) ```python import fz -fz.fzc(input_path, input_variables, model, output_dir) +fz.fzc(input_path, input_variables, model, output_dir, input_static=None) ``` **Parameters**: @@ -181,6 +183,8 @@ fz.fzc(input_path, input_variables, model, output_dir) - `input_variables` (dict): Variable values (scalar or list) - `model` (dict or str): Model definition or alias - `output_dir` (str): Output directory path +- `input_static` (list of str, optional): Files identical across every case (see `fzr`'s + `input_static`); symlinked into `output_dir` rather than templated/duplicated **Returns**: None (writes files to output_dir) @@ -386,6 +390,29 @@ results_df = fz.fzr( single `cases.csv` manifest is written at the results root mapping each case directory to its variables (each case's own `info.txt` also has them, as a fallback). Defaults to the `FZ_CASE_NAMING` env var, or `"path"`. +- `input_static` (list of str, optional): Files identical across every case (e.g. a + shared weather CSV or a large reference dataset) that are never templated/ + substituted, never re-hashed per case, and (for relative paths) not duplicated on + disk per case: + - **Absolute path** entries are assumed already present at that same path on the + calculator side too (shared/mounted storage); fz never copies, symlinks, or + transfers them - only hashes them once per `fzr()`/`fzd()` call, so `cache://` + still reacts if the shared file's content changes. The calculator command/script + must reference the absolute path directly. + - **Relative path** entries are resolved against the cwd `fzr()`/`fzd()` was called + from, identified by their **basename** (not the full declared path, which may + contain `..` to reach outside `input_path`), symlinked into every case's + directory (falling back to a real copy if the platform disallows symlinks, e.g. + Windows without developer mode/admin), and explicitly transferred to `ssh://`, + `slurm://` (remote), and `funz://` calculators, since they live outside + `input_path` and the generic per-case file transfer never finds them. + - Either way, `fzi()` never scans them for `$variables`, and `.fz_hash` always + includes them so `cache://` matching stays correct. + - **Detection helper**: if a file under `input_path` has no variables and is at + least `FZ_STATIC_CANDIDATE_MIN_SIZE` bytes (default 1 MiB), `fzr()` logs a + one-time warning suggesting it be passed via `input_static` instead - it's + otherwise re-read/re-copied and re-hashed on every case. Set + `FZ_STATIC_CANDIDATE_MIN_SIZE=0` to disable. **Returns**: pandas DataFrame with all results and metadata @@ -537,7 +564,8 @@ result = fz.fzd( algorithm, calculators=None, algorithm_options=None, - analysis_dir="analysis" + analysis_dir="analysis", + input_static=None ) ``` @@ -550,6 +578,9 @@ result = fz.fzd( - `calculators` (str, list, or int): Calculator URI(s) (default: `["sh://"]`); when `model` is a callable, must be an `int` (default: `1`), accepted for API compatibility — calls are always run sequentially, never in parallel (see below) - `algorithm_options` (dict, str, or None): Algorithm-specific options (dict, JSON string, or JSON file path) - `analysis_dir` (str): Analysis results directory (default: `"analysis"`) +- `input_static` (list of str, optional): Files identical across every case (see `fzr`'s + `input_static`); passed through unchanged to each iteration's internal `fzr()` call + for file-based models **Returns**: Dictionary with keys: - `XY`: pandas DataFrame with all input and output values diff --git a/fz/cli.py b/fz/cli.py index e7f0768..2a7cc58 100644 --- a/fz/cli.py +++ b/fz/cli.py @@ -241,6 +241,26 @@ def _add_calculators_arg(parser): help="Calculator URI, alias, JSON file, or JSON list (repeatable)") +def _add_input_static_arg(parser): + parser.add_argument("--input_static", dest="input_static", action="append", default=None, + help="Static file path, identical across every case and never " + "templated (see docs); or an inline JSON list of paths. Repeatable.") + + +def _resolve_input_static(args): + values = getattr(args, "input_static", None) + if not values: + return None + result = [] + for item in values: + stripped = item.strip() + if stripped.startswith("["): + result.extend(json.loads(stripped)) + else: + result.append(item) + return result + + def _resolve_calculators(args): if not args.calculators: return None @@ -508,6 +528,7 @@ def fzi_main(): parser.add_argument("--version", action="version", version=f"fzi {get_version()}") _add_input_path_args(parser) _add_model_args(parser) + _add_input_static_arg(parser) _add_format_arg(parser) args = parser.parse_args() @@ -515,7 +536,7 @@ def fzi_main(): try: input_path = _resolve_path(parser, args.input_path, args.input_path_pos, "input_path") model = _resolve_model(parser, args) - result = fzi_func(input_path, model) + result = fzi_func(input_path, model, input_static=_resolve_input_static(args)) print(format_output(result, args.format)) return 0 except TypeError as e: @@ -540,6 +561,7 @@ def fzc_main(): _add_input_path_args(parser) _add_model_args(parser) _add_variables_arg(parser) + _add_input_static_arg(parser) parser.add_argument("--output_dir", "--output", "-o", dest="output_dir", default="output", help="Output directory (default: output)") @@ -549,7 +571,8 @@ def fzc_main(): input_path = _resolve_path(parser, args.input_path, args.input_path_pos, "input_path") model = _resolve_model(parser, args) variables = parse_variables(args.input_variables) - fzc_func(input_path, variables, model, output_dir=args.output_dir) + fzc_func(input_path, variables, model, output_dir=args.output_dir, + input_static=_resolve_input_static(args)) print(f"Compiled input saved to {args.output_dir}") return 0 except TypeError as e: @@ -613,6 +636,7 @@ def fzr_main(): "'hash' (short content hash, avoids filename length limits), or " "'index' (case_). Defaults to FZ_CASE_NAMING env var, or 'path'.") _add_calculators_arg(parser) + _add_input_static_arg(parser) _add_format_arg(parser) args = parser.parse_args() @@ -626,7 +650,8 @@ def fzr_main(): result = fzr_func(input_path, variables, model, results_dir=args.results_dir, calculators=calculators, - case_naming=args.case_naming) + case_naming=args.case_naming, + input_static=_resolve_input_static(args)) print(format_output(result, args.format)) # Exit non-zero when no case succeeded, so shell scripts and agents # can detect total failure without parsing the per-case status column @@ -665,6 +690,7 @@ def fzd_main(): parser.add_argument("--results_dir", "-r", default="results_fzd", help="Results directory (default: results_fzd)") parser.add_argument("--calculators", "-c", help="Calculator specifications (JSON file or inline JSON)") parser.add_argument("--options", "-o", help="Algorithm options (JSON file or inline JSON)") + _add_input_static_arg(parser) args = parser.parse_args() @@ -684,6 +710,7 @@ def fzd_main(): calculators=calculators, algorithm_options=(algo_options if isinstance(algo_options, dict) else {}), analysis_dir=args.results_dir, + input_static=_resolve_input_static(args), ) # Print summary @@ -722,6 +749,7 @@ def main(): parser_input = subparsers.add_parser("input", help="Parse input to find variables") _add_input_path_args(parser_input) _add_model_args(parser_input) + _add_input_static_arg(parser_input) _add_format_arg(parser_input) # compile command (fzc) @@ -729,6 +757,7 @@ def main(): _add_input_path_args(parser_compile) _add_model_args(parser_compile) _add_variables_arg(parser_compile) + _add_input_static_arg(parser_compile) parser_compile.add_argument("--output_dir", "--output", "-o", dest="output_dir", default="output", help="Output directory (default: output)") @@ -751,6 +780,7 @@ def main(): "'hash' (short content hash, avoids filename length limits), or " "'index' (case_). Defaults to FZ_CASE_NAMING env var, or 'path'.") _add_calculators_arg(parser_run) + _add_input_static_arg(parser_run) _add_format_arg(parser_run) # design command (fzd) @@ -766,6 +796,7 @@ def main(): parser_design.add_argument("--results_dir", "-r", default="results_fzd", help="Results directory (default: results_fzd)") parser_design.add_argument("--calculators", "-c", help="Calculator specifications (JSON file or inline JSON)") parser_design.add_argument("--options", "-o", help="Algorithm options (JSON file or inline JSON)") + _add_input_static_arg(parser_design) # list command (fzl) parser_list = subparsers.add_parser("list", help="List installed models and calculators") @@ -821,14 +852,15 @@ def main(): if args.command == "input": input_path = _resolve_path(parser, args.input_path, args.input_path_pos, "input_path") model = _resolve_model(parser, args) - result = fzi_func(input_path, model) + result = fzi_func(input_path, model, input_static=_resolve_input_static(args)) print(format_output(result, args.format)) elif args.command == "compile": input_path = _resolve_path(parser, args.input_path, args.input_path_pos, "input_path") model = _resolve_model(parser, args) variables = parse_variables(args.input_variables) - fzc_func(input_path, variables, model, output_dir=args.output_dir) + fzc_func(input_path, variables, model, output_dir=args.output_dir, + input_static=_resolve_input_static(args)) print(f"Compiled input saved to {args.output_dir}") elif args.command == "output": @@ -846,7 +878,8 @@ def main(): result = fzr_func(input_path, variables, model, results_dir=args.results_dir, calculators=calculators, - case_naming=args.case_naming) + case_naming=args.case_naming, + input_static=_resolve_input_static(args)) print(format_output(result, args.format)) elif args.command == "design": @@ -874,6 +907,7 @@ def main(): calculators=calculators, algorithm_options=algo_options, analysis_dir=args.results_dir, + input_static=_resolve_input_static(args), ) # Print summary diff --git a/fz/config.py b/fz/config.py index cce485d..df27027 100755 --- a/fz/config.py +++ b/fz/config.py @@ -94,6 +94,12 @@ def _load_from_environment(self): case_naming = 'path' self.case_naming = case_naming + # Size threshold (bytes) above which an input_path file with no + # variables triggers a one-time warning suggesting input_static + # instead (it's otherwise re-read/re-copied and re-hashed per case). + # Set to 0 to disable the warning entirely. + self.static_candidate_min_size = self._parse_int_env('FZ_STATIC_CANDIDATE_MIN_SIZE', 1_048_576) + def _parse_int_env(self, key: str, default: Optional[int]) -> Optional[int]: """Parse integer environment variable""" value = os.getenv(key) @@ -140,7 +146,8 @@ def get_summary(self) -> dict: 'ssh_keepalive': self.ssh_keepalive, 'run_timeout': self.run_timeout, 'shell_path': self.shell_path, - 'case_naming': self.case_naming + 'case_naming': self.case_naming, + 'static_candidate_min_size': self.static_candidate_min_size } @@ -236,6 +243,10 @@ def print_config(): print("\n📁 CASE DIRECTORY NAMING:") print(f" FZ_CASE_NAMING = {summary['case_naming']}") + print("\n📦 STATIC FILE DETECTION:") + print(f" FZ_STATIC_CANDIDATE_MIN_SIZE = {summary['static_candidate_min_size']} bytes " + f"(0 = disabled)") + print("\n" + "=" * 60) print("Set environment variables to customize these defaults") print("Example: export FZ_LOG_LEVEL=INFO FZ_MAX_RETRIES=3") diff --git a/fz/core.py b/fz/core.py index c239769..88c967e 100644 --- a/fz/core.py +++ b/fz/core.py @@ -948,13 +948,15 @@ def fzl(models: str = "*", calculators: str = "*", check: bool = False) -> Dict[ @with_helpful_errors -def fzi(input_path: str, model: Union[str, Dict]) -> Dict[str, Any]: +def fzi(input_path: str, model: Union[str, Dict], input_static: Optional[List[str]] = None) -> Dict[str, Any]: """ Parse input file(s) to find variables, formulas, and static objects Args: input_path: Path to input file or directory model: Model definition dict or alias string + input_static: Files identical across every case (see fzr()'s input_static); + never scanned for variables, since they're never templated Returns: Dict with static objects, variable names, and formula expressions as keys, with their values (or None) @@ -975,6 +977,9 @@ def fzi(input_path: str, model: Union[str, Dict]) -> Dict[str, Any]: if not isinstance(input_path, (str, Path)): raise TypeError(f"input_path must be a string or Path, got {type(input_path).__name__}") + from .helpers import _validate_input_static + _validate_input_static(input_static) + # This represents the directory from which the function was launched working_dir = os.getcwd() @@ -999,8 +1004,10 @@ def fzi(input_path: str, model: Union[str, Dict]) -> Dict[str, Any]: if not input_path.exists(): raise FileNotFoundError(f"Input path '{input_path}' not found") - # Parse variables - variables = parse_variables_from_path(input_path, varprefix, var_delim) + # Parse variables (input_static files are never templated, so excluded from the scan) + from .helpers import resolve_static_file_paths + static_paths = resolve_static_file_paths(input_static, working_dir) + variables = parse_variables_from_path(input_path, varprefix, var_delim, exclude_paths=static_paths) # Read content to extract defaults and formulas if input_path.is_file(): @@ -1133,6 +1140,7 @@ def fzc( input_variables: Dict, model: Union[str, Dict], output_dir: str = "output", + input_static: Optional[List[str]] = None, ) -> None: """ Compile input file(s) replacing variables with values @@ -1142,6 +1150,8 @@ def fzc( input_variables: Dict of variable values or lists/numpy arrays of values for grid model: Model definition dict or alias string output_dir: Output directory for compiled files + input_static: Files identical across every case (see fzr()'s input_static); + symlinked into output_dir rather than templated/duplicated Raises: TypeError: If arguments have invalid types @@ -1159,6 +1169,9 @@ def fzc( if not isinstance(output_dir, (str, Path)): raise TypeError(f"output_dir must be a string or Path, got {type(output_dir).__name__}") + from .helpers import _validate_input_static + _validate_input_static(input_static) + # This represents the directory from which the function was launched working_dir = os.getcwd() @@ -1172,7 +1185,7 @@ def fzc( raise FileNotFoundError(f"Input path '{input_path}' not found") # Check if any input_variable keys are missing in input files - found_variables = fzi(str(input_path), model) + found_variables = fzi(str(input_path), model, input_static=input_static) missing_vars = set(input_variables.keys()) - set(found_variables.keys()) if missing_vars: log_warning(f"⚠️ Warning: The following input variables are not found in input files: {', '.join(sorted(missing_vars))}") @@ -1181,12 +1194,14 @@ def fzc( output_dir, _ = ensure_unique_directory(output_dir) # Generate all combinations if lists are provided - from .helpers import generate_variable_combinations + from .helpers import generate_variable_combinations, resolve_static_files var_combinations = generate_variable_combinations(input_variables) + static_entries = resolve_static_files(input_static, working_dir) # Use compile_to_result_directories helper to avoid code duplication compile_to_result_directories( - input_path, model, input_variables, var_combinations, output_dir + input_path, model, input_variables, var_combinations, output_dir, + static_entries=static_entries, ) # Always restore the original working directory @@ -1497,6 +1512,7 @@ def fzr( callbacks: Optional[Dict[str, callable]] = None, timeout: int = None, case_naming: str = None, + input_static: Optional[List[str]] = None, ) -> Union[Dict[str, List[Any]], "pandas.DataFrame"]: """ Run full parametric calculations @@ -1525,6 +1541,18 @@ def fzr( Regardless of scheme, the exact variable values are always recoverable from each case's info.txt, and fzo() falls back to reading it when the directory name doesn't parse as "key=val,...". Defaults to FZ_CASE_NAMING env var, or "path". + input_static: Files identical across every case (e.g. a shared weather CSV or a large + reference dataset) that are never templated/substituted, never re-hashed per + case, and (for relative paths) not duplicated on disk per case: + - Absolute path entries are assumed already present at that same path on the + calculator side too (shared/mounted storage); never copied/symlinked/ + transferred, only hashed (so cache:// still reacts to content changes). + - Relative path entries are resolved against cwd at call time, identified by + basename, symlinked into every case's directory, and explicitly transferred + to ssh://, slurm:// (remote), and funz:// calculators (they live outside + input_path, so the generic per-case file transfer never finds them). + fzi() excludes them from variable discovery; .fz_hash always includes them. + See doc/core-functions.md ("fzr" -> input_static) for the full write-up. Returns: DataFrame with variable values and results (if pandas available), otherwise Dict with lists @@ -1554,6 +1582,9 @@ def fzr( if not isinstance(results_dir, (str, Path)): raise TypeError(f"results_dir must be a string or Path, got {type(results_dir).__name__}") + from .helpers import _validate_input_static + _validate_input_static(input_static) + # Resolve case_naming: explicit arg > FZ_CASE_NAMING env var (via config) > "path" if case_naming is None: case_naming = get_config().case_naming @@ -1687,9 +1718,16 @@ def fzr( resolved_calculators.append(calc) calculators = resolved_calculators + # Resolve input_static once for this whole fzr() call (not per case): + # relative entries symlinked/hashed once and reused everywhere, absolute + # entries hashed once and assumed already present on the calculator side + from .helpers import resolve_static_files + static_entries = resolve_static_files(input_static, original_cwd) + # Compile all combinations directly to result directories, then prepare temp directories compile_to_result_directories( - input_path, model, input_variables, var_combinations, results_dir, case_naming + input_path, model, input_variables, var_combinations, results_dir, case_naming, + static_entries=static_entries, ) # Create temp directories and copy from result directories (excluding .fz_hash) @@ -1711,6 +1749,7 @@ def fzr( callbacks, timeout, case_naming, + static_entries, ) # Collect results in the correct order, filtering out None (interrupted/incomplete cases) @@ -2041,7 +2080,8 @@ def fzd( algorithm: str, calculators: Union[str, List[str], int] = None, algorithm_options: Union[Dict[str, Any], str] = None, - analysis_dir: str = "analysis" + analysis_dir: str = "analysis", + input_static: Optional[List[str]] = None, ) -> Dict[str, Any]: """ Run iterative design of experiments with algorithms @@ -2092,6 +2132,8 @@ def fzd( Each iteration's cases live in "/iter/case_/" — file-based models are run via fzr() internally with case_naming="index", since design points from an algorithm can carry many variables/long float values. + input_static: Files identical across every case (see fzr()'s input_static); passed + through unchanged to each iteration's internal fzr() call for file-based models. Returns: Dict with algorithm results including: @@ -2136,6 +2178,9 @@ def fzd( ... algorithm_options="algo_config.json" # Path to JSON file ... ) """ + from .helpers import _validate_input_static + _validate_input_static(input_static) + # This represents the directory from which the function was launched working_dir = os.getcwd() @@ -2144,7 +2189,7 @@ def fzd( _interrupt_requested = False _install_signal_handler() - + try: is_function_model = callable(model) and not isinstance(model, (str, dict)) @@ -2310,6 +2355,7 @@ def fzd( # (cache:// matching is by .fz_hash content, not directory name, # so this doesn't affect cross-iteration cache reuse above). case_naming="index", + input_static=input_static, ) # Expand result_df back to full current_design length (re-map duplicates) diff --git a/fz/helpers.py b/fz/helpers.py index c281958..b11e168 100644 --- a/fz/helpers.py +++ b/fz/helpers.py @@ -361,6 +361,22 @@ def _cleanup_fzr_resources(): _calculator_manager = None +def _validate_input_static(input_static: Optional[List[str]]) -> None: + """ + Validate the input_static argument of fzr()/fzc()/fzi()/fzd(). + + Raises: + TypeError: If input_static is not a list of strings + """ + if input_static is None: + return + if not isinstance(input_static, list): + raise TypeError(f"input_static must be a list, got {type(input_static).__name__}") + for entry in input_static: + if not isinstance(entry, str): + raise TypeError(f"input_static entries must be strings, got {type(entry).__name__}") + + def _validate_model(model: Dict) -> None: """ Validate model dictionary structure and required fields @@ -603,7 +619,8 @@ def try_calculators_with_retry(non_cache_calculator_ids: List[str], case_index: tmp_dir: Path, model: Dict, original_input_was_dir: bool, thread_id: int, start_time: float, original_cwd: str = None, input_files_list: List[str] = None, timeout: int = None, - history: 'CaseHistory | None' = None) -> Tuple[Dict[str, Any], str]: + history: 'CaseHistory | None' = None, + static_entries: List[Dict[str, Any]] = None) -> Tuple[Dict[str, Any], str]: """ Try calculators with retry mechanism for failed calculations @@ -618,6 +635,8 @@ def try_calculators_with_retry(non_cache_calculator_ids: List[str], case_index: original_cwd: Original working directory input_files_list: List of input file names in order timeout: Timeout in seconds (None uses FZ_RUN_TIMEOUT from config, default 600) + static_entries: Pre-resolved input_static entries (see resolve_static_files), + forwarded to remote calculators for explicit transfer of the relative ones Returns: Tuple of (calculation result dict, used calculator ID) @@ -694,7 +713,8 @@ def try_calculators_with_retry(non_cache_calculator_ids: List[str], case_index: if history: history.append(f"Running command: {selected_calculator_uri}") calc_result = run_single_case_calculation( - tmp_dir, selected_calculator_uri, model, timeout, original_input_was_dir, original_cwd, input_files_list + tmp_dir, selected_calculator_uri, model, timeout, original_input_was_dir, original_cwd, + input_files_list, static_entries=static_entries ) calc_elapsed = time.time() - calc_start @@ -866,6 +886,7 @@ def run_single_case(case_info: Dict) -> Dict[str, Any]: callbacks = case_info.get("callbacks") # Optional callbacks for progress monitoring timeout = case_info.get("timeout") # Optional timeout for calculations case_naming = case_info.get("case_naming", "path") # Case directory naming scheme + static_entries = case_info.get("static_entries") # Pre-resolved input_static entries # Get thread ID for debugging thread_id = threading.get_ident() @@ -1030,7 +1051,7 @@ def run_single_case(case_info: Dict) -> Dict[str, Any]: calc_result, used_calculator_id = try_calculators_with_retry( non_cache_calculator_ids, case_index, tmp_dir, model, original_input_was_dir, thread_id, start_time, original_cwd, input_files_list, timeout, - history=history + history=history, static_entries=static_entries ) # Use calculator ID directly (includes #n suffix for duplicate URIs) used_calculator = used_calculator_id @@ -1386,7 +1407,8 @@ def run_cases_parallel(var_combinations: List[Dict], temp_path: Path, resultsdir calculators: List[str], model: Dict, original_input_was_dir: bool, var_names: List[str], output_keys: List[str], original_cwd: str = None, has_input_variables: bool = True, callbacks: Optional[Dict[str, callable]] = None, - timeout: int = None, case_naming: str = "path") -> List[Dict[str, Any]]: + timeout: int = None, case_naming: str = "path", + static_entries: List[Dict[str, Any]] = None) -> List[Dict[str, Any]]: """ Run multiple cases in parallel across available calculators @@ -1403,6 +1425,10 @@ def run_cases_parallel(var_combinations: List[Dict], temp_path: Path, resultsdir callbacks: Optional dict of callback functions for progress monitoring timeout: Timeout in seconds for each calculation (None uses FZ_RUN_TIMEOUT from config, default 600) case_naming: Case directory naming scheme - "path", "hash", or "index" (see _case_subdir_name) + static_entries: Pre-resolved input_static entries (see resolve_static_files), + forwarded to remote calculators (ssh/slurm/funz) so they can explicitly + transfer the relative ones, which live outside input_path and wouldn't + otherwise be found by the per-case file transfer Returns: List of case results in the same order as var_combinations @@ -1446,6 +1472,7 @@ def run_cases_parallel(var_combinations: List[Dict], temp_path: Path, resultsdir "output_keys": output_keys, "total_cases": var_combinations, "original_cwd": original_cwd, + "static_entries": static_entries, # Add resolved static_files entries "spinner": spinner, # Add spinner instance "has_input_variables": has_input_variables, # Add flag for directory structure "callbacks": callbacks, # Add callbacks for progress monitoring @@ -1644,9 +1671,109 @@ def run_cases_parallel(var_combinations: List[Dict], temp_path: Path, resultsdir +def resolve_static_file_paths(input_static: Optional[List[str]], base_dir: Union[str, Path]) -> set: + """ + Resolve "input_static" declarations to a set of absolute source paths, + without hashing them (used by fzi() to exclude them from variable + discovery - cheap enough to call on every fzi() even for large assets). + """ + entries = input_static or [] + base_dir = Path(base_dir) + resolved = set() + for entry in entries: + entry_path = Path(entry) + source = entry_path if entry_path.is_absolute() else (base_dir / entry_path).resolve() + resolved.add(source) + return resolved + + +def resolve_static_files(input_static: Optional[List[str]], base_dir: Union[str, Path]) -> List[Dict[str, Any]]: + """ + Resolve "input_static" declarations (fzr()/fzd()/fzc()/fzi()'s input_static + argument) and hash each one once. + + input_static is a list of paths to files that are identical across every + case (e.g. a shared weather CSV or a large reference dataset) and are + therefore never templated/substituted, never re-hashed per case, and + (for relative paths) not duplicated on disk per case - they're symlinked + instead: + + - Absolute path entries are assumed already present at that same absolute + path on the calculator side too (shared/mounted storage); fz never + copies, symlinks, or transfers them - it only hashes them (by that + absolute path, from `base_dir`) so cache matching still reacts if the + shared file's content changes. + - Relative path entries are resolved against `base_dir` (the cwd fzr()/ + fzd() was called from), then identified by their **basename** (not the + full declared path, which may contain ".." to reach outside input_path + and would otherwise escape the case directory when used as a symlink + destination). fz symlinks them into every case's result_dir and tmp_dir + under that basename (so local `sh://` calculators find them + transparently by filename), and explicitly transfers them to remote + calculators (ssh://, slurm:// remote, funz://) since they live outside + input_path and the generic per-case file walk won't find them. + + Args: + input_static: List of static file paths (fzr()/fzd()/fzc()/fzi()'s + input_static argument) + base_dir: Base directory relative paths are resolved against + + Returns: + List of {"name": str, "source": Path, "is_absolute": bool, "hash": str} + dicts, one per input_static entry that could be read; unreadable + entries are skipped with a warning. + """ + entries = input_static or [] + base_dir = Path(base_dir) + resolved = [] + seen_names = set() + for entry in entries: + entry_path = Path(entry) + is_absolute = entry_path.is_absolute() + source = entry_path if is_absolute else (base_dir / entry_path).resolve() + # Absolute entries are identified by their full path (never placed in + # the case directory, so no collision risk); relative entries by + # basename only (used as an actual filesystem symlink name). + name = str(source) if is_absolute else source.name + if not source.is_file(): + log_warning(f"⚠️ input_static entry '{entry}' not found (resolved to {source}), skipping") + continue + if name in seen_names: + log_warning(f"⚠️ input_static entry '{entry}' has the same name '{name}' as another entry, skipping") + continue + seen_names.add(name) + try: + from .io import md5_file + file_hash = md5_file(source) + except Exception as e: + log_warning(f"⚠️ Could not hash input_static entry '{entry}' ({source}): {e}") + continue + resolved.append({"name": name, "source": source, "is_absolute": is_absolute, "hash": file_hash}) + return resolved + + +def _symlink_static_files(static_entries: List[Dict[str, Any]], target_dir: Path) -> None: + """Symlink each relative input_static entry into target_dir under its declared name.""" + for entry in static_entries: + if entry["is_absolute"]: + continue + link_path = target_dir / entry["name"] + link_path.parent.mkdir(parents=True, exist_ok=True) + if link_path.exists() or link_path.is_symlink(): + continue + try: + link_path.symlink_to(entry["source"]) + except OSError as e: + # Symlinks require developer mode/admin on some Windows setups; + # fall back to a real copy so the case still runs. + log_warning(f"⚠️ Could not symlink static file '{entry['name']}' ({e}), copying instead") + shutil.copy2(entry["source"], link_path) + + def compile_to_result_directories(input_path: str, model: Dict, input_variables: Dict, var_combinations: List[Dict], - resultsdir: Path, case_naming: str = "path") -> None: + resultsdir: Path, case_naming: str = "path", + static_entries: List[Dict[str, Any]] = None) -> None: """ Compile input files directly to result directories for each case @@ -1657,6 +1784,9 @@ def compile_to_result_directories(input_path: str, model: Dict, input_variables: var_combinations: List of variable combinations (cases) resultsdir: Results directory case_naming: Case directory naming scheme - "path", "hash", or "index" (see _case_subdir_name) + static_entries: Pre-resolved input_static entries (see resolve_static_files), + computed once by the caller (from fzr()'s/fzc()'s input_static argument) + rather than per case """ from .interpreter import replace_variables_in_content, evaluate_formulas from .io import create_hash_file @@ -1688,6 +1818,37 @@ def compile_to_result_directories(input_path: str, model: Dict, input_variables: if case_naming in ("hash", "index") and has_input_variables: write_case_naming_manifest(var_combinations, resultsdir, case_naming) + # static_entries is pre-resolved once by the caller (fzr()/fzc(), from + # their input_static argument) rather than per case + static_entries = static_entries or [] + static_hash_pairs = [(e["name"], e["hash"]) for e in static_entries] + + # Warn (once per file, not per case) about large input_path files with no + # variables - they're re-read/re-copied and re-hashed on every case, when + # passing them via input_static instead would symlink and hash them once. + static_candidate_min_size = get_config().static_candidate_min_size + _static_candidates_warned = set() + + def _maybe_warn_static_candidate(src_path: Path, has_variables: bool): + if static_candidate_min_size <= 0 or has_variables: + return + resolved = str(src_path.resolve()) + if resolved in _static_candidates_warned: + return + try: + size = src_path.stat().st_size + except OSError: + return + if size < static_candidate_min_size: + return + _static_candidates_warned.add(resolved) + log_warning( + f"⚠️ '{src_path.name}' ({size / 1_048_576:.1f} MB) has no variables and is " + f"re-copied/re-hashed for every case. Consider passing it via input_static " + f"instead, so it's symlinked and hashed once (see doc/core-functions.md → " + f"'fzr' → input_static)." + ) + for case_index, var_combo in enumerate(var_combinations): # Use dedicated result directory function to avoid any temp_path contamination result_dir, case_name = _get_result_directory( @@ -1703,19 +1864,21 @@ def compile_file(src_path: Path, dst_path: Path): content = f.read() eol = f.newlines if f.newlines else '\n' except UnicodeDecodeError: - # Copy binary files as-is + # Copy binary files as-is - inherently "no variables" + _maybe_warn_static_candidate(src_path, has_variables=False) shutil.copy2(src_path, dst_path) return # Replace variables - content = replace_variables_in_content(content, var_combo, varprefix, delim) + substituted = replace_variables_in_content(content, var_combo, varprefix, delim) # Evaluate formulas - content = evaluate_formulas(content, model, var_combo, interpreter) + substituted = evaluate_formulas(substituted, model, var_combo, interpreter) + _maybe_warn_static_candidate(src_path, has_variables=(substituted != content)) # Write compiled content with open(dst_path, 'w', newline=eol) as f: - f.write(content) + f.write(substituted) # Compile files to result directory and track input file names in order input_files_list = [] @@ -1733,9 +1896,14 @@ def compile_file(src_path: Path, dst_path: Path): compile_file(src_file, dst_file) input_files_list.append(str(rel_path)) - # Create hash file of compiled input files with input files in order + # Symlink relative static_files into the result directory under their + # declared name (absolute entries are never placed in the case directory) + _symlink_static_files(static_entries, result_dir) + + # Create hash file of compiled input files with input files in order, + # plus the pre-hashed static_files entries try: - create_hash_file(result_dir, input_files_list) + create_hash_file(result_dir, input_files_list, static_file_hashes=static_hash_pairs) log_info(f"Created result hash file: {result_dir}/.fz_hash") except Exception as e: log_warning(f"Warning: Could not create hash file for case {var_combo}: {e}") @@ -1773,14 +1941,20 @@ def prepare_temp_directories(var_combinations: List[Dict], temp_path: Path, resu # Copy files from result directory to temp directory (excluding .fz_hash). # Subdirectories are copied recursively so directory-tree inputs (e.g. an # OpenFOAM case with system/, constant/, 0/) reach the calculator intact. + # Symlinks (static_files) are recreated as symlinks rather than dereferenced - + # copying their target content here would defeat the point of not duplicating + # a large static file on disk per case. try: if result_dir.exists(): files_copied = 0 for item in result_dir.iterdir(): if item.name == ".fz_hash": continue - if item.is_dir(): - shutil.copytree(item, tmp_dir / item.name, dirs_exist_ok=True) + if item.is_symlink(): + (tmp_dir / item.name).symlink_to(os.readlink(item)) + files_copied += 1 + elif item.is_dir(): + shutil.copytree(item, tmp_dir / item.name, dirs_exist_ok=True, symlinks=True) files_copied += 1 elif item.is_file(): shutil.copy2(item, tmp_dir) diff --git a/fz/interpreter.py b/fz/interpreter.py index ccb27aa..d063f99 100755 --- a/fz/interpreter.py +++ b/fz/interpreter.py @@ -5,7 +5,7 @@ import json import ast from pathlib import Path -from typing import Dict, List, Union, Any, Set +from typing import Dict, List, Union, Any, Set, Optional def _get_comment_char(model: Dict) -> str: @@ -166,7 +166,8 @@ def parse_variables_from_file(filepath: Path, varprefix: str = "$", delim: str = return parse_variables_from_content(content, varprefix, delim) -def parse_variables_from_path(input_path: Path, varprefix: str = "$", delim: str = "()") -> Set[str]: +def parse_variables_from_path(input_path: Path, varprefix: str = "$", delim: str = "()", + exclude_paths: Optional[Set[Path]] = None) -> Set[str]: """ Parse variables from file or directory @@ -174,17 +175,22 @@ def parse_variables_from_path(input_path: Path, varprefix: str = "$", delim: str input_path: Path to input file or directory varprefix: Variable prefix (e.g., "$") delim: Delimiter characters (e.g., "()") + exclude_paths: Optional set of resolved absolute paths to skip (e.g. a + model's static_files - not templated, so not worth scanning for + variables, and possibly too large to read cheaply) Returns: Set of variable names found """ variables = set() + exclude_paths = exclude_paths or set() if input_path.is_file(): - variables.update(parse_variables_from_file(input_path, varprefix, delim)) + if input_path.resolve() not in exclude_paths: + variables.update(parse_variables_from_file(input_path, varprefix, delim)) elif input_path.is_dir(): for filepath in input_path.rglob("*"): - if filepath.is_file(): + if filepath.is_file() and filepath.resolve() not in exclude_paths: variables.update(parse_variables_from_file(filepath, varprefix, delim)) else: raise FileNotFoundError(f"Input path '{input_path}' not found") diff --git a/fz/io.py b/fz/io.py index cd04479..a6ea8da 100644 --- a/fz/io.py +++ b/fz/io.py @@ -50,7 +50,17 @@ def ensure_unique_directory(directory_path: Path) -> tuple[Path, Optional[Path]] return directory_path, new_path -def create_hash_file(directory: Path, input_files_order: List[str] = None) -> None: +def md5_file(file_path: Path) -> str: + """Compute the MD5 hex digest of a file's content.""" + hasher = hashlib.md5() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(4096), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def create_hash_file(directory: Path, input_files_order: List[str] = None, + static_file_hashes: List[tuple] = None) -> None: """ Create .fz_hash file containing MD5 checksums of all files in the directory The input files are listed first in the order they were provided @@ -58,14 +68,30 @@ def create_hash_file(directory: Path, input_files_order: List[str] = None) -> No Args: directory: Directory to hash all files in input_files_order: Optional list of input file names in the order they should appear + static_file_hashes: Optional list of (name, hash) pairs for static_files entries + (see helpers.resolve_static_files), precomputed once per fzr() call rather + than per case. "name" is the declared relative path for relative static_files + (may not physically exist in `directory` - a symlink is placed there + separately), or the absolute path itself for absolute static_files (which are + never copied/symlinked into the case directory at all). """ hash_file = directory / ".fz_hash" - # Get all files in directory (excluding .fz_hash itself and subdirectories) - all_files = [f for f in directory.iterdir() if f.is_file() and f.name != ".fz_hash"] + static_names = {name for name, _ in static_file_hashes} if static_file_hashes else set() + + # Get all files in directory (excluding .fz_hash itself, subdirectories, and + # static_files - those are hashed once via static_file_hashes, not re-read per case) + all_files = [ + f for f in directory.iterdir() + if f.is_file() and f.name != ".fz_hash" and f.name not in static_names + ] hash_content = [] + if static_file_hashes: + for name, file_hash in static_file_hashes: + hash_content.append(f"{file_hash} {name}") + # If input_files_order is provided, process those files first in order processed_files = set() if input_files_order: diff --git a/fz/runners.py b/fz/runners.py index 5fc858e..cc2526b 100644 --- a/fz/runners.py +++ b/fz/runners.py @@ -1248,6 +1248,7 @@ def run_calculation( original_input_was_dir: bool = False, original_cwd: str = None, input_files_list: List[str] = None, + static_entries: List[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Run a single calculation on a calculator @@ -1259,6 +1260,12 @@ def run_calculation( timeout: Timeout in seconds (None uses FZ_RUN_TIMEOUT from config, default 600) original_input_was_dir: Whether original input was a directory input_files_list: List of input file names in order (from .fz_hash) + static_entries: Pre-resolved input_static entries (see + helpers.resolve_static_files). Relative entries live outside + input_path/working_dir (only symlinked there for local execution), so + remote calculators (ssh/slurm-remote/funz) transfer them explicitly from + their real source path. Absolute entries are never transferred - assumed + already present at that path on the calculator side. Returns: Dict containing calculation results and status @@ -1274,7 +1281,8 @@ def run_calculation( return {"status": "cache_miss"} elif base_uri.startswith("sh://") or base_uri == "sh:": - # Local shell execution + # Local shell execution - static_files are already symlinked into + # working_dir (same filesystem), nothing extra to transfer command = base_uri[5:] if base_uri.startswith("sh://") else "" return run_local_calculation( working_dir, @@ -1289,19 +1297,19 @@ def run_calculation( elif base_uri.startswith("ssh://"): # Remote SSH execution return run_ssh_calculation( - working_dir, base_uri, model, timeout, input_files_list + working_dir, base_uri, model, timeout, input_files_list, static_entries=static_entries ) elif base_uri.startswith("slurm://"): # SLURM execution (local or remote) return run_slurm_calculation( - working_dir, base_uri, model, timeout, input_files_list + working_dir, base_uri, model, timeout, input_files_list, static_entries=static_entries ) elif base_uri.startswith("funz://"): # Funz server execution return run_funz_calculation( - working_dir, base_uri, model, timeout, input_files_list + working_dir, base_uri, model, timeout, input_files_list, static_entries=static_entries ) else: @@ -1802,6 +1810,7 @@ def run_ssh_calculation( model: Dict, timeout: int = None, input_files_list: List[str] = None, + static_entries: List[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Run calculation via SSH @@ -1812,6 +1821,8 @@ def run_ssh_calculation( model: Model definition dict timeout: Timeout in seconds (None uses FZ_RUN_TIMEOUT from config, default 600) input_files_list: List of input file names in order (from .fz_hash) + static_entries: Pre-resolved input_static entries, explicitly + transferred (relative ones only - see transfer_static_files_to_remote_sftp) Returns: Dict containing calculation results and status @@ -1943,6 +1954,7 @@ def run_ssh_calculation( try: # Transfer input files to remote _transfer_files_to_remote(sftp, working_dir, remote_temp_dir) + transfer_static_files_to_remote_sftp(sftp, static_entries, remote_temp_dir) # Execute command on remote result = _execute_remote_command( @@ -2023,6 +2035,7 @@ def run_slurm_calculation( model: Dict, timeout: int = None, input_files_list: List[str] = None, + static_entries: List[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Run calculation via SLURM workload manager @@ -2033,6 +2046,9 @@ def run_slurm_calculation( model: Model definition dict timeout: Timeout in seconds (None uses FZ_RUN_TIMEOUT from config, default 600) input_files_list: List of input file names in order (from .fz_hash) + static_entries: Pre-resolved input_static entries; only used for + remote SLURM execution (local execution shares the filesystem, so the + local symlink already resolves) Returns: Dict containing calculation results and status @@ -2083,7 +2099,7 @@ def run_slurm_calculation( return _run_remote_slurm_calculation( working_dir, host, port or 22, username, password, partition, script, - model, timeout, start_time, env_info, input_files_list + model, timeout, start_time, env_info, input_files_list, static_entries=static_entries ) except Exception as e: @@ -2297,6 +2313,7 @@ def _run_remote_slurm_calculation( start_time: datetime, env_info: Dict, input_files_list: List[str] = None, + static_entries: List[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Run SLURM calculation on remote host via SSH @@ -2314,6 +2331,8 @@ def _run_remote_slurm_calculation( start_time: Calculation start time env_info: Local environment information input_files_list: List of input file names in order + static_entries: Pre-resolved input_static entries, explicitly + transferred (relative ones only - see transfer_static_files_to_remote_sftp) Returns: Dict containing calculation results and status @@ -2401,6 +2420,7 @@ def _run_remote_slurm_calculation( try: # Transfer input files to remote _transfer_files_to_remote(sftp, working_dir, remote_temp_dir) + transfer_static_files_to_remote_sftp(sftp, static_entries, remote_temp_dir) # Execute SLURM command on remote result = _execute_remote_slurm_command( @@ -2763,6 +2783,7 @@ def run_funz_calculation( model: Dict, timeout: int = None, input_files_list: List[str] = None, + static_entries: List[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Run calculation via Funz server protocol @@ -2773,6 +2794,9 @@ def run_funz_calculation( model: Model definition dict timeout: Timeout in seconds (None uses FZ_RUN_TIMEOUT from config, default 600) input_files_list: List of input file names in order (from .fz_hash) + static_entries: Pre-resolved input_static entries; relative ones are + explicitly uploaded (they live outside working_dir - only symlinked there + for local execution), absolute ones are assumed already present server-side Returns: Dict containing calculation results and status @@ -3093,14 +3117,21 @@ def read_response(): # Step 3: Upload input files (after NEW_CASE) log_info("📤 Step 3: Uploading input files...") - files_to_upload = [item for item in working_dir.iterdir() if item.is_file()] + # (relative_path, real_path) pairs: files physically in working_dir, + # plus relative static_files entries (uploaded from their real source + # path, since they live outside working_dir - only symlinked there + # for local execution). Absolute static_files are assumed already + # present server-side and are not uploaded. + files_to_upload = [(item.name, item) for item in working_dir.iterdir() if item.is_file()] + files_to_upload += [ + (e["name"], e["source"]) for e in (static_entries or []) if not e["is_absolute"] + ] log_debug(f"Found {len(files_to_upload)} files to upload") uploaded_count = 0 - for item in files_to_upload: + for relative_path, real_path in files_to_upload: # Send PUT_FILE request - file_size = item.stat().st_size - relative_path = item.name + file_size = real_path.stat().st_size log_info(f" 📄 Uploading {relative_path} ({file_size} bytes)") log_debug(f"Sending {METHOD_PUT_FILE} request for {relative_path}") @@ -3115,7 +3146,7 @@ def read_response(): log_debug(f"Server ready to receive {relative_path}") # Send file content - with open(item, 'rb') as f: + with open(real_path, 'rb') as f: file_data = f.read() bytes_sent = sock.sendall(file_data) log_debug(f"Sent {len(file_data)} bytes of file data") @@ -3361,6 +3392,43 @@ def _transfer_files_to_remote(sftp, local_dir: Path, remote_dir: str) -> None: sftp.put(local_path, remote_path) +def _sftp_mkdir_p(sftp, remote_dir: str) -> None: + """Create a remote directory (and parents) via SFTP, ignoring "already exists".""" + parts = remote_dir.strip("/").split("/") + path = "" + for part in parts: + path = f"{path}/{part}" if path else f"/{part}" + try: + sftp.mkdir(path) + except IOError: + pass # Already exists + + +def transfer_static_files_to_remote_sftp(sftp, static_entries: Optional[List[Dict[str, Any]]], remote_dir: str) -> None: + """ + Explicitly transfer a model's relative static_files entries to a remote + directory via SFTP. + + Relative static_files (see helpers.resolve_static_files) live outside + input_path/working_dir - only a local symlink is placed there for local + execution - so the generic per-case file transfer (_transfer_files_to_remote, + which only sees what's physically in working_dir) never finds them. This + uploads them explicitly from their real source path instead. Absolute entries + are skipped: they're assumed already present at that same path on the + calculator side. + """ + if not static_entries: + return + for entry in static_entries: + if entry["is_absolute"]: + continue + remote_path = f"{remote_dir}/{entry['name']}" + if "/" in entry["name"]: + _sftp_mkdir_p(sftp, str(Path(remote_path).parent).replace("\\", "/")) + log_info(f"Transferring static file {entry['name']} from {entry['source']} to remote ({remote_path})") + sftp.put(str(entry["source"]), remote_path) + + def _execute_remote_command( ssh_client, command: str, @@ -3610,6 +3678,7 @@ def run_single_case_calculation( original_input_was_dir: bool = False, original_cwd: str = None, input_files_list: List[str] = None, + static_entries: List[Dict[str, Any]] = None, ) -> Dict[str, Any]: """ Run calculation for a single case on a specific calculator @@ -3622,6 +3691,8 @@ def run_single_case_calculation( original_input_was_dir: Whether original input was a directory original_cwd: Original working directory input_files_list: List of input file names in order + static_entries: Pre-resolved input_static entries (see + helpers.resolve_static_files), force-transferred to remote calculators Returns: Dict containing calculation results and status @@ -3635,6 +3706,7 @@ def run_single_case_calculation( original_input_was_dir, original_cwd, input_files_list, + static_entries=static_entries, ) # Always add calculator URI to result diff --git a/skills/fz/reference.md b/skills/fz/reference.md index c6e7b3a..b63e197 100644 --- a/skills/fz/reference.md +++ b/skills/fz/reference.md @@ -12,17 +12,18 @@ import fz ### fz.fzi — parse input, discover variables ```python -fz.fzi(input_path: str, model: str | dict) -> dict +fz.fzi(input_path: str, model: str | dict, input_static: list[str] = None) -> dict ``` Returns a dict whose keys are the variables, formulas, and static objects found in `input_path` (file or directory); variable values are `None` (or their `~default`). +`input_static` entries (see `fz.fzr` below) are never scanned for variables. ### fz.fzc — compile input files ```python fz.fzc(input_path: str, input_variables: dict, model: str | dict, - output_dir: str = "output") -> None + output_dir: str = "output", input_static: list[str] = None) -> None ``` Substitutes variables and evaluates formulas. Scalar values produce a single compiled @@ -62,7 +63,8 @@ fz.fzr(input_path: str, calculators: str | list[str] = None, # default "sh://" callbacks: dict = None, timeout: int = None, - case_naming: str = None) -> pandas.DataFrame # "path" (default), "hash", "index" + case_naming: str = None, # "path" (default), "hash", "index" + input_static: list[str] = None) -> pandas.DataFrame ``` - dict `input_variables` ⇒ factorial (Cartesian product); DataFrame ⇒ one case per row. @@ -74,6 +76,16 @@ fz.fzr(input_path: str, `"index"` (`case_`). With `"hash"`/`"index"`, a single `cases.csv` manifest is written at the results root (case dir name → variables); each case's own `info.txt` also has them, as a fallback. Defaults to the `FZ_CASE_NAMING` env var, or `"path"`. +- `input_static`: files identical across every case (a shared weather CSV, a large + reference dataset), never templated, never re-hashed per case. Absolute path entries + are assumed already present at that path on the calculator too (no copy/symlink/ + transfer, just hashed once for cache busting); relative entries (resolved against + cwd at call time, identified by basename) are symlinked into every case directory and + explicitly transferred to `ssh://`/`slurm://` (remote)/`funz://` calculators. `fzi()` + never scans them for variables. See `doc/core-functions.md` → "fzr" → `input_static` + for the full write-up. A large (`FZ_STATIC_CANDIDATE_MIN_SIZE`, default 1 MiB) + variable-free file left in `input_path` instead triggers a one-time warning + suggesting `input_static`. - `callbacks` supports `on_start(total_cases, calculators)`, plus per-case progress callbacks (see docstring of `fz.fzr`). - Ctrl+C interrupts gracefully; completed cases stay in `results_dir` and can be reused @@ -89,7 +101,8 @@ fz.fzd(input_path: str | None, algorithm: str, # name or path to .py algorithm calculators: str | list[str] | int = None, algorithm_options: dict | str = None, # dict, JSON string, or JSON file path - analysis_dir: str = "analysis") -> dict # CLI default: results_fzd + analysis_dir: str = "analysis", # CLI default: results_fzd + input_static: list[str] = None) -> dict # passed through to each iteration's fzr() ``` Returns `{"XY": DataFrame, "analysis": ..., "iterations": int, @@ -144,16 +157,19 @@ plus calculator script/alias. `fz uninstall model|algorithm ` removes them Flags per command: ``` -fzi [input_path] --input_path/-i --model/-m --format/-f -fzc [input_path] --input_path/-i --model/-m --input_variables/-v --output_dir/-o +fzi [input_path] --input_path/-i --model/-m --input_static --format/-f +fzc [input_path] --input_path/-i --model/-m --input_variables/-v --input_static --output_dir/-o fzo [output_path] --output_path/-o --model/-m --format/-f fzr [input_path] --input_path/-i --model/-m --input_variables/-v --results_dir/-r - --calculators/-c --format/-f --case_naming {path,hash,index} + --calculators/-c --format/-f --case_naming {path,hash,index} --input_static fzl --models/-m --calculators/-c --check --format/-f fzd --input_dir/-i --input_vars/-v --model/-m --output_expression/-e - --algorithm/-a --results_dir/-r --calculators/-c --options/-o + --algorithm/-a --results_dir/-r --calculators/-c --options/-o --input_static ``` +`--input_static` (fzi/fzc/fzr/fzd): a static file path, or an inline JSON list of paths; +repeatable to add several. See `input_static` in `fz.fzr`'s signature above. + > **`fzd` flag divergence (easy to trip on):** `fzd`'s canonical input flags are > `--input_dir`/`-i` and `--input_vars`/`-v` (fz ≥ 1.1 also accepts the `fzi`/`fzc`/`fzr` > names `--input_path` and `--input_variables` as aliases; fz 1.0 took only the canonical @@ -203,6 +219,10 @@ All fields optional except `output` (required to parse results). `id` links the calculator alias files. Search path for aliases: `./.fz/models/.json` then `~/.fz/models/.json`. +Static files identical across every case (never templated) are declared via `fzr`'s +`input_static` argument, not the model — see `fz.fzr` above and `doc/core-functions.md` +→ "fzr" → `input_static`. + ## Calculator JSON schema ```json @@ -246,6 +266,7 @@ FZ_SSH_AUTO_ACCEPT_HOSTKEYS 1 to skip interactive host-key prompt (CI; use with FZ_SSH_KEEPALIVE SSH keepalive seconds FZ_SHELL_PATH bash location on Windows (MSYS2/Git Bash bin dirs) FZ_CASE_NAMING fzr case dir naming: path (default) | hash | index +FZ_STATIC_CANDIDATE_MIN_SIZE bytes threshold for the input_static warning (default 1048576; 0 disables) ``` ## Variable syntax in input files diff --git a/tests/test_funz_udp_fallback.py b/tests/test_funz_udp_fallback.py index 6b7fd72..3b40373 100644 --- a/tests/test_funz_udp_fallback.py +++ b/tests/test_funz_udp_fallback.py @@ -29,7 +29,7 @@ def _sh_success(calc_uri): def _mock_run(tmp_dir, calculator_uri, model, timeout, - original_input_was_dir, original_cwd, input_files_list): + original_input_was_dir, original_cwd, input_files_list, static_entries=None): if calculator_uri.startswith("funz://"): return _udp_miss(calculator_uri) if calculator_uri.startswith("sh://"): diff --git a/tests/test_static_files.py b/tests/test_static_files.py new file mode 100644 index 0000000..c35ccd7 --- /dev/null +++ b/tests/test_static_files.py @@ -0,0 +1,207 @@ +""" +Tests for fzr()/fzi()'s "input_static" argument: files identical across every +case (e.g. a shared weather CSV or a large reference dataset) that are never +templated/substituted, never re-hashed per case, and (for relative paths) not +duplicated on disk per case - symlinked into each case directory instead. + +- Absolute path entries: assumed already present at that same path on the + calculator side too; never copied/symlinked/transferred, only hashed (so + cache matching still reacts if the shared file's content changes). +- Relative path entries: resolved against cwd at fzr() call time, identified + by basename, symlinked into every case's result_dir/tmp_dir, and explicitly + transferred to remote calculators (see tests/test_static_files_ssh.py for + the ssh:// remote-transfer coverage). +""" +import os +from pathlib import Path + +import pytest + +import fz + + +def _write_input(tmp_path): + input_file = tmp_path / "input.txt" + input_file.write_text("x=$x\n") + return input_file + + +def test_static_files_relative_symlinked_and_hashed(tmp_path, monkeypatch): + assets_dir = tmp_path / "assets" + assets_dir.mkdir() + weather = assets_dir / "weather.csv" + weather.write_text("weather-v1") + + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + model = {"output": {"echo": "cat weather.csv"}} + res = fz.fzr(str(input_file), {"x": [1, 2]}, model, + results_dir="results", calculators="sh://true", + input_static=["../assets/weather.csv"]) + + for p in res["path"]: + link = Path(p) / "weather.csv" + # Symlinked where the platform allows it; falls back to a real copy + # on Windows without developer mode/admin privileges - either way the + # content must be correct. + if link.is_symlink(): + assert link.resolve() == weather.resolve() + assert link.read_text() == "weather-v1" + + # Hashed with its basename, not the declared "../assets/weather.csv" path + # (which would otherwise escape the case directory as a symlink name) + hash_content = (Path(res["path"][0]) / ".fz_hash").read_text() + assert "weather.csv" in hash_content + assert "../assets/weather.csv" not in hash_content + + +def test_static_files_absolute_not_transferred_but_hashed(tmp_path, monkeypatch): + shared = tmp_path / "shared_ref.bin" + shared.write_text("shared-content") + + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + model = {"output": {"echo": "echo done"}} + res = fz.fzr(str(input_file), {"x": [1]}, model, + results_dir="results", calculators="sh://true", + input_static=[str(shared)]) + + case_dir = Path(res["path"][0]) + # Never copied or symlinked into the case directory + assert not (case_dir / "shared_ref.bin").exists() + assert not (case_dir / shared.name).is_symlink() + + # But still hashed, identified by its absolute path + hash_content = (case_dir / ".fz_hash").read_text() + assert str(shared) in hash_content + + +def test_static_files_command_can_read_both_kinds(tmp_path, monkeypatch): + """Symlinked relative + absolute-referenced static files are both usable by + a calculator script running in the case's working directory.""" + assets_dir = tmp_path / "assets" + assets_dir.mkdir() + weather = assets_dir / "weather.csv" + weather.write_text("weather-data\n") + + shared = tmp_path / "shared_ref.bin" + shared.write_text("shared-data") + + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + calc_script = study_dir / "calc.sh" + # bash needs forward slashes: a literal Windows "C:\Users\..." path would + # have its backslashes misread as escape characters + calc_script.write_text(f"#!/bin/bash\ncat weather.csv {shared.as_posix()} > combined.txt\n") + calc_script.chmod(0o755) + + model = {"output": {"echo": "cat combined.txt"}} + res = fz.fzr(str(input_file), {"x": [1]}, model, + results_dir="results", calculators="sh://bash calc.sh", + input_static=["../assets/weather.csv", str(shared)]) + + assert res["status"][0] == "done" + assert res["echo"][0] == "weather-data\nshared-data" + + +def test_static_files_excluded_from_fzi_variable_scan(tmp_path, monkeypatch): + assets_dir = tmp_path / "assets" + assets_dir.mkdir() + # This file contains something that looks like a $variable - it must NOT + # be picked up as an fz variable since it's declared static + weather = assets_dir / "weather.csv" + weather.write_text("station=$not_a_real_var\n") + + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + model = {"output": {}} + variables = fz.fzi(str(input_file), model, input_static=["../assets/weather.csv"]) + assert "x" in variables + assert "not_a_real_var" not in variables + + +def test_static_files_cache_invalidated_on_content_change(tmp_path, monkeypatch): + assets_dir = tmp_path / "assets" + assets_dir.mkdir() + weather = assets_dir / "weather.csv" + weather.write_text("v1") + + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + model = {"output": {"echo": "cat weather.csv"}} + fz.fzr(str(input_file), {"x": [1]}, model, results_dir="results1", + calculators="sh://true", input_static=["../assets/weather.csv"]) + + # Re-run via cache with unchanged static file -> cache hit, same content + res_hit = fz.fzr(str(input_file), {"x": [1]}, model, results_dir="results2", + calculators=["cache://results1", "sh://true"], + input_static=["../assets/weather.csv"]) + assert res_hit["echo"][0] == "v1" + + # Change the shared static file, re-run via cache -> must NOT reuse stale cache + weather.write_text("v2-changed") + res_miss = fz.fzr(str(input_file), {"x": [1]}, model, results_dir="results3", + calculators=["cache://results1", "sh://true"], + input_static=["../assets/weather.csv"]) + assert res_miss["echo"][0] == "v2-changed" + + +def test_static_files_missing_entry_skipped_with_warning(tmp_path, monkeypatch, caplog): + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + model = {"output": {"echo": "echo done"}} + # Should not raise - the missing entry is skipped, case still runs + res = fz.fzr(str(input_file), {"x": [1]}, model, + results_dir="results", calculators="sh://true", + input_static=["does_not_exist.csv"]) + assert res["status"][0] == "done" + + +def test_static_files_invalid_type_raises(): + with pytest.raises(TypeError): + fz.fzr("x", {}, {"output": {}}, input_static="not-a-list") + + with pytest.raises(TypeError): + fz.fzr("x", {}, {"output": {}}, input_static=[123]) + + +def test_static_files_name_collision_skipped(tmp_path, monkeypatch): + """Two relative entries resolving to the same basename: only the first is kept.""" + dir_a = tmp_path / "a" + dir_a.mkdir() + (dir_a / "config.xml").write_text("from-a") + + dir_b = tmp_path / "b" + dir_b.mkdir() + (dir_b / "config.xml").write_text("from-b") + + study_dir = tmp_path / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + input_file = _write_input(study_dir) + + model = {"output": {"echo": "cat config.xml"}} + res = fz.fzr(str(input_file), {"x": [1]}, model, + results_dir="results", calculators="sh://true", + input_static=["../a/config.xml", "../b/config.xml"]) + # Whichever was kept, the case must still run and read a consistent file + assert res["status"][0] == "done" + assert res["echo"][0] in ("from-a", "from-b") diff --git a/tests/test_static_files_ssh.py b/tests/test_static_files_ssh.py new file mode 100644 index 0000000..1141229 --- /dev/null +++ b/tests/test_static_files_ssh.py @@ -0,0 +1,175 @@ +""" +Test fzr()'s "input_static" argument (see tests/test_static_files.py) over a +real ssh:// calculator connecting to localhost. + +Unlike sh:// (same filesystem, symlinks just resolve), ssh:// exercises the +actual remote-transfer code path: relative input_static entries are +explicitly uploaded via SFTP from their real source path +(runners.transfer_static_files_to_remote_sftp), since they live outside +input_path and the generic per-case file transfer only sees what's physically +in the local working directory. + +This requires an SSH server reachable at localhost with key-based auth to the +current user, set up the same way as tests/test_ssh_many_cases.py. +""" +import os +import subprocess +import time +from pathlib import Path + +import pytest +import getpass + +from conftest import SSH_AVAILABLE + +try: + import paramiko + PARAMIKO_AVAILABLE = True +except ImportError: + PARAMIKO_AVAILABLE = False + + +def _setup_ssh_key(test_dir: Path): + """Generate a dedicated SSH key pair and register it for localhost auth. + + Returns (key_path, ssh_config_path, cleanup) where cleanup() removes the + key from authorized_keys again. + """ + ssh_dir = test_dir / ".ssh" + ssh_dir.mkdir(mode=0o700, exist_ok=True) + + key_path = ssh_dir / "test_key" + pub_key_path = ssh_dir / "test_key.pub" + + result = subprocess.run( + ["ssh-keygen", "-t", "rsa", "-b", "2048", "-f", str(key_path), + "-N", "", "-C", "fz-static-files-test-key"], + capture_output=True, text=True, + ) + assert result.returncode == 0, f"Failed to generate SSH key: {result.stderr}" + key_path.chmod(0o600) + pub_key_path.chmod(0o644) + + pub_key_content = pub_key_path.read_text().strip() + home_ssh_dir = Path.home() / ".ssh" + home_ssh_dir.mkdir(mode=0o700, exist_ok=True) + authorized_keys_path = home_ssh_dir / "authorized_keys" + + key_marker = f"# FZ-STATIC-FILES-TEST-KEY-MARKER-{os.getpid()}" + original_authorized_keys = authorized_keys_path.read_text() if authorized_keys_path.exists() else None + with open(authorized_keys_path, "a") as f: + f.write(f"{key_marker}\n{pub_key_content}\n") + authorized_keys_path.chmod(0o600) + time.sleep(0.5) + + ssh_config_path = ssh_dir / "config" + ssh_config_path.write_text( + "Host localhost\n" + " StrictHostKeyChecking no\n" + " UserKnownHostsFile /dev/null\n" + " LogLevel ERROR\n" + ) + ssh_config_path.chmod(0o600) + + def cleanup(): + if not authorized_keys_path.exists(): + return + lines = authorized_keys_path.read_text().split("\n") + cleaned, skip_next = [], False + for line in lines: + if key_marker in line: + skip_next = True + continue + if skip_next and line.strip() == pub_key_content: + skip_next = False + continue + cleaned.append(line) + if original_authorized_keys is not None: + authorized_keys_path.write_text(original_authorized_keys) + else: + authorized_keys_path.write_text("\n".join(cleaned)) + + return key_path, ssh_config_path, cleanup + + +@pytest.mark.requires_ssh +@pytest.mark.requires_paramiko +@pytest.mark.skipif(not SSH_AVAILABLE, reason="SSH server not available on localhost") +@pytest.mark.skipif(not PARAMIKO_AVAILABLE, reason="paramiko library not installed") +def test_static_files_over_ssh_localhost(tmp_path, monkeypatch): + """Relative static_files are uploaded via SFTP; absolute ones are assumed + already present on the "remote" side (true here since it's localhost).""" + import fz + + test_dir = tmp_path + key_path, ssh_config_path, cleanup = _setup_ssh_key(test_dir) + + try: + # Verify the key actually authenticates before trusting the fz test below + check = subprocess.run( + ["ssh", "-i", str(key_path), "-F", str(ssh_config_path), + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", + f"{getpass.getuser()}@localhost", "echo ok"], + capture_output=True, text=True, timeout=10, + ) + assert check.returncode == 0, f"SSH connection failed: {check.stderr}" + + # fz's own ssh:// calculator (paramiko) needs localhost in the user's + # known_hosts, unlike the -F ssh_config check above + home_ssh_dir = Path.home() / ".ssh" + home_ssh_dir.mkdir(mode=0o700, exist_ok=True) + known_hosts_path = home_ssh_dir / "known_hosts" + keyscan = subprocess.run(["ssh-keyscan", "-H", "localhost"], capture_output=True, text=True) + if keyscan.returncode == 0 and keyscan.stdout: + with open(known_hosts_path, "a") as f: + f.write(keyscan.stdout) + known_hosts_path.chmod(0o644) + + assets_dir = test_dir / "assets" + assets_dir.mkdir() + weather = assets_dir / "weather.csv" + weather.write_text("weather-over-ssh\n") + + shared_absolute = test_dir / "shared_ref.bin" + shared_absolute.write_text("shared-over-ssh") + + study_dir = test_dir / "study" + study_dir.mkdir() + monkeypatch.chdir(study_dir) + + input_file = study_dir / "input.txt" + input_file.write_text("x=$x\n") + + calc_script = study_dir / "calc.sh" + # Reference the relative (uploaded/symlinked) file by name, and the + # absolute one by its full path - it's genuinely present remotely + # here since "remote" is localhost. + calc_script.write_text( + f"#!/bin/bash\ncat weather.csv {shared_absolute} > combined.txt\n" + ) + calc_script.chmod(0o755) + + model = {"output": {"echo": "cat combined.txt"}} + + # fz's ssh:// URI doesn't take custom ssh options; rely on default + # agent/key discovery plus the authorized_keys entry set up above, + # matching tests/test_ssh_many_cases.py's approach. + ssh_calculator = f"ssh://{getpass.getuser()}@localhost/bash {calc_script.resolve()}" + + result = subprocess.run(["ssh-add", str(key_path)], capture_output=True, text=True) + if result.returncode != 0: + std_key = Path.home() / ".ssh" / "id_rsa_fz_static_files_test" + std_key.write_bytes(key_path.read_bytes()) + std_key.chmod(0o600) + (Path.home() / ".ssh" / "id_rsa_fz_static_files_test.pub").write_bytes( + (key_path.with_suffix(".pub")).read_bytes() + ) + + res = fz.fzr(str(input_file), {"x": [1]}, model, + results_dir="results", calculators=[ssh_calculator], + input_static=["../assets/weather.csv", str(shared_absolute)]) + + assert res["status"][0] == "done", res.get("error", [None])[0] + assert res["echo"][0] == "weather-over-ssh\nshared-over-ssh" + finally: + cleanup() diff --git a/tests/test_static_files_warning.py b/tests/test_static_files_warning.py new file mode 100644 index 0000000..137c181 --- /dev/null +++ b/tests/test_static_files_warning.py @@ -0,0 +1,104 @@ +""" +Tests for the one-time warning that suggests moving a large, variable-free +input_path file into input_static instead (see fz/helpers.py's +_maybe_warn_static_candidate, used by compile_to_result_directories). + +Threshold is fz.config's FZ_STATIC_CANDIDATE_MIN_SIZE (default 1 MiB). +""" +from pathlib import Path + +import fz +from fz import set_log_level +from fz.logging import LogLevel +from fz.config import get_config + + +def _write_template(tmp_path, extra_files): + template_dir = tmp_path / "template" + template_dir.mkdir() + (template_dir / "input.txt").write_text("x=$x\n") + for name, content in extra_files.items(): + (template_dir / name).write_bytes(content) + return template_dir + + +def test_warns_once_for_large_variable_free_file(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(get_config(), "static_candidate_min_size", 1000) + set_log_level(LogLevel.WARNING) + try: + template_dir = _write_template(tmp_path, {"big_static.dat": b"A" * 2000}) + run_dir = tmp_path / "run" + run_dir.mkdir() + monkeypatch.chdir(run_dir) + + model = {"output": {"echo": "echo done"}} + fz.fzr(str(template_dir), {"x": [1, 2, 3]}, model, + results_dir="results", calculators="sh://true") + + stderr = capsys.readouterr().err + assert "big_static.dat" in stderr + assert "input_static" in stderr + # Only once, not once per case + assert stderr.count("big_static.dat") == 1 + finally: + set_log_level(LogLevel.ERROR) + + +def test_no_warning_for_small_file(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(get_config(), "static_candidate_min_size", 1_048_576) + set_log_level(LogLevel.WARNING) + try: + template_dir = _write_template(tmp_path, {"small_static.dat": b"A" * 1000}) + run_dir = tmp_path / "run" + run_dir.mkdir() + monkeypatch.chdir(run_dir) + + model = {"output": {"echo": "echo done"}} + fz.fzr(str(template_dir), {"x": [1]}, model, + results_dir="results", calculators="sh://true") + + stderr = capsys.readouterr().err + assert "small_static.dat" not in stderr + finally: + set_log_level(LogLevel.ERROR) + + +def test_no_warning_for_templated_file(tmp_path, monkeypatch, capsys): + """A large file that does contain a variable is not a static candidate.""" + monkeypatch.setattr(get_config(), "static_candidate_min_size", 1000) + set_log_level(LogLevel.WARNING) + try: + template_dir = _write_template( + tmp_path, {"big_templated.dat": b"val=$x " + b"A" * 2000} + ) + run_dir = tmp_path / "run" + run_dir.mkdir() + monkeypatch.chdir(run_dir) + + model = {"output": {"echo": "echo done"}} + fz.fzr(str(template_dir), {"x": [1, 2]}, model, + results_dir="results", calculators="sh://true") + + stderr = capsys.readouterr().err + assert "big_templated.dat" not in stderr + finally: + set_log_level(LogLevel.ERROR) + + +def test_warning_disabled_when_threshold_is_zero(tmp_path, monkeypatch, capsys): + monkeypatch.setattr(get_config(), "static_candidate_min_size", 0) + set_log_level(LogLevel.WARNING) + try: + template_dir = _write_template(tmp_path, {"big_static.dat": b"A" * 2_000_000}) + run_dir = tmp_path / "run" + run_dir.mkdir() + monkeypatch.chdir(run_dir) + + model = {"output": {"echo": "echo done"}} + fz.fzr(str(template_dir), {"x": [1]}, model, + results_dir="results", calculators="sh://true") + + stderr = capsys.readouterr().err + assert "big_static.dat" not in stderr + finally: + set_log_level(LogLevel.ERROR)