diff --git a/aviary/core/aviary_group.py b/aviary/core/aviary_group.py index 7f56f0de51..3c0f6a5567 100644 --- a/aviary/core/aviary_group.py +++ b/aviary/core/aviary_group.py @@ -1021,10 +1021,6 @@ def link_phases(self, verbosity=None, comm=None): phase_info2 = self.mission_info[phase2]['user_options'] vars2 = link_vars_dict[phase2] - if self.reserve_phases and phase2 == self.reserve_phases[0]: - # Don't link to first reserve phase. - continue - # Find common vars across 1-2 boundary common = vars1.intersection(vars2) upstream_analytic = [item for item in vars1 if item.startswith('initial_')] diff --git a/aviary/mission/energy_state_problem_configurator.py b/aviary/mission/energy_state_problem_configurator.py index 5dd67c79a9..208e16dcdb 100644 --- a/aviary/mission/energy_state_problem_configurator.py +++ b/aviary/mission/energy_state_problem_configurator.py @@ -493,11 +493,16 @@ def set_phase_initial_guesses( altitude_final = wrapped_convert_units(options['altitude_final'], 'ft') if altitude_initial is None: - # TODO: Pull from downstream phase. + # TODO: Pull from upstream phase. altitude_initial = wrapped_convert_units(options['altitude_bounds'], 'ft')[0] - if altitude_final is None: - # TODO: Pull from downstream phase. + if altitude_initial is None and altitude_final is None: + # No way to get an IC, so just some nominal values. + altitude_initial = 0.0 + altitude_final = 0.0 + elif altitude_initial is None: + altitude_initial = altitude_final + elif altitude_final is None: altitude_final = altitude_initial guess_dict['altitude'] = ([altitude_initial, altitude_final], 'ft') @@ -508,11 +513,16 @@ def set_phase_initial_guesses( mach_final = wrapped_convert_units(options['mach_final'], 'unitless') if mach_initial is None: - # TODO: Pull from downstream phase. + # TODO: Pull from upstream phase. mach_initial = wrapped_convert_units(options['mach_bounds'], 'unitless')[0] - if mach_final is None: - # TODO: Pull from downstream phase. + if mach_initial is None and mach_final is None: + # No way to get an IC, so just some nominal values. + mach_initial = 0.1 + mach_final = 0.1 + elif mach_initial is None: + mach_initial = mach_final + elif mach_final is None: mach_final = mach_initial guess_dict['mach'] = ([mach_initial, mach_final], 'unitless') diff --git a/aviary/run_all_benchmarks.py b/aviary/run_all_benchmarks.py index 81be3f2f74..53f9685466 100644 --- a/aviary/run_all_benchmarks.py +++ b/aviary/run_all_benchmarks.py @@ -1,3 +1,39 @@ import subprocess -status = subprocess.run(['testflo', '--testmatch=bench_test*']) +returns = subprocess.run( + ['testflo', '--nocapture', '--testmatch=bench_test*'], + capture_output=subprocess.PIPE, + text=True, +) + +DEBUG = False +if DEBUG: + print(returns.stdout) + +lines = returns.stdout.split('\n') + +print('\n\n') +print('Benchmark Results') +print('\n') + +j = 0 +results = {} +for j, line in enumerate(lines): + if 'BENCH:' in line: + line = line.partition('BENCH: ')[-1] + if line in results: + raise RuntimeError(f'Use a unique name for test {line}!') + results[line] = lines[j + 1] + +for name, bench_data in sorted(results.items()): + print(name) + print(bench_data) + print('\n') + +# Summary +print('\n') +print('Testflo Summary') +print('\n') + +for line in lines[-11:]: + print(line) diff --git a/aviary/validation_cases/benchmark_tests/test_FLOPS_balanced_field_length.py b/aviary/validation_cases/benchmark_tests/test_FLOPS_balanced_field_length.py index bc7303ecff..c716c40854 100644 --- a/aviary/validation_cases/benchmark_tests/test_FLOPS_balanced_field_length.py +++ b/aviary/validation_cases/benchmark_tests/test_FLOPS_balanced_field_length.py @@ -24,6 +24,7 @@ from aviary.utils.functions import set_aviary_initial_values, set_aviary_input_defaults from aviary.utils.preprocessors import preprocess_options from aviary.utils.test_utils.default_subsystems import get_default_mission_subsystems +from aviary.validation_cases.benchmark_utils import print_benchmark_results from aviary.variable_info.functions import setup_model_options from aviary.variable_info.variables import Aircraft, Dynamic @@ -43,7 +44,8 @@ def bench_test_IPOPT(self): driver.opt_settings['tol'] = 1e-3 driver.opt_settings['print_level'] = 4 - self._do_run(driver, optimizer) + prob = self._do_run(driver, optimizer) + print_benchmark_results(prob) @require_pyoptsparse(optimizer='SNOPT') def bench_test_SNOPT(self): @@ -57,7 +59,8 @@ def bench_test_SNOPT(self): driver.opt_settings['Major feasibility tolerance'] = 1e-6 driver.opt_settings['iSumm'] = 6 - self._do_run(driver, optimizer) + prob = self._do_run(driver, optimizer) + print_benchmark_results(prob) def _do_run(self, driver: Driver, optimizer, *args): aviary_options = _inputs.deepcopy() @@ -154,6 +157,8 @@ def _do_run(self, driver: Driver, optimizer, *args): actual = takeoff.model.get_val('traj.balanced_rotate.states:velocity', units='kn')[-1] assert_near_equal(actual, desired, 2e-2) + return takeoff + if __name__ == '__main__': use_SNOPT = False diff --git a/aviary/validation_cases/benchmark_tests/test_FLOPS_based_sizing_N3CC.py b/aviary/validation_cases/benchmark_tests/test_FLOPS_based_sizing_N3CC.py index e2e5534d13..2d09555e21 100644 --- a/aviary/validation_cases/benchmark_tests/test_FLOPS_based_sizing_N3CC.py +++ b/aviary/validation_cases/benchmark_tests/test_FLOPS_based_sizing_N3CC.py @@ -19,7 +19,10 @@ from aviary.core.aviary_problem import AviaryProblem from aviary.models.aircraft.advanced_single_aisle.phase_info import phase_info from aviary.utils.test_utils.assert_utils import warn_timeseries_near_equal -from aviary.validation_cases.benchmark_utils import compare_against_expected_values +from aviary.validation_cases.benchmark_utils import ( + compare_against_expected_values, + print_benchmark_results, +) from aviary.variable_info.variables import Aircraft, Mission @@ -47,7 +50,7 @@ def run_trajectory(sim=True): prob.check_and_preprocess_inputs() prob.build_model() - prob.add_driver('SNOPT', max_iter=50, verbosity=1) + prob.add_driver('SNOPT', max_iter=50, verbosity=0) ########################## # Design Variables # @@ -79,6 +82,7 @@ class ProblemPhaseTestCase(unittest.TestCase): def bench_test_sizing_N3CC(self): prob = run_trajectory(sim=False) + print_benchmark_results(prob) # self.assertTrue(prob.result.success) times_climb = prob.get_val('traj.climb.timeseries.time', units='s') @@ -515,7 +519,6 @@ def bench_test_sizing_N3CC(self): ) compare_against_expected_values(prob, self.expected_dict) - # self.assertTrue(prob.result.success) if __name__ == '__main__': diff --git a/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_landing.py b/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_landing.py index 180af38d81..cd5951aa2f 100644 --- a/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_landing.py +++ b/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_landing.py @@ -22,6 +22,7 @@ from aviary.utils.functions import set_aviary_initial_values, set_aviary_input_defaults from aviary.utils.preprocessors import preprocess_options from aviary.utils.test_utils.default_subsystems import get_default_mission_subsystems +from aviary.validation_cases.benchmark_utils import print_benchmark_results from aviary.variable_info.functions import setup_model_options from aviary.variable_info.variables import Aircraft, Dynamic @@ -42,7 +43,8 @@ def bench_test_IPOPT(self): driver.opt_settings['print_level'] = 4 driver.opt_settings['mu_init'] = 1e-5 - self._do_run(driver, optimizer) + prob = self._do_run(driver, optimizer) + print_benchmark_results(prob) @require_pyoptsparse(optimizer='SNOPT') def bench_test_SNOPT(self): @@ -56,7 +58,8 @@ def bench_test_SNOPT(self): driver.opt_settings['Major feasibility tolerance'] = 1e-6 driver.opt_settings['iSumm'] = 6 - self._do_run(driver, optimizer) + prob = self._do_run(driver, optimizer) + print_benchmark_results(prob) def _do_run(self, driver: Driver, optimizer, *args): aviary_options = _inputs.deepcopy() @@ -151,6 +154,7 @@ def _do_run(self, driver: Driver, optimizer, *args): actual = landing.model.get_val('traj.landing_fullstop.t', units='s')[-1] assert_near_equal(actual, desired, 0.05) + return landing if __name__ == '__main__': diff --git a/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_takeoff.py b/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_takeoff.py index 38082653c4..ebdd8fcbc2 100644 --- a/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_takeoff.py +++ b/aviary/validation_cases/benchmark_tests/test_FLOPS_detailed_takeoff.py @@ -22,6 +22,7 @@ from aviary.utils.functions import set_aviary_initial_values, set_aviary_input_defaults from aviary.utils.preprocessors import preprocess_options from aviary.utils.test_utils.default_subsystems import get_default_mission_subsystems +from aviary.validation_cases.benchmark_utils import print_benchmark_results from aviary.variable_info.functions import setup_model_options from aviary.variable_info.variables import Aircraft, Dynamic @@ -46,7 +47,8 @@ def bench_test_IPOPT(self): # driver.opt_settings['nlp_scaling_method'] = None # driver.opt_settings['linear_solver'] = 'mumps' - self._do_run(driver, optimizer) + prob = self._do_run(driver, optimizer) + print_benchmark_results(prob) @require_pyoptsparse(optimizer='SNOPT') def bench_test_SNOPT(self): @@ -60,7 +62,8 @@ def bench_test_SNOPT(self): driver.opt_settings['Major feasibility tolerance'] = 1e-6 driver.opt_settings['iSumm'] = 6 - self._do_run(driver, optimizer) + prob = self._do_run(driver, optimizer) + print_benchmark_results(prob) def _do_run(self, driver: Driver, optimizer, *args): aviary_options = _inputs.deepcopy() @@ -158,6 +161,8 @@ def _do_run(self, driver: Driver, optimizer, *args): assert_near_equal(actual, desired, 2e-2) + return takeoff + if __name__ == '__main__': use_SNOPT = True diff --git a/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py b/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py index 772a19a5f0..fa67909b3d 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_FwGm.py @@ -7,6 +7,7 @@ from aviary.models.missions.two_dof_default import phase_info from aviary.interface.run_aviary import run_aviary +from aviary.validation_cases.benchmark_utils import print_benchmark_results from aviary.variable_info.variables import Aircraft, Mission @@ -32,6 +33,7 @@ def bench_test_swap_3_FwGm_IPOPT(self): optimizer='IPOPT', ) + print_benchmark_results(prob) # TODO: This problem does not always converge. # self.assertTrue(prob.result.success) @@ -59,11 +61,12 @@ def bench_test_swap_3_FwGm_SNOPT(self): prob = run_aviary( 'validation_cases/validation_data/test_models/aircraft_for_bench_FwGm.csv', local_phase_info, - verbosity=1, + verbosity=0, optimizer='SNOPT', max_iter=60, ) + print_benchmark_results(prob) self.assertTrue(prob.result.success) rtol = 1e-2 @@ -84,8 +87,6 @@ def bench_test_swap_3_FwGm_SNOPT(self): else: assert_near_equal(prob.get_val(var_name), expected_val, tolerance=rtol) - self.assertTrue(prob.result.success) - if __name__ == '__main__': test = ProblemPhaseTestCase() diff --git a/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py b/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py index b1c300ee01..c7e8fe6625 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_GwFm.py @@ -16,7 +16,10 @@ from aviary.interface.run_aviary import run_aviary from aviary.validation_cases.validation_data.test_models.GwFm_phase_info import phase_info -from aviary.validation_cases.benchmark_utils import compare_against_expected_values +from aviary.validation_cases.benchmark_utils import ( + compare_against_expected_values, + print_benchmark_results, +) @use_tempdirs @@ -126,6 +129,7 @@ def bench_test_swap_1_GwFm_IPOPT(self): optimizer='IPOPT', verbosity=0, ) + print_benchmark_results(prob) # self.assertTrue(prob.result.success) compare_against_expected_values(prob, self.expected_dict) @@ -136,8 +140,9 @@ def bench_test_swap_1_GwFm_SNOPT(self): self.phase_info, max_iter=50, optimizer='SNOPT', - verbosity=1, + verbosity=0, ) + print_benchmark_results(prob) # self.assertTrue(prob.result.success) compare_against_expected_values(prob, self.expected_dict) self.assertTrue(prob.result.success) diff --git a/aviary/validation_cases/benchmark_tests/test_bench_solved2dof.py b/aviary/validation_cases/benchmark_tests/test_bench_solved2dof.py index ffb22aefb1..252b5647a9 100644 --- a/aviary/validation_cases/benchmark_tests/test_bench_solved2dof.py +++ b/aviary/validation_cases/benchmark_tests/test_bench_solved2dof.py @@ -5,14 +5,15 @@ from aviary.models.missions.solved2dof_landing_default import phase_info as phase_info_landing from openmdao.utils.assert_utils import assert_near_equal from openmdao.utils.testing_utils import require_pyoptsparse, use_tempdirs +from aviary.validation_cases.benchmark_utils import print_benchmark_results @use_tempdirs class TestBenchSolved2DOF(unittest.TestCase): """Run the model in serial that is setup in ProblemPhaseTestCase class.""" - @require_pyoptsparse(optimizer='IPOPT') - def test_bench_Solved2DOF(self): + @require_pyoptsparse(optimizer='SNOPT') + def bench_test_Solved2DOF(self): subsystem_options = { 'aerodynamics': { 'method': 'low_speed', @@ -97,18 +98,20 @@ def test_bench_Solved2DOF(self): prob = av.run_aviary( aircraft_data='validation_cases/validation_data/test_models/aircraft_for_bench_solved2dof.csv', phase_info=phase_info, - optimizer='IPOPT', + optimizer='SNOPT', objective_type='time', max_iter=100, ) + print_benchmark_results(prob) self.assertTrue(prob.result.success) + tol = 1e-2 assert_near_equal(prob.get_val(av.Mission.FINAL_TIME, units='s'), 108.84030411, tol) assert_near_equal(prob.get_val(av.Mission.FUEL_MASS, units='lbm'), 459.3830223, tol) - @require_pyoptsparse(optimizer='IPOPT') - def test_bench_Solved2DOF_landing(self): + @require_pyoptsparse(optimizer='SNOPT') + def bench_test_Solved2DOF_landing(self): # This problem solves better with a reduced ref for objective time, therefore need to call add_objectve() subsystem_options = { 'aerodynamics': { @@ -204,17 +207,21 @@ def test_bench_Solved2DOF_landing(self): ) prob.check_and_preprocess_inputs() prob.build_model() - prob.add_driver('IPOPT', max_iter=100) + prob.add_driver('SNOPT', max_iter=100) prob.add_design_variables() prob.add_objective('time', ref=1e2) prob.setup() prob.run_aviary_problem() - self.assertTrue(prob.result.success) + print_benchmark_results(prob) + # self.assertTrue(prob.result.success) + tol = 1e-2 assert_near_equal(prob.get_val(av.Mission.FINAL_TIME, units='s'), 68.30353617, tol) assert_near_equal(prob.get_val(av.Mission.FUEL_MASS, units='lbm'), 98.91566618, tol) if __name__ == '__main__': - unittest.main() + # unittest.main() + z = TestBenchSolved2DOF() + z.bench_test_Solved2DOF_landing() diff --git a/aviary/validation_cases/benchmark_tests/test_problem_types_GwGm.py b/aviary/validation_cases/benchmark_tests/test_problem_types_GwGm.py index e28b8ccfba..5ac80f27fe 100644 --- a/aviary/validation_cases/benchmark_tests/test_problem_types_GwGm.py +++ b/aviary/validation_cases/benchmark_tests/test_problem_types_GwGm.py @@ -6,7 +6,8 @@ import aviary.api as av from aviary.models.missions.two_dof_default import phase_info -from aviary.variable_info.enums import ProblemType, Verbosity +from aviary.validation_cases.benchmark_utils import print_benchmark_results +from aviary.variable_info.enums import ProblemType class TwoDOFTestCase(unittest.TestCase): @@ -31,7 +32,7 @@ def test_off_design_IPOPT(self): prob_off_design_max_range.load_inputs( 'validation_cases/validation_data/test_models/aircraft_for_bench_GwGm.csv', self.phase_info, - verbosity=Verbosity.BRIEF, + verbosity=0, ) prob_off_design_max_range.problem_type = ProblemType.OFF_DESIGN_MAX_RANGE @@ -62,7 +63,7 @@ def test_off_design_IPOPT(self): prob_off_design_min_fuel.load_inputs( 'validation_cases/validation_data/test_models/aircraft_for_bench_GwGm.csv', self.phase_info, - verbosity=Verbosity.BRIEF, + verbosity=0, ) prob_off_design_min_fuel.problem_type = ProblemType.OFF_DESIGN_MIN_FUEL prob_off_design_min_fuel.aviary_inputs.set_val( @@ -96,7 +97,7 @@ def test_off_design_SNOPT(self): prob_off_design_max_range.load_inputs( 'validation_cases/validation_data/test_models/aircraft_for_bench_GwGm.csv', self.phase_info, - verbosity=Verbosity.BRIEF, + verbosity=0, ) prob_off_design_max_range.problem_type = ProblemType.OFF_DESIGN_MAX_RANGE @@ -123,7 +124,7 @@ def test_off_design_SNOPT(self): prob_off_design_min_fuel.load_inputs( 'validation_cases/validation_data/test_models/aircraft_for_bench_GwGm.csv', self.phase_info, - verbosity=Verbosity.BRIEF, + verbosity=0, ) prob_off_design_min_fuel.problem_type = ProblemType.OFF_DESIGN_MIN_FUEL prob_off_design_min_fuel.aviary_inputs.set_val( diff --git a/aviary/validation_cases/benchmark_utils.py b/aviary/validation_cases/benchmark_utils.py index 577a91f965..1a5c6ce4aa 100644 --- a/aviary/validation_cases/benchmark_utils.py +++ b/aviary/validation_cases/benchmark_utils.py @@ -1,9 +1,38 @@ +from pathlib import Path +import sys + import numpy as np from openmdao.utils.assert_utils import assert_near_equal from aviary.utils.test_utils.assert_utils import warn_timeseries_near_equal +def print_benchmark_results(prob): + """ + Prints summary of results for a completed benchmark. + """ + + frame = sys._getframe(1) + test_name = frame.f_code.co_name + file_name = frame.f_code.co_filename + file_name = Path(file_name).name + + print(f'BENCH: {file_name}:{test_name} -- {prob.driver.options["optimizer"]}') + + pyopt = prob.driver.pyopt_solution + code = pyopt.optInform['value'] + msg = pyopt.optInform['text'] + dt = pyopt.optTime + nobj = pyopt.userObjCalls + nsen = pyopt.userSensCalls + data = prob.list_driver_vars(driver_scaling=False, out_stream=None) + obj = data['objectives'].pop()[1]['val'][0] + print( + f' Obj: {obj:.4f} Time: {dt:.2f} s Obj Calls: {nobj} Sens Calls: {nsen} Status: {code} - {msg}' + ) + print('', flush=True) + + def compare_against_expected_values(prob, expected_dict): """Compare values in prob with the ones in expected_dict.""" expected_times = expected_dict['times']