test/select/test_select.py::test_problem_select - AssertionError

[dweindl]

https://github.com/ICB-DCM/pyPESTO/actions/runs/6903540764/job/18782434139

_____________________________ test_problem_select ______________________________

pypesto_select_problem = <pypesto.select.problem.Problem object at 0x7f43f12e5810>

    def test_problem_select(pypesto_select_problem):
        """Test the `Problem.select` method."""
        expected_results = [
            {
                'candidates_model_subspace_ids': ['M1_0'],
                'best_model_subspace_id': 'M1_0',
                'best_model_aic': 36.97,
            },
            {
                'candidates_model_subspace_ids': ['M1_1', 'M1_2', 'M1_3'],
                'best_model_subspace_id': 'M1_3',
                'best_model_aic': -4.71,
            },
            {
                'candidates_model_subspace_ids': ['M1_5', 'M1_6'],
                'best_model_subspace_id': 'M1_6',
                'best_model_aic': -4.15,
            },
            {
                'candidates_model_subspace_ids': ['M1_7'],
                'best_model_subspace_id': 'M1_7',
                'best_model_aic': -4.06,
            },
        ]
    
        candidate_space = (
            pypesto_select_problem.petab_select_problem.new_candidate_space(
                method=Method.FORWARD
            )
        )
        criterion = Criterion.AIC
    
        best_model = None
        for expected_result in expected_results:
            best_model, _ = pypesto_select_problem.select(
                criterion=criterion,
                minimize_options=minimize_options,
                predecessor_model=best_model,
                candidate_space=candidate_space,
            )
    
            test_candidates_model_subspace_ids = [
                model.model_subspace_id for model in candidate_space.models
            ]
    
            test_best_model_subspace_id = best_model.model_subspace_id
            test_best_model_aic = best_model.get_criterion(Criterion.AIC)
    
            test_result = {
                'candidates_model_subspace_ids': test_candidates_model_subspace_ids,
                'best_model_subspace_id': test_best_model_subspace_id,
                'best_model_aic': test_best_model_aic,
            }
    
            # The expected "forward" models were found.
            assert (
                test_result['candidates_model_subspace_ids']
                == expected_result['candidates_model_subspace_ids']
            )
    
            # The best model is as expected.
            assert (
                test_result["best_model_subspace_id"]
                == expected_result["best_model_subspace_id"]
            )
    
            # The best model has its criterion value set and is the expected value.
>           assert np.isclose(
                [test_result["best_model_aic"]],
                [expected_result['best_model_aic']],
                **tolerances,
            )
E           AssertionError: assert array([False])
E            +  where array([False]) = <function isclose at 0x7f441653cf70>([-4.000321187135173], [-4.06], **{'atol': 0.01, 'rtol': 0.01})
E            +    where <function isclose at 0x7f441653cf70> = np.isclose

test/select/test_select.py:171: AssertionError

@dilpath : Could you please check - is this expected to fail occasionally?

[dilpath]

@dilpath : Could you please check - is this expected to fail occasionally?

I think so, only 10 starts were used. Since the test takes < 4 mins with 10 starts, I doubled it to 20 starts in #1240