diff --git a/activitysim/abm/models/location_choice.py b/activitysim/abm/models/location_choice.py index c955f4a606..f4320a0b16 100644 --- a/activitysim/abm/models/location_choice.py +++ b/activitysim/abm/models/location_choice.py @@ -463,6 +463,9 @@ def location_presample( n_total_alts=n_total_alts, ) + if state.settings.skip_failed_choices and taz_sample.empty: + return taz_sample.rename(columns={DEST_TAZ: alt_dest_col_name}) + # print(f"taz_sample\n{taz_sample}") # dest_TAZ prob pick_count # person_id @@ -869,6 +872,12 @@ def run_location_choice( ), ) + if state.settings.skip_failed_choices: + # Failed sampling choices have no alternatives for logsums or simulation. + choosers = choosers[choosers.index.isin(location_sample_df.index)] + if choosers.empty: + continue + # - location_logsums # skip logsum calculations if LOGSUM_SETTINGS is None if model_settings.LOGSUM_SETTINGS: diff --git a/activitysim/abm/test/test_location_choice.py b/activitysim/abm/test/test_location_choice.py index 9bcdd7711f..65a945c061 100644 --- a/activitysim/abm/test/test_location_choice.py +++ b/activitysim/abm/test/test_location_choice.py @@ -5,6 +5,7 @@ import pandas as pd import pandas.testing as pdt +import pytest from activitysim.abm.models import location_choice @@ -58,6 +59,7 @@ def test_estimation_override_preserves_destination_choice_logsum(monkeypatch): state = SimpleNamespace( settings=SimpleNamespace( sample_method="inverse_cdf", + skip_failed_choices=False, trace_hh_id=None, use_explicit_error_terms=False, ) @@ -88,3 +90,69 @@ def test_estimation_override_preserves_destination_choice_logsum(monkeypatch): ) pdt.assert_frame_equal(choices, expected) assert sample is None + + +@pytest.mark.parametrize("sampled_person_ids", [[], [2, 2]]) +def test_location_choice_skips_failed_samples(monkeypatch, sampled_person_ids): + """Only persons with sampled alternatives reach logsums and final choice.""" + persons = pd.DataFrame( + {"segment": [1, 1]}, index=pd.Index([1, 2], name="person_id") + ) + location_sample = pd.DataFrame( + {"alt_dest": [101, 202][: len(sampled_person_ids)]}, + index=pd.Index(sampled_person_ids, name="person_id"), + ) + expected_persons = persons.loc[persons.index.isin(sampled_person_ids)] + sample_model = Mock(return_value=location_sample) + logsums_model = Mock( + return_value=location_sample.assign(**{location_choice.ALT_LOGSUM: 0.0}) + ) + simulate_model = Mock( + return_value=pd.DataFrame({"choice": 101}, index=expected_persons.index) + ) + monkeypatch.setattr(location_choice, "run_location_sample", sample_model) + monkeypatch.setattr(location_choice, "run_location_logsums", logsums_model) + monkeypatch.setattr(location_choice, "run_location_simulate", simulate_model) + shadow_price_calculator = Mock() + shadow_price_calculator.dest_size_terms.return_value = pd.Series( + [1.0, 1.0], index=[101, 202] + ) + model_settings = SimpleNamespace( + ALT_DEST_COL_NAME="alt_dest", + CHOOSER_SEGMENT_COLUMN_NAME="segment", + LOGSUM_SETTINGS="tour_mode_choice.yaml", + SEGMENT_IDS={"workers": 1}, + ) + state = SimpleNamespace( + settings=SimpleNamespace( + skip_failed_choices=True, use_explicit_error_terms=True + ) + ) + + choices, sample = location_choice.run_location_choice( + state=state, + persons_merged_df=persons, + network_los=Mock(), + shadow_price_calculator=shadow_price_calculator, + want_logsums=False, + want_sample_table=True, + estimator=None, + model_settings=model_settings, + chunk_size=0, + chunk_tag="workplace_location", + trace_label="workplace_location", + ) + + if expected_persons.empty: + assert choices.empty + assert sample is None + logsums_model.assert_not_called() + simulate_model.assert_not_called() + else: + pdt.assert_index_equal(choices.index, expected_persons.index) + pdt.assert_frame_equal(logsums_model.call_args.args[2], expected_persons) + pdt.assert_frame_equal(simulate_model.call_args.args[2], expected_persons) + pdt.assert_index_equal( + sample.index.get_level_values("person_id").unique(), + expected_persons.index, + ) diff --git a/activitysim/abm/test/test_misc/test_location_choice_sampling.py b/activitysim/abm/test/test_misc/test_location_choice_sampling.py index 6b21e0329c..262781c793 100644 --- a/activitysim/abm/test/test_misc/test_location_choice_sampling.py +++ b/activitysim/abm/test/test_misc/test_location_choice_sampling.py @@ -1,5 +1,8 @@ from __future__ import annotations +from types import SimpleNamespace +from unittest.mock import Mock + import pandas as pd from activitysim.abm.models import location_choice @@ -23,6 +26,58 @@ def get_skim_dict(self, layer): return _DummySkimDict() +def test_location_presample_skips_empty_taz_sample(monkeypatch): + state = workflow.State().default_settings() + state.settings.skip_failed_choices = True + state.settings.use_explicit_error_terms = True + taz_sample = pd.DataFrame( + columns=[location_choice.DEST_TAZ, "prob", "pick_count"], + index=pd.Index([], name="person_id"), + ) + monkeypatch.setattr( + location_choice, "_location_sample", Mock(return_value=taz_sample) + ) + monkeypatch.setattr( + location_choice.shadow_pricing, + "load_shadow_price_calculator", + Mock(return_value=SimpleNamespace(use_shadow_pricing=False)), + ) + choose_maz = Mock() + monkeypatch.setattr( + location_choice.tour_destination, "choose_MAZ_for_TAZ", choose_maz + ) + persons = pd.DataFrame( + {location_choice.HOME_MAZ: [9001], location_choice.HOME_TAZ: [90]}, + index=pd.Index([1001], name="person_id"), + ) + size_terms = pd.DataFrame( + { + "size_term": [1.0, 2.0], + "shadow_price_size_term_adjustment": [1.0, 1.0], + "shadow_price_utility_adjustment": [0.0, 0.0], + }, + index=pd.Index([101, 102], name="zone_id"), + ) + + out = location_choice.location_presample( + state, + "workers", + persons, + _DummyNetworkLos({101: 1, 102: 2}), + size_terms, + estimator=None, + model_settings=SimpleNamespace(ALT_DEST_COL_NAME="zone_id"), + chunk_size=0, + chunk_tag="workplace_location", + trace_label="workplace_location", + ) + + pd.testing.assert_frame_equal( + out, taz_sample.rename(columns={location_choice.DEST_TAZ: "zone_id"}) + ) + choose_maz.assert_not_called() + + def test_location_presample_uses_taz_stable_mapping(monkeypatch): captured = {} diff --git a/activitysim/core/interaction_sample.py b/activitysim/core/interaction_sample.py index d8bfe945db..99c970b484 100644 --- a/activitysim/core/interaction_sample.py +++ b/activitysim/core/interaction_sample.py @@ -394,7 +394,7 @@ def make_sample_choices( assert isinstance(alternatives, pd.DataFrame) assert len(alternatives) == alternative_count - if allow_zero_probs: + if allow_zero_probs or state.settings.skip_failed_choices: zero_probs = probs.sum(axis=1) == 0 if zero_probs.all(): return pd.DataFrame( @@ -897,7 +897,7 @@ def _interaction_sample( else: # eet and poisson: optionally trim choosers with all-zero probs. The # inverse-CDF path handles this inside make_sample_choices - if allow_zero_probs: + if allow_zero_probs or state.settings.skip_failed_choices: non_zero = probs.sum(axis=1) != 0 if not non_zero.any(): return pd.DataFrame( @@ -1159,9 +1159,13 @@ def interaction_sample( # http://pandas.pydata.org/pandas-docs/stable/io.html#id2 if len(result_list) > 1: choices = pd.concat(result_list) + elif result_list: + choices = result_list[0] - assert allow_zero_probs or ( - len(choosers.index) == len(np.unique(choices.index.values)) + assert ( + allow_zero_probs + or state.settings.skip_failed_choices + or len(choosers.index) == len(np.unique(choices.index.values)) ) # keep alts in canonical order so choices based on their probs are stable across runs diff --git a/activitysim/core/test/test_interaction_sample.py b/activitysim/core/test/test_interaction_sample.py index 9efae1f53d..89ce126fa3 100644 --- a/activitysim/core/test/test_interaction_sample.py +++ b/activitysim/core/test/test_interaction_sample.py @@ -8,6 +8,7 @@ import numpy as np import pandas as pd import pytest +import sharrow as sh from activitysim.core import interaction_sample, workflow from activitysim.core.configuration.base import ComputeSettings @@ -20,6 +21,67 @@ def state() -> workflow.State: return state +@pytest.mark.parametrize("sample_method", [None, "eet", "inverse_cdf"]) +@pytest.mark.parametrize("all_failed", [False, True]) +@pytest.mark.parametrize("explicit_chunk_size", [0, 1]) +def test_interaction_sample_skips_underflowed_probs( + state, tmp_path, sample_method, all_failed, explicit_chunk_size +): + (tmp_path / "configs").mkdir() + (tmp_path / "data").mkdir() + state.initialize_filesystem(working_dir=tmp_path, configs_dir=("configs",)) + state.filesystem.sharrow_cache_dir = tmp_path / "sharrow-cache" + state.set("skim_dataset", sh.Dataset()) + state.settings.sharrow = True + state.settings.chunk_training_mode = "explicit" + state.settings.skip_failed_choices = True + state.settings.use_explicit_error_terms = True + choosers = pd.DataFrame( + { + "household_id": [101, 102, 103], + # exp(-120) underflows in float32, but not in float64. + "utility": [-120.0 if all_failed else 0.0, -120.0, -120.0], + }, + index=pd.Index([1, 2, 3], name="person_id"), + ) + alternatives = pd.DataFrame( + {"alt_utility": [0.0, -1.0]}, + index=pd.Index([10, 20], name="alt_id"), + ) + spec = pd.DataFrame( + {"coefficient": [1.0, 1.0]}, + index=pd.Index(["utility", "alt_utility"], name="Expression"), + ) + state.rng().add_channel("person_id", choosers) + state.rng().begin_step("test_underflowed_probs") + + sample = interaction_sample.interaction_sample( + state, + choosers, + alternatives, + spec, + sample_size=2, + alt_col_name="alt_id", + explicit_chunk_size=explicit_chunk_size, + compute_settings=ComputeSettings(sample_method=sample_method), + stable_alt_positions=np.array([0, 2]), + n_total_alts=3, + ) + + assert sample.index.name == "person_id" + assert set(sample.index) == (set() if all_failed else {1}) + assert (sample["prob"] > 0).all() + assert sample["alt_id"].isin(alternatives.index).all() + assert (sample["pick_count"] > 0).all() + skipped_households = { + household_id + for ids in state.get("skipped_household_ids").values() + for household_id in ids + } + assert skipped_households == ({101, 102, 103} if all_failed else {102, 103}) + assert state.get("num_skipped_households") == len(skipped_households) + + def test_interaction_sample_ignores_stable_positions_without_global_eet( state, monkeypatch ):