diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index b45f3501e..14da930ea 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -76,12 +76,8 @@ jobs: run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py apply --create-missing-roles - name: Verify the managed v2 roles hold their declared privileges run: uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify - - name: Verify the installed PolicyEngine.py catalog interface - run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py - env: - RUN_V2_CATALOG_COMPATIBILITY: "1" - name: Test v2 metadata publication, resource routes and the countries list - run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py + run: uv run --frozen --extra dev coverage run --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations @@ -92,10 +88,10 @@ jobs: run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_household_persistence.py tests/integration/test_v1_household_dual_write.py tests/integration/test_mysql_household_spm_dual_write.py - name: Test the row counts v1 ORM updates report on MySQL run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_mysql_update_row_counts.py - - name: Qualify production-scale v2 metadata publication - run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication_qualification.py - env: - RUN_V2_CATALOG_PUBLICATION_QUALIFICATION: "1" + # The real US and UK models, reduced to a few parameter folders: the + # complete installed catalog is never loaded in CI. + - name: Publish selected parameter folders of the installed PolicyEngine.py models + run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q --durations=10 tests/integration/test_v2_catalog_installed_folders.py - name: Test real Redis cross-instance semantics run: uv run --frozen --extra dev coverage run -a --branch -m pytest -q tests/integration/test_runtime_cache_redis.py - name: Export v2 integration coverage diff --git a/changelog.d/4149.changed.md b/changelog.d/4149.changed.md new file mode 100644 index 000000000..0d08d41f9 --- /dev/null +++ b/changelog.d/4149.changed.md @@ -0,0 +1 @@ +CI no longer loads the complete installed PolicyEngine.py catalog. The cross-database integration tests publish a few parameter folders of the installed US and UK models, and a small catalog of edge-case values, and check that every value comes back from PostgreSQL unchanged. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 7331988d9..a6155d82d 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -203,6 +203,24 @@ Otherwise the release run runs them, and `publish-git-tag` requires lint, both Alembic checks and the integration tests to pass, as before. The release test suite always runs. +CI never loads the complete installed PolicyEngine.py catalog (about 1.2 +million parameter values). Two tests in the cross-database check cover +publishing it instead, on every run: + +- `test_v2_catalog_installed_folders.py` runs the real US and UK models through + extraction and publication, reduced to the parameter folders in its + `SELECTED_FOLDERS`, and reads every value back. Between them the folders + hold every value shape the installed models hold, and the test fails when + one is missing: replace a renamed folder with one that holds the same + values. +- `test_v2_catalog_publication.py` publishes + `tests/fixtures/v2_catalog_value_shapes.py`, which holds one example of each + value shape extraction accepts, and reads every value back. When extraction + accepts a new shape, add an example there. + +Each release publishes and validates the complete catalog against staging, +then production, in `seed-v2-database.yml`. + Staging deployment checks run the live integration suite against the tagged Cloud Run staging URL before promoting the exact tested revision to the service URL. No production deployment may begin until the diff --git a/docs/migration/stage-9-v2-metadata.md b/docs/migration/stage-9-v2-metadata.md index 8ee9dc4b4..243826d98 100644 --- a/docs/migration/stage-9-v2-metadata.md +++ b/docs/migration/stage-9-v2-metadata.md @@ -158,17 +158,13 @@ regions. Publication took 24.120 seconds and added 7,968,103 bytes of measured peak publisher memory. The US fallback summary reported 436 congressional districts, 333 places, and 51 states. -Re-run the production-scale test only against disposable Postgres: - -```bash -RUN_V2_CATALOG_PUBLICATION_QUALIFICATION=1 \ -V2_ALEMBIC_DISPOSABLE_TEST=1 \ -V2_MIGRATION_DATABASE_URL="" \ -uv run pytest -q tests/integration/test_v2_catalog_publication_qualification.py -``` - -Do not point this qualification command at a persistent staging or production -target. +The production-scale test that measured this, +`tests/integration/test_v2_catalog_publication_qualification.py`, was removed +with PolicyEngine/policyengine-api#4149: CI no longer loads the complete +catalog. Each release publishes and validates it against staging, then +production (`seed-v2-database.yml`), and the cross-database integration tests +publish selected parameter folders of the installed models and a catalog of +edge-case values (`docs/engineering/skills/testing.md`). ## Preview verification diff --git a/tests/fixtures/v2_catalog_value_shapes.py b/tests/fixtures/v2_catalog_value_shapes.py new file mode 100644 index 000000000..09dd8990e --- /dev/null +++ b/tests/fixtures/v2_catalog_value_shapes.py @@ -0,0 +1,430 @@ +"""A small catalog with one example of each value shape extraction accepts. + +CI never loads the complete installed catalog (about 1.2 million parameter +values). tests/integration/test_v2_catalog_installed_folders.py publishes a few +parameter folders of the installed models, which hold only the shapes those +models use; publishing this catalog checks that every shape extraction accepts +comes back from PostgreSQL unchanged (tests/integration/test_v2_catalog_publication.py): +every JSON value type a parameter value may hold, date-only and naive start +dates, year-1, one-day and open-ended date ranges, bracket parameters and +their nodes, and the text, number and membership-list region filters. + +It is the deterministic fixture catalog from tests/fixtures/v2_catalog.py with +these cases added, so that catalog's counts stay as its own tests assert. +""" + +from __future__ import annotations + +import math +from dataclasses import KW_ONLY, dataclass +from datetime import date, datetime +from enum import Enum +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import numpy as np + +from policyengine_api.data.v2.catalog.extraction import extract_catalog +from tests.fixtures.v2_catalog import ( + POLICYENGINE_VERSION, + bundle, + installed_version, + source_models, +) + +if TYPE_CHECKING: + from policyengine_api.data.v2.catalog.records import NormalizedCatalog + from policyengine_api.json_types import JSONValue + + +class _FilingStatus(Enum): + SINGLE = "SINGLE" + JOINT = "JOINT" + + +@dataclass(frozen=True) +class ValueCase: + """One source parameter value and the JSON value the catalog stores for it. + + ``start`` and ``end`` are as the source gives them; the catalog stores + them as UTC datetimes. + """ + + start: date | datetime + end: date | datetime | None + _: KW_ONLY + value: object + stored_value: JSONValue + + +@dataclass(frozen=True) +class ParameterCase: + """A US parameter whose values, oldest first, each test one shape.""" + + name: str + data_type: type | None + values: tuple[ValueCase, ...] + + def source(self) -> SimpleNamespace: + """Return the parameter as the public model lists it.""" + return SimpleNamespace( + name=self.name, + label=f"Value shape {self.name.rsplit('.', maxsplit=1)[-1]}", + description=None, + data_type=self.data_type, + unit=None, + parameter_values=[ + SimpleNamespace( + value=value.value, start_date=value.start, end_date=value.end + ) + for value in self.values + ], + ) + + +# .py reads Core's earliest instant, 0000-01-01, as year 1, and gives naive +# datetimes; extraction reads both, and dates, as UTC. +EARLIEST = datetime(1, 1, 1) +TEXT_WITH_COPY_ESCAPES = 'Tab\there, new\nline, back\\slash, "quotes", café €' +BRACKET_PARAMETER_NODES = ( + "gov.value_shapes.scale[0]", + "gov.value_shapes.scale[1]", +) +PARAMETER_CASES = ( + ParameterCase( + name="gov.value_shapes.flag", + data_type=bool, + values=( + ValueCase(EARLIEST, datetime(2024, 12, 31), value=True, stored_value=True), + ValueCase(datetime(2025, 1, 1), None, value=False, stored_value=False), + ), + ), + ParameterCase( + name="gov.value_shapes.count", + data_type=int, + values=( + ValueCase( + datetime(2025, 1, 1), datetime(2025, 6, 30), value=0, stored_value=0 + ), + # In force for one day only. + ValueCase( + datetime(2025, 7, 1), datetime(2025, 7, 1), value=-3, stored_value=-3 + ), + ValueCase(datetime(2025, 7, 2), None, value=12, stored_value=12), + ), + ), + ParameterCase( + name="gov.value_shapes.amount", + data_type=float, + values=( + ValueCase( + date(2024, 1, 1), + date(2024, 12, 31), + value=0.30000000000000004, + stored_value=0.30000000000000004, + ), + # A whole number that must stay a float. + ValueCase(date(2025, 1, 1), None, value=12000.0, stored_value=12000.0), + ), + ), + ParameterCase( + name="gov.value_shapes.cap", + data_type=float, + values=( + ValueCase( + datetime(2024, 1, 1), + datetime(2024, 12, 31), + value=math.inf, + stored_value="Infinity", + ), + ValueCase( + datetime(2025, 1, 1), None, value=-math.inf, stored_value="-Infinity" + ), + ), + ), + ParameterCase( + name="gov.value_shapes.numpy_scalars", + data_type=float, + values=( + # numpy.float64 is a float; numpy.int64 and numpy.True_ are read + # through their item(). + ValueCase( + datetime(2024, 1, 1), + datetime(2024, 12, 31), + value=np.float64(0.25), + stored_value=0.25, + ), + ValueCase( + datetime(2025, 1, 1), + datetime(2025, 12, 31), + value=np.int64(7), + stored_value=7, + ), + ValueCase(datetime(2026, 1, 1), None, value=np.True_, stored_value=True), + ), + ), + ParameterCase( + name="gov.value_shapes.text", + data_type=str, + values=( + ValueCase( + datetime(2025, 1, 1), + datetime(2025, 12, 31), + value="single", + stored_value="single", + ), + ValueCase( + datetime(2026, 1, 1), + None, + value=TEXT_WITH_COPY_ESCAPES, + stored_value=TEXT_WITH_COPY_ESCAPES, + ), + ), + ), + ParameterCase( + name="gov.value_shapes.filing_status", + data_type=str, + values=( + ValueCase( + datetime(2025, 1, 1), + None, + value=_FilingStatus.JOINT, + stored_value="JOINT", + ), + ), + ), + ParameterCase( + name="gov.value_shapes.effective_date", + data_type=date, + values=( + ValueCase( + datetime(2024, 1, 1), + datetime(2024, 12, 31), + value=date(2024, 4, 6), + stored_value="2024-04-06", + ), + ValueCase( + datetime(2025, 1, 1), + None, + value=datetime(2025, 4, 6, 12, 30), + stored_value="2025-04-06T12:30:00", + ), + ), + ), + ParameterCase( + name="gov.value_shapes.sources", + data_type=list, + values=( + ValueCase( + datetime(2024, 1, 1), + datetime(2024, 12, 31), + value=["employment_income", "self_employment_income"], + stored_value=["employment_income", "self_employment_income"], + ), + ValueCase( + datetime(2025, 1, 1), + datetime(2025, 12, 31), + value=("pension_income",), + stored_value=["pension_income"], + ), + ValueCase(datetime(2026, 1, 1), None, value=[], stored_value=[]), + ), + ), + ParameterCase( + name="gov.value_shapes.amounts_by_filing_status", + data_type=dict, + values=( + ValueCase( + datetime(2025, 1, 1), + datetime(2025, 12, 31), + value={ + "SINGLE": 12000.0, + "JOINT": 24000, + "NESTED": {"enabled": True, "rates": (0.1, None)}, + }, + stored_value={ + "SINGLE": 12000.0, + "JOINT": 24000, + "NESTED": {"enabled": True, "rates": [0.1, None]}, + }, + ), + ValueCase(datetime(2026, 1, 1), None, value={}, stored_value={}), + ), + ), + ParameterCase( + name="gov.value_shapes.unset", + data_type=None, + values=(ValueCase(datetime(2025, 1, 1), None, value=None, stored_value=None),), + ), + ParameterCase( + name="gov.value_shapes.scale[0].rate", + data_type=float, + values=(ValueCase(datetime(2025, 1, 1), None, value=0.1, stored_value=0.1),), + ), + ParameterCase( + name="gov.value_shapes.scale[0].threshold", + data_type=int, + values=(ValueCase(datetime(2025, 1, 1), None, value=0, stored_value=0),), + ), + ParameterCase( + name="gov.value_shapes.scale[1].rate", + data_type=float, + values=(ValueCase(datetime(2025, 1, 1), None, value=0.2, stored_value=0.2),), + ), + ParameterCase( + name="gov.value_shapes.scale[1].threshold", + data_type=float, + values=( + ValueCase(datetime(2025, 1, 1), None, value=50000.0, stored_value=50000.0), + ), + ), +) +# The filter value the catalog stores for each region this catalog filters, +# by country and region code. +REGION_FILTER_VALUES = { + ("us", "state/ca"): "6", + ("us", "congressional_district/CA-01"): "601", + ("uk", "country/england"): '["NORTH_EAST","LONDON"]', + ("uk", "country/wales"): "WALES", + ("uk", "constituency/W07000081"): "W07000081", +} +US_ACS_LOCAL_URI = "hf://policyengine/us/populace_us_2024_acs_local.h5@fixture" +UK_DENSE_URI = "hf://policyengine/uk/microcosm_uk_2024_25_dense.h5@fixture" + + +@dataclass(frozen=True, kw_only=True) +class _FilteredRegion: + """A region that selects its households with one row filter.""" + + code: str + label: str + region_type: str + parent_code: str + filter_field: str + filter_value: object + dataset_path: str | None = None + state_code: str | None = None + state_name: str | None = None + + def source(self) -> SimpleNamespace: + """Return the region as the public region registry holds it.""" + return SimpleNamespace( + code=self.code, + label=self.label, + region_type=self.region_type, + parent_code=self.parent_code, + dataset_path=self.dataset_path, + scoping_strategy=SimpleNamespace( + strategy_type="row_filter", + variable_name=self.filter_field, + variable_value=self.filter_value, + additional_filters={}, + ), + requires_filter=True, + state_code=self.state_code, + state_name=self.state_name, + ) + + +def _variables() -> list[SimpleNamespace]: + """Return US variables with each default, possible-value and sum shape.""" + return [ + SimpleNamespace( + name="filing_status", + label="Filing status", + entity="tax_unit", + description="The tax unit's filing status", + data_type=str, + possible_values=["SINGLE", "JOINT"], + default_value=_FilingStatus.SINGLE, + adds=None, + subtracts=None, + ), + SimpleNamespace( + name="is_blind", + label=None, + entity="person", + description=None, + data_type=bool, + possible_values=None, + default_value=False, + adds=None, + subtracts=None, + ), + SimpleNamespace( + name="household_net_income", + label="Household net income", + entity="household", + description="Income after taxes and benefits", + data_type=float, + possible_values=None, + default_value=0.0, + adds=["employment_income"], + subtracts=["income_tax"], + ), + ] + + +def value_shape_catalog() -> NormalizedCatalog: + """Return the fixture catalog with every value-shape case added to it.""" + models = source_models() + us_model = models["us"] + for variable in _variables(): + us_model.variables.append(variable) + us_model.variables_by_name[variable.name] = variable + for node_name in ("gov.value_shapes", *BRACKET_PARAMETER_NODES): + node = SimpleNamespace(name=node_name, label=None, description=None) + us_model.parameter_nodes.append(node) + us_model.parameter_nodes_by_name[node_name] = node + for case in PARAMETER_CASES: + parameter = case.source() + us_model.parameters.append(parameter) + us_model.parameters_by_name[parameter.name] = parameter + us_model.region_registry.append( + _FilteredRegion( + code="congressional_district/CA-01", + label="California's 1st congressional district", + region_type="congressional_district", + parent_code="state/ca", + filter_field="congressional_district_geoid", + filter_value=601, + dataset_path=US_ACS_LOCAL_URI, + state_code="CA", + state_name="California", + ).source() + ) + + uk_model = models["uk"] + england = next( + region + for region in uk_model.region_registry + if region.code == "country/england" + ) + england.scoping_strategy.variable_value = ("NORTH_EAST", "LONDON") + uk_model.region_registry.extend( + [ + _FilteredRegion( + code="country/wales", + label="Wales", + region_type="country", + parent_code="uk", + filter_field="country", + filter_value="WALES", + ).source(), + _FilteredRegion( + code="constituency/W07000081", + label="Ynys Môn", + region_type="constituency", + parent_code="uk", + filter_field="constituency_code_oa", + filter_value="W07000081", + dataset_path=UK_DENSE_URI, + ).source(), + ] + ) + return extract_catalog( + bundle=bundle(), + policyengine_version=POLICYENGINE_VERSION, + models=models, + installed_version=installed_version, + ) diff --git a/tests/integration/catalog_rows.py b/tests/integration/catalog_rows.py new file mode 100644 index 000000000..f44ab0865 --- /dev/null +++ b/tests/integration/catalog_rows.py @@ -0,0 +1,130 @@ +"""Read published catalog rows back from PostgreSQL and compare them with a catalog. + +Every value is compared as strict JSON text, which tells 1, 1.0 and true +apart, so a value PostgreSQL stores in another form fails the comparison. +""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING + +from sqlalchemy import text + +if TYPE_CHECKING: + from sqlalchemy.engine import Engine + + from policyengine_api.data.v2.catalog.records import NormalizedCatalog + +# The columns each catalog table stores, named as on the catalog's records: +# each table's key column first. Each country catalog names its records after +# the table that stores them. +CATALOG_COLUMNS = { + "variables": ( + "name", + "label", + "entity", + "description", + "data_type", + "possible_values", + "default_value", + "adds", + "subtracts", + ), + "parameter_nodes": ("name", "label", "description"), + "parameters": ("name", "label", "description", "data_type", "unit"), + "regions": ( + "code", + "label", + "region_type", + "requires_filter", + "filter_field", + "filter_value", + "filter_strategy", + "parent_code", + "state_code", + "state_name", + "default_dataset_id", + ), +} + + +def strict_json(value: object) -> str: + """Return ``value`` as JSON text, which tells 1, 1.0 and true apart.""" + return json.dumps(value, sort_keys=True, default=str) + + +def stored_catalog_rows(engine: Engine, table_name: str) -> dict[tuple, str]: + """Return each row of a catalog table, keyed by model and name or code.""" + key_column, *columns = CATALOG_COLUMNS[table_name] + selected = ", ".join(f"catalog_row.{column}" for column in (key_column, *columns)) + with engine.connect() as connection: + rows = connection.execute( + text( + f""" + SELECT model.name, {selected} + FROM {table_name} AS catalog_row + JOIN tax_benefit_model_versions AS version + ON version.id = catalog_row.tax_benefit_model_version_id + JOIN tax_benefit_models AS model ON model.id = version.model_id + """ + ) + ).all() + return {(row[0], row[1]): strict_json(row[2:]) for row in rows} + + +def catalog_rows(catalog: NormalizedCatalog, table_name: str) -> dict[tuple, str]: + """Return what ``stored_catalog_rows`` should read for ``catalog``.""" + key_column, *columns = CATALOG_COLUMNS[table_name] + return { + (country.model.name, getattr(record, key_column)): strict_json( + [getattr(record, column) for column in columns] + ) + for country in catalog.countries + for record in getattr(country, table_name) + } + + +def stored_parameter_values(engine: Engine) -> dict[tuple, tuple]: + """Return each parameter value's JSON and end date, by model, name and start.""" + with engine.connect() as connection: + rows = connection.execute( + text( + """ + SELECT model.name, parameter.name, parameter_value.start_date, + parameter_value.value_json, parameter_value.end_date + FROM parameter_values AS parameter_value + JOIN parameters AS parameter + ON parameter.id = parameter_value.parameter_id + JOIN tax_benefit_model_versions AS version + ON version.id = parameter.tax_benefit_model_version_id + JOIN tax_benefit_models AS model ON model.id = version.model_id + """ + ) + ).all() + return { + (model_name, parameter_name, start_date): (strict_json(value_json), end_date) + for model_name, parameter_name, start_date, value_json, end_date in rows + } + + +def catalog_parameter_values(catalog: NormalizedCatalog) -> dict[tuple, tuple]: + """Return what ``stored_parameter_values`` should read for ``catalog``.""" + return { + (country.model.name, parameter.name, value.start_date): ( + strict_json(value.value_json), + value.end_date, + ) + for country in catalog.countries + for parameter in country.parameters + for value in parameter.values + } + + +def assert_catalog_stored_unchanged(engine: Engine, catalog: NormalizedCatalog) -> None: + """Assert that the database holds exactly ``catalog``'s values and records.""" + assert stored_parameter_values(engine) == catalog_parameter_values(catalog) + for table_name in CATALOG_COLUMNS: + assert stored_catalog_rows(engine, table_name) == catalog_rows( + catalog, table_name + ), table_name diff --git a/tests/integration/test_v2_catalog_installed.py b/tests/integration/test_v2_catalog_installed.py deleted file mode 100644 index 62a5afbef..000000000 --- a/tests/integration/test_v2_catalog_installed.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Compatibility coverage for the complete installed PolicyEngine.py catalog.""" - -from __future__ import annotations - -import json -import os -from importlib import metadata as importlib_metadata - -import pytest -from policyengine.bundle import get_current_bundle - -from policyengine_api.data.v2.catalog.extraction import extract_installed_catalog - -pytestmark = pytest.mark.skipif( - os.environ.get("RUN_V2_CATALOG_COMPATIBILITY") != "1", - reason=("loads the complete public catalog; set RUN_V2_CATALOG_COMPATIBILITY=1"), -) - - -def test_installed_policyengine_catalog_is_complete_and_bounded() -> None: - catalog = extract_installed_catalog() - bundle = get_current_bundle() - policyengine_version = importlib_metadata.version("policyengine") - expected_dependencies = tuple( - (name, bundle["packages"][name]["version"]) - for name in ( - "policyengine-core", - "policyengine-us", - "policyengine-uk", - ) - ) - - assert catalog.policyengine_version == policyengine_version - assert catalog.dependency_versions == expected_dependencies - assert all( - importlib_metadata.version(name) == expected - for name, expected in expected_dependencies - ) - # Counts for the 6.11.1 bundle's models: US 2.2.1 and UK 2.151.0. Keep - # this exact assertion so later package changes require explicit review. - # Since 6.10.0 the UK registry also lists its 650 constituencies and 361 - # local authorities, because the bundle routes them to a local-area - # release, so regions grow by 1,011 from 6.7.5's 826 and the catalog - # adds that release's dataset. - assert catalog.entity_counts() == { - "models": 2, - "model_versions": 2, - "variables": 7_438, - "parameter_nodes": 29_367, - "parameters": 104_673, - "parameter_values": 1_196_139, - "datasets": 4, - "regions": 1_837, - } - - for country_id in ("us", "uk"): - country = catalog.country(country_id) - assert country.model.name == f"policyengine-{country_id}" - assert country.model_version.version == policyengine_version - assert country.model_version.version not in dict(expected_dependencies).values() - assert country.variables - assert country.parameter_nodes - assert all("__pycache__" not in node.name for node in country.parameter_nodes) - assert country.parameters - assert all( - not dataset.is_output_dataset and dataset.storage_path is None - for dataset in country.datasets - ) - - total_values = 0 - for batch in country.parameter_value_batches(batch_size=10_000): - assert 0 < len(batch) <= 10_000 - total_values += len(batch) - assert total_values == country.entity_counts()["parameter_values"] - - assert {dataset.name for dataset in catalog.country("us").datasets} == { - "populace_us_2024", - "populace_us_2024_acs_local", - } - assert {dataset.name for dataset in catalog.country("uk").datasets} == { - "microcosm_uk_2024_25", - "microcosm_uk_2024_25_dense", - } - # The national file has no area codes, so UK constituencies and local - # authorities use the local-area dataset .py routes them to, while the - # UK countries filter the national file. - uk_catalog = catalog.country("uk") - uk_dataset_names = {dataset.id: dataset.name for dataset in uk_catalog.datasets} - uk_region_type_datasets: dict[str, set[str]] = {} - for region in uk_catalog.regions: - uk_region_type_datasets.setdefault(region.region_type, set()).add( - uk_dataset_names[region.default_dataset_id] - ) - assert uk_region_type_datasets == { - "national": {"microcosm_uk_2024_25"}, - "country": {"microcosm_uk_2024_25"}, - "constituency": {"microcosm_uk_2024_25_dense"}, - "local_authority": {"microcosm_uk_2024_25_dense"}, - } - assert uk_catalog.fallback_summaries == () - england = next( - region - for region in catalog.country("uk").regions - if region.code == "country/england" - ) - assert england.filter_field == "region" - assert england.filter_strategy == "row_filter" - assert england.filter_value is not None - assert json.loads(england.filter_value) == [ - "NORTH_EAST", - "NORTH_WEST", - "YORKSHIRE", - "EAST_MIDLANDS", - "WEST_MIDLANDS", - "EAST_OF_ENGLAND", - "LONDON", - "SOUTH_EAST", - "SOUTH_WEST", - ] - - # Since 6.7.5, US state and congressional-district regions use the - # ACS-local dataset again, on the PolicyEngine/microcosm#1263 rebuild - # (PolicyEngine/policyengine.py#649, ending #604's national-file filter). - # The national region uses the national dataset, and places fall back to it. - us_catalog = catalog.country("us") - us_dataset_names = {dataset.id: dataset.name for dataset in us_catalog.datasets} - region_type_datasets: dict[str, set[str]] = {} - for region in us_catalog.regions: - region_type_datasets.setdefault(region.region_type, set()).add( - us_dataset_names[region.default_dataset_id] - ) - assert region_type_datasets == { - "national": {"populace_us_2024"}, - "state": {"populace_us_2024_acs_local"}, - "congressional_district": {"populace_us_2024_acs_local"}, - "place": {"populace_us_2024"}, - } - assert [ - (summary.region_type, summary.count) - for summary in us_catalog.fallback_summaries - ] == [ - ("place", 333), - ] diff --git a/tests/integration/test_v2_catalog_installed_folders.py b/tests/integration/test_v2_catalog_installed_folders.py new file mode 100644 index 000000000..2b4de803d --- /dev/null +++ b/tests/integration/test_v2_catalog_installed_folders.py @@ -0,0 +1,245 @@ +"""Publish a few parameter folders of the installed PolicyEngine.py models. + +CI doesn't load the complete installed catalog (about 1.2 million parameter +values). This test runs the real US and UK models through extraction and +publication instead, each reduced to the parameters under the folders in +SELECTED_FOLDERS, and checks that every extracted value comes back from +PostgreSQL unchanged. Extraction gets only what it needs besides those +parameters: each model's identity, package and release manifest, its national +region, and no variables. + +tests/fixtures/v2_catalog_value_shapes.py covers the value shapes extraction +accepts that the installed models don't hold, such as text and mappings. +""" + +from __future__ import annotations + +import math +from datetime import UTC, datetime +from importlib import metadata as importlib_metadata +from pathlib import Path +from types import SimpleNamespace +from typing import TYPE_CHECKING + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool + +from policyengine_api.data.v2.catalog.extraction import extract_catalog +from policyengine_api.data.v2.catalog.publication import publish_catalog +from tests.fixtures.v2_catalog import RegionRegistry +from tests.integration.catalog_rows import assert_catalog_stored_unchanged + +if TYPE_CHECKING: + from collections.abc import Iterable, Iterator, Mapping + + from sqlalchemy.engine import Engine + + from policyengine_api.data.v2.catalog.records import ( + NormalizedCatalog, + ParameterValueRecord, + ) + +REPO = Path(__file__).parents[2] +# The parameter folders this test publishes, by country, with why each is +# here. Between them they hold every value shape the installed models hold. +SELECTED_FOLDERS = { + "us": { + "gov.irs.income.bracket": ( + "income tax rates and thresholds broken down by bracket number and " + "by filing status (SINGLE, JOINT, SEPARATE, HEAD_OF_HOUSEHOLD, " + "SURVIVING_SPOUSE); the top bracket's thresholds are infinite" + ), + "gov.irs.payroll.social_security": ( + "plain rates and a cap, with a long history of values" + ), + "gov.aca.family_tier_states": ( + "booleans broken down by state, some starting in year 1, which .py " + "reads from Core's 0000-01-01" + ), + "gov.aca.ineligible_immigration_statuses": "a list of enum names", + "gov.irs.credits.premium_tax_credit.phase_out": ( + "bracket scales (rate[0].amount, rate[1].threshold, ...) whose " + "upper brackets hold null values" + ), + "gov.states.ne.dhhs.child_care_subsidy.fpg_fraction": ( + "values in force for a single day, the only ones in the installed models" + ), + }, + "uk": { + "gov.hmrc.income_tax.rates.uk": ( + "the UK income tax bracket scale: a rate and a threshold per bracket" + ), + "gov.dwp.state_pension.triple_lock": "UK booleans and a null value", + }, +} +EARLIEST = datetime(1, 1, 1, tzinfo=UTC) + + +def _in_folders(name: str, folders: Iterable[str]) -> bool: + """Whether ``name`` is one of ``folders`` or lies under one.""" + return any( + name == folder or name.startswith((f"{folder}.", f"{folder}[")) + for folder in folders + ) + + +def selected_model(country_id: str, model: object, folders: Iterable[str]) -> object: + """Return ``model`` reduced to what extraction needs and ``folders``' parameters. + + Raise LookupError naming every folder the model no longer has, so a renamed + or removed folder fails the test instead of shrinking it. + """ + folders = tuple(folders) + parameter_names = tuple(model.parameters_by_name) + missing = [ + folder + for folder in folders + if not any(_in_folders(name, (folder,)) for name in parameter_names) + ] + if missing: + msg = ( + f"The installed {country_id} model has no parameters under " + f"{', '.join(missing)}. A folder was renamed or removed: choose one " + "that holds the same values and update SELECTED_FOLDERS in " + "tests/integration/test_v2_catalog_installed_folders.py." + ) + raise LookupError(msg) + return SimpleNamespace( + model=model.model, + model_package=model.model_package, + release_manifest=model.release_manifest, + region_registry=RegionRegistry( + country_id, + [region for region in model.region_registry if region.code == country_id], + ), + variables_by_name={}, + parameter_nodes_by_name={ + name: node + for name, node in model.parameter_nodes_by_name.items() + if _in_folders(name, folders) + }, + parameters_by_name={ + name: parameter + for name, parameter in model.parameters_by_name.items() + if _in_folders(name, folders) + }, + ) + + +@pytest.fixture(scope="module") +def installed_models() -> dict[str, object]: + """Return the installed US and UK models; importing policyengine builds both.""" + import policyengine + + return { + country_id: getattr(policyengine, country_id).model + for country_id in SELECTED_FOLDERS + } + + +@pytest.fixture(scope="module") +def selected_catalog(installed_models: Mapping[str, object]) -> NormalizedCatalog: + """Return the catalog extracted from the installed models' selected folders.""" + from policyengine.bundle import get_current_bundle + + return extract_catalog( + bundle=get_current_bundle(), + policyengine_version=importlib_metadata.version("policyengine"), + models={ + country_id: selected_model(country_id, model, SELECTED_FOLDERS[country_id]) + for country_id, model in installed_models.items() + }, + ) + + +@pytest.fixture +def publication_engine(disposable_v2_database_url: str) -> Iterator[Engine]: + """Return an engine for the disposable v2 database, with no catalog in it.""" + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + with engine.begin() as connection: + connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) + yield engine + with engine.begin() as connection: + connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) + engine.dispose() + + +def _values(catalog: NormalizedCatalog) -> list[ParameterValueRecord]: + return [ + value + for country in catalog.countries + for parameter in country.parameters + for value in parameter.values + ] + + +def test_a_folder_missing_from_the_installed_model_fails_by_name() -> None: + model = SimpleNamespace(parameters_by_name={"gov.irs.payroll.rate": object()}) + + with pytest.raises( + LookupError, match=r"no parameters under gov\.renamed\. A folder" + ): + selected_model("us", model, ("gov.irs.payroll", "gov.renamed")) + + +def test_the_installed_models_are_the_us_and_uk_models( + installed_models: Mapping[str, object], +) -> None: + # The first test to use installed_models, so pytest --durations reports + # its setup as the cost of importing .py and building both models, apart + # from extraction, which is the next test's setup. + assert { + country_id: model.model.id for country_id, model in installed_models.items() + } == {"us": "policyengine-us", "uk": "policyengine-uk"} + + +def test_selected_folders_hold_every_value_shape_the_installed_models_hold( + selected_catalog: NormalizedCatalog, +) -> None: + values = _values(selected_catalog) + stored = [value.value_json for value in values] + names = [ + parameter.name + for country in selected_catalog.countries + for parameter in country.parameters + ] + shapes = { + "a boolean": any(isinstance(value, bool) for value in stored), + "an integer": any(type(value) is int for value in stored), + "a fraction": any( + isinstance(value, float) and not value.is_integer() for value in stored + ), + "an infinity, stored as text": "Infinity" in stored, + "a null": None in stored, + "a list": any(isinstance(value, list) and value for value in stored), + "a bracket scale": any("[" in name for name in names), + "a breakdown by state": any(name.endswith(".CA") for name in names), + "a breakdown by filing status": any(name.endswith(".JOINT") for name in names), + "a value in force for one day": any( + value.start_date == value.end_date for value in values + ), + "a start in year 1": any(value.start_date == EARLIEST for value in values), + "an open-ended value": any(value.end_date is None for value in values), + } + + assert [shape for shape, held in shapes.items() if not held] == [], ( + "The selected folders no longer hold these shapes; add a folder that " + "does to SELECTED_FOLDERS." + ) + assert all(not isinstance(value, float) or math.isfinite(value) for value in stored) + + +def test_selected_folders_publish_and_come_back_unchanged( + selected_catalog: NormalizedCatalog, + publication_engine: Engine, +) -> None: + publish_catalog(publication_engine, selected_catalog) + # Publishing the same version again compares every stored row with the + # catalog in both directions and fails on any difference. + publish_catalog(publication_engine, selected_catalog) + + assert_catalog_stored_unchanged(publication_engine, selected_catalog) diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index 25bc9178f..aeab2e5b6 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os from concurrent.futures import ThreadPoolExecutor from dataclasses import replace @@ -31,6 +32,18 @@ ) from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from tests.fixtures.v2_catalog import normalized_catalog +from tests.fixtures.v2_catalog_value_shapes import ( + PARAMETER_CASES, + REGION_FILTER_VALUES, + value_shape_catalog, +) +from tests.integration.catalog_rows import ( + CATALOG_COLUMNS, + assert_catalog_stored_unchanged, + stored_catalog_rows, + stored_parameter_values, + strict_json, +) REPO = Path(__file__).parents[2] DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" @@ -340,3 +353,34 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "reports": 0, "report_runs": 0, } + + +def test_publication_stores_every_value_shape_unchanged( + publication_engine: Engine, +) -> None: + catalog = value_shape_catalog() + + publish_catalog(publication_engine, catalog) + # Publishing the same version again compares every stored row with the + # catalog in both directions and fails on any difference, so this also + # checks that the comparison reads each shape back as it was written. + publish_catalog(publication_engine, catalog) + + assert_catalog_stored_unchanged(publication_engine, catalog) + stored_values = stored_parameter_values(publication_engine) + us_model = catalog.country("us").model.name + for case in PARAMETER_CASES: + assert [ + value_json + for (model_name, parameter_name, _start), (value_json, _end) in sorted( + stored_values.items() + ) + if (model_name, parameter_name) == (us_model, case.name) + ] == [strict_json(value.stored_value) for value in case.values], case.name + + stored_regions = stored_catalog_rows(publication_engine, "regions") + filter_value_index = CATALOG_COLUMNS["regions"].index("filter_value") - 1 + for (country_id, code), filter_value in REGION_FILTER_VALUES.items(): + model_name = catalog.country(country_id).model.name + stored_columns = json.loads(stored_regions[(model_name, code)]) + assert stored_columns[filter_value_index] == filter_value, code diff --git a/tests/integration/test_v2_catalog_publication_qualification.py b/tests/integration/test_v2_catalog_publication_qualification.py deleted file mode 100644 index 8400c4ac9..000000000 --- a/tests/integration/test_v2_catalog_publication_qualification.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Production-scale qualification for the installed PolicyEngine.py catalog.""" - -from __future__ import annotations - -import gc -import json -import os -import tracemalloc -from pathlib import Path - -import pytest -from alembic import command -from alembic.config import Config -from sqlalchemy import create_engine, text -from sqlalchemy.engine import make_url -from sqlalchemy.pool import NullPool - -from policyengine_api.data.v2.catalog.extraction import extract_installed_catalog -from policyengine_api.data.v2.catalog.publication import publish_catalog -from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL - -REPO = Path(__file__).parents[2] -DISPOSABLE_DATABASE = "policyengine_v2_alembic_test" -MAX_ADDITIONAL_PUBLISHER_MEMORY_BYTES = 256 * 1024 * 1024 - -pytestmark = pytest.mark.skipif( - os.environ.get("RUN_V2_CATALOG_PUBLICATION_QUALIFICATION") != "1", - reason=( - "publishes the complete catalog; set RUN_V2_CATALOG_PUBLICATION_QUALIFICATION=1" - ), -) - - -def _disposable_url() -> str: - database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") - if not database_url: - pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") - url = make_url(database_url) - if url.database != DISPOSABLE_DATABASE or url.host not in { - "localhost", - "127.0.0.1", - "::1", - "postgres", - }: - pytest.fail("catalog qualification requires disposable local Postgres") - return database_url - - -def test_complete_installed_catalog_bulk_publication() -> None: - database_url = _disposable_url() - command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") - engine = create_engine(database_url, poolclass=NullPool) - with engine.begin() as connection: - connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) - - try: - catalog = extract_installed_catalog() - gc.collect() - tracemalloc.start() - evidence = publish_catalog(engine, catalog) - _, peak_bytes = tracemalloc.get_traced_memory() - tracemalloc.stop() - - assert peak_bytes < MAX_ADDITIONAL_PUBLISHER_MEMORY_BYTES - assert evidence.entity_counts == catalog.entity_counts() - with engine.connect() as connection: - persisted_counts = connection.execute( - text( - """ - SELECT - (SELECT count(*) FROM tax_benefit_models), - (SELECT count(*) FROM tax_benefit_model_versions), - (SELECT count(*) FROM variables), - (SELECT count(*) FROM parameter_nodes), - (SELECT count(*) FROM parameters), - (SELECT count(*) FROM parameter_values - WHERE policy_id IS NULL AND dynamic_id IS NULL), - (SELECT count(*) FROM datasets - WHERE NOT is_output_dataset AND storage_path IS NULL), - (SELECT count(*) FROM regions) - """ - ) - ).one() - representative = connection.execute( - text( - """ - SELECT - EXISTS (SELECT 1 FROM variables - WHERE name = 'employment_income'), - EXISTS (SELECT 1 FROM parameters - WHERE name = 'gov.benefit_uprating_cpi'), - EXISTS (SELECT 1 FROM regions WHERE code = 'us'), - EXISTS (SELECT 1 FROM regions WHERE code = 'uk') - """ - ) - ).one() - assert tuple(persisted_counts) == ( - evidence.entity_counts["models"], - evidence.entity_counts["model_versions"], - evidence.entity_counts["variables"], - evidence.entity_counts["parameter_nodes"], - evidence.entity_counts["parameters"], - evidence.entity_counts["parameter_values"], - evidence.entity_counts["datasets"], - evidence.entity_counts["regions"], - ) - assert all(representative) - print( - json.dumps( - { - **evidence.as_dict(), - "peak_additional_publisher_memory_bytes": peak_bytes, - }, - sort_keys=True, - ) - ) - finally: - if tracemalloc.is_tracing(): - tracemalloc.stop() - with engine.begin() as connection: - connection.execute(text("TRUNCATE tax_benefit_models CASCADE")) - engine.dispose() diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index de029defa..eb520f527 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -218,8 +218,7 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "alembic -c alembic-v1.ini upgrade head" in workflow assert "alembic -c alembic-v2.ini upgrade head" in workflow assert "test_alembic_v2_lifecycle.sh" not in workflow - assert "test_v2_catalog_installed.py" in workflow - assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow + assert "test_v2_catalog_installed_folders.py" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow assert "test_v2_country_routes.py" in workflow @@ -235,12 +234,10 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "test_v1_household_dual_write.py" in workflow assert "test_mysql_household_spm_dual_write.py" in workflow assert "test_mysql_update_row_counts.py" in workflow - assert "test_v2_catalog_publication_qualification.py" in workflow - assert "RUN_V2_CATALOG_PUBLICATION_QUALIFICATION" in workflow assert "test_runtime_cache_redis.py" in workflow assert "uv sync --frozen" in workflow assert workflow.count("coverage run --branch") == 1 - assert workflow.count("coverage run -a --branch") == 8 + assert workflow.count("coverage run -a --branch") == 7 assert "coverage xml -i -o coverage-v2.xml" in workflow assert "codecov/codecov-action@v5" in workflow assert "files: coverage-v2.xml" in workflow diff --git a/tests/unit/test_v2_integration_workflow.py b/tests/unit/test_v2_integration_workflow.py new file mode 100644 index 000000000..725e999a1 --- /dev/null +++ b/tests/unit/test_v2_integration_workflow.py @@ -0,0 +1,92 @@ +"""The cross-database integration check, .github/workflows/v2-integration-check.yml. + +The check has one job, catalog-runtime-integration, which never loads the +complete installed PolicyEngine.py catalog. It publishes a few parameter +folders of the installed models instead +(tests/integration/test_v2_catalog_installed_folders.py), on every run. +""" + +from __future__ import annotations + +import yaml + +from tests.unit.cloud_run_deploy_support import REPO, job_step + +WORKFLOW_PATH = REPO / ".github/workflows/v2-integration-check.yml" +RUNTIME_JOB = "catalog-runtime-integration" +INSTALLED_FOLDERS_STEP = ( + "Publish selected parameter folders of the installed PolicyEngine.py models" +) +INSTALLED_FOLDERS_TEST = "tests/integration/test_v2_catalog_installed_folders.py" +REMOVED_FULL_CATALOG_FILES = ( + "tests/integration/test_v2_catalog_installed.py", + "tests/integration/test_v2_catalog_publication_qualification.py", + "tests/fixtures/installed_catalog.py", +) + + +def _workflow_text() -> str: + return WORKFLOW_PATH.read_text(encoding="utf-8") + + +def _workflow() -> dict: + return yaml.safe_load(_workflow_text()) + + +def _triggers() -> dict: + workflow = _workflow() + # PyYAML reads the bare key "on" as True. + return workflow.get("on", workflow.get(True)) + + +def _runtime_job() -> dict: + return _workflow()["jobs"][RUNTIME_JOB] + + +def _step_commands(job: dict) -> list[str]: + return [step["run"] for step in job["steps"] if "run" in step] + + +def test_the_workflow_runs_when_called_or_dispatched() -> None: + triggers = _triggers() + + assert set(triggers) == {"workflow_call", "workflow_dispatch"} + assert triggers["workflow_call"]["secrets"] == { + "CODECOV_TOKEN": {"required": False} + } + + +def test_the_check_has_one_job_that_always_runs_every_step() -> None: + jobs = _workflow()["jobs"] + + assert set(jobs) == {RUNTIME_JOB} + assert "if" not in jobs[RUNTIME_JOB] + for step in jobs[RUNTIME_JOB]["steps"]: + assert "if" not in step, step["name"] + + +def test_no_step_loads_the_complete_installed_catalog() -> None: + workflow_text = _workflow_text() + + for removed_path in REMOVED_FULL_CATALOG_FILES: + assert removed_path not in workflow_text + assert not (REPO / removed_path).exists(), removed_path + assert "RUN_V2_CATALOG_COMPATIBILITY" not in workflow_text + assert "RUN_V2_CATALOG_PUBLICATION_QUALIFICATION" not in workflow_text + + +def test_the_job_publishes_selected_folders_of_the_installed_models() -> None: + job = _runtime_job() + step = job_step(job, INSTALLED_FOLDERS_STEP) + commands = _step_commands(job) + + assert step["run"] == ( + "uv run --frozen --extra dev coverage run -a --branch -m pytest -q " + f"--durations=10 {INSTALLED_FOLDERS_TEST}" + ) + assert "env" not in step + # After the schema upgrade and the role checks, like the other test steps. + assert commands.index( + "uv run --frozen --extra dev python scripts/reconcile_v2_privileges.py verify" + ) < commands.index(step["run"]) + assert (REPO / INSTALLED_FOLDERS_TEST).exists() diff --git a/tests/unit/v2/test_catalog_value_shapes.py b/tests/unit/v2/test_catalog_value_shapes.py new file mode 100644 index 000000000..0319f9277 --- /dev/null +++ b/tests/unit/v2/test_catalog_value_shapes.py @@ -0,0 +1,131 @@ +"""The value-shape catalog holds an example of each shape extraction accepts. + +The cross-database integration tests publish it to PostgreSQL and read every +value back (tests/integration/test_v2_catalog_publication.py). These tests fail +when extraction accepts a shape the value-shape catalog leaves out, or stores +one differently from what the catalog says it stores. +""" + +from __future__ import annotations + +import json +from datetime import UTC, date, datetime + +from policyengine_api.data.v2.catalog.extraction_values import JSON_VALUE_NORMALIZERS +from tests.fixtures.v2_catalog_value_shapes import ( + BRACKET_PARAMETER_NODES, + PARAMETER_CASES, + REGION_FILTER_VALUES, + value_shape_catalog, +) + + +def _source_values() -> list[object]: + return [value.value for case in PARAMETER_CASES for value in case.values] + + +def _strict_json(value: object) -> str: + """Return ``value`` as JSON text, which tells 1, 1.0 and true apart.""" + return json.dumps(value, sort_keys=True) + + +def test_holds_a_value_for_each_json_normalizer() -> None: + source_values = _source_values() + + for value_types, _normalize in JSON_VALUE_NORMALIZERS: + assert any(isinstance(value, value_types) for value in source_values), ( + value_types + ) + + +def test_holds_each_plain_json_value_and_scalar_wrapper() -> None: + source_values = _source_values() + plain_types = (type(None), str, bool, int, float, list, tuple, dict) + + assert any(value is None for value in source_values) + assert any(value is True for value in source_values) + assert any(value is False for value in source_values) + assert any(type(value) is int for value in source_values) + assert any(type(value) is str for value in source_values) + assert any(type(value) is float and value.is_integer() for value in source_values) + assert any(value == float("inf") for value in source_values) + assert any(value == float("-inf") for value in source_values) + # NumPy scalars that are no Python number, read through their item(). + assert any( + not isinstance(value, plain_types) and callable(getattr(value, "item", None)) + for value in source_values + ) + + +def test_holds_each_date_form_and_range() -> None: + values = [value for case in PARAMETER_CASES for value in case.values] + us = value_shape_catalog().country("us") + stored_values = [value for parameter in us.parameters for value in parameter.values] + + assert any(type(value.start) is date for value in values) + assert any( + isinstance(value.start, datetime) and value.start.tzinfo is None + for value in values + ) + # Core writes its earliest instant as 0000-01-01, which .py reads as year 1. + assert any( + value.start_date == datetime(1, 1, 1, tzinfo=UTC) for value in stored_values + ) + # A value in force for a single day, then one in force from the next day. + assert any(value.start_date == value.end_date for value in stored_values) + assert all(parameter.values[-1].end_date is None for parameter in us.parameters) + + +def test_extraction_stores_what_each_case_says() -> None: + us = value_shape_catalog().country("us") + parameters = {parameter.name: parameter for parameter in us.parameters} + + for case in PARAMETER_CASES: + assert [ + _strict_json(value.value_json) for value in parameters[case.name].values + ] == [_strict_json(value.stored_value) for value in case.values], case.name + + +def test_holds_bracket_parameters_and_their_nodes() -> None: + us = value_shape_catalog().country("us") + node_names = {node.name for node in us.parameter_nodes} + bracket_parameters = [ + parameter.name for parameter in us.parameters if "[" in parameter.name + ] + + assert set(BRACKET_PARAMETER_NODES) <= node_names + for node_name in BRACKET_PARAMETER_NODES: + assert f"{node_name}.rate" in bracket_parameters + assert f"{node_name}.threshold" in bracket_parameters + + +def test_holds_each_region_filter_form() -> None: + catalog = value_shape_catalog() + filter_values = { + (country.country_id, region.code): region.filter_value + for country in catalog.countries + for region in country.regions + } + filtered = [ + region + for country in catalog.countries + for region in country.regions + if region.requires_filter + ] + unfiltered_subnational = [ + region + for country in catalog.countries + for region in country.regions + if not region.requires_filter and region.region_type != "national" + ] + + assert {key: filter_values[key] for key in REGION_FILTER_VALUES} == ( + REGION_FILTER_VALUES + ) + # A scalar text filter, a number written as text and a JSON membership list. + stored_forms = set(REGION_FILTER_VALUES.values()) + assert any(value.isdigit() for value in stored_forms) + assert any(value.startswith("[") for value in stored_forms) + assert any(value.isalpha() for value in stored_forms) + assert all(region.filter_strategy == "row_filter" for region in filtered) + assert unfiltered_subnational