diff --git a/.github/workflows/CDA-testing.yml b/.github/workflows/CDA-testing.yml index b37856a..7b9afc8 100644 --- a/.github/workflows/CDA-testing.yml +++ b/.github/workflows/CDA-testing.yml @@ -25,9 +25,9 @@ jobs: fail-fast: false max-parallel: 6 matrix: - python-version: ['3.9', '3.13', '3.x'] - # Ordinary PRs run two production combinations. Release Please PRs, - # weekly runs, and manual runs exercise all 18 combinations. + python-version: ['3.9', '3.13', '3.14'] + # Ordinary PRs run three production combinations. Release Please PRs, + # weekly runs, and manual runs exercise all 27 combinations. cda: ${{ fromJSON(github.event_name == 'pull_request' && !startsWith(github.head_ref, 'release-please--') && '["production"]' || '["latest", "production", "test"]') }} schema: ${{ fromJSON(github.event_name == 'pull_request' && !startsWith(github.head_ref, 'release-please--') && '["production"]' || '["latest", "production", "test"]') }} diff --git a/.github/workflows/pypi-deploy.yml b/.github/workflows/pypi-deploy.yml index 7539afe..9daf6eb 100644 --- a/.github/workflows/pypi-deploy.yml +++ b/.github/workflows/pypi-deploy.yml @@ -47,7 +47,7 @@ jobs: - name: Set Up Python uses: actions/setup-python@v7 with: - python-version: '3.x' + python-version: '3.14' - name: Install Poetry uses: abatilo/actions-poetry@v4 - name: Install Dependencies diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 2a1cf3a..cc01c80 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ['3.9', '3.13', '3.x'] + python-version: ['3.9', '3.13', '3.14'] steps: - uses: actions/checkout@v7 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1034758..ec9cba5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,8 +103,8 @@ passed, bearer token auth takes precedence. Stop the previous stack before starting another one using the same host ports. The integration workflow in `.github/workflows/CDA-testing.yml` tests every - combination of three CDA images and three database versions on Python 3.9 - and latest stable (`3.x`) in exhaustive runs (18 jobs): + combination of three CDA images and three database versions on Python 3.9, + 3.13, and the latest validated stable (`3.14`) in exhaustive runs (27 jobs): | Lane | CDA tag | Database and schema-installer tag | | --- | --- | --- | @@ -200,24 +200,24 @@ merged commits. ### CI coverage -Unit tests and strict type checks run on Python **3.9** and **latest stable** -(`3.x`) for PRs targeting any branch and pushes to `main`. Formatting and CodeQL -also run on PRs and main. Feature-branch pushes do not start duplicate checks; -new PR commits cancel superseded runs. Draft and documentation-only PRs use the -same straightforward checks as other PRs. +Unit tests and strict type checks run on Python **3.9**, **3.13**, and the latest +validated stable version (**3.14**) for PRs targeting any branch and pushes to +`main`. Formatting and CodeQL also run on PRs and main. Feature-branch pushes do +not start duplicate checks; new PR commits cancel superseded runs. Draft and +documentation-only PRs use the same straightforward checks as other PRs. The **CDA integration (PR production, weekly exhaustive)** workflow runs on every -PR. Ordinary PRs run two integration jobs: production CDA with the production -database/schema on Python **3.9** and **latest stable** (`3.x`). These jobs use +PR. Ordinary PRs run three integration jobs: production CDA with the production +database/schema on Python **3.9**, **3.13**, and **3.14**. These jobs use disposable local containers with production version pins, not deployed production services. New PR commits cancel superseded integration runs. -The full **18-job matrix** runs weekly on **Monday at 08:17 UTC**, on manual +The full **27-job matrix** runs weekly on **Monday at 08:17 UTC**, on manual dispatch, and on Release Please PRs (branches beginning with `release-please--`). -It covers two Python versions, three CDA versions, and three schema versions, +It covers three Python versions, three CDA versions, and three schema versions, including mixed CDA/schema combinations. -**Before merging a Release Please PR, all 18 integration jobs must pass for its +**Before merging a Release Please PR, all 27 integration jobs must pass for its current revision.** If the bot-created PR has no checks, close and reopen it as described below, or manually run the full matrix against its head branch. You can also run the full matrix for any other PR when broader coverage is useful: @@ -233,11 +233,11 @@ run in Actions before approving. Inspect failed combinations and backend logs, reproduce using the Compose overrides above, and fix the cause rather than dropping failing combinations. -Release publishing still tests the exact tag on latest stable Python, verifies +Release publishing still tests the exact tag on Python 3.14, verifies the package version, and builds before publishing. CodeQL retains its weekly run. -Repository administrators should require the two unit-test checks, the two +Repository administrators should require the three unit-test checks, the three production CDA integration checks, formatting, and CodeQL (plus existing external -requirements). The other 16 CDA combinations do not run on ordinary PRs and +requirements). The other 24 CDA combinations do not run on ordinary PRs and should not be required globally. Maintainers must verify the full release matrix before merging a release PR; these workflow changes do not modify repository merge rules or enforce a separate release-only merge gate. @@ -249,7 +249,7 @@ merge rules or enforce a separate release-only merge gate. proposed `pyproject.toml` version, `CHANGELOG.md`, and `.release-please-manifest.json`. 3. Review the proposed version and release notes, verify the required checks and - all 18 CDA integration jobs pass for the current revision, and merge the release + all 27 CDA integration jobs pass for the current revision, and merge the release PR when ready. 4. The same workflow creates the `vX.Y.Z` tag and GitHub release, checks out that exact tag, tests and builds the distribution, publishes to PyPI, signs the diff --git a/cwms/timeseries/timeseries.py b/cwms/timeseries/timeseries.py index 42acba3..066a7fb 100644 --- a/cwms/timeseries/timeseries.py +++ b/cwms/timeseries/timeseries.py @@ -310,7 +310,7 @@ def fetch_timeseries_chunks( return results -def combine_timeseries_results(results: List[Data]) -> Data: +def combine_timeseries_results(results: List[Data], trim: bool = False) -> Data: """ Combines the results from multiple chunks into a single cwms Data object. @@ -335,13 +335,27 @@ def combine_timeseries_results(results: List[Data]) -> Data: # Drop duplicate rows based on 'date-time' (if necessary) combined_df.drop_duplicates(subset="date-time", inplace=True) + combined_df.reset_index(drop=True, inplace=True) + + # CDA trimming applies to the requested range. Applying it to each chunk + # independently can remove scheduled missing values at internal boundaries. + if trim and not combined_df.empty: + populated = combined_df["value"].notna() + if populated.any(): + populated_positions = populated.to_numpy().nonzero()[0] + first = int(populated_positions[0]) + last = int(populated_positions[-1]) + 1 + combined_df = combined_df.iloc[first:last].copy() + else: + combined_df = combined_df.iloc[0:0].copy() # Extract metadata from the first result (assuming all chunks share the same metadata) combined_json = results[0].json # Update metadata to reflect the combined time range - combined_json["begin"] = combined_df["date-time"].min().isoformat() - combined_json["end"] = combined_df["date-time"].max().isoformat() + if not combined_df.empty: + combined_json["begin"] = combined_df["date-time"].min().isoformat() + combined_json["end"] = combined_df["date-time"].max().isoformat() combined_json["total"] = len(combined_df) combined_df["date-time"] = combined_df["date-time"].apply( @@ -385,7 +399,7 @@ def get_timeseries( datum: Optional[str] = None, begin: Optional[datetime] = None, end: Optional[datetime] = None, - page_size: Optional[int] = 300000, + page_size: Optional[int] = -1, version_date: Optional[datetime] = None, trim: Optional[bool] = True, multithread: Optional[bool] = True, @@ -422,8 +436,8 @@ def get_timeseries( not specified, any required time window ends at the current time. Any timezone information should be passed within the datetime object. If no timezone information is given, default will be UTC. - page_size: int, optional, default is 300000: Specifies the number of records to obtain in - a single call. + page_size: int, optional, default is -1: Specifies the number of records to obtain in + a single call. A value of -1 disables paging. version_date: datetime, optional, default is None Version date of time series values being requested. If this field is not specified and the timeseries is versioned, the query will return the max aggregate for the time period. @@ -491,16 +505,18 @@ def get_timeseries( f"Fetching {len(chunks)} chunks of timeseries data with {max_workers} threads" ) # fetch the data + chunk_params = params.copy() + chunk_params["trim"] = False result_list = fetch_timeseries_chunks( chunks, - params, + chunk_params, selector, endpoint, max_workers, ) # combine the results - results = combine_timeseries_results(result_list) + results = combine_timeseries_results(result_list, trim=trim is not False) return results diff --git a/tests/mock/timeseries/timeseries_test.py b/tests/mock/timeseries/timeseries_test.py index 8fe964e..c13097c 100644 --- a/tests/mock/timeseries/timeseries_test.py +++ b/tests/mock/timeseries/timeseries_test.py @@ -13,6 +13,7 @@ import cwms.api import cwms.timeseries.timeseries as timeseries import cwms.timeseries.timeseries_group as timeseries_group +from cwms.cwms_types import Data from tests._test_utils import read_resource_file _MOCK_ROOT = "https://mockwebserver.cwms.gov" @@ -158,7 +159,7 @@ def test_get_timeseries_unversioned_default(requests_mock, monkeypatch): "unit=EN&" "begin=2008-05-01T15%3A00%3A00%2B00%3A00&" "end=2008-05-01T17%3A00%3A00%2B00%3A00&" - "page-size=300000", + "page-size=-1", json=_UNVERS_TS_JSON, ) @@ -196,7 +197,7 @@ def test_get_empty_ts_df(requests_mock, monkeypatch): "unit=EN&" "begin=2008-05-01T15%3A00%3A00%2B00%3A00&" "end=2008-05-01T17%3A00%3A00%2B00%3A00&" - "page-size=300000&" + "page-size=-1&" "trim=true", json=_EMPTY_TS_JSON, ) @@ -336,6 +337,31 @@ def test_chunk_timeseries_time_range_rejects_nonpositive_size(): ) +def test_combine_timeseries_results_trims_only_outer_missing_values(): + first_json = _UNVERS_TS_JSON.copy() + first_json["values"] = [[0, None, 0], [1000, 1.0, 0], [2000, None, 0]] + second_json = _UNVERS_TS_JSON.copy() + second_json["values"] = [ + [2000, None, 0], + [3000, None, 0], + [4000, 2.0, 0], + [5000, None, 0], + ] + + combined = timeseries.combine_timeseries_results( + [Data(first_json, selector="values"), Data(second_json, selector="values")], + trim=True, + ) + + assert combined.json["values"] == [ + [1000, 1.0, 0], + [2000, None, 0], + [3000, None, 0], + [4000, 2.0, 0], + ] + assert combined.json["total"] == 4 + + @pytest.mark.parametrize( ("ts_id", "expected_days"), [ @@ -354,11 +380,14 @@ def test_get_timeseries_uses_interval_chunk_size_by_default( def fetch_chunks(chunks, params, selector, endpoint, max_workers): captured["chunks"] = chunks captured["max_workers"] = max_workers + captured["trim"] = params["trim"] return [object()] monkeypatch.setattr(timeseries, "fetch_timeseries_chunks", fetch_chunks) monkeypatch.setattr( - timeseries, "combine_timeseries_results", lambda results: expected + timeseries, + "combine_timeseries_results", + lambda results, trim=False: expected, ) result = timeseries.get_timeseries( @@ -373,6 +402,7 @@ def fetch_chunks(chunks, params, selector, endpoint, max_workers): begin, end, timedelta(days=expected_days) ) assert captured["max_workers"] == 3 + assert captured["trim"] is False def test_get_timeseries_max_days_per_chunk_overrides_interval(monkeypatch): @@ -386,7 +416,9 @@ def fetch_chunks(chunks, params, selector, endpoint, max_workers): monkeypatch.setattr(timeseries, "fetch_timeseries_chunks", fetch_chunks) monkeypatch.setattr( - timeseries, "combine_timeseries_results", lambda results: object() + timeseries, + "combine_timeseries_results", + lambda results, trim=False: object(), ) timeseries.get_timeseries( @@ -468,7 +500,7 @@ def test_get_multi_timeseries_default(requests_mock): "unit=EN&" "begin=2008-05-01T15%3A00%3A00%2B00%3A00&" "end=2008-05-01T17%3A00%3A00%2B00%3A00&" - "page-size=300000&" + "page-size=-1&" "version-date=2021-06-20T08%3A00%3A00%2B00%3A00", json=_VERS_TS_JSON, ) @@ -480,7 +512,7 @@ def test_get_multi_timeseries_default(requests_mock): "unit=EN&" "begin=2008-05-01T15%3A00%3A00%2B00%3A00&" "end=2008-05-01T17%3A00%3A00%2B00%3A00&" - "page-size=300000", + "page-size=-1", json=_UNVERS_TS_JSON, ) @@ -533,7 +565,7 @@ def test_get_timeseries_versioned_default(requests_mock, monkeypatch): "unit=EN&" "begin=2008-05-01T15%3A00%3A00%2B00%3A00&" "end=2008-05-01T17%3A00%3A00%2B00%3A00&" - "page-size=300000&" + "page-size=-1&" "version-date=2021-06-20T08%3A00%3A00%2B00%3A00", json=_VERS_TS_JSON, )