diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index 53a465ba..ed4fab79 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -21,7 +21,7 @@ jobs: UV_PYTHON: $(pythonVersion) UV_CACHE_DIR: $(Pipeline.Workspace)/uv-cache strategy: - maxParallel: '1' + maxParallel: '5' matrix: Python39: pythonVersion: '3.9' @@ -92,7 +92,14 @@ jobs: echo "##vso[task.logissue type=error]APPLICATION_INSIGHTS_TEST_DATABASE is required when APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING is configured." exit 1 fi - poe test --ci + poe test --ci --no-e2e + timeout --verbose --signal=TERM --kill-after=30s 25m poe test-e2e --ci || { + exit_code=$? + if [[ "$exit_code" -eq 124 ]]; then + echo "##vso[task.logissue type=error]Kusto E2E tests exceeded the 25-minute process timeout." + fi + exit "$exit_code" + } env: TEST_DATABASE: $(TEST_DATABASE) TEST_BLOB: $(TEST_BLOB) @@ -121,7 +128,7 @@ jobs: displayName: Upload test results condition: always() inputs: - targetPath: pytest.xml + targetPath: test-results artifact: Test Results (Python $(pythonVersion)) - task: PublishTestResults@2 @@ -129,7 +136,8 @@ jobs: condition: always() inputs: testResultsFormat: JUnit - testResultsFiles: pytest*.xml + testResultsFiles: test-results/pytest*.xml + mergeTestResults: true testRunTitle: Python $(pythonVersion) failTaskOnMissingResultsFile: true diff --git a/.gitignore b/.gitignore index 374c28ad..f50b997b 100644 --- a/.gitignore +++ b/.gitignore @@ -300,6 +300,7 @@ build/ # pytest temp files .pytest_cache* +test-results/ # VS code files .vscode* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fce591d..01cc4a8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,12 +115,34 @@ To test without E2E tests, you can run: poe test --no-e2e ``` +To run only E2E tests with eight workers: + +```bash +poe test-e2e +``` + ### Continuous integration The [Azure pipeline](.azuredevops/build.yml) uses the Python 3.9-3.13 matrix, uv dependency caching, formatting, linting, optional type checking, and test/documentation artifacts. -Azure matrix jobs run sequentially to reduce shared-test-data conflicts. +Azure matrix jobs run in parallel, with up to five jobs at a time. +Each trusted job runs unit tests with eight pytest workers, followed by E2E tests +with eight workers, allowing up to 40 E2E workers per pipeline run. +E2E-only pytest fixtures retry individual Kusto requests up to five attempts with +randomized exponential backoff capped at 30 seconds, honoring a longer +`Retry-After` delay when supplied. Throttled requests are retried only when their +bodies can be replayed; other transient HTTP and network failures are retried +only for queries and read-only management commands. Ambiguous ingestion failures, +permanent errors, and failed assertions are not retried. +These fixtures also cover E2E class setup but do not change unit tests or the +SDK's production behavior. Existing higher-level SDK retry and fallback policies +remain in effect. +On the Ubuntu CI agents, the entire E2E invocation has a 25-minute process timeout. +Expiry fails the job and terminates the process group, with a forced kill after +a further 30 seconds if needed. A hard timeout can interrupt table cleanup. +Concurrent pipeline runs still share backend capacity. +The suites write separate JUnit reports under `test-results/`, published together. Python versions are provisioned with `uv python install`, rather than the hosted agent's Python tool cache, so Python 3.9 does not depend on an Ubuntu-specific package in the GitHub Actions Python registry. diff --git a/azure-kusto-ingest/tests/test_e2e_ingest.py b/azure-kusto-ingest/tests/test_e2e_ingest.py index cf1cecaf..3ba4013f 100644 --- a/azure-kusto-ingest/tests/test_e2e_ingest.py +++ b/azure-kusto-ingest/tests/test_e2e_ingest.py @@ -10,6 +10,8 @@ import sys import time import uuid +from unittest.mock import AsyncMock, Mock +from requests import Response from azure.identity import DefaultAzureCredential from typing import Optional, ClassVar, Callable @@ -242,13 +244,10 @@ async def assert_rows_added(cls, table_name: str, expected: int, timeout: int = time.sleep(1) timeout -= 1 - try: - async with await cls.get_async_client() as async_client: - command = "{} | count".format(table_name) - response = cls.client.execute(cls.test_db, command) - response_from_async = await async_client.execute(cls.test_db, command) - except KustoServiceError: - continue + async with await cls.get_async_client() as async_client: + command = "{} | count".format(table_name) + response = cls.client.execute(cls.test_db, command) + response_from_async = await async_client.execute(cls.test_db, command) if response is not None: row = response.primary_results[0][0] @@ -278,6 +277,23 @@ async def assert_rows_unchanged(cls, table_name: str, expected: int, timeout: in return await asyncio.sleep(min(1, remaining)) + @pytest.mark.asyncio + async def test_row_count_retry_exhaustion_is_not_swallowed(self, monkeypatch): + response = Response() + response.status_code = 503 + response._content = b"Service unavailable" + post = Mock(return_value=response) + with KustoClient("https://dummy.kusto.windows.net") as client: + monkeypatch.setattr(client._session, "post", post) + monkeypatch.setattr(type(self), "client", client) + monkeypatch.setattr(type(self), "get_async_client", AsyncMock()) + monkeypatch.setattr(time, "sleep", lambda _: None) + + with pytest.raises(KustoServiceError): + await self.assert_rows_added("table", 1) + + assert post.call_count == 5 + @pytest.mark.asyncio async def test_csv_ingest_existing_table(self, is_managed_streaming): table = await self.init_table("csv_ingest_existing_table") diff --git a/azure-kusto-ingest/tests/test_e2e_retry.py b/azure-kusto-ingest/tests/test_e2e_retry.py new file mode 100644 index 00000000..36a71564 --- /dev/null +++ b/azure-kusto-ingest/tests/test_e2e_retry.py @@ -0,0 +1,184 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime +from io import BytesIO +from unittest.mock import AsyncMock, Mock, patch + +import pytest +from aioresponses import aioresponses +from requests import Response + +from azure.kusto.data import ClientRequestProperties, KustoClient +from azure.kusto.data.aio import KustoClient as AsyncKustoClient +from azure.kusto.data.client_base import ExecuteRequestParams +from azure.kusto.data.exceptions import KustoApiError, KustoNetworkError, KustoServiceError, KustoThrottlingError +from e2e_retry import retry_request, retry_request_async + + +@pytest.fixture(autouse=True) +def retry_sleeps(): + with patch("time.sleep") as sync_sleep, patch("asyncio.sleep", new_callable=AsyncMock) as async_sleep: + yield sync_sleep, async_sleep + + +def make_request(query="table | count", stream=None): + with KustoClient("https://dummy.kusto.windows.net") as client: + return ExecuteRequestParams( + stream, + {"csl": query} if stream is None else None, + {}, + "test;", + ClientRequestProperties(), + timedelta(seconds=1), + timedelta(seconds=1), + timedelta(0), + client.client_details, + ) + + +def make_error(status, headers=None): + response = Response() + response.status_code = status + response.headers.update(headers or {}) + if status == 429: + return KustoThrottlingError("Throttled", response) + return KustoServiceError("Request failed", response) + + +@pytest.mark.parametrize("status", [429, 503]) +def test_retry_transient_query(status, caplog): + operation = Mock(side_effect=[make_error(status), "ok"]) + + assert retry_request(make_request(), operation) == "ok" + + assert operation.call_count == 2 + assert "attempt 2/5" in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [429, 503]) +async def test_retry_transient_query_async(status): + operation = AsyncMock(side_effect=[make_error(status), "ok"]) + + assert await retry_request_async(make_request(), operation) == "ok" + + assert operation.await_count == 2 + + +@pytest.mark.asyncio +async def test_async_client_throttling_response_respects_retry_after(retry_sleeps): + endpoint = "https://dummy.kusto.windows.net/v1/rest/mgmt" + with aioresponses() as responses: + responses.post(endpoint, status=429, payload={}, headers={"Retry-After": "60"}) + responses.post(endpoint, payload={"Tables": [{"TableName": "Table_0", "Columns": [{"ColumnName": "TableName", "DataType": "String"}], "Rows": []}]}) + async with AsyncKustoClient("https://dummy.kusto.windows.net") as client: + await retry_request_async(make_request(".show tables"), lambda: client.execute_mgmt("db", ".show tables")) + + assert sum(len(calls) for calls in responses.requests.values()) == 2 + assert retry_sleeps[1].await_args.args[0] == 60 + + +def test_retry_exhaustion_preserves_failure_and_exponential_delays(retry_sleeps): + error = make_error(429) + operation = Mock(side_effect=error) + with patch("random.uniform", side_effect=lambda low, high: high): + with pytest.raises(KustoThrottlingError) as raised: + retry_request(make_request(), operation) + + assert raised.value is error + assert operation.call_count == 5 + assert [call.args[0] for call in retry_sleeps[0].call_args_list] == [1, 2, 4, 8] + + +@pytest.mark.parametrize("status", [400, 401, 403]) +def test_permanent_failure_is_not_retried(status): + error = make_error(status) + operation = Mock(side_effect=error) + + with pytest.raises(KustoServiceError) as raised: + retry_request(make_request(), operation) + + assert raised.value is error + assert operation.call_count == 1 + + +def test_permanent_api_failure_is_not_retried(): + response = Response() + response.status_code = 500 + error = KustoApiError({"error": {"code": "Permanent", "message": "Permanent failure", "@permanent": True}}, http_response=response) + operation = Mock(side_effect=error) + + with pytest.raises(KustoApiError): + retry_request(make_request(), operation) + + assert operation.call_count == 1 + + +@pytest.mark.parametrize("query", [".ingest inline into table t <| 1", ".create table t (x:int)"]) +def test_ambiguous_write_failure_is_not_retried(query): + operation = Mock(side_effect=make_error(503)) + + with pytest.raises(KustoServiceError): + retry_request(make_request(query), operation) + + assert operation.call_count == 1 + + +@pytest.mark.parametrize("query, retryable", [("table | count", True), (".ingest inline into table t <| 1", False)]) +def test_network_failure_is_retried_only_for_reads(query, retryable): + operation = Mock(side_effect=[KustoNetworkError("https://dummy.kusto.windows.net"), "ok"]) + + if retryable: + assert retry_request(make_request(query), operation) == "ok" + assert operation.call_count == 2 + else: + with pytest.raises(KustoNetworkError): + retry_request(make_request(query), operation) + assert operation.call_count == 1 + + +def test_throttled_stream_is_rewound_to_original_position(): + stream = BytesIO(b"prefix-data") + stream.seek(7) + bodies = [] + + def ingest(): + bodies.append(stream.read()) + if len(bodies) == 1: + raise make_error(429) + return "ok" + + assert retry_request(make_request(stream=stream), ingest) == "ok" + assert bodies == [b"data", b"data"] + + +def test_non_replayable_stream_is_not_retried(): + class NonReplayableStream(BytesIO): + def seekable(self): + return False + + operation = Mock(side_effect=make_error(429)) + + with pytest.raises(KustoThrottlingError): + retry_request(make_request(stream=NonReplayableStream(b"data")), operation) + + assert operation.call_count == 1 + + +@pytest.mark.parametrize("http_date", [False, True]) +def test_retry_after_is_respected(http_date, retry_sleeps): + header = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=60), usegmt=True) if http_date else "60" + operation = Mock(side_effect=[make_error(429, {"Retry-After": header}), "ok"]) + + assert retry_request(make_request(), operation) == "ok" + + assert 58 <= retry_sleeps[0].call_args.args[0] <= 60 + + +def test_invalid_retry_after_warns_and_uses_backoff(caplog): + operation = Mock(side_effect=[make_error(429, {"Retry-After": "invalid"}), "ok"]) + + assert retry_request(make_request(), operation) == "ok" + + assert "invalid Retry-After" in caplog.text diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..08fdb6d5 --- /dev/null +++ b/conftest.py @@ -0,0 +1,40 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License +from typing import Iterator, Optional + +import pytest + +from azure.kusto.data import ClientRequestProperties, KustoClient +from azure.kusto.data.aio import KustoClient as AsyncKustoClient +from azure.kusto.data.client_base import ExecuteRequestParams +from e2e_retry import retry_request, retry_request_async + + +@pytest.fixture(scope="module", autouse=True) +def e2e_request_retries(request: pytest.FixtureRequest) -> Iterator[None]: + if request.path.name not in ("test_e2e_data.py", "test_e2e_ingest.py"): + yield + return + + # Module scope covers the requests made by E2E class setup as well as tests. + sync_execute = KustoClient._execute + async_execute = AsyncKustoClient._execute + + def execute( + client: KustoClient, endpoint: str, request: ExecuteRequestParams, properties: Optional[ClientRequestProperties] = None, stream_response: bool = False + ): + return retry_request(request, lambda: sync_execute(client, endpoint, request, properties, stream_response)) + + async def execute_async( + client: AsyncKustoClient, + endpoint: str, + request: ExecuteRequestParams, + properties: Optional[ClientRequestProperties] = None, + stream_response: bool = False, + ): + return await retry_request_async(request, lambda: async_execute(client, endpoint, request, properties, stream_response)) + + with pytest.MonkeyPatch.context() as monkeypatch: + monkeypatch.setattr(KustoClient, "_execute", execute) + monkeypatch.setattr(AsyncKustoClient, "_execute", execute_async) + yield diff --git a/e2e_retry.py b/e2e_retry.py new file mode 100644 index 00000000..fda32b2e --- /dev/null +++ b/e2e_retry.py @@ -0,0 +1,110 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License +import logging +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +from io import IOBase +from typing import Awaitable, Callable, Optional, TypeVar, Union + +from aiohttp import ClientResponse +from requests import Response +from tenacity import AsyncRetrying, Retrying, RetryCallState, stop_after_attempt, wait_random_exponential + +from azure.kusto.data.client_base import ExecuteRequestParams +from azure.kusto.data.exceptions import KustoApiError, KustoNetworkError, KustoServiceError, KustoThrottlingError + +_LOGGER = logging.getLogger(__name__) +_MAX_ATTEMPTS = 5 +_T = TypeVar("_T") + + +def _http_response(error: Optional[BaseException]) -> Optional[Union[Response, ClientResponse]]: + if isinstance(error, KustoServiceError): + return error.get_raw_http_response() + if isinstance(error, KustoThrottlingError) and len(error.args) > 1 and isinstance(error.args[1], (Response, ClientResponse)): + return error.args[1] + return None + + +def _retry_after(error: Optional[BaseException]) -> float: + response = _http_response(error) + if response is None: + return 0 + header = response.headers.get("Retry-After") + if header is None: + return 0 + try: + return max(0, float(int(header))) + except (ValueError, OverflowError): + try: + retry_at = parsedate_to_datetime(header) + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + return max(0, (retry_at - datetime.now(timezone.utc)).total_seconds()) + except (ValueError, TypeError, OverflowError): + _LOGGER.warning("Ignoring invalid Retry-After header on an E2E Kusto response.") + return 0 + + +class _RequestRetry: + def __init__(self, request: ExecuteRequestParams): + payload = request.payload + self._stream = payload if isinstance(payload, IOBase) else None + self._position = self._stream.tell() if self._stream is not None and self._stream.seekable() else None + self._replayable = payload is None or isinstance(payload, (bytes, str)) or self._position is not None + query = request.json_payload.get("csl") if isinstance(request.json_payload, dict) else None + self._read_only = ( + isinstance(query, str) and bool(query.strip()) and (not query.lstrip().startswith(".") or query.lstrip().lower().startswith((".show ", ".get "))) + ) + self._backoff = wait_random_exponential(multiplier=1, max=30) + + def should_retry(self, state: RetryCallState) -> bool: + error = state.outcome.exception() if state.outcome is not None else None + if not self._replayable or (self._stream is not None and self._stream.closed): + if isinstance(error, KustoThrottlingError): + _LOGGER.warning("Cannot retry a throttled E2E Kusto request because its body cannot be replayed.") + return False + if isinstance(error, KustoThrottlingError): + return True + if not self._read_only: + return False + if isinstance(error, KustoApiError) and error.get_api_error().permanent: + return False + if isinstance(error, KustoNetworkError): + return True + response = _http_response(error) + if response is None: + return False + status = response.status_code if isinstance(response, Response) else response.status + return status in (408, 500, 502, 503, 504) + + def wait(self, state: RetryCallState) -> float: + error = state.outcome.exception() if state.outcome is not None else None + return max(self._backoff(state), _retry_after(error)) + + def before_sleep(self, state: RetryCallState) -> None: + if self._stream is not None and self._position is not None: + self._stream.seek(self._position) + error = state.outcome.exception() if state.outcome is not None else None + delay = state.next_action.sleep if state.next_action is not None else 0 + _LOGGER.warning( + "Retrying E2E Kusto request after %s; attempt %d/%d in %.2fs.", + type(error).__name__, + state.attempt_number + 1, + _MAX_ATTEMPTS, + delay, + ) + + +def retry_request(request: ExecuteRequestParams, operation: Callable[[], _T]) -> _T: + policy = _RequestRetry(request) + return Retrying(stop=stop_after_attempt(_MAX_ATTEMPTS), retry=policy.should_retry, wait=policy.wait, before_sleep=policy.before_sleep, reraise=True)( + operation + ) + + +async def retry_request_async(request: ExecuteRequestParams, operation: Callable[[], Awaitable[_T]]) -> _T: + policy = _RequestRetry(request) + return await AsyncRetrying( + stop=stop_after_attempt(_MAX_ATTEMPTS), retry=policy.should_retry, wait=policy.wait, before_sleep=policy.before_sleep, reraise=True + )(operation) diff --git a/pyproject.toml b/pyproject.toml index 0a6045ff..b4609955 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,12 +68,18 @@ lint-fix-unsafe = "ruff check --fix --unsafe-fixes" fix = ["format", "lint-fix"] show-version = "uv version" [tool.poe.tasks.test] -sequence = ["sync", { cmd = 'pytest ${NO_E2E:+ --ignore=azure-kusto-ingest/tests/test_e2e_ingest.py --ignore=azure-kusto-data/tests/test_e2e_data.py } ${CI:+ -n8 --junit-xml pytest.xml}'}] +sequence = ["sync", { cmd = 'pytest ${NO_E2E:+ --ignore=azure-kusto-ingest/tests/test_e2e_ingest.py --ignore=azure-kusto-data/tests/test_e2e_data.py } ${CI:+ -n8 --junit-xml test-results/pytest.xml}'}] [tool.poe.tasks.test.args] NO_E2E = { help = "Skip E2E tests", type = "boolean", options = ["--no-e2e"] } CI = { help = "Run tests in CI mode", type = "boolean", options = ["--ci"] } +[tool.poe.tasks.test-e2e] +sequence = ["sync", { cmd = 'pytest -n8 azure-kusto-data/tests/test_e2e_data.py azure-kusto-ingest/tests/test_e2e_ingest.py ${CI:+ --junit-xml test-results/pytest-e2e.xml}'}] + +[tool.poe.tasks.test-e2e.args] +CI = { help = "Run tests in CI mode", type = "boolean", options = ["--ci"] } + [tool.poe.tasks.bump-version] sequence = [{ cmd = "uv version --bump $bump" }, { cmd = "uv version --bump $bump --package azure-kusto-data" }, { cmd = "uv version --bump $bump --package azure-kusto-ingest" }, { cmd = "uv version --bump $bump --package quick-start" }] args = { bump = { help = "Bump version [possible values: major, minor, patch, stable, alpha, beta, rc, post, dev]", positional = true, default = "patch" } }