Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions .azuredevops/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
UV_PYTHON: $(pythonVersion)
UV_CACHE_DIR: $(Pipeline.Workspace)/uv-cache
strategy:
maxParallel: '1'
maxParallel: '5'
matrix:
Python39:
pythonVersion: '3.9'
Expand Down Expand Up @@ -92,7 +92,14 @@ jobs:
echo "##vso[task.logissue type=error]APPLICATION_INSIGHTS_TEST_DATABASE is required when APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING is configured."
exit 1
fi
poe test --ci
poe test --ci --no-e2e
timeout --verbose --signal=TERM --kill-after=30s 25m poe test-e2e --ci || {
exit_code=$?
if [[ "$exit_code" -eq 124 ]]; then
echo "##vso[task.logissue type=error]Kusto E2E tests exceeded the 25-minute process timeout."
fi
exit "$exit_code"
}
env:
TEST_DATABASE: $(TEST_DATABASE)
TEST_BLOB: $(TEST_BLOB)
Expand Down Expand Up @@ -121,15 +128,16 @@ jobs:
displayName: Upload test results
condition: always()
inputs:
targetPath: pytest.xml
targetPath: test-results
artifact: Test Results (Python $(pythonVersion))

- task: PublishTestResults@2
displayName: Publish test results
condition: always()
inputs:
testResultsFormat: JUnit
testResultsFiles: pytest*.xml
testResultsFiles: test-results/pytest*.xml
mergeTestResults: true
testRunTitle: Python $(pythonVersion)
failTaskOnMissingResultsFile: true

Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,7 @@ build/

# pytest temp files
.pytest_cache*
test-results/

# VS code files
.vscode*
Expand Down
24 changes: 23 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,34 @@ To test without E2E tests, you can run:
poe test --no-e2e
```

To run only E2E tests with eight workers:

```bash
poe test-e2e
```

### Continuous integration

The [Azure pipeline](.azuredevops/build.yml) uses the Python 3.9-3.13 matrix,
uv dependency caching, formatting, linting, optional type checking,
and test/documentation artifacts.
Azure matrix jobs run sequentially to reduce shared-test-data conflicts.
Azure matrix jobs run in parallel, with up to five jobs at a time.
Each trusted job runs unit tests with eight pytest workers, followed by E2E tests
with eight workers, allowing up to 40 E2E workers per pipeline run.
E2E-only pytest fixtures retry individual Kusto requests up to five attempts with
randomized exponential backoff capped at 30 seconds, honoring a longer
`Retry-After` delay when supplied. Throttled requests are retried only when their
bodies can be replayed; other transient HTTP and network failures are retried
only for queries and read-only management commands. Ambiguous ingestion failures,
permanent errors, and failed assertions are not retried.
These fixtures also cover E2E class setup but do not change unit tests or the
SDK's production behavior. Existing higher-level SDK retry and fallback policies
remain in effect.
On the Ubuntu CI agents, the entire E2E invocation has a 25-minute process timeout.
Expiry fails the job and terminates the process group, with a forced kill after
a further 30 seconds if needed. A hard timeout can interrupt table cleanup.
Concurrent pipeline runs still share backend capacity.
The suites write separate JUnit reports under `test-results/`, published together.
Python versions are provisioned with `uv python install`, rather than the hosted
agent's Python tool cache, so Python 3.9 does not depend on an Ubuntu-specific
package in the GitHub Actions Python registry.
Expand Down
30 changes: 23 additions & 7 deletions azure-kusto-ingest/tests/test_e2e_ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import sys
import time
import uuid
from unittest.mock import AsyncMock, Mock
from requests import Response
from azure.identity import DefaultAzureCredential
from typing import Optional, ClassVar, Callable

Expand Down Expand Up @@ -242,13 +244,10 @@ async def assert_rows_added(cls, table_name: str, expected: int, timeout: int =
time.sleep(1)
timeout -= 1

try:
async with await cls.get_async_client() as async_client:
command = "{} | count".format(table_name)
response = cls.client.execute(cls.test_db, command)
response_from_async = await async_client.execute(cls.test_db, command)
except KustoServiceError:
continue
async with await cls.get_async_client() as async_client:
command = "{} | count".format(table_name)
response = cls.client.execute(cls.test_db, command)
response_from_async = await async_client.execute(cls.test_db, command)

if response is not None:
row = response.primary_results[0][0]
Expand Down Expand Up @@ -278,6 +277,23 @@ async def assert_rows_unchanged(cls, table_name: str, expected: int, timeout: in
return
await asyncio.sleep(min(1, remaining))

@pytest.mark.asyncio
async def test_row_count_retry_exhaustion_is_not_swallowed(self, monkeypatch):
response = Response()
response.status_code = 503
response._content = b"Service unavailable"
post = Mock(return_value=response)
with KustoClient("https://dummy.kusto.windows.net") as client:
monkeypatch.setattr(client._session, "post", post)
monkeypatch.setattr(type(self), "client", client)
monkeypatch.setattr(type(self), "get_async_client", AsyncMock())
monkeypatch.setattr(time, "sleep", lambda _: None)

with pytest.raises(KustoServiceError):
await self.assert_rows_added("table", 1)

assert post.call_count == 5

@pytest.mark.asyncio
async def test_csv_ingest_existing_table(self, is_managed_streaming):
table = await self.init_table("csv_ingest_existing_table")
Expand Down
184 changes: 184 additions & 0 deletions azure-kusto-ingest/tests/test_e2e_retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from datetime import datetime, timedelta, timezone
from email.utils import format_datetime
from io import BytesIO
from unittest.mock import AsyncMock, Mock, patch

import pytest
from aioresponses import aioresponses
from requests import Response

from azure.kusto.data import ClientRequestProperties, KustoClient
from azure.kusto.data.aio import KustoClient as AsyncKustoClient
from azure.kusto.data.client_base import ExecuteRequestParams
from azure.kusto.data.exceptions import KustoApiError, KustoNetworkError, KustoServiceError, KustoThrottlingError
from e2e_retry import retry_request, retry_request_async


@pytest.fixture(autouse=True)
def retry_sleeps():
with patch("time.sleep") as sync_sleep, patch("asyncio.sleep", new_callable=AsyncMock) as async_sleep:
yield sync_sleep, async_sleep


def make_request(query="table | count", stream=None):
with KustoClient("https://dummy.kusto.windows.net") as client:
return ExecuteRequestParams(
stream,
{"csl": query} if stream is None else None,
{},
"test;",
ClientRequestProperties(),
timedelta(seconds=1),
timedelta(seconds=1),
timedelta(0),
client.client_details,
)


def make_error(status, headers=None):
response = Response()
response.status_code = status
response.headers.update(headers or {})
if status == 429:
return KustoThrottlingError("Throttled", response)
return KustoServiceError("Request failed", response)


@pytest.mark.parametrize("status", [429, 503])
def test_retry_transient_query(status, caplog):
operation = Mock(side_effect=[make_error(status), "ok"])

assert retry_request(make_request(), operation) == "ok"

assert operation.call_count == 2
assert "attempt 2/5" in caplog.text


@pytest.mark.asyncio
@pytest.mark.parametrize("status", [429, 503])
async def test_retry_transient_query_async(status):
operation = AsyncMock(side_effect=[make_error(status), "ok"])

assert await retry_request_async(make_request(), operation) == "ok"

assert operation.await_count == 2


@pytest.mark.asyncio
async def test_async_client_throttling_response_respects_retry_after(retry_sleeps):
endpoint = "https://dummy.kusto.windows.net/v1/rest/mgmt"
with aioresponses() as responses:
responses.post(endpoint, status=429, payload={}, headers={"Retry-After": "60"})
responses.post(endpoint, payload={"Tables": [{"TableName": "Table_0", "Columns": [{"ColumnName": "TableName", "DataType": "String"}], "Rows": []}]})
async with AsyncKustoClient("https://dummy.kusto.windows.net") as client:
await retry_request_async(make_request(".show tables"), lambda: client.execute_mgmt("db", ".show tables"))

assert sum(len(calls) for calls in responses.requests.values()) == 2
assert retry_sleeps[1].await_args.args[0] == 60


def test_retry_exhaustion_preserves_failure_and_exponential_delays(retry_sleeps):
error = make_error(429)
operation = Mock(side_effect=error)
with patch("random.uniform", side_effect=lambda low, high: high):
with pytest.raises(KustoThrottlingError) as raised:
retry_request(make_request(), operation)

assert raised.value is error
assert operation.call_count == 5
assert [call.args[0] for call in retry_sleeps[0].call_args_list] == [1, 2, 4, 8]


@pytest.mark.parametrize("status", [400, 401, 403])
def test_permanent_failure_is_not_retried(status):
error = make_error(status)
operation = Mock(side_effect=error)

with pytest.raises(KustoServiceError) as raised:
retry_request(make_request(), operation)

assert raised.value is error
assert operation.call_count == 1


def test_permanent_api_failure_is_not_retried():
response = Response()
response.status_code = 500
error = KustoApiError({"error": {"code": "Permanent", "message": "Permanent failure", "@permanent": True}}, http_response=response)
operation = Mock(side_effect=error)

with pytest.raises(KustoApiError):
retry_request(make_request(), operation)

assert operation.call_count == 1


@pytest.mark.parametrize("query", [".ingest inline into table t <| 1", ".create table t (x:int)"])
def test_ambiguous_write_failure_is_not_retried(query):
operation = Mock(side_effect=make_error(503))

with pytest.raises(KustoServiceError):
retry_request(make_request(query), operation)

assert operation.call_count == 1


@pytest.mark.parametrize("query, retryable", [("table | count", True), (".ingest inline into table t <| 1", False)])
def test_network_failure_is_retried_only_for_reads(query, retryable):
operation = Mock(side_effect=[KustoNetworkError("https://dummy.kusto.windows.net"), "ok"])

if retryable:
assert retry_request(make_request(query), operation) == "ok"
assert operation.call_count == 2
else:
with pytest.raises(KustoNetworkError):
retry_request(make_request(query), operation)
assert operation.call_count == 1


def test_throttled_stream_is_rewound_to_original_position():
stream = BytesIO(b"prefix-data")
stream.seek(7)
bodies = []

def ingest():
bodies.append(stream.read())
if len(bodies) == 1:
raise make_error(429)
return "ok"

assert retry_request(make_request(stream=stream), ingest) == "ok"
assert bodies == [b"data", b"data"]


def test_non_replayable_stream_is_not_retried():
class NonReplayableStream(BytesIO):
def seekable(self):
return False

operation = Mock(side_effect=make_error(429))

with pytest.raises(KustoThrottlingError):
retry_request(make_request(stream=NonReplayableStream(b"data")), operation)

assert operation.call_count == 1


@pytest.mark.parametrize("http_date", [False, True])
def test_retry_after_is_respected(http_date, retry_sleeps):
header = format_datetime(datetime.now(timezone.utc) + timedelta(seconds=60), usegmt=True) if http_date else "60"
operation = Mock(side_effect=[make_error(429, {"Retry-After": header}), "ok"])

assert retry_request(make_request(), operation) == "ok"

assert 58 <= retry_sleeps[0].call_args.args[0] <= 60


def test_invalid_retry_after_warns_and_uses_backoff(caplog):
operation = Mock(side_effect=[make_error(429, {"Retry-After": "invalid"}), "ok"])

assert retry_request(make_request(), operation) == "ok"

assert "invalid Retry-After" in caplog.text
40 changes: 40 additions & 0 deletions conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License
from typing import Iterator, Optional

import pytest

from azure.kusto.data import ClientRequestProperties, KustoClient
from azure.kusto.data.aio import KustoClient as AsyncKustoClient
from azure.kusto.data.client_base import ExecuteRequestParams
from e2e_retry import retry_request, retry_request_async


@pytest.fixture(scope="module", autouse=True)
def e2e_request_retries(request: pytest.FixtureRequest) -> Iterator[None]:
if request.path.name not in ("test_e2e_data.py", "test_e2e_ingest.py"):
yield
return

# Module scope covers the requests made by E2E class setup as well as tests.
sync_execute = KustoClient._execute
async_execute = AsyncKustoClient._execute

def execute(
client: KustoClient, endpoint: str, request: ExecuteRequestParams, properties: Optional[ClientRequestProperties] = None, stream_response: bool = False
):
return retry_request(request, lambda: sync_execute(client, endpoint, request, properties, stream_response))

async def execute_async(
client: AsyncKustoClient,
endpoint: str,
request: ExecuteRequestParams,
properties: Optional[ClientRequestProperties] = None,
stream_response: bool = False,
):
return await retry_request_async(request, lambda: async_execute(client, endpoint, request, properties, stream_response))

with pytest.MonkeyPatch.context() as monkeypatch:
monkeypatch.setattr(KustoClient, "_execute", execute)
monkeypatch.setattr(AsyncKustoClient, "_execute", execute_async)
yield
Loading
Loading