From 5e3dae39a6afb50fe2a39d97f3e605f2aea57d0e Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Thu, 8 Oct 2026 12:59:01 +0530 Subject: [PATCH 01/10] Migrate build and E2E tests to Azure Pipelines Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 019ed4af-f34c-4cc8-8465-a038be55d9f6 --- .azuredevops/build.yml | 95 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 89 insertions(+), 6 deletions(-) diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index ce01168f..527ee373 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -1,14 +1,97 @@ -# Temporary Azure Pipelines bootstrap for .github/workflows/build.yml - trigger: - master pr: -- master + branches: + include: + - '*' pool: vmImage: ubuntu-latest -steps: -- script: echo "Hello from Azure Pipelines" - displayName: Hello world +variables: +- group: azure-kusto-python-e2e + +jobs: +- job: BuildAndTest + displayName: Build and test + timeoutInMinutes: 60 + strategy: + failFast: false + matrix: + Python39: + pythonVersion: '3.9' + Python310: + pythonVersion: '3.10' + Python311: + pythonVersion: '3.11' + Python312: + pythonVersion: '3.12' + Python313: + pythonVersion: '3.13' + steps: + - checkout: self + clean: true + + - task: UsePythonVersion@0 + displayName: Use Python $(pythonVersion) + inputs: + versionSpec: $(pythonVersion) + + - bash: | + set -euo pipefail + python -m pip install --disable-pip-version-check uv poethepoet + uv --version + poe --version + displayName: Install build tools + + - bash: poe test --ci --no-e2e + displayName: Run tests without Azure credentials + condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'True')) + + - task: AzureCLI@2 + displayName: Run unit and Kusto E2E tests + condition: and(succeeded(), ne(variables['System.PullRequest.IsFork'], 'True')) + inputs: + azureSubscription: $(AZURE_SERVICE_CONNECTION) + scriptType: bash + scriptLocation: inlineScript + inlineScript: poe test --ci + env: + TEST_DATABASE: $(TEST_DATABASE) + ENGINE_CONNECTION_STRING: $(ENGINE_CONNECTION_STRING) + APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: $(APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING) + APPLICATION_INSIGHTS_TEST_DATABASE: $(APPLICATION_INSIGHTS_TEST_DATABASE) + + - bash: poe check-format + displayName: Run Ruff formatter check + condition: succeededOrFailed() + + - bash: poe lint + displayName: Run Ruff linter + condition: succeededOrFailed() + + - bash: poe types + displayName: Run type checking + condition: succeededOrFailed() + continueOnError: true + + - bash: poe docs --ci + displayName: Generate documentation + condition: and(succeededOrFailed(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: PublishTestResults@2 + displayName: Publish test results + condition: succeededOrFailed() + inputs: + testResultsFormat: JUnit + testResultsFiles: pytest*.xml + testRunTitle: Python $(pythonVersion) + failTaskOnMissingResultsFile: true + + - task: PublishPipelineArtifact@1 + displayName: Publish documentation + condition: and(succeededOrFailed(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + inputs: + targetPath: docs + artifact: documentation From 311bd78a8e7e1363e4f5952b8557ddd38f21a474 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Thu, 8 Oct 2026 13:01:16 +0530 Subject: [PATCH 02/10] Fix Azure Pipelines matrix strategy Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 019ed4af-f34c-4cc8-8465-a038be55d9f6 --- .azuredevops/build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index 527ee373..7a4bb3a2 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -17,7 +17,6 @@ jobs: displayName: Build and test timeoutInMinutes: 60 strategy: - failFast: false matrix: Python39: pythonVersion: '3.9' From 82d6b9ec196f1e493a0443b2250316c8c921a452 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Thu, 8 Oct 2026 13:24:13 +0530 Subject: [PATCH 03/10] Stabilize governed Python E2E tests Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 019ed4af-f34c-4cc8-8465-a038be55d9f6 --- .azuredevops/build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index 7a4bb3a2..2e2cfffe 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -17,6 +17,7 @@ jobs: displayName: Build and test timeoutInMinutes: 60 strategy: + maxParallel: 1 matrix: Python39: pythonVersion: '3.9' @@ -59,8 +60,6 @@ jobs: env: TEST_DATABASE: $(TEST_DATABASE) ENGINE_CONNECTION_STRING: $(ENGINE_CONNECTION_STRING) - APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: $(APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING) - APPLICATION_INSIGHTS_TEST_DATABASE: $(APPLICATION_INSIGHTS_TEST_DATABASE) - bash: poe check-format displayName: Run Ruff formatter check From 793db65bbcf375b863a1466e746796a8aa6e1ab0 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Tue, 13 Oct 2026 11:20:27 +0530 Subject: [PATCH 04/10] Use actual Azure service connection name Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4ce38852-3d06-4124-b043-c7591a9f49f7 --- .azuredevops/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index 2e2cfffe..2c9ddfad 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -53,7 +53,7 @@ jobs: displayName: Run unit and Kusto E2E tests condition: and(succeeded(), ne(variables['System.PullRequest.IsFork'], 'True')) inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) + azureSubscription: kusto-connectors-arm-sc scriptType: bash scriptLocation: inlineScript inlineScript: poe test --ci From 1fb7481f63e8d291f5b2b10b7d6307bd762a71e0 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Mon, 9 Nov 2026 15:18:35 +0530 Subject: [PATCH 05/10] Fix Azure pipeline Python provisioning and variable-group configuration Provision matrix interpreters with uv, preserve sequential E2E jobs, and validate variable-group inputs. Document pipeline setup requirements. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .azuredevops/build.yml | 78 +++++++++++++++++++++++++++++++++--------- CONTRIBUTING.md | 32 +++++++++++++++++ 2 files changed, 93 insertions(+), 17 deletions(-) diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index 2c9ddfad..3bfcc910 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -15,9 +15,13 @@ variables: jobs: - job: BuildAndTest displayName: Build and test - timeoutInMinutes: 60 + timeoutInMinutes: '60' + variables: + PYTHON: $(pythonVersion) + UV_PYTHON: $(pythonVersion) + UV_CACHE_DIR: $(Pipeline.Workspace)/uv-cache strategy: - maxParallel: 1 + maxParallel: '1' matrix: Python39: pythonVersion: '3.9' @@ -31,19 +35,29 @@ jobs: pythonVersion: '3.13' steps: - checkout: self - clean: true + clean: 'true' - - task: UsePythonVersion@0 - displayName: Use Python $(pythonVersion) + - task: Cache@2 + displayName: Cache uv dependencies inputs: - versionSpec: $(pythonVersion) + key: '"uv" | "$(Agent.OS)" | "$(pythonVersion)" | uv.lock' + restoreKeys: | + "uv" | "$(Agent.OS)" | "$(pythonVersion)" + path: $(UV_CACHE_DIR) - bash: | set -euo pipefail - python -m pip install --disable-pip-version-check uv poethepoet + python -m pip install --disable-pip-version-check uv + uv python install "$(pythonVersion)" + python_bin="$(dirname "$(uv python find --managed-python "$(pythonVersion)")")" + export PATH="$python_bin:$PATH" + echo "##vso[task.prependpath]$python_bin" + python --version + python -c "import sys; assert '.'.join(map(str, sys.version_info[:2])) == '$(pythonVersion)'" + uv tool install --python "$(pythonVersion)" poethepoet + echo "##vso[task.prependpath]$(uv tool dir --bin)" uv --version - poe --version - displayName: Install build tools + displayName: Install Python $(pythonVersion) and build tools - bash: poe test --ci --no-e2e displayName: Run tests without Azure credentials @@ -56,31 +70,61 @@ jobs: azureSubscription: kusto-connectors-arm-sc scriptType: bash scriptLocation: inlineScript - inlineScript: poe test --ci + inlineScript: | + set -euo pipefail + for name in TEST_DATABASE ENGINE_CONNECTION_STRING; do + value="${!name:-}" + if [[ -z "$value" || "$value" == "\$($name)" ]]; then + echo "##vso[task.logissue type=error]Missing required variable in azure-kusto-python-e2e: $name" + exit 1 + fi + done + for name in TEST_BLOB APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING APPLICATION_INSIGHTS_TEST_DATABASE; do + value="${!name:-}" + if [[ -z "$value" || "$value" == "\$($name)" ]]; then + echo "##vso[task.logissue type=warning]Optional variable $name is not configured; related E2E coverage may be skipped." + unset "$name" + fi + done + if [[ -n "${APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING:-}" && -z "${APPLICATION_INSIGHTS_TEST_DATABASE:-}" ]]; then + echo "##vso[task.logissue type=error]APPLICATION_INSIGHTS_TEST_DATABASE is required when APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING is configured." + exit 1 + fi + poe test --ci env: TEST_DATABASE: $(TEST_DATABASE) + TEST_BLOB: $(TEST_BLOB) ENGINE_CONNECTION_STRING: $(ENGINE_CONNECTION_STRING) + APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: $(APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING) + APPLICATION_INSIGHTS_TEST_DATABASE: $(APPLICATION_INSIGHTS_TEST_DATABASE) - bash: poe check-format displayName: Run Ruff formatter check - condition: succeededOrFailed() + condition: always() - bash: poe lint displayName: Run Ruff linter - condition: succeededOrFailed() + condition: always() - bash: poe types displayName: Run type checking - condition: succeededOrFailed() - continueOnError: true + condition: always() + continueOnError: 'true' - bash: poe docs --ci displayName: Generate documentation - condition: and(succeededOrFailed(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + condition: and(succeeded(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + + - task: PublishPipelineArtifact@1 + displayName: Upload test results + condition: always() + inputs: + targetPath: pytest.xml + artifact: Test Results (Python $(pythonVersion)) - task: PublishTestResults@2 displayName: Publish test results - condition: succeededOrFailed() + condition: always() inputs: testResultsFormat: JUnit testResultsFiles: pytest*.xml @@ -89,7 +133,7 @@ jobs: - task: PublishPipelineArtifact@1 displayName: Publish documentation - condition: and(succeededOrFailed(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) + condition: and(succeeded(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master')) inputs: targetPath: docs artifact: documentation diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 00b7c32c..fca601de 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -115,6 +115,38 @@ To test without E2E tests, you can run: poe test --no-e2e ``` +### Continuous integration + +The [Azure pipeline](.azuredevops/build.yml) uses the Python 3.9-3.13 matrix, +uv dependency caching, formatting, linting, optional type checking, +and test/documentation artifacts from the [GitHub Actions workflow](.github/workflows/build.yml). +Azure matrix jobs run sequentially to reduce shared-test-data conflicts. +Python versions are provisioned with `uv python install`, rather than the hosted +agent's Python tool cache, so Python 3.9 does not depend on an Ubuntu-specific +package in the GitHub Actions Python registry. +GitHub Actions remains responsible for suggested-change PR reviews and GitHub Pages deployment; +the Azure documentation artifact alone does not deploy the site. + +The Azure pipeline imports the `azure-kusto-python-e2e` variable group. +Authorize this pipeline to use the group in Azure DevOps Library > Pipeline permissions. +Configure these variables in the group: + +- `TEST_DATABASE` and `ENGINE_CONNECTION_STRING` are required. +- Secret `TEST_BLOB` (a blob SAS URI for `dataset.csv`) enables blob-ingestion coverage. +- `APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING` enables Application Insights coverage; + `APPLICATION_INSIGHTS_TEST_DATABASE` is required when that connection string is configured. + +The YAML explicitly maps these group variables into the test process environment, +including variables marked secret. Missing required inputs fail with an error. +Missing optional inputs generate warnings and disable their related coverage. +Trusted builds authenticate through the `kusto-connectors-arm-sc` service connection, +whose identity needs access to both the Kusto test database and the Application Insights resource. +Authorize the pipeline to use this service connection and configure its workload identity +federation in Azure DevOps. +Fork builds, if enabled in the Azure project, run `poe test --ci --no-e2e` without Azure login. +Project-level fork restrictions, same-repository PR approvals, and service-connection +approvals must be configured in Azure DevOps; the YAML does not enforce those policies. + ### Style This project uses [ruff](https://docs.astral.sh/ruff/) for linting and formatting. From ed57f1f823f32de9eea04ffda6cf392bf163d5d0 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Tue, 10 Nov 2026 14:36:22 +0530 Subject: [PATCH 06/10] Fix tag-based duplicate ingestion E2E test setup Seed tagged data before conditional ingestion and observe the row count for delayed duplicates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-kusto-ingest/tests/test_e2e_ingest.py | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/azure-kusto-ingest/tests/test_e2e_ingest.py b/azure-kusto-ingest/tests/test_e2e_ingest.py index 278b9ddc..bee7cf16 100644 --- a/azure-kusto-ingest/tests/test_e2e_ingest.py +++ b/azure-kusto-ingest/tests/test_e2e_ingest.py @@ -260,6 +260,24 @@ async def assert_rows_added(cls, table_name: str, expected: int, timeout: int = break assert actual == expected, "Row count expected = {0}, while actual row count = {1}".format(expected, actual) + @classmethod + async def assert_rows_unchanged(cls, table_name: str, expected: int, timeout: int = 120): + deadline = time.monotonic() + timeout + async with await cls.get_async_client() as async_client: + while True: + command = "{} | count".format(table_name) + response = await asyncio.to_thread(cls.client.execute, cls.test_db, command) + response_from_async = await async_client.execute(cls.test_db, command) + actual = int(response.primary_results[0][0]["Count"]) + actual_async = int(response_from_async.primary_results[0][0]["Count"]) + assert actual == expected, f"Row count expected = {expected}, while actual row count = {actual}" + assert actual_async == expected, f"Async row count expected = {expected}, while actual row count = {actual_async}" + + remaining = deadline - time.monotonic() + if remaining <= 0: + return + await asyncio.sleep(min(1, remaining)) + @pytest.mark.asyncio async def test_csv_ingest_existing_table(self, is_managed_streaming): table = await self.init_table("csv_ingest_existing_table") @@ -424,15 +442,20 @@ async def test_json_ingestion_ingest_by_tag(self): data_format=DataFormat.JSON, column_mappings=self.table_json_mappings(), ingest_if_not_exists=["ingestByTag"], + ingest_by_tags=["ingestByTag"], report_level=ReportLevel.FailuresAndSuccesses, drop_by_tags=["drop", "drop-by"], flush_immediately=True, ) + self.ingest_client.ingest_from_file(self.json_file_path, json_ingestion_props) + await self.assert_rows_added(table, 2) + for f in [self.json_file_path, self.zipped_json_file_path]: self.ingest_client.ingest_from_file(f, json_ingestion_props) - await self.assert_rows_added(table, 0) + # Observe the full window so delayed duplicate ingestion cannot pass an immediate count check. + await self.assert_rows_unchanged(table, 2) finally: await self.drop_table(table) From 0b5710d64165887d326d466ac603d7ef99dd74f3 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Tue, 10 Nov 2026 15:18:11 +0530 Subject: [PATCH 07/10] Fix 24 type-check errors without changing client behavior Correct optional and response annotations, narrow synchronous client types, and fix pandas test typing. Leave warning-only diagnostics and the type-check baseline unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../azure/kusto/data/_cloud_settings.py | 2 +- azure-kusto-data/azure/kusto/data/client.py | 4 ++-- azure-kusto-data/azure/kusto/data/client_base.py | 7 ++++--- azure-kusto-data/azure/kusto/data/env_utils.py | 12 ++++++++++-- azure-kusto-data/azure/kusto/data/exceptions.py | 13 ++++++------- azure-kusto-data/tests/test_e2e_data.py | 2 ++ azure-kusto-data/tests/test_helpers.py | 3 +-- azure-kusto-ingest/tests/test_e2e_ingest.py | 6 +++--- 8 files changed, 29 insertions(+), 20 deletions(-) diff --git a/azure-kusto-data/azure/kusto/data/_cloud_settings.py b/azure-kusto-data/azure/kusto/data/_cloud_settings.py index 0c97cd37..c1d2c1b9 100644 --- a/azure-kusto-data/azure/kusto/data/_cloud_settings.py +++ b/azure-kusto-data/azure/kusto/data/_cloud_settings.py @@ -56,7 +56,7 @@ class CloudSettings: @classmethod @distributed_trace(name_of_span="CloudSettings.get_cloud_info", kind=SpanKind.CLIENT) - def get_cloud_info_for_cluster(cls, kusto_uri: str, proxies: Optional[Dict[str, str]] = None, session: requests.Session = None) -> CloudInfo: + def get_cloud_info_for_cluster(cls, kusto_uri: str, proxies: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None) -> CloudInfo: normalized_authority = cls._normalize_uri(kusto_uri) # tracing attributes for cloud info diff --git a/azure-kusto-data/azure/kusto/data/client.py b/azure-kusto-data/azure/kusto/data/client.py index c1a36e09..d6b956b3 100644 --- a/azure-kusto-data/azure/kusto/data/client.py +++ b/azure-kusto-data/azure/kusto/data/client.py @@ -3,7 +3,7 @@ import socket import sys from datetime import timedelta -from typing import AnyStr, IO, List, Optional, TYPE_CHECKING, Tuple, Union +from typing import AnyStr, IO, List, Optional, TYPE_CHECKING, Tuple, Union, cast import requests import requests.adapters @@ -323,7 +323,7 @@ def _execute( request_headers["Authorization"] = self._aad_helper.acquire_authorization_header() # trace http post call for response - invoker = lambda: self._session.post( + invoker = lambda: cast(requests.Session, self._session).post( endpoint, headers=request_headers, json=request.json_payload, diff --git a/azure-kusto-data/azure/kusto/data/client_base.py b/azure-kusto-data/azure/kusto/data/client_base.py index cffd4674..abe0db82 100644 --- a/azure-kusto-data/azure/kusto/data/client_base.py +++ b/azure-kusto-data/azure/kusto/data/client_base.py @@ -4,7 +4,7 @@ import json import uuid from datetime import timedelta -from typing import Union, Optional, Any, NoReturn, ClassVar, TYPE_CHECKING +from typing import Union, Optional, Any, NoReturn, ClassVar, TYPE_CHECKING, cast from urllib.parse import urljoin from requests import Response, Session @@ -78,15 +78,16 @@ def set_proxy(self, proxy_url: str): def validate_endpoint(self): if not self._endpoint_validated and self._aad_helper is not None: + kusto_cluster = cast(str, self._kusto_cluster) # Trusted-endpoint validation must run for every authentication method. Gating it on the # token provider type let token-based and callback-based flows send the Authorization # header to an arbitrary host named in the connection string. # The login endpoint is resolved lazily because doing so contacts the cluster itself. well_known_kusto_endpoints.validate_trusted_endpoint( - self._kusto_cluster, + kusto_cluster, lambda: ( CloudSettings.get_cloud_info_for_cluster( - self._kusto_cluster, + kusto_cluster, self._aad_helper.token_provider._proxy_dict, self._session if isinstance(self._session, Session) else None, ).login_endpoint diff --git a/azure-kusto-data/azure/kusto/data/env_utils.py b/azure-kusto-data/azure/kusto/data/env_utils.py index 7fa6c614..9864c1e0 100644 --- a/azure-kusto-data/azure/kusto/data/env_utils.py +++ b/azure-kusto-data/azure/kusto/data/env_utils.py @@ -1,9 +1,17 @@ import os from dataclasses import dataclass, astuple -from typing import Optional +from typing import Literal, Optional, overload -def get_env(*args, optional=False, default=None): +@overload +def get_env(*args: Optional[str], optional: Literal[False] = False, default: None = None) -> str: ... + + +@overload +def get_env(*args: Optional[str], optional: bool = False, default: Optional[str] = None) -> Optional[str]: ... + + +def get_env(*args, optional: bool = False, default: Optional[str] = None) -> Optional[str]: """Return the first environment variable that is defined.""" for arg in args: if arg in os.environ: diff --git a/azure-kusto-data/azure/kusto/data/exceptions.py b/azure-kusto-data/azure/kusto/data/exceptions.py index 3da752fe..38ea160b 100644 --- a/azure-kusto-data/azure/kusto/data/exceptions.py +++ b/azure-kusto-data/azure/kusto/data/exceptions.py @@ -7,12 +7,7 @@ if TYPE_CHECKING: import requests - try: - from aiohttp import ClientResponse - except ImportError: - # No aio installed, ignore - ClientResponse = None - pass + from aiohttp import ClientResponse class KustoError(Exception): @@ -121,7 +116,11 @@ class KustoApiError(KustoServiceError): """ def __init__( - self, error_dict: dict, message: Optional[str] = None, http_response: "Union[requests.Response, ClientResponse, None]" = None, kusto_response=None + self, + error_dict: dict[str, Any], + message: Optional[str] = None, + http_response: "Union[requests.Response, ClientResponse, None]" = None, + kusto_response=None, ): self.error = OneApiError.from_dict(error_dict["error"]) service_error_message = message or self.error.description or "Unknown Kusto service error" diff --git a/azure-kusto-data/tests/test_e2e_data.py b/azure-kusto-data/tests/test_e2e_data.py index 9e86597b..2076eec3 100644 --- a/azure-kusto-data/tests/test_e2e_data.py +++ b/azure-kusto-data/tests/test_e2e_data.py @@ -13,6 +13,7 @@ from urllib.parse import urljoin import pytest +from requests import Response from azure.identity import DefaultAzureCredential from azure.kusto.data import KustoClient, KustoConnectionStringBuilder from azure.kusto.data._cloud_settings import CloudSettings, DEFAULT_DEV_KUSTO_SERVICE_RESOURCE_ID @@ -360,6 +361,7 @@ def test_no_redirects_fail_in_client(self, code): ) as client: with pytest.raises(KustoServiceError) as ex: client.execute("db", "table") + assert isinstance(ex.value.http_response, Response) assert ex.value.http_response.status_code == code @pytest.mark.asyncio diff --git a/azure-kusto-data/tests/test_helpers.py b/azure-kusto-data/tests/test_helpers.py index 6ee4043c..a5898c21 100644 --- a/azure-kusto-data/tests/test_helpers.py +++ b/azure-kusto-data/tests/test_helpers.py @@ -162,7 +162,6 @@ def test_all_null_datetime_column(): ) assert len(df) == 2 - assert pandas.isnull(df["timestamp"][0]) - assert pandas.isnull(df["timestamp"][1]) + assert df["timestamp"].isna().to_numpy().all() assert df["value"][0] == 10 assert df["value"][1] == 11 diff --git a/azure-kusto-ingest/tests/test_e2e_ingest.py b/azure-kusto-ingest/tests/test_e2e_ingest.py index bee7cf16..cf1cecaf 100644 --- a/azure-kusto-ingest/tests/test_e2e_ingest.py +++ b/azure-kusto-ingest/tests/test_e2e_ingest.py @@ -54,7 +54,7 @@ class TestE2E: input_folder_path: ClassVar[str] engine_cs: ClassVar[Optional[str]] dm_cs: ClassVar[Optional[str]] - test_db: ClassVar[Optional[str]] + test_db: ClassVar[str] client: ClassVar[KustoClient] test_table: ClassVar[str] current_count: ClassVar[int] @@ -599,7 +599,7 @@ async def test_streaming_ingest_from_json_io_streams(self): async def test_streaming_ingest_from_dataframe(self): table = await self.init_table("streaming_ingest_from_dataframe") try: - from pandas import DataFrame + from pandas import DataFrame, Index fields = [ "rownumber", @@ -627,7 +627,7 @@ async def test_streaming_ingest_from_dataframe(self): dynamic_value = ["me@dummy.com", "you@dummy.com", "them@dummy.com"] rows = [[0, str(guid), 0.0, 0.0, 0, 0, 0, 0, 0, 0, 0, 0, "2014-01-01T01:01:01Z", "Zero", "Zero", "0", "00:00:00", None, dynamic_value]] - df = DataFrame(data=rows, columns=fields) + df = DataFrame(data=rows, columns=Index(fields)) ingestion_properties = IngestionProperties(database=self.test_db, table=table, flush_immediately=True) self.ingest_client.ingest_from_dataframe(df, ingestion_properties) From 2e190be1e94c28837fa30f9493974e5904493408 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Tue, 10 Nov 2026 15:36:53 +0530 Subject: [PATCH 08/10] Trigger CI rerun Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> From e9276927fa0e669b0b82957559fcacf92dfedaa9 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Tue, 10 Nov 2026 16:12:33 +0530 Subject: [PATCH 09/10] Use Azure CI and make type warnings non-blocking Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/build.yml | 127 ------------------------------------ CONTRIBUTING.md | 10 ++- pyproject.toml | 2 +- 3 files changed, 8 insertions(+), 131 deletions(-) delete mode 100644 .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index cfbd5607..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,127 +0,0 @@ -# This workflow will install the dependencies, run tests and lint every push - -name: Python package -permissions: - checks: write - pull-requests: write - id-token: write - contents: read - -on: - push: - branches: [ master ] - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: [ '3.9', '3.10', '3.11', '3.12', '3.13' ] - env: - PYTHON: ${{ matrix.python-version }} - - steps: - - name: Azure login - uses: azure/login@7184910d9eb2b1c5e48f7073824a90609bb9b6d6 # v2.3.1 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - name: Install uv and set the python version - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6.8.0 - with: - python-version: ${{ matrix.python-version }} - enable-cache: true - - name: Install Poe - run: uv tool install poethepoet - - name: EtoE Test with pytest - env: - TEST_DATABASE: ${{ secrets.TEST_DATABASE }} - TEST_BLOB: ${{ secrets.TEST_BLOB }} - ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }} - APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: https://ade.applicationinsights.io/subscriptions/12534eb3-8109-4d84-83ad-576c0d5e1d06/resourcegroups/clients_e2e_test/providers/microsoft.insights/components/kusto-e2e-app-insights - APPLICATION_INSIGHTS_TEST_DATABASE: kusto-e2e-app-insights - run: poe test --ci - - - name: "Run Ruff Formatter" - run: poe check-format - id: ruff-formatter - if: always() - - - name: "Run Ruff Linter" - run: poe lint - id: ruff-linter - if: always() - - - name: "Run Ruff Fixes" - id: fix-ruff - if: matrix.python-version == '3.9' && failure() && (steps.ruff-formatter.outcome != 'success' || steps.ruff-linter.outcome != 'success') - run: poe fix - - - uses: parkerbxyz/suggest-changes@f09e88bbf08621677fa3f4d1f0dd8732ccfd7d70 # v2.0.7 - if: matrix.python-version == '3.9' && failure() && steps.fix-ruff.outcome == 'success' - with: - comment: 'Please commit the suggested changes from markdownlint.' - event: 'REQUEST_CHANGES' - - - name: "Run Type Checking (Optional to passing)" - if: always() - run: poe types - continue-on-error: true - - - name: "Generate Docs" - if: matrix.python-version == '3.9' && github.ref == 'refs/heads/master' - run: poe docs --ci - - - name: "Upload Docs" - uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1 - if: matrix.python-version == '3.9' && github.ref == 'refs/heads/master' - with: - path: docs/ - - - name: Upload Unit Test Results - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: Unit Test Results (Python ${{ matrix.python-version }}) - path: pytest*.xml - - publish-test-results: - name: "Publish Unit Tests Results" - needs: build - runs-on: ubuntu-latest - if: always() - permissions: - checks: write - pull-requests: write - - steps: - - name: Download Artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 - with: - path: artifacts - - - name: Publish Unit Test Results - uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0 - with: - files: artifacts/**/*.xml - - publish-docs: - name: "Publish Docs" - needs: build - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/master' - permissions: - pages: write - id-token: write - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - steps: - - id: deployment - uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fca601de..1fce591d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -119,13 +119,13 @@ poe test --no-e2e The [Azure pipeline](.azuredevops/build.yml) uses the Python 3.9-3.13 matrix, uv dependency caching, formatting, linting, optional type checking, -and test/documentation artifacts from the [GitHub Actions workflow](.github/workflows/build.yml). +and test/documentation artifacts. Azure matrix jobs run sequentially to reduce shared-test-data conflicts. Python versions are provisioned with `uv python install`, rather than the hosted agent's Python tool cache, so Python 3.9 does not depend on an Ubuntu-specific package in the GitHub Actions Python registry. -GitHub Actions remains responsible for suggested-change PR reviews and GitHub Pages deployment; -the Azure documentation artifact alone does not deploy the site. +The GitHub Actions build workflow has been removed, including its suggested-change PR reviews +and GitHub Pages deployment. The Azure documentation artifact alone does not deploy the site. The Azure pipeline imports the `azure-kusto-python-e2e` variable group. Authorize this pipeline to use the group in Azure DevOps Library > Pipeline permissions. @@ -177,6 +177,10 @@ To check the types, run: poe types ``` +The task reports only errors and exits successfully when there are no errors; warnings are non-blocking. +To include warnings, run `uv run basedpyright` directly. +The Azure pipeline currently treats type checking as optional, so even errors do not fail the overall job. + Note that we have a `baseline.json` file that is used to ignore old type errors, as we gradually add types to the codebase. ## PRs diff --git a/pyproject.toml b/pyproject.toml index 0514387c..0a6045ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ pythonpath = ["."] envfile = ".env" [tool.poe.tasks] -types = "uv run basedpyright" +types = "uv run basedpyright --level error" docs = {cmd = "uv run pdoc ./azure-kusto-data/azure/kusto/data ./azure-kusto-data/azure/kusto/data/aio ./azure-kusto-ingest/azure/kusto/ingest ${CI:+ -o ./docs}", args = { CI = { help = "Run in CI mode", type = "boolean", options = ["--ci"] } } } sync = "uv sync --all-packages --all-extras --dev --locked" build = "uv build --all-packages" From 5e28d05de3b8871d54c0ec2f1f2c4e3452d666d5 Mon Sep 17 00:00:00 2001 From: komal-rani522 Date: Tue, 10 Nov 2026 17:16:15 +0530 Subject: [PATCH 10/10] Provide dummy engine URL for credential-free test collection Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .azuredevops/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.azuredevops/build.yml b/.azuredevops/build.yml index 3bfcc910..53a465ba 100644 --- a/.azuredevops/build.yml +++ b/.azuredevops/build.yml @@ -62,6 +62,8 @@ jobs: - bash: poe test --ci --no-e2e displayName: Run tests without Azure credentials condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'True')) + env: + ENGINE_CONNECTION_STRING: https://dummy.kusto.windows.net - task: AzureCLI@2 displayName: Run unit and Kusto E2E tests