Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
139 changes: 133 additions & 6 deletions .azuredevops/build.yml
Original file line number Diff line number Diff line change
@@ -1,14 +1,141 @@
# Temporary Azure Pipelines bootstrap for .github/workflows/build.yml

trigger:
- master

pr:
- master
branches:
include:
- '*'

pool:
vmImage: ubuntu-latest

steps:
- script: echo "Hello from Azure Pipelines"
displayName: Hello world
variables:
- group: azure-kusto-python-e2e

jobs:
- job: BuildAndTest
displayName: Build and test
timeoutInMinutes: '60'
variables:
PYTHON: $(pythonVersion)
UV_PYTHON: $(pythonVersion)
UV_CACHE_DIR: $(Pipeline.Workspace)/uv-cache
strategy:
maxParallel: '1'
matrix:
Python39:
pythonVersion: '3.9'
Python310:
pythonVersion: '3.10'
Python311:
pythonVersion: '3.11'
Python312:
pythonVersion: '3.12'
Python313:
pythonVersion: '3.13'
steps:
- checkout: self
clean: 'true'

- task: Cache@2
displayName: Cache uv dependencies
inputs:
key: '"uv" | "$(Agent.OS)" | "$(pythonVersion)" | uv.lock'
restoreKeys: |
"uv" | "$(Agent.OS)" | "$(pythonVersion)"
path: $(UV_CACHE_DIR)

- bash: |
set -euo pipefail
python -m pip install --disable-pip-version-check uv
uv python install "$(pythonVersion)"
python_bin="$(dirname "$(uv python find --managed-python "$(pythonVersion)")")"
export PATH="$python_bin:$PATH"
echo "##vso[task.prependpath]$python_bin"
python --version
python -c "import sys; assert '.'.join(map(str, sys.version_info[:2])) == '$(pythonVersion)'"
uv tool install --python "$(pythonVersion)" poethepoet
echo "##vso[task.prependpath]$(uv tool dir --bin)"
uv --version
displayName: Install Python $(pythonVersion) and build tools

- bash: poe test --ci --no-e2e
displayName: Run tests without Azure credentials
condition: and(succeeded(), eq(variables['System.PullRequest.IsFork'], 'True'))
env:
ENGINE_CONNECTION_STRING: https://dummy.kusto.windows.net

- task: AzureCLI@2
displayName: Run unit and Kusto E2E tests
condition: and(succeeded(), ne(variables['System.PullRequest.IsFork'], 'True'))
inputs:
azureSubscription: kusto-connectors-arm-sc
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -euo pipefail
for name in TEST_DATABASE ENGINE_CONNECTION_STRING; do
value="${!name:-}"
if [[ -z "$value" || "$value" == "\$($name)" ]]; then
echo "##vso[task.logissue type=error]Missing required variable in azure-kusto-python-e2e: $name"
exit 1
fi
done
for name in TEST_BLOB APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING APPLICATION_INSIGHTS_TEST_DATABASE; do
value="${!name:-}"
if [[ -z "$value" || "$value" == "\$($name)" ]]; then
echo "##vso[task.logissue type=warning]Optional variable $name is not configured; related E2E coverage may be skipped."
unset "$name"
fi
done
if [[ -n "${APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING:-}" && -z "${APPLICATION_INSIGHTS_TEST_DATABASE:-}" ]]; then
echo "##vso[task.logissue type=error]APPLICATION_INSIGHTS_TEST_DATABASE is required when APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING is configured."
exit 1
fi
poe test --ci
env:
TEST_DATABASE: $(TEST_DATABASE)
TEST_BLOB: $(TEST_BLOB)
ENGINE_CONNECTION_STRING: $(ENGINE_CONNECTION_STRING)
APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING: $(APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING)
APPLICATION_INSIGHTS_TEST_DATABASE: $(APPLICATION_INSIGHTS_TEST_DATABASE)

- bash: poe check-format
displayName: Run Ruff formatter check
condition: always()

- bash: poe lint
displayName: Run Ruff linter
condition: always()

- bash: poe types
displayName: Run type checking
condition: always()
continueOnError: 'true'

- bash: poe docs --ci
displayName: Generate documentation
condition: and(succeeded(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master'))

- task: PublishPipelineArtifact@1
displayName: Upload test results
condition: always()
inputs:
targetPath: pytest.xml
artifact: Test Results (Python $(pythonVersion))

- task: PublishTestResults@2
displayName: Publish test results
condition: always()
inputs:
testResultsFormat: JUnit
testResultsFiles: pytest*.xml
testRunTitle: Python $(pythonVersion)
failTaskOnMissingResultsFile: true

- task: PublishPipelineArtifact@1
displayName: Publish documentation
condition: and(succeeded(), eq(variables['pythonVersion'], '3.9'), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
inputs:
targetPath: docs
artifact: documentation
127 changes: 0 additions & 127 deletions .github/workflows/build.yml

This file was deleted.

36 changes: 36 additions & 0 deletions CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,38 @@ To test without E2E tests, you can run:
poe test --no-e2e
```

### Continuous integration

The [Azure pipeline](.azuredevops/build.yml) uses the Python 3.9-3.13 matrix,
uv dependency caching, formatting, linting, optional type checking,
and test/documentation artifacts.
Azure matrix jobs run sequentially to reduce shared-test-data conflicts.
Python versions are provisioned with `uv python install`, rather than the hosted
agent's Python tool cache, so Python 3.9 does not depend on an Ubuntu-specific
package in the GitHub Actions Python registry.
The GitHub Actions build workflow has been removed, including its suggested-change PR reviews
and GitHub Pages deployment. The Azure documentation artifact alone does not deploy the site.

The Azure pipeline imports the `azure-kusto-python-e2e` variable group.
Authorize this pipeline to use the group in Azure DevOps Library > Pipeline permissions.
Configure these variables in the group:

- `TEST_DATABASE` and `ENGINE_CONNECTION_STRING` are required.
- Secret `TEST_BLOB` (a blob SAS URI for `dataset.csv`) enables blob-ingestion coverage.
- `APPLICATION_INSIGHTS_ENGINE_CONNECTION_STRING` enables Application Insights coverage;
`APPLICATION_INSIGHTS_TEST_DATABASE` is required when that connection string is configured.

The YAML explicitly maps these group variables into the test process environment,
including variables marked secret. Missing required inputs fail with an error.
Missing optional inputs generate warnings and disable their related coverage.
Trusted builds authenticate through the `kusto-connectors-arm-sc` service connection,
whose identity needs access to both the Kusto test database and the Application Insights resource.
Authorize the pipeline to use this service connection and configure its workload identity
federation in Azure DevOps.
Fork builds, if enabled in the Azure project, run `poe test --ci --no-e2e` without Azure login.
Project-level fork restrictions, same-repository PR approvals, and service-connection
approvals must be configured in Azure DevOps; the YAML does not enforce those policies.

### Style

This project uses [ruff](https://docs.astral.sh/ruff/) for linting and formatting.
Expand Down Expand Up @@ -145,6 +177,10 @@ To check the types, run:
poe types
```

The task reports only errors and exits successfully when there are no errors; warnings are non-blocking.
To include warnings, run `uv run basedpyright` directly.
The Azure pipeline currently treats type checking as optional, so even errors do not fail the overall job.

Note that we have a `baseline.json` file that is used to ignore old type errors, as we gradually add types to the codebase.

## PRs
Expand Down
2 changes: 1 addition & 1 deletion azure-kusto-data/azure/kusto/data/_cloud_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ class CloudSettings:

@classmethod
@distributed_trace(name_of_span="CloudSettings.get_cloud_info", kind=SpanKind.CLIENT)
def get_cloud_info_for_cluster(cls, kusto_uri: str, proxies: Optional[Dict[str, str]] = None, session: requests.Session = None) -> CloudInfo:
def get_cloud_info_for_cluster(cls, kusto_uri: str, proxies: Optional[Dict[str, str]] = None, session: Optional[requests.Session] = None) -> CloudInfo:
normalized_authority = cls._normalize_uri(kusto_uri)

# tracing attributes for cloud info
Expand Down
4 changes: 2 additions & 2 deletions azure-kusto-data/azure/kusto/data/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import socket
import sys
from datetime import timedelta
from typing import AnyStr, IO, List, Optional, TYPE_CHECKING, Tuple, Union
from typing import AnyStr, IO, List, Optional, TYPE_CHECKING, Tuple, Union, cast

import requests
import requests.adapters
Expand Down Expand Up @@ -323,7 +323,7 @@ def _execute(
request_headers["Authorization"] = self._aad_helper.acquire_authorization_header()

# trace http post call for response
invoker = lambda: self._session.post(
invoker = lambda: cast(requests.Session, self._session).post(
endpoint,
headers=request_headers,
json=request.json_payload,
Expand Down
7 changes: 4 additions & 3 deletions azure-kusto-data/azure/kusto/data/client_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import json
import uuid
from datetime import timedelta
from typing import Union, Optional, Any, NoReturn, ClassVar, TYPE_CHECKING
from typing import Union, Optional, Any, NoReturn, ClassVar, TYPE_CHECKING, cast
from urllib.parse import urljoin

from requests import Response, Session
Expand Down Expand Up @@ -78,15 +78,16 @@ def set_proxy(self, proxy_url: str):

def validate_endpoint(self):
if not self._endpoint_validated and self._aad_helper is not None:
kusto_cluster = cast(str, self._kusto_cluster)
# Trusted-endpoint validation must run for every authentication method. Gating it on the
# token provider type let token-based and callback-based flows send the Authorization
# header to an arbitrary host named in the connection string.
# The login endpoint is resolved lazily because doing so contacts the cluster itself.
well_known_kusto_endpoints.validate_trusted_endpoint(
self._kusto_cluster,
kusto_cluster,
lambda: (
CloudSettings.get_cloud_info_for_cluster(
self._kusto_cluster,
kusto_cluster,
self._aad_helper.token_provider._proxy_dict,
self._session if isinstance(self._session, Session) else None,
).login_endpoint
Expand Down
Loading
Loading