Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ RUN poetry config virtualenvs.create false \
# Build and install the package
RUN pip install dist/*.whl

# pandas is imported lazily by the CDK, so verify it is importable at build time
RUN python -c "import pandas"

# Recreate the original structure
RUN mkdir -p source_declarative_manifest \
&& echo 'from source_declarative_manifest.run import run\n\nif __name__ == "__main__":\n run()' > main.py \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
from math import nan
from typing import Any, Dict, Iterable, Mapping, Optional, Tuple

import pandas as pd
import requests

from airbyte_cdk.sources.declarative.extractors.record_extractor import RecordExtractor
Expand Down Expand Up @@ -139,6 +138,13 @@ def _read_with_chunks(
ValueError: If an IO/Error occurs while reading the temporary data.
"""

# Imported lazily: pandas (and numpy) are slow to import and only needed for CSV parsing.
try:
import pandas as pd
except ImportError:
os.remove(path)
raise

try:
with open(path, "r", encoding=file_encoding) as data:
# When preserving NA values, disable pandas' default NA parsing so strings like "NA"/"N/A" are kept as-is,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# Copyright (c) 2026 Airbyte, Inc., all rights reserved.

import json
import subprocess
import sys
from pathlib import Path

SCRIPT = """
import json
import sys
from unittest.mock import patch

import requests_mock
import yaml

from airbyte_cdk.cli.source_declarative_manifest import _run

manifest_path, config_path = sys.argv[1:]
with open(manifest_path) as manifest_file:
manifest = yaml.safe_load(manifest_file)

with patch(
"airbyte_cdk.cli.source_declarative_manifest._run._is_local_manifest_command",
return_value=True,
):
with patch(
"airbyte_cdk.cli.source_declarative_manifest._run.YamlDeclarativeSource._read_and_parse_yaml_file",
return_value=manifest,
):
with requests_mock.Mocker() as mocker:
mocker.get(
"https://pokeapi.co/api/v2/pokemon/blastoise",
json={"name": "blastoise"},
)
_run.handle_command(["spec"])
_run.handle_command(["check", "--config", config_path])
_run.handle_command(["discover", "--config", config_path])

print("LOADED_HEAVY_MODULES=" + json.dumps(sorted(m for m in ("pandas", "numpy") if m in sys.modules)))
"""


def test_spec_check_discover_do_not_import_pandas_or_numpy(
valid_local_manifest, valid_local_config_file
) -> None:
"""Declarative/SDM start-up must not import pandas or numpy. Import heavy libraries inside the function that needs them; `python -X importtime -c "import airbyte_cdk.cli.source_declarative_manifest._run"` shows which module pulls one in."""
result = subprocess.run(
[
sys.executable,
"-c",
SCRIPT,
str(valid_local_manifest),
str(valid_local_config_file),
],
capture_output=True,
text=True,
timeout=120,
cwd=Path(__file__).resolve().parents[2],
)

output = f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}"
assert result.returncode == 0, output
assert '"connectionStatus":{"status":"SUCCEEDED"}' in result.stdout, output
assert '"catalog"' in result.stdout, output
marker = next(
(line for line in result.stdout.splitlines() if line.startswith("LOADED_HEAVY_MODULES=")),
None,
)
assert marker is not None, output
assert json.loads(marker.split("=", 1)[1]) == [], output
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from pathlib import Path
from unittest import TestCase

import pandas # noqa: F401 # loaded here so limit_memory measures CSV parsing, not the extractor's lazy import
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
import pytest
import requests
import requests_mock
Expand Down
Loading