diff --git a/specs/llmwhisperer.json b/specs/llmwhisperer.json index 1cd64d2..1664cd9 100644 --- a/specs/llmwhisperer.json +++ b/specs/llmwhisperer.json @@ -1589,6 +1589,15 @@ }, "x-internal": true }, + { + "in": "query", + "name": "include_cell_borders", + "required": false, + "schema": { + "default": true, + "type": "boolean" + } + }, { "in": "query", "name": "include_line_confidence", diff --git a/src/unstract/llmwhisperer/_sdk_llmwhisperer/api/whisper/extract.py b/src/unstract/llmwhisperer/_sdk_llmwhisperer/api/whisper/extract.py index e534036..f006a8d 100644 --- a/src/unstract/llmwhisperer/_sdk_llmwhisperer/api/whisper/extract.py +++ b/src/unstract/llmwhisperer/_sdk_llmwhisperer/api/whisper/extract.py @@ -25,6 +25,7 @@ def _get_kwargs( gaussian_blur_radius: float | Unset = 0.0, horizontal_stretch_factor: float | Unset = 1.0, ignore_vertical_text: bool | Unset = False, + include_cell_borders: bool | Unset = True, include_line_confidence: bool | Unset = False, lang: str | Unset = "eng", line_splitter_strategy: ExtractLineSplitterStrategy | Unset = "left-priority", @@ -66,6 +67,8 @@ def _get_kwargs( params["ignore_vertical_text"] = ignore_vertical_text + params["include_cell_borders"] = include_cell_borders + params["include_line_confidence"] = include_line_confidence params["lang"] = lang @@ -211,6 +214,7 @@ def sync_detailed( gaussian_blur_radius: float | Unset = 0.0, horizontal_stretch_factor: float | Unset = 1.0, ignore_vertical_text: bool | Unset = False, + include_cell_borders: bool | Unset = True, include_line_confidence: bool | Unset = False, lang: str | Unset = "eng", line_splitter_strategy: ExtractLineSplitterStrategy | Unset = "left-priority", @@ -243,6 +247,7 @@ def sync_detailed( gaussian_blur_radius (float | Unset): Default: 0.0. horizontal_stretch_factor (float | Unset): Default: 1.0. ignore_vertical_text (bool | Unset): Default: False. + include_cell_borders (bool | Unset): Default: True. include_line_confidence (bool | Unset): Default: False. lang (str | Unset): Default: 'eng'. line_splitter_strategy (ExtractLineSplitterStrategy | Unset): Default: 'left-priority'. @@ -282,6 +287,7 @@ def sync_detailed( gaussian_blur_radius=gaussian_blur_radius, horizontal_stretch_factor=horizontal_stretch_factor, ignore_vertical_text=ignore_vertical_text, + include_cell_borders=include_cell_borders, include_line_confidence=include_line_confidence, lang=lang, line_splitter_strategy=line_splitter_strategy, @@ -323,6 +329,7 @@ def sync( gaussian_blur_radius: float | Unset = 0.0, horizontal_stretch_factor: float | Unset = 1.0, ignore_vertical_text: bool | Unset = False, + include_cell_borders: bool | Unset = True, include_line_confidence: bool | Unset = False, lang: str | Unset = "eng", line_splitter_strategy: ExtractLineSplitterStrategy | Unset = "left-priority", @@ -355,6 +362,7 @@ def sync( gaussian_blur_radius (float | Unset): Default: 0.0. horizontal_stretch_factor (float | Unset): Default: 1.0. ignore_vertical_text (bool | Unset): Default: False. + include_cell_borders (bool | Unset): Default: True. include_line_confidence (bool | Unset): Default: False. lang (str | Unset): Default: 'eng'. line_splitter_strategy (ExtractLineSplitterStrategy | Unset): Default: 'left-priority'. @@ -395,6 +403,7 @@ def sync( gaussian_blur_radius=gaussian_blur_radius, horizontal_stretch_factor=horizontal_stretch_factor, ignore_vertical_text=ignore_vertical_text, + include_cell_borders=include_cell_borders, include_line_confidence=include_line_confidence, lang=lang, line_splitter_strategy=line_splitter_strategy, @@ -430,6 +439,7 @@ async def asyncio_detailed( gaussian_blur_radius: float | Unset = 0.0, horizontal_stretch_factor: float | Unset = 1.0, ignore_vertical_text: bool | Unset = False, + include_cell_borders: bool | Unset = True, include_line_confidence: bool | Unset = False, lang: str | Unset = "eng", line_splitter_strategy: ExtractLineSplitterStrategy | Unset = "left-priority", @@ -462,6 +472,7 @@ async def asyncio_detailed( gaussian_blur_radius (float | Unset): Default: 0.0. horizontal_stretch_factor (float | Unset): Default: 1.0. ignore_vertical_text (bool | Unset): Default: False. + include_cell_borders (bool | Unset): Default: True. include_line_confidence (bool | Unset): Default: False. lang (str | Unset): Default: 'eng'. line_splitter_strategy (ExtractLineSplitterStrategy | Unset): Default: 'left-priority'. @@ -501,6 +512,7 @@ async def asyncio_detailed( gaussian_blur_radius=gaussian_blur_radius, horizontal_stretch_factor=horizontal_stretch_factor, ignore_vertical_text=ignore_vertical_text, + include_cell_borders=include_cell_borders, include_line_confidence=include_line_confidence, lang=lang, line_splitter_strategy=line_splitter_strategy, @@ -540,6 +552,7 @@ async def asyncio( gaussian_blur_radius: float | Unset = 0.0, horizontal_stretch_factor: float | Unset = 1.0, ignore_vertical_text: bool | Unset = False, + include_cell_borders: bool | Unset = True, include_line_confidence: bool | Unset = False, lang: str | Unset = "eng", line_splitter_strategy: ExtractLineSplitterStrategy | Unset = "left-priority", @@ -572,6 +585,7 @@ async def asyncio( gaussian_blur_radius (float | Unset): Default: 0.0. horizontal_stretch_factor (float | Unset): Default: 1.0. ignore_vertical_text (bool | Unset): Default: False. + include_cell_borders (bool | Unset): Default: True. include_line_confidence (bool | Unset): Default: False. lang (str | Unset): Default: 'eng'. line_splitter_strategy (ExtractLineSplitterStrategy | Unset): Default: 'left-priority'. @@ -613,6 +627,7 @@ async def asyncio( gaussian_blur_radius=gaussian_blur_radius, horizontal_stretch_factor=horizontal_stretch_factor, ignore_vertical_text=ignore_vertical_text, + include_cell_borders=include_cell_borders, include_line_confidence=include_line_confidence, lang=lang, line_splitter_strategy=line_splitter_strategy, diff --git a/src/unstract/llmwhisperer/client_v2.py b/src/unstract/llmwhisperer/client_v2.py index 11502ca..8a49e20 100644 --- a/src/unstract/llmwhisperer/client_v2.py +++ b/src/unstract/llmwhisperer/client_v2.py @@ -87,6 +87,7 @@ "derotate_threshold", "checkbox_confidence_threshold", "min_table_width", + "include_cell_borders", # In URL mode the URL travels in the body; it is not also a query # parameter, so `url` is deliberately absent here. "url_in_post", @@ -703,6 +704,7 @@ def whisper( # noqa: C901 derotate_threshold: float | None = None, checkbox_confidence_threshold: float | None = None, min_table_width: float | None = None, + include_cell_borders: bool | None = None, ) -> Any: """Sends a request to the LLMWhisperer API to process a document. Refer to https://docs.unstract.com/llm_whisperer/apis/llm_whisperer_text_extraction_api. @@ -770,6 +772,12 @@ def whisper( # noqa: C901 [0.0, 1.0]. Defaults to 0.3. min_table_width (float, optional): The minimum width a table must span, as a fraction of the page width, to be extracted as a table. Defaults to 0. + include_cell_borders (bool, optional): Whether an Excel workbook's own cell borders + are drawn into the extracted text as box-drawing characters. Turning it off drops + every border, leaving columns aligned and no cell value lost, which costs fewer + tokens downstream. This keys off the uploaded file type, not the mode: it applies + to any XLS or XLSX upload whatever mode was requested, the default "form" mode + included, and is ignored for every other kind of document. Defaults to True. Returns: Dict[Any, Any]: The response from the API as a dictionary. @@ -834,6 +842,7 @@ def whisper( # noqa: C901 ("derotate_threshold", derotate_threshold), ("checkbox_confidence_threshold", checkbox_confidence_threshold), ("min_table_width", min_table_width), + ("include_cell_borders", include_cell_borders), ) if value is not None } diff --git a/tests/unit/client_v2_test.py b/tests/unit/client_v2_test.py index 2296673..428b881 100644 --- a/tests/unit/client_v2_test.py +++ b/tests/unit/client_v2_test.py @@ -282,6 +282,27 @@ def test_whisper_rejects_both_spellings( _whisper_query(mocker, client_v2, **{deprecated_name: "x", name: "y"}) +# --- include_cell_borders tests --- + + +def test_whisper_omits_include_cell_borders_by_default(mocker: MockerFixture, client_v2: LLMWhispererClientV2) -> None: + """Unset means absent, so the service picks, and a deployment that predates + the parameter is unaffected.""" + query = _whisper_query(mocker, client_v2) + + assert "include_cell_borders" not in query + + +@pytest.mark.parametrize(("value", "expected"), [(False, "False"), (True, "True")]) +def test_whisper_sends_include_cell_borders_when_set( + mocker: MockerFixture, client_v2: LLMWhispererClientV2, value: bool, expected: str +) -> None: + """Either value reaches the query string once the caller asks for it.""" + query = _whisper_query(mocker, client_v2, include_cell_borders=value) + + assert query["include_cell_borders"] == [expected] + + # --- Retry behavior tests --- diff --git a/tools/gen_sdk.sh b/tools/gen_sdk.sh index d1299f4..45b1859 100755 --- a/tools/gen_sdk.sh +++ b/tools/gen_sdk.sh @@ -15,7 +15,7 @@ REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # here, so the next refresher can tell what they are replacing. SPEC_SOURCE_REPO="Zipstack/unstract-llm-whisperer" SPEC_SOURCE_PATH="specs/llmwhisperer.json" -SPEC_SOURCE_REV="c4cc7f1426612a5489f40c5948bb198f89f19c9f" +SPEC_SOURCE_REV="9fe7085b44da6ca37e635fba3ab082c725ef8aef" VENV="$REPO/.gen-venv" # The generator shells out to ruff for its own post-processing. Without this it