From c35621fb576d807f38459bb8822b44dafa5547f3 Mon Sep 17 00:00:00 2001 From: Sriram-Nambiar Date: Tue, 20 Oct 2026 14:38:08 +0530 Subject: [PATCH] Support multi-archive search and SearchResult with snippets --- .gitignore | 1 + CHANGELOG.md | 9 +++ README.md | 5 +- libzim/libwrapper.h | 10 ++- libzim/libzim.pyx | 134 ++++++++++++++++++++++++++--------- libzim/search.pyi | 13 +++- libzim/zim.pxd | 3 + pyproject.toml | 2 +- tests/test_libzim_creator.py | 57 ++++++++++++++- 9 files changed, 194 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index e95ed3f5..d6abd155 100644 --- a/.gitignore +++ b/.gitignore @@ -241,6 +241,7 @@ libzim.dylib libzim.*.dylib zim*.dll libicu*.dll +icu*.dll zim.lib # pkg-config file diff --git a/CHANGELOG.md b/CHANGELOG.md index b77fb54b..29ecaee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Support multi-archive searching via `Searcher.add_archive()` (#230) +- Add `SearchResult` object with `archive`, `path`, `title`, and `snippet` (#230, #226) + +### Changed + +- Breaking: `SearchResultSet` now yields `SearchResult` objects instead of entry path strings (#230, #226) + ## [3.13.1] - 2026-10-05 ### Changed diff --git a/README.md b/README.md index 551ee797..ed2941fe 100644 --- a/README.md +++ b/README.md @@ -61,11 +61,12 @@ print(bytes(entry.get_item().content).decode("UTF-8")) # searching using full-text index search_string = "Welcome" query = Query().set_query(search_string) -searcher = Searcher(zim) +searcher = Searcher(zim) # or Searcher() followed by searcher.add_archive(...) search = searcher.search(query) search_count = search.getEstimatedMatches() print(f"there are {search_count} matches for {search_string}") -print(list(search.getResults(0, search_count))) +for result in search.getResults(0, search_count): + print(result.archive, result.path, result.title, result.snippet) # accessing suggestions search_string = "kiwix" diff --git a/libzim/libwrapper.h b/libzim/libwrapper.h index c66eba65..974370ee 100644 --- a/libzim/libwrapper.h +++ b/libzim/libwrapper.h @@ -202,10 +202,18 @@ class Search : public Wrapper class Searcher : public Wrapper { public: - Searcher() = default; + Searcher() : Wrapper(zim::Searcher(std::vector())) {}; Searcher(const wrapper::Archive& a) : Wrapper(zim::Searcher(*a)) {}; Searcher(const zim::Searcher& o) : Wrapper(o) {}; + void addArchive(const wrapper::Archive& a) + { + if (!mp_base) { + mp_base.reset(new zim::Searcher(std::vector())); + } + mp_base->addArchive(*a); + } + FORWARD(void, setVerbose) FORWARD(wrapper::Search, search) }; diff --git a/libzim/libzim.pyx b/libzim/libzim.pyx index db63e4c8..c2c8e659 100644 --- a/libzim/libzim.pyx +++ b/libzim/libzim.pyx @@ -1454,55 +1454,93 @@ cdef class Query: return self +cdef class SearchResult: + """Identifiable search result containing archive, path, title, and snippet.""" + __module__ = search_module_name + cdef readonly Archive archive + cdef readonly str path + cdef readonly str title + cdef readonly str snippet + + def __init__(self, Archive archive, str path, str title, str snippet): + self.archive = archive + self.path = path + self.title = title + self.snippet = snippet + + def __repr__(self) -> str: + return f"" + + def __str__(self) -> str: + return self.path + + def __eq__(self, other): + if isinstance(other, str): + return self.path == other + if isinstance(other, SearchResult): + return self.archive == (other).archive and self.path == (other).path + return False + + def __hash__(self): + try: + return hash((self.archive.filename.expanduser().resolve(), self.path)) + except Exception: + return hash(self.path) + + cdef class SearchResultSet: - """Iterator over a Search result: entry paths.""" + """Iterator over search results.""" __module__ = search_module_name cdef zim.SearchResultSet c_resultset + cdef list _archives @staticmethod - cdef from_resultset(zim.SearchResultSet _resultset): + cdef from_resultset(zim.SearchResultSet _resultset, list archives): cdef SearchResultSet resultset = SearchResultSet() resultset.c_resultset = move(_resultset) + resultset._archives = archives return resultset - def __iter__(self) -> Iterator[str]: - """Entry paths found in Archive for Search""" + def __iter__(self) -> Iterator[SearchResult]: + """Search results found in Archive(s) for Search""" cdef zim.SearchIterator current = self.c_resultset.begin() cdef zim.SearchIterator end = self.c_resultset.end() + cdef int archive_idx + cdef Archive archive + cdef str path, title, snippet while current != end: - yield current.getPath().decode('UTF-8') + archive_idx = current.getFileIndex() + archive = self._archives[archive_idx] + path = current.getPath().decode('UTF-8') + title = current.getTitle().decode('UTF-8') + snippet = current.getSnippet().decode('UTF-8') + yield SearchResult(archive, path, title, snippet) preincrement(current) + cdef class Search: - """Search request over a ZIM Archive.""" + """Search request over ZIM Archive(s).""" __module__ = search_module_name cdef zim.Search c_search + cdef list _archives - # Factory functions - Currently Cython can't use classmethods @staticmethod - cdef from_search(zim.Search _search): - """Creates a python ReadArticle from a C++ Article (zim::) -> ReadArticle - - Args: - _item (Item): A C++ Item - - Returns: - (Item): Casted item""" + cdef from_search(zim.Search _search, list archives): cdef Search search = Search() search.c_search = move(_search) + search._archives = archives return search def getEstimatedMatches(self) -> pyint: - """Estimated number of results in Archive for the search. - + """Estimated number of results in Archive(s) for the search. Returns: - The number of estimeated results for this search. + The number of estimated results for this search. """ return self.c_search.getEstimatedMatches() def getResults(self, start: pyint, count: pyint) -> SearchResultSet: - """Iterator over Entry paths found in Archive for the search. + """Iterator over search results found in Archive(s) for the search. Args: start (int): The beginning of the range to get (offset of the first result). @@ -1511,14 +1549,18 @@ cdef class Search: Returns: A set of results for this search. """ - return SearchResultSet.from_resultset(move(self.c_search.getResults(start, count))) + return SearchResultSet.from_resultset( + move(self.c_search.getResults(start, count)), + self._archives + ) cdef class Searcher: - """ZIM Archive Searcher. + """ZIM Archive Searcher supporting single and multi-archive search. Args: - archive (Archive): ZIM Archive to search. + archive (Archive, optional): Initial ZIM Archive to search. + *extra_archives (Archive): Additional ZIM Archives to search across. Attributes: *c_searcher (zim.Searcher): a pointer to a C++ Searcher object. @@ -1526,28 +1568,53 @@ cdef class Searcher: __module__ = search_module_name cdef zim.Searcher c_searcher + cdef list _archives - def __cinit__(self, object archive: Archive): - """Constructs a Searcher from a ZIM Archive. + def __cinit__(self, object archive = None, *extra_archives): + """Constructs a Searcher, optionally bound to one or more ZIM Archives. Args: - archive (Archive): ZIM Archive to search.. - """ + archive (Archive or list of Archives, optional): Initial Archive(s). + *extra_archives (Archive): Additional Archives. + """ + self.c_searcher = zim.Searcher() + self._archives = [] + if archive is not None: + if isinstance(archive, (list, tuple)): + for a in archive: + self.add_archive(a) + else: + self.add_archive(archive) + for a in extra_archives: + self.add_archive(a) + + def add_archive(self, Archive archive): + """Bind an Archive to this Searcher. - self.c_searcher = move(zim.Searcher(archive.c_archive)) + Args: + archive (Archive): ZIM Archive to add. + """ + self.c_searcher.addArchive(archive.c_archive) + self._archives.append(archive) def search(self, object query: Query) -> Search: - """Create a Search object for a query of this Searcher's ZIM Archive. + """Create a Search object for a query across all bound ZIM Archives. Returns: - A Search object for querying this Searcher's ZIM Archive. + A Search object for querying this Searcher's ZIM Archive(s). """ - return Search.from_search(move(self.c_searcher.search(query.c_query))) + return Search.from_search( + move(self.c_searcher.search((query).c_query)), + self._archives + ) + search_module_doc = """libzim search module - Query to prepare a query from a string -- Searcher to perform a search over a libzim.reader.Archive +- Searcher to perform a search over one or more libzim.reader.Archive objects +- Search to execute the query +- SearchResult containing archive, path, title, and snippet Usage: @@ -1556,11 +1623,12 @@ archive = libzim.reader.Archive(fpath) searcher = Searcher(archive) query = Query().set_query("foo") search = searcher.search(query) -for path in search.getResults(10, 10) # get result from 10 to 20 (10 results) - print(path, archive.get_entry_by_path(path).title) +for result in search.getResults(10, 10): + print(result.archive, result.path, result.title, result.snippet) ```""" search_public_objects = [ Query, + SearchResult, SearchResultSet, Search, Searcher, diff --git a/libzim/search.pyi b/libzim/search.pyi index b0d7136c..b848242c 100644 --- a/libzim/search.pyi +++ b/libzim/search.pyi @@ -8,13 +8,22 @@ from libzim.reader import Archive class Query: def set_query(self, query: str) -> Self: ... +class SearchResult: + archive: Archive + path: str + title: str + snippet: str + class SearchResultSet: - def __iter__(self) -> Iterator[str]: ... + def __iter__(self) -> Iterator[SearchResult]: ... class Search: def getEstimatedMatches(self) -> int: ... # noqa: N802 def getResults(self, start: int, count: int) -> SearchResultSet: ... # noqa: N802 class Searcher: - def __init__(self, archive: Archive) -> None: ... + def __init__( + self, archive: Archive | list[Archive] | None = None, *extra_archives: Archive + ) -> None: ... + def add_archive(self, archive: Archive) -> None: ... def search(self, query: Query) -> Search: ... diff --git a/libzim/zim.pxd b/libzim/zim.pxd index 691f0e9a..dc25e0f9 100644 --- a/libzim/zim.pxd +++ b/libzim/zim.pxd @@ -90,6 +90,8 @@ cdef extern from "zim/search_iterator.h" namespace "zim": bint operator!=(SearchIterator) string getPath() string getTitle() + string getSnippet() + int getFileIndex() # Import the python wrappers (ObjWrapper) from libwrapper. @@ -185,6 +187,7 @@ cdef extern from "libwrapper.h" namespace "wrapper": cdef cppclass Searcher: Searcher() Searcher(const Archive& archive) except + + void addArchive(const Archive& archive) except + setVerbose(bool verbose) Search search(Query query) except + diff --git a/pyproject.toml b/pyproject.toml index fd096fa1..4b4a45f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ build-backend = "setuptools.build_meta" [project] name = "libzim" -version = "3.13.2-dev0" +version = "4.0.0-dev0" requires-python = ">=3.10,<3.15" description = "A python-facing API for creating and interacting with ZIM files" authors = [ diff --git a/tests/test_libzim_creator.py b/tests/test_libzim_creator.py index 360048e2..942d322a 100644 --- a/tests/test_libzim_creator.py +++ b/tests/test_libzim_creator.py @@ -15,7 +15,7 @@ import libzim.writer # pyright: ignore [reportMissingModuleSource] from libzim.reader import Archive # pyright: ignore [reportMissingModuleSource] -from libzim.search import Query, Searcher # pyright: ignore [reportMissingModuleSource] +from libzim.search import Query, SearchResult, Searcher # pyright: ignore [reportMissingModuleSource] from libzim.suggestion import ( # pyright: ignore [reportMissingModuleSource] SuggestionSearcher, ) @@ -894,3 +894,58 @@ def test_accented_search_from_libzim(fpath): accented_search = accented_searcher.search(accented_query) assert accented_search.getEstimatedMatches() == zim.article_count assert list(accented_search.getResults(0, zim.article_count)) == ["path0", "path1"] + + +def test_multi_archive_search_and_snippets(tmpdir): + p1 = pathlib.Path(tmpdir / "multi1.zim") + p2 = pathlib.Path(tmpdir / "multi2.zim") + + item1 = StaticItem( + path="python_doc", + title="Python Guide", + content="Python is an open source programming language with clean syntax.", + mimetype="text/html", + ) + item2 = StaticItem( + path="ruby_doc", + title="Ruby Guide", + content="Ruby is an open source programming language with elegant blocks.", + mimetype="text/html", + ) + + with Creator(p1).config_indexing(True, "eng") as c: + c.add_item(item1) + + with Creator(p2).config_indexing(True, "eng") as c: + c.add_item(item2) + + zim1 = Archive(p1) + zim2 = Archive(p2) + + searcher = Searcher() + searcher.add_archive(zim1) + searcher.add_archive(zim2) + + query = Query().set_query("programming") + search = searcher.search(query) + + assert search.getEstimatedMatches() == 2 + results = list(search.getResults(0, 10)) + assert len(results) == 2 + + for r in results: + assert isinstance(r, SearchResult) + assert r.path in ("python_doc", "ruby_doc") + assert isinstance(r.snippet, str) + assert len(r.snippet) > 0 + + archives_found = [r.archive for r in results] + assert zim1 in archives_found + assert zim2 in archives_found + assert len(set(results)) == 2 + + paths_found = {r.path for r in results} + assert paths_found == {"python_doc", "ruby_doc"} + + multi_searcher = Searcher([zim1, zim2]) + assert multi_searcher.search(query).getEstimatedMatches() == 2