From 2acfe096b1e8c6c97082ebf3fa5af49bc3010d35 Mon Sep 17 00:00:00 2001 From: John Rogers Date: Wed, 23 Sep 2026 12:45:21 +0100 Subject: [PATCH] Fix 500 on instrument sets with no Latin-script text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit generate_cluster_topics tokenises with ASCII-only patterns: the `re_tokenise` regex `(?i)([a-z][a-z]+)` and CountVectorizer's `token_pattern=r'(?u)\b[a-zA-Z][a-zA-Z]+\b'`. For text in any non-Latin script these match nothing, so the vectoriser is fitted on an empty vocabulary and sklearn raises: ValueError: empty vocabulary; perhaps the documents only contain stop words That propagates out of match_instruments_with_function and the API returns HTTP 500. It fires whenever *every* question in the request is non-Latin — Hebrew, Kannada, Cyrillic, Greek, Arabic, Devanagari. Mixed sets survive only because the Latin items supply the vocabulary, and their non-Latin keywords are meaningless. All four clustering algorithms call generate_cluster_topics, so no algorithm avoids it. Switch both patterns to Unicode-aware `\w`, which matches those scripts, and make the fit best-effort: scripts with no whitespace word boundaries (Chinese, Japanese, Thai) and all-numeric items can still produce an empty vocabulary, and losing cluster keywords is the correct degradation there — failing the whole match is not. Verified against a deployed build: Hebrew+Kannada, Russian and Chinese instrument pairs all returned 500 before and 200 after, with real keywords extracted for the space-delimited scripts (волноваться/могли/перестать, בשבועיים/חרדה). English-only and mixed English+Hebrew results are unchanged. Co-Authored-By: Claude Opus 5 --- src/harmony/matching/generate_cluster_topics.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/harmony/matching/generate_cluster_topics.py b/src/harmony/matching/generate_cluster_topics.py index 5074e92..13a1010 100644 --- a/src/harmony/matching/generate_cluster_topics.py +++ b/src/harmony/matching/generate_cluster_topics.py @@ -70,7 +70,10 @@ def generate_cluster_topics( A list of the top k keywords representing each cluster. """ # tokenise and count tokens - re_tokenise = re.compile(r'(?i)([a-z][a-z]+)') + # \w under (?u) covers Hebrew, Kannada, Cyrillic, Greek, Arabic and + # Devanagari as well as Latin. An [a-z]-only pattern silently yields no + # tokens for those scripts, which left the vocabulary empty below. + re_tokenise = re.compile(r'(?u)(\w\w+)') token_counter = Counter() for cluster in clusters: tokens_in_cluster = set() @@ -89,7 +92,7 @@ def generate_cluster_topics( idf[word] = np.log(num_clusters/count) # fit a multinomial naive bayes classifier - vectoriser = CountVectorizer(lowercase=True, token_pattern=r'(?u)\b[a-zA-Z][a-zA-Z]+\b') + vectoriser = CountVectorizer(lowercase=True, token_pattern=r'(?u)\b\w\w+\b') transformer = TfidfTransformer() nb = MultinomialNB() model = make_pipeline(vectoriser, transformer, nb) @@ -101,7 +104,15 @@ def generate_cluster_topics( X.append(item.question_text) y.append(cluster_id) - model.fit(X, y) + try: + model.fit(X, y) + except ValueError: + # Topic generation is best-effort. CountVectorizer raises + # "empty vocabulary" when nothing tokenises — scripts without + # whitespace word boundaries (Chinese, Japanese, Thai), or items + # that are entirely digits or punctuation. Returning no topics is + # correct here; failing the whole match is not. + return [[] for _ in clusters] # detect langauge of the questions languages = set()