diff --git a/app/(tabs)/lists/study.tsx b/app/(tabs)/lists/study.tsx index 3ce1098..008e325 100644 --- a/app/(tabs)/lists/study.tsx +++ b/app/(tabs)/lists/study.tsx @@ -1651,14 +1651,17 @@ function StudyScreen() { // Store in pools — refillDeque will pull from these duePoolRef.current = dueItems; newPoolRef.current = newItems; - dueIdsRef.current = new Set(dueItems.map((item) => item.srsCard!.id)); + // A flagged-card review is built fresh as all-new cards, so its + // progress counts every card rather than only the due ones. + const countedItems = isEphemeralListId(listId) ? [...dueItems, ...newItems] : dueItems; + dueIdsRef.current = new Set(countedItems.map((item) => item.srsCard!.id)); // Seed the deque: refill from empty state pulls first ~10 const seeded = refillDeque([], 0); setCards(seeded); setCursor(0); setOriginalCardCount(seeded.length); - setDueAtStart(dueRows.length); + setDueAtStart(countedItems.length); completedSrsIdsRef.current.clear(); setSessionPhase("studying"); } else { @@ -1908,7 +1911,7 @@ function StudyScreen() { shouldReQueue = true; reQueueItem = { ...item, srsCard: rated.card }; } else if (dueIdsRef.current.has(card.id)) { - // Only count due cards toward progress (not new cards) + // Only cards counted at session start advance progress completedSrsIdsRef.current.add(card.id); } } else if (card) { @@ -2295,9 +2298,13 @@ function StudyScreen() { if (list?.confusionDetection === false) return; if (sessionMode === "add_order") return; - // Check cooldown: skip if we checked this card recently - if (card.lastConfusionCheck) { - const lastCheck = new Date(card.lastConfusionCheck).getTime(); + // Check cooldown against the DB: a re-queued copy's row predates this session's checks. + const checked = await userDb.getFirstAsync<{ last_confusion_check: string | null }>( + "SELECT last_confusion_check FROM srs_cards WHERE id = ?", + [card.id], + ); + if (checked?.last_confusion_check) { + const lastCheck = new Date(checked.last_confusion_check).getTime(); const cooldownMs = CONFUSION_COOLDOWN_HOURS * 60 * 60 * 1000; if (Date.now() - lastCheck < cooldownMs) return; } @@ -2886,7 +2893,8 @@ function StudyScreen() { > {isBrowsingHistory ? (() => { - const frontier = cards.findIndex((c) => c.status === "pending"); + // A flipped-but-unanswered card is the frontier too, not only a pending one. + const frontier = cards.findIndex((c) => c.status !== "rated"); const stepsBack = (frontier === -1 ? cards.length : frontier) - cursor; return `\u2190 ${stepsBack} back`; })() diff --git a/components/SmartReviewModal.tsx b/components/SmartReviewModal.tsx index 06c1425..a5afa5b 100644 --- a/components/SmartReviewModal.tsx +++ b/components/SmartReviewModal.tsx @@ -5,11 +5,23 @@ import { Text } from "@/components/ui/text"; import { Button } from "@/components/ui/button"; import { useUserDb } from "@/db/user-provider"; import { dayResetHourAtom, smartReviewDaysAtom } from "@/stores/settings"; -import { countMarkedInWindow, getOrCreateSmartList } from "@/lib/smart-review"; +import { + countMarkedInWindow, + getOrCreateSmartList, + type SmartReviewWindow, +} from "@/lib/smart-review"; import { useRouter } from "expo-router"; import type { WordList } from "@/db/types"; -const DAY_OPTIONS = [3, 7, 14, 30] as const; +const WINDOW_OPTIONS: SmartReviewWindow[] = [3, 7, 14, 30, null]; + +function windowLabel(days: SmartReviewWindow): string { + return days == null ? "All" : `${days}d`; +} + +function windowPhrase(days: SmartReviewWindow): string { + return days == null ? "in total" : `in the last ${days} days`; +} interface SmartReviewModalProps { visible: boolean; @@ -22,24 +34,27 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo const userDb = useUserDb(); const [days, setDays] = useAtom(smartReviewDaysAtom); const [resetHour] = useAtom(dayResetHourAtom); - const [count, setCount] = useState(null); + const [counts, setCounts] = useState | null>(null); const [starting, setStarting] = useState(false); useEffect(() => { if (!visible || !userDb || !sourceList) return; let cancelled = false; - setCount(null); - countMarkedInWindow(userDb, sourceList.id, days, resetHour) - .then((n) => { - if (!cancelled) setCount(n); + setCounts(null); + Promise.all(WINDOW_OPTIONS.map((d) => countMarkedInWindow(userDb, sourceList.id, d, resetHour))) + .then((ns) => { + if (!cancelled) setCounts(new Map(WINDOW_OPTIONS.map((d, i) => [d, ns[i]]))); }) - .catch(() => { - if (!cancelled) setCount(0); + .catch((err) => { + console.error("[SmartReview] count failed:", err); + if (!cancelled) setCounts(new Map()); }); return () => { cancelled = true; }; - }, [visible, userDb, sourceList?.id, days, resetHour]); + }, [visible, userDb, sourceList?.id, resetHour]); + + const count = counts ? (counts.get(days) ?? 0) : null; async function handleStart() { if (!userDb || !sourceList || starting) return; @@ -71,15 +86,15 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo > Smart Review - Review cards you flagged on this list, prioritized by how often and how recently you - flagged or failed them. + Review every card you flagged on this list in the window, most-flagged and most-failed + first. Progress here stays separate from the list. Lookback window - {DAY_OPTIONS.map((d) => ( + {WINDOW_OPTIONS.map((d) => ( setDays(d)} className={`flex-1 items-center rounded-lg border py-2 ${ days === d ? "border-primary bg-primary/10" : "border-border" @@ -90,7 +105,10 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo days === d ? "text-primary" : "text-muted-foreground" }`} > - {d}d + {windowLabel(d)} + + + {counts ? (counts.get(d) ?? 0) : "–"} ))} @@ -104,11 +122,11 @@ export function SmartReviewModal({ visible, onClose, sourceList }: SmartReviewMo ) : count === 0 ? ( - No flagged cards in the last {days} days. + No flagged cards {windowPhrase(days)}. ) : ( - {count} flagged {count === 1 ? "card" : "cards"} in the last {days} days. + {count} flagged {count === 1 ? "card" : "cards"} {windowPhrase(days)}. )} diff --git a/docs/reader-lookup-decisions.md b/docs/reader-lookup-decisions.md index 96c58b3..d0268ae 100644 --- a/docs/reader-lookup-decisions.md +++ b/docs/reader-lookup-decisions.md @@ -38,6 +38,13 @@ of argued about. `it.fails`, so the suite is green today and goes red the moment one is fixed. +**None of this runs in CI.** The dictionaries are build artifacts and the +workflow does not fetch them, so `test/dictionary-db.ts` makes every suite that +queries them `describe.skipIf` — three of the five files added for the lookup +work skip entirely there, and `yarn check:tap-consistency` is not a CI step at +all. The sweeps above and those suites are local gates. What CI does gate is +the pure rule table: `deinflect.*.test.ts` and the scoring helpers. + A change is shippable when the diff is enumerable and every entry in it is an improvement, neutral, or a regression named and accepted in this document. A regression nobody wrote down is not accepted, it is unnoticed. @@ -861,7 +868,376 @@ take a past. That was reading **any** entry with an `exp` sense as classless, and 棒 is `n,exp` — so ぼって was painted across the corpus as a te-form of the noun 棒. `exp` now has to be the only class recorded. -### Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way +### Negative ～ぬ, ～なく and ～なくて, and the guess they are + +Reported from a novel: 広げた風呂敷を**畳まぬ**ようなもの printed じょう over 畳 +and answered the tap with the mat たたみ, and 散歩に**誘わなく**なった printed +いざな — a girl's name out of JMnedict — and answered nothing at all. Neither +form was in the rule table. + +- **～ぬ** is the ～ない of literary prose, parallel to the ～ず rules above and + spelled out the same way per godan row, plus ichidan, する and くる. +- **～なく / ～なくて** is that negative's adverbial and its te-form, the half of + 誘わなくなった that is a verb. The adjective rule く→い already reaches 誘わない + and stops there: nothing is spelled that way, and it types the result ADJ so + no verb rule can follow it. + +**Every one of them is a guess, and that is the load-bearing part.** The first +version of this change shipped the rules unmarked, and an adversarial review +found what that costs, over a population the corpus cannot reach — all 4,155 +～ない-adverbial and ～ぬ dictionary surfaces, where **237 taps moved and 53 were +regressions**: + +| | was | became | +| -------------------------- | ----------------- | ------ | +| 問題**でなく**単なる誤解だ | で無い | 出る | +| **弛まぬ**努力 | 弛まぬ (たゆまぬ) | 緩む | +| **卒なく**こなす | 卒なく | 終わる | +| 馬が**いななく** | 嘶く | 犬 | +| **せわしなく**歩き回る | 忙しない | 世話 | + +Each of those surfaces is itself a dictionary entry, and the manufactured verb +beat it because `hasCommon` pays 100 against five points a reason. The repo +already had the answer — **a reading that needs no guess wins first**, the rule +that keeps にしては from being answered with にしても — so the rules now carry +`guess: true`, `deinflect` propagates a `guessed` flag (merging it, because a +word reachable without a guess is not one), and the gate was extended to the +three walks that never had it: the drag-selection expansion, `findFirstWord` +and `findBoundaryWord`. The furigana resolver asks the same question as a +**preference, not a weight** — commonness pays +120 there against −80 for a +deinflection, so a score could not express it. + +Checked against every one of the 53: all fixed, and none of 畳まぬ, 知らぬ, +誘わなく, 出なく, 心配しなく, 消えぬ, 知れぬ is itself an entry, so every win +still speaks. Ordering cannot defeat the gate — the surface is candidate 0 and +the ADJ block precedes the negative block, measured over 747,558 surfaces with +zero counterexamples. + +Three further guards, each from a measured failure: + +- **`typeIn: RAW` on every rule.** ～ぬ is also a verb ending — the te-form rule + んで→ぬ makes たくさんで a ぬ-verb — so without it a second pass strips the + ending the first one produced: たくさんで answered 託す, んです answered 酢. +- **`stemEnd: V1_STEM` on the three bare ichidan rules**, a new field holding + what the character before the ending has to be. お小遣い**が**なくて passes a + length floor, so the bare rule reached がる. +- **`minStem: 1` on the row rules, and none on せぬ / こぬ / しなく / こなく**, + which are whole words with no stem at all. + +**A bit leak the same review found, fixed here:** `ANY` is `0xff` and `RAW` is +`0x80`, so a rule whose output is typed `ANY` re-granted RAW — よぎなくさせる +reached 余儀る through the suru-noun rule, three rules deep. Rules that emit +`ANY` now emit `ANY_OUTPUT`, which is `ANY & ~RAW`. Latent today (余儀る and +程る are not words) and found by sweeping 827,842 dictionary forms. + +**Not shipped here, deliberately:** ～ん (読まん), ～ざる (行かざる) and ～なくちゃ +resolve to nothing at all. ～ん needs its own collision sweep before it goes +near the table — it collides with sentence-final ん, with のだ→んだ and with +every noun ending in ん, which is the over-broad shape that produced +たくさんで→託す in the first place. + +### What a guess is allowed to land on, and the branch measured whole + +A reviewer pointed out that the sections above each measure a 12,000-character +slice, not the corpus, and that the whole file is four times that. Measured +whole — **37,411 taps**, every character of `test/corpus/bocchan.txt`, this +branch against `origin/master`: + +- **Word mode: 232 change**, all read. +- **Auto mode: 2,050 change** — 1,766 a kana run that was answered with a + JMnedict name and is now answered with a word, 220 the word-mode changes, 60 + a real name the tap used to answer with a single kanji, 2 lateral. +- **`yarn check:tap-consistency`: 128 disagreeing pairs → 125.** The per-commit + numbers above (126, 123) were each measured on their own commit; the branch + as a whole lands at 125, the two pairs being the entry filter below refusing + a span the gate had counted as agreeing. + +That sweep also found the last defect, which the slice could not: the ～なく +rules reach する, the suru-noun rule then strips it, and what came back was +whatever stood in front — 釣や猟を**しなく**っちゃ answered the particle を, +足音も**しなく** answered 喪, ものでなく**っちゃ** answered ので, and 何にもせぬ +swallowed せぬ to answer the adverb 何にも. A guess claims the surface is an +inflected word, so the entry it lands on now has to be one (`entryInflects` or +`entryTakesSuru`). + +**The flag is a question about the tap, not about one span.** It was declared +inside the per-length loop, so a guess was only ever compared with a literal at +the _same_ characters — and a common guess one character in then outscores a +rare longer literal, because commonness pays 120 and the extra character 100. +要求に**こたえられなく** answered 耐える off たえられなく, having thrown away +こたえる (応える, the reading the sentence wants) at its own span. Hoisted to tap +scope, where the walk is longest-first so the longer literal is always seen +first: **one corpus tap changes** (かもしれぬ → もし, which is what master +answers), and the gate reads 125. + +A flat −150 score penalty instead of the skip was measured and is the wrong +shape: it reverts six of the ～ぬ wins and breaks 手はない, 訳はない and 気はせく, +gate 131. The scope was the bug, not the strength. + +**The filter runs in every walk**, not only the tap: a drag answered +釣や猟を**しなく**っちゃ with the particle を long after the tap stopped doing it. + +**That filter is on the literary negatives only, and the scope was measured +twice.** Over every suru-noun strip it moves 869 taps and takes +正直**にしていれば** from 直に to the broth 煮汁, 出たり**はいったり**して from +入る to 配流. Over the contractions and particle swaps — which have their own +filters already — it takes 虫の好かない from the phrase to 良く and すると to the +particle と. Narrowed to candidates this branch introduced, it moves 16. + +**The costs in those 233, named.** All but one are the suru-noun proof closing +a position that is genuinely ambiguous, which is the trade `11eeebc` already +records: + +- 安心した**らしかった** → しかった (4 taps). 安心したら is the conditional and + proves ら closed; here した is past and らしかった is 〜らしい. Nothing in the + kana separates them. +- 遠慮**するがいい** → がい (6), 説明**しようがない** → がない (3), + 勘当**すると** → と (4), 心配**しようと** → と (2), **しろと** → と (2), + **したく** → くらい, **したに** → に, **したて** → て (3 between them). Same + shape: a suru-verb's own kana closing a position a longer entry wanted. +- まだ**寝ぬ** → 寝ぬ/いぬ, an all-`arch` classical entry, where it used to reach + 寝る. The sole case in 34,527 synthetic ～ぬ/～なく surfaces built from every + v5/v1 entry; the other ten literal-entry wins (叶わぬ, 思わぬ, 変わらぬ, + 漏れなく) are improvements. Letting an archaic literal lose to a guess would + clear it, and nobody has measured that rule. +- 足音**もしなく** → 模する (2). もする is a real suru verb, so both filters pass + it; `origin/master` answered もし there, which is no better. A commonness + requirement would clear it and would cost 出揃わぬ, which is uncommon and + right. +- なくなった → なった/ナウ (9), **くっちゃ寝** → ちゃ (3), なくてもいい → いいん + (3), なくては → は (2). Tails left over where a longer correct span now wins + in front of them. + +Against roughly 185 taps that are the point: 知らぬ, 入らぬ, 知れぬ, 切れぬ, +消えぬ, 思えぬ, 心得ぬ, 似ぬ, 劣らぬ, 分らぬ, わからぬ, 出揃わぬ, もせぬ, はせぬ +(馳せる, where it was 蓮), 差支えなく, 行かなく, 聞かなく, 聞かなくて, 心配しなく, +鳴らなくて, 及ぼさなくて, 仰がれなくて, 待ってなくて, 動かなく, 殺さなく, 笑わなく, +寝なく, 癪に障らなく, 出来なく, 出なく, and the suru-verb splits やる, やろう, +やった, くれ, くれた, から, もの. + +### A kanji followed by a particle is the particle + +花のつぼみのような printed **あや** over 花, and 水の流れ printed **にず** over +水. JMnedict holds 花の (Ayano) and 水の (Nizuno); the furigana path takes a +name outright when no word covers the surface, the renderer takes the longest +surface, and two characters beat one. Both spellings have no observed +frequency at all — nobody has been seen reading 水の as にずの. + +The ruby that came out is the tell: `stripOkurigana` had already cut the の +off, so the page got a reading over 花 alone that only the whole name has. + +**Refusing every kana-tailed name was too wide, and review caught it before +this shipped.** Two separate costs, neither visible in the corpus: + +- **33 place and person names lost their ruby entirely** — 十勝ダム, 一戸トンネル, + 三輪ひとみ — because their kanji prefix is also a `counter_readings` form, and + a counter rejected by the `showCounters` setting shadows the whole span. With + counters on, 17 of them printed the counter instead: 一戸【いっこ】. +- **The ◯◯通り street class was garbled, which is worse than absent.** + 三条通【さんじょうどお】り became 三条通【さんじょうどおり】り — the reading + absorbs the り and the page reprints it. 94 of the 101 通り spellings changed. + +So the guard is what the bug actually was: a name is refused only when the +surface ends in a **single-kana particle** (の, が, を, に, は, へ, と, も, や, か) +directly after a kanji. り, め and み are the name. + +Measured over the corpus: **13 of 67,299 surfaces change, every one a junk name +reading going away** — 上の, 仲の, 坂の, 夢か, 木の, 東の, 松の, 水の, 浜の, 理の, +育の, 高の, 魚の. Nothing gains or reads differently. Over the whole kana-tailed +population (7,467 spellings) the narrowed guard changes 308 and loses **none**, +against 2,785 changed and 33 lost for the wide one; 帝国ホテル, 龍ヶ嶽トンネル, +三条通り, お染め, 晴み and 鉞り are all back to what they read before. + +**ね, よ and さ are in the set too**, which review measured at 17 more junk name +readings going away (多ね, 晴さ, 嘉よ, 寧ね, 理さ, 里よ) and none lost. で earns no +place: no name spelling ends in a kanji plus で. The corpus is neutral to all of +them — the 13 surfaces it moves are the の cases. + +**Accepted, and inherited rather than introduced:** a handful of single-kanji +given names spelled with one trailing particle still lose — 秋を (Akio) reads +秋【とき】を. All carry a frequency of 1 or 2, and the fallback is the kanji's +own word reading. + +**One thing this does not reach:** a name entry short-circuits the JLPT level +filters and the word entry that replaces it does not, so under a restricted +level config 165 surfaces lose ruby that the name was carrying past the filter. +That is `entry.isName ||` in `shouldShowFuriganaForSurface`, not this guard. + +五十嵐, 杏子, 丸木, 後味 and 西條 are unmoved — nothing written in kanji is +touched. + +### A two-kanji name the counts have seen may override a one-kanji word + +演じた**西條**さんがそばに来て: the furigana read さいじょう and the tap, on the +same two characters, answered 西/せい — "Spain". Tapping the second character +gave the name; tapping the first gave the word. + +`AUTO_NAME_OVERRIDE_MIN_LENGTH` is 3, and the note beside it says what that +costs: "two-kanji surnames (渋沢, 山田), which still resolve to a single kanji". +The floor is there for two-kanji spans straddling a word boundary — 田先 out of +山田先生, 中電 out of 食事中電話 — and `names.name_freq` now separates those +cleanly: 西條 is さいじょう in **24** sightings, 山田 やまだ in 935, 渋沢 しぶさわ +in 34, while 田先 and 中電 have never been observed as names at all. A +two-kanji name with any observed frequency may now override +(`nameMayOverrideShorterWord`). + +Measured over the first **11,296** auto-mode corpus taps: **12 change, all of +them names in the book that a tap used to answer with a single kanji** — 茂作 (was +作), 箱根 (箱, 根), 小倉 (小, 倉), 古賀 (古, 賀), 浅井 (井, three times) and +堀田 (堀, 田). Nothing else moves, in either direction. + +The floor itself is untouched above two characters, and an uncounted two-kanji +spelling is still refused — which is the whole of what keeps 田先 out. + +**A number is a count until the counts say otherwise.** Review found the +relaxation handing 鳥が**二羽**いた to ふたば, 全**三巻** to みつまき, 犬が**二戸** +to にと and 第**三章** to さんしょう — each a real span of the app's own +`counter_readings` table, each also a name somebody has been seen with once or +twice, and each returned as the _only_ result with no word to fall back to. A +span carrying a kanji numeral now needs **three** sightings rather than one +(`AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ`). That clears every one of those at 1–2 +and costs nothing real: 一郎 is 393, 三郎 334, 七海 38, and the corpus diff is +unmoved at 12. The floor is asked **before** the three-character shortcut, or +千万人 (no sightings at all) and 東洋一 ride in on length alone; over the corpus +that takes 五畳敷, 二十歩, 五十人 and 五万石 off the name walk, counts in all 14 +of their taps. 1,334 of the 2,242 numeral-carrying two-kanji spellings lose the +override; 908 keep it. + +**A name may not cut a word in half.** Reviewed against 236,778 characters of +modern prose rather than Bocchan — Wikipedia, bibliography-dense, the hardest +case for a name walk — the counted override answered 東京府と東京**市** with the +given name きょういち (15 taps), 外来**音** with らいね, 提出**元** with でもと and +分野**別** with のべつ: 45 wrong taps in 2,556, every one a span cutting a word +the walk already knew about (東京|市, 外来|音, 提出|元). The counts cannot +separate those from a surname — 京市 and 定家 are both one sighting and only one +is a name — but the surrounding text can, which is the question +`findOkuriganaStarts` asks of kana asked of kanji: a **common** entry that +covers the span's first character and starts before it, or its last and ends +after it, three characters out either way (`spanCutsAWord`). + +Measured on the reviewer's 20 real-prose straddles and 42 confirmed wins: it +refuses **11 of 20** straddles and loses **0 of 42** wins — 西條, 堀田, 華山, +吉川, 浅井, 茂作, 箱根, 小倉, 古賀, 池上, 定家, 鈴木, 佐藤 all keep theirs. Only +common entries answer, because any entry at reach 3 refuses 14 and costs 滝山 +(to 山城) and 家康 (to 徳川家). The nine it still allows are a name cutting +another name — 杉[藤]美代子, 桜[井]茂治 — where the neighbour is in no dictionary +at all, and catching those would be one name vouching for another. + +**Only for the two-kanji override.** A longer name overlaps an ordinary word all +the time: asking the same question of every override takes フランクリン to +フランク and ゴルキ to ゴ, 125 taps of real names in this corpus alone. + +**The bound, recorded because it is much wider than the two straddles above.** +**36,405** two-kanji spellings that are not dictionary words now outrank a +one-character word, **18,212 of them on a single sighting**. A synthetic frame +— `その{A}{B}を見た。` over the 11,248 pairs whose characters are each a common +standalone word and whose pair is not a word — flips **21,598 of 22,496 taps** +from word to name. Taken as shipped, because the frame is the thing being +measured there, not the language: two adjacent single-kanji words with no kana +between them is rare in real prose, which is why the same relaxation moves 12 +taps in 11,296 of Bocchan and every one is a name in the book. If this ever +reads wrong in the wild, the fix is a frequency floor set from the +distribution, not another guard. `yarn check:tap-consistency` is word-mode and +cannot see any of this. + +### A suru-verb noun proves its own kana tail, and the name walk refuses the same starts + +海外へ赴任した**ため**だった answered したため — 認む, "to write down", in its +imperative — out of a span starting inside 赴任した. + +"A span may not start inside a word" proves a position closed by looking the +kanji run and its kana up together and finding an **inflecting** word. 赴任した +deinflects to 赴任する, which no dictionary spells, and then to 赴任, which every +dictionary does — and that second step was counted as a chain through an +auxiliary and dropped. Stripping する off a suru-verb noun is not a second word, +so it no longer counts towards that bound, and an entry reached that way proves +the tail when it is tagged `vs`. + +**Closing those starts exposed the other half.** The name walk had no such +guard, so with the word span refused, 注文したから came back as the surname +たから. Auto mode now computes the starts once and hands them to both walks +(`okuriganaStartsForTap`). + +**And NAME mode, which review caught twice.** `autoLookupWithOffset` +computes the starts and hands them to both walks, but the reader calls +`nameLookupWithOffset` directly when the user switches to Names, and that call +had no starts at all — so name mode went on answering 注文したから with たから. +It computes them too now, when the word dictionary is open. + +**And then the name walk's own hole.** A name may override a shorter word +outright when the surface is "an exact name match", and that accepted a match +on the name's **reading**: ためだった holds the reading of 為田, したから holds +多可良's. The override now asks that the surface be how the name is _written_ — +its kanji form, or the whole spelling for a name written in kana, so ひろし and +たけし still resolve (`nameSpellingIsSurface`). + +Measured over the first 11,296 corpus taps. **Word mode: 15 change, all of +them a span that was cutting into a suru-verb** — 注文したから→から, 勉強してやろう→やろう, +合点したもの→もの, 紹介してやる→やる, 心配しなくって→って, 珍重してく→くれた. +**Auto mode: 489 change — 472 of them a kana run that was being answered with a +JMnedict name and is now answered with a word**: たから→から (24 taps), +いから→から (14), たのだ→のだ, ものか→もの, ぬから→から, なるの→なる, plus a +tail that reaches real words — いなが→云い, きなが→泣き, きでな→好き, +みもな→望み, しもなか→少しも, うがい→使う. Two are lateral (いなひと→ひとは, +junk either way). Gate 126 disagreeing pairs → **123**. + +**Open, and pre-existing: a deinflected candidate that is not a guess is still +accepted on the dictionary alone.** Nothing requires the entry to be a word that +could have carried the inflection, so a past or a negative lands on a noun: +なった → ナウ (adj-na), 安心したらしかった → 死活, 選ばなくてはならない → 離る +(a classical bigrade verb that cannot take modern ない), といった → と言う (exp). +The obvious filter is in the file already (`entryInflects`) and the obvious +scope — every candidate with a reason — was measured here and rejected, because +an adverb reached by stripping する is a perfectly good answer. A narrower scope +that exempts the suru-noun step has not been measured. + +**What this does not fix:** めだった still answers 目立つ. ため is a noun, so +its め is not okurigana and nothing proves the position closed; separating that +from a real word boundary needs segmentation, not a rule. + +### A drag selection is answered with what it selected, first + +Selecting **展開** in チェーン展開の新古書店 answered チェーン展開. The expansion +step — which tries substrings of the selection plus ten characters either side, +longest first — runs before anything else and returns on its first hit, so a +selection that is itself a word was never asked about. Reported as: "when I +drag EXACTLY a region I want the lookup restricted to that region, the opposite +of greedy walking both directions when I just tap." + +A tap has no boundary, so it has to guess one. A drag is the boundary. The +selection's own spelling is now looked up first and emitted first, and the +expansion still runs and lands under it: + +| selected | prefix / suffix | was | now | +| ---------------------- | --------------------- | ---------------------------- | --------------------------- | +| 展開 | チェーン / の新古書店 | チェーン展開 | **展開**, then チェーン展開 | +| べた | 食 / 。 | 食べた | **べた**, then 食べた | +| 色い | 茶 / シャツ | 茶色い | 茶色い | +| くもない | 若 / という | 若くもない | 若くもない | +| 若くもないというのに姿 | / 勢がよく | 若くもない・というのに・姿勢 | unchanged | + +Nothing is lost: a selection that is not a word still reaches past its own +edges, and a long selection still walks word by word with the suffix attached, +which is what 姿 → 姿勢 depends on. + +**The selection cases in `lib/smart-lookup.test.ts` re-implement the walk** in +the test file rather than calling it, so they cannot see this change at all — +they pass before and after. `lookup.selection.test.ts` runs the shipping +function — including `autoSelectionLookup`, which is what the reader's default +mode actually calls and which nothing covered until review said so. + +**A one-character drag does not count.** The reader emits a selection for a +single character (`touch.ts` fires on any non-empty range), and "the selection +first" there answers 意 ahead of 注意, 本 ahead of 日本, 気 ahead of 人気のある — +and in auto mode the primary result loses its name alternate too, because those +are attached per matched text. Two characters is the length at which a drag +says something the tap walk could not. + +**The expansion step had no literal-first gate either**, which is where the +negative rules above would have spoken over an entry on a drag. It has one now, +as do `findFirstWord` and `findBoundaryWord`. + +## Rejected: scoring a kana-spelled kanji word below one the dictionary spells that way The fourth case above, 敷きこんで, is not the guard. The dictionary has no 敷き込む, and きこんで (four characters, squared: 16) beats 敷き + こんで diff --git a/lib/deinflect.test.ts b/lib/deinflect.test.ts index fa3ab78..2d27c22 100644 --- a/lib/deinflect.test.ts +++ b/lib/deinflect.test.ts @@ -667,7 +667,9 @@ describe("Deinflection algorithm properties", () => { test("empty string returns just itself", () => { const result = deinflect(""); - expect(result).toEqual([{ word: "", typeMask: 0xff, entryMask: 0xff, reasons: [] }]); + expect(result).toEqual([ + { word: "", typeMask: 0xff, entryMask: 0xff, reasons: [], guessed: false }, + ]); }); test("non-Japanese text returns just itself", () => { diff --git a/lib/smart-review.test.ts b/lib/smart-review.test.ts index 2631f91..eacd4bd 100644 --- a/lib/smart-review.test.ts +++ b/lib/smart-review.test.ts @@ -163,6 +163,12 @@ describe("countMarkedInWindow", () => { expect(await countMarkedInWindow(db, "src-1", 60, 3)).toBe(1); }); + test("a null window counts every flag", async () => { + await insertMark(db, "m-old", 100, "src-1", daysAgoIso(80)); + await insertMark(db, "m-new", 200, "src-1", new Date().toISOString()); + expect(await countMarkedInWindow(db, "src-1", null, 3)).toBe(2); + }); + test("filters by sourceListId", async () => { const now = new Date().toISOString(); await insertMark(db, "m1", 100, "src-1", now); @@ -366,67 +372,73 @@ describe("getOrCreateSmartList", () => { expect(cards).toHaveLength(2); }); - test("additive refresh — new marks added, existing cards preserved", async () => { + test("rebuild resets studied cards to new, so every flagged card is in the session", async () => { const source = makeSourceList(); await insertSourceListRow(db, source); - const now = new Date().toISOString(); - await insertMark(db, "m1", 100, "src-1", now); - + await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); + await insertMark(db, "m2", 200, "src-1", new Date().toISOString()); await getOrCreateSmartList(db, source, 7, 3); - await insertMark(db, "m2", 200, "src-1", now); + await db.runAsync( + `UPDATE srs_cards SET simple_stage = 1, simple_n = 99999, simple_interval = 5 + WHERE list_id = ? AND entry_id = 100`, + ["_smart_src-1"], + ); + await getOrCreateSmartList(db, source, 7, 3); - const entries = await db.getAllAsync<{ entry_id: number }>( - `SELECT entry_id FROM list_entries WHERE list_id = ? ORDER BY entry_id`, + const cards = await db.getAllAsync<{ entry_id: number; simple_stage: number | null }>( + `SELECT entry_id, simple_stage FROM srs_cards WHERE list_id = ? ORDER BY entry_id`, ["_smart_src-1"], ); - expect(entries.map((e) => e.entry_id)).toEqual([100, 200]); + expect(cards).toEqual([ + { entry_id: 100, simple_stage: null }, + { entry_id: 200, simple_stage: null }, + ]); }); - test("cards past 'new' (simple_stage NOT NULL) keep their created_at on refresh", async () => { + test("entries that leave the window are dropped on rebuild", async () => { const source = makeSourceList(); await insertSourceListRow(db, source); await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); + await insertMark(db, "m2", 200, "src-1", new Date().toISOString()); await getOrCreateSmartList(db, source, 7, 3); - // Simulate the user studying the card past the new pool - const studied = "2099-12-31T00:00:00.000Z"; - await db.runAsync( - `UPDATE srs_cards SET simple_stage = 1, simple_n = 5, simple_interval = 5, - created_at = ? WHERE list_id = ? AND entry_id = ?`, - [studied, "_smart_src-1", 100], - ); - - // Refresh + await db.runAsync(`UPDATE review_marks SET marked_at = ? WHERE id = 'm1'`, [daysAgoIso(30)]); await getOrCreateSmartList(db, source, 7, 3); - const row = await db.getFirstAsync<{ created_at: string; simple_stage: number | null }>( - `SELECT created_at, simple_stage FROM srs_cards WHERE list_id = ? AND entry_id = 100`, + const entries = await db.getAllAsync<{ entry_id: number }>( + `SELECT entry_id FROM list_entries WHERE list_id = ?`, ["_smart_src-1"], ); - expect(row?.simple_stage).toBe(1); - expect(row?.created_at).toBe(studied); + expect(entries.map((e) => e.entry_id)).toEqual([200]); }); - test("entries no longer in window are NOT removed (additive)", async () => { + test("an 'all' window takes every flag, however old", async () => { const source = makeSourceList(); await insertSourceListRow(db, source); - await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); - await getOrCreateSmartList(db, source, 7, 3); + await insertMark(db, "m1", 100, "src-1", daysAgoIso(80)); + await insertMark(db, "m2", 200, "src-1", new Date().toISOString()); - // Backdate the mark so it leaves the window - await db.runAsync( - `UPDATE review_marks SET marked_at = datetime('now', '-30 days') WHERE id = 'm1'`, - ); + await getOrCreateSmartList(db, source, null, 3); + + const cards = await db.getAllAsync(`SELECT * FROM srs_cards WHERE list_id = ?`, [ + "_smart_src-1", + ]); + expect(cards).toHaveLength(2); + }); + test("leaves the source list's own cards untouched", async () => { + const source = makeSourceList(); + await insertSourceListRow(db, source); + await insertMark(db, "m1", 100, "src-1", new Date().toISOString()); + await insertSrsCard(db, "src-card-100", 100, "src-1"); + + await getOrCreateSmartList(db, source, 7, 3); await getOrCreateSmartList(db, source, 7, 3); - const entries = await db.getAllAsync<{ entry_id: number }>( - `SELECT entry_id FROM list_entries WHERE list_id = ?`, - ["_smart_src-1"], - ); - expect(entries.map((e) => e.entry_id)).toEqual([100]); + const own = await db.getAllAsync(`SELECT * FROM srs_cards WHERE list_id = 'src-1'`); + expect(own).toHaveLength(1); }); test("bumps updated_at and clears deleted_at", async () => { diff --git a/lib/smart-review.ts b/lib/smart-review.ts index f8b7f8c..8d13f31 100644 --- a/lib/smart-review.ts +++ b/lib/smart-review.ts @@ -36,7 +36,12 @@ export function isEphemeralListId(id: string | null | undefined): boolean { return isSmartListId(id) || isMarkedListId(id); } -function windowStartIso(days: number, resetHour: number): string { +/** Lookback window in logical days; `null` means every retained flag. */ +export type SmartReviewWindow = number | null; + +function windowStartIso(days: SmartReviewWindow, resetHour: number): string { + // Sorts before every ISO timestamp, so `marked_at >= ?` keeps every row. + if (days == null) return ""; const today = getLogicalToday(resetHour); const start = new Date(`${today}T00:00:00Z`); // Inclusive window: "7 days" means today + the previous 6 logical days. @@ -65,7 +70,7 @@ export interface SmartCandidate { export async function countMarkedInWindow( userDb: WrappedUserDb, sourceListId: string, - days: number, + days: SmartReviewWindow, resetHour: number, ): Promise { const start = windowStartIso(days, resetHour); @@ -92,7 +97,7 @@ export async function countMarkedInWindow( export async function rankSmartCandidates( userDb: WrappedUserDb, sourceListId: string, - days: number, + days: SmartReviewWindow, resetHour: number, nowMs: number = Date.now(), ): Promise { @@ -165,27 +170,21 @@ function priorityTimestamp(rank: number): string { } /** - * Idempotent create/refresh of a per-source smart-review list. + * Create or rebuild the per-source smart-review list. * - * Behavior: * - Creates `_smart_${sourceListId}` if missing, inheriting visual settings * (front/back faces, audio, animations) from the source list but always * using `simple_srs` mode so the pass/fail UX is available. - * - Computes priority ranking via {@link rankSmartCandidates}, then upserts - * list_entries and srs_cards so the new-card pool surfaces highest-priority - * cards first. `list_entries.added_at` and `srs_cards.created_at` get - * synthesised timestamps ordered by rank. - * - Cards already past the "new" stage (simple_stage IS NOT NULL) are not - * re-ranked — their SRS state and queue position is preserved. - * - Cards present in the smart list but no longer in the current window remain; - * they just don't get re-ranked. (Additive refresh.) + * - Every call rebuilds the cards from scratch: each card flagged in the window + * enters as new, ordered by {@link rankSmartCandidates}. The source list's own + * cards are never touched, and these rows never sync. * * Returns the smart list id. */ export async function getOrCreateSmartList( userDb: WrappedUserDb, sourceList: WordList, - days: number, + days: SmartReviewWindow, resetHour: number, ): Promise { const smartId = smartListIdFor(sourceList.id); @@ -229,96 +228,45 @@ export async function getOrCreateSmartList( ]); } - const ranked = await rankSmartCandidates(userDb, sourceList.id, days, resetHour); - if (ranked.length === 0) return smartId; - - const existingEntries = await userDb.getAllAsync<{ - entry_id: number; - kanji_literal: string | null; - }>( - `SELECT entry_id, kanji_literal FROM list_entries - WHERE list_id = ? AND deleted_at IS NULL`, - [smartId], - ); - const existingEntryKeys = new Set( - existingEntries.map((r) => `${r.entry_id}|${r.kanji_literal ?? ""}`), - ); + await userDb.runAsync(`DELETE FROM srs_cards WHERE list_id = ?`, [smartId]); + await userDb.runAsync(`DELETE FROM list_entries WHERE list_id = ?`, [smartId]); - const existingCards = await userDb.getAllAsync<{ - id: string; - entry_id: number; - kanji_literal: string | null; - simple_stage: number | null; - }>( - `SELECT id, entry_id, kanji_literal, simple_stage - FROM srs_cards WHERE list_id = ? AND deleted_at IS NULL`, - [smartId], - ); - const cardByKey = new Map( - existingCards.map((c) => [`${c.entry_id}|${c.kanji_literal ?? ""}`, c]), - ); + const ranked = await rankSmartCandidates(userDb, sourceList.id, days, resetHour); for (let rank = 0; rank < ranked.length; rank++) { const cand = ranked[rank]; - const key = `${cand.entryId}|${cand.kanjiLiteral ?? ""}`; + // created_at carries priority: the new-card pool reads ORDER BY created_at. const ts = priorityTimestamp(rank); - - if (existingEntryKeys.has(key)) { - await userDb.runAsync( - `UPDATE list_entries SET added_at = ?, position = ?, updated_at = ?, deleted_at = NULL - WHERE list_id = ? AND entry_id = ? - AND ((kanji_literal IS NULL AND ? IS NULL) OR kanji_literal = ?)`, - [ts, rank, now, smartId, cand.entryId, cand.kanjiLiteral, cand.kanjiLiteral], - ); - } else { - await userDb.runAsync( - `INSERT OR IGNORE INTO list_entries (id, list_id, entry_id, kanji_literal, added_at, position, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?)`, - [ - makeId(smartId, cand.entryId, cand.kanjiLiteral), - smartId, - cand.entryId, - cand.kanjiLiteral, - ts, - rank, - now, - ], - ); - } - - const card = cardByKey.get(key); - if (!card) { - // New card — insert as "new" (simple_stage NULL). created_at carries - // priority so the new-card pool's ORDER BY created_at ASC surfaces - // highest priority first. - await userDb.runAsync( - `INSERT OR IGNORE INTO srs_cards ( - id, entry_id, kanji_literal, list_id, - due, stability, difficulty, elapsed_days, scheduled_days, - reps, lapses, state, front_mode, back_mode, - created_at, updated_at - ) VALUES (?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0, 'kanji', 'english', ?, ?)`, - [ - makeId(`${smartId}-srs`, cand.entryId, cand.kanjiLiteral), - cand.entryId, - cand.kanjiLiteral, - smartId, - ts, - ts, - now, - ], - ); - } else if (card.simple_stage == null) { - // Existing but still in "new" pool — update created_at to new rank so - // priority reflects the latest signal. - await userDb.runAsync(`UPDATE srs_cards SET created_at = ?, updated_at = ? WHERE id = ?`, [ + await userDb.runAsync( + `INSERT OR IGNORE INTO list_entries (id, list_id, entry_id, kanji_literal, added_at, position, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + [ + makeId(smartId, cand.entryId, cand.kanjiLiteral), + smartId, + cand.entryId, + cand.kanjiLiteral, + ts, + rank, + now, + ], + ); + await userDb.runAsync( + `INSERT OR IGNORE INTO srs_cards ( + id, entry_id, kanji_literal, list_id, + due, stability, difficulty, elapsed_days, scheduled_days, + reps, lapses, state, front_mode, back_mode, + created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, 0, 0, 0, 0, 0, 0, 0, 'kanji', 'english', ?, ?)`, + [ + makeId(`${smartId}-srs`, cand.entryId, cand.kanjiLiteral), + cand.entryId, + cand.kanjiLiteral, + smartId, + ts, ts, now, - card.id, - ]); - } - // Past-new cards (simple_stage NOT NULL): leave alone. Their queue - // position is now driven by simple_n / due time. + ], + ); } return smartId; diff --git a/lib/version.ts b/lib/version.ts index 86c4bbe..fabaac3 100644 --- a/lib/version.ts +++ b/lib/version.ts @@ -4,8 +4,8 @@ export const APP_VERSION = { version: "1.0.0", // Marketing version for app stores buildNumber: 23, // Native build number (increments only for native builds) - otaVersion: 31, // OTA update version (increments for JS updates) - timestamp: "2026-10-04T08:40:35Z", // Last update timestamp + otaVersion: 32, // OTA update version (increments for JS updates) + timestamp: "2026-10-05T00:59:34Z", // Last update timestamp channel: "testflight", // Release channel }; diff --git a/packages/japanese-reader/src/auto-name.override.test.ts b/packages/japanese-reader/src/auto-name.override.test.ts new file mode 100644 index 0000000..91c090f --- /dev/null +++ b/packages/japanese-reader/src/auto-name.override.test.ts @@ -0,0 +1,105 @@ +/** + * A two-kanji name the counts have seen may override a one-kanji word. + * + * 西條さんがそばに来て: tapping 西 answered 西/せい, "Spain", while the furigana + * over the same two characters read さいじょう. The override floor is three + * characters, and it is there for spans straddling a word boundary — 田先 out + * of 山田先生 — which the frequency column now separates: 西條 has been seen 24 + * times, 田先 and 中電 never. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { nameMayOverrideShorterWord } from "./auto-name"; +import { autoLookupWithOffset } from "./lookup"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +async function tap(text: string, at: number) { + return (await autoLookupWithOffset(text, at, wrap(dict!), wrap(ext!)))[0]; +} + +describe("the override floor", () => { + it("lets a counted two-kanji name through", () => { + expect(nameMayOverrideShorterWord("西條", 24)).toBe(true); + expect(nameMayOverrideShorterWord("箱根", 1)).toBe(true); + }); + + it("keeps an uncounted one out", () => { + // An extended DB built before the frequency column has no counts at all, + // and every two-kanji override silently switches off rather than guessing. + expect(nameMayOverrideShorterWord("田先", null)).toBe(false); + expect(nameMayOverrideShorterWord("中電", 0)).toBe(false); + }); + + it("leaves three characters where they were", () => { + expect(nameMayOverrideShorterWord("大泉学園", null)).toBe(true); + }); + + // 二羽 is two birds, 三巻 is volume three, 三章 is chapter three — each also + // a name somebody has been seen with once. A number is a count until the + // counts say otherwise. + it("asks a span carrying a numeral for more than one sighting", () => { + expect(nameMayOverrideShorterWord("二羽", 2)).toBe(false); + expect(nameMayOverrideShorterWord("三章", 1)).toBe(false); + expect(nameMayOverrideShorterWord("計三", 2)).toBe(false); + expect(nameMayOverrideShorterWord("一郎", 393)).toBe(true); + expect(nameMayOverrideShorterWord("七海", 38)).toBe(true); + }); +}); + +describe.skipIf(!hasBothDbs)("a two-kanji surname under the finger", () => { + it("answers the name from either character", async () => { + for (const at of [12, 13]) { + const hit = await tap("ずいぶん多くの役を演じた西條さんがそばに来て、こんな", at); + expect(hit?.matchedText).toBe("西條"); + expect(hit?.nameMatches?.[0]?.kana).toBe("さいじょう"); + } + }); + + // The corpus's own names, each previously answered as a single-kanji word. + it("answers the names the corpus holds", async () => { + expect((await tap("の角をつれて、茂作の人参畠をあらした", 8))?.matchedText).toBe("茂作"); + expect((await tap("と云うと「箱根のさきですか", 6))?.matchedText).toBe("箱根"); + expect((await tap("同じ数学の教師に堀田というのが居た", 8))?.matchedText).toBe("堀田"); + }); + + // A guard, not a gate: this is the pre-existing floor's behaviour, kept + // under the relaxation rather than proved by it. + it("still refuses a span straddling a word boundary", async () => { + expect((await tap("山田先生のお宅へ行った", 1))?.matchedText).not.toBe("田先"); + expect((await tap("食事中電話が鳴った", 2))?.matchedText).not.toBe("中電"); + }); + + // 東京|市 is not the given name きょういち. The counts cannot tell a straddle + // from a surname — 京市 and 定家 are both one sighting — but the text can. + it("refuses a two-kanji name that cuts a word in half", async () => { + expect((await tap("これにより東京府と東京市が廃止され", 11))?.lookupKind).not.toBe("name"); + expect((await tap("その他の外来音を含める場合は", 6))?.lookupKind).not.toBe("name"); + }); + + it("leaves a longer name alone, which overlaps words all the time", async () => { + expect((await tap("云うならフランクリンの自伝", 4))?.matchedText).toBe("フランクリン"); + }); + + it("answers a count with the counter, not with a name", async () => { + expect((await tap("鳥が二羽いた。", 3))?.lookupKind).not.toBe("name"); + expect((await tap("全三巻を買った。", 2))?.lookupKind).not.toBe("name"); + expect((await tap("第三章まで進んだ。", 2))?.lookupKind).not.toBe("name"); + }); +}); diff --git a/packages/japanese-reader/src/auto-name.ts b/packages/japanese-reader/src/auto-name.ts index 1c0f939..94de666 100644 --- a/packages/japanese-reader/src/auto-name.ts +++ b/packages/japanese-reader/src/auto-name.ts @@ -20,6 +20,38 @@ export const AUTO_NAME_DUAL_CONFIDENCE = 45; */ export const AUTO_NAME_OVERRIDE_MIN_LENGTH = 3; +/** + * What that floor costs is two-kanji surnames, and the counts can pay it back: + * 西條 is さいじょう in 24 sightings, while 田先 and 中電 — the straddles the + * floor exists for — have never been observed as names at all. + */ +export const AUTO_NAME_OVERRIDE_COUNTED_LENGTH = 2; +export const AUTO_NAME_OVERRIDE_MIN_FREQ = 1; +/** + * What a span carrying a numeral has to show instead. 二羽 is two birds, 三巻 + * is volume three and 三章 is chapter three, and each is also a name somebody + * has been seen with once or twice — so a number is a count until the counts + * say otherwise. 一郎 (393), 七海 (38) and 三郎 (334) clear this comfortably. + */ +export const AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ = 3; + +const KANJI_NUMERALS = new Set([..."一二三四五六七八九十百千万億〇零壱弐参"]); + +export function nameMayOverrideShorterWord( + matchedText: string, + topFreq: number | null | undefined, +): boolean { + const chars = [...matchedText]; + if (chars.length < AUTO_NAME_OVERRIDE_COUNTED_LENGTH) return false; + // Asked before the length shortcut, or 千万人 and 東洋一 ride in on it with + // no sightings at all. + if (chars.some((ch) => KANJI_NUMERALS.has(ch))) { + return (topFreq ?? 0) >= AUTO_NAME_OVERRIDE_NUMERAL_MIN_FREQ; + } + if (chars.length >= AUTO_NAME_OVERRIDE_MIN_LENGTH) return true; + return (topFreq ?? 0) >= AUTO_NAME_OVERRIDE_MIN_FREQ; +} + export interface AutoNameWordCandidate { matchedText: string; exactSurface: boolean; diff --git a/packages/japanese-reader/src/deinflect.negative-nu.test.ts b/packages/japanese-reader/src/deinflect.negative-nu.test.ts new file mode 100644 index 0000000..9d40f3b --- /dev/null +++ b/packages/japanese-reader/src/deinflect.negative-nu.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; +import { deinflect } from "./deinflect"; + +function bases(word: string): string[] { + return deinflect(word).map((c) => c.word); +} + +// ～ぬ is the ～ない of literary prose and the table only had ～ず. 広げた風呂敷を +// 畳まぬ therefore resolved nothing: the tap answered the mat 畳/たたみ and the +// furigana wrote じょう, the counter, over a verb. +describe("negative ～ぬ", () => { + it("recovers godan dictionary forms", () => { + expect(bases("畳まぬ")).toContain("畳む"); + expect(bases("知らぬ")).toContain("知る"); + expect(bases("言わぬ")).toContain("言う"); + expect(bases("書かぬ")).toContain("書く"); + expect(bases("急がぬ")).toContain("急ぐ"); + expect(bases("出さぬ")).toContain("出す"); + expect(bases("待たぬ")).toContain("待つ"); + expect(bases("死なぬ")).toContain("死ぬ"); + expect(bases("呼ばぬ")).toContain("呼ぶ"); + }); + + it("recovers ichidan, suru and kuru dictionary forms", () => { + expect(bases("見ぬ")).toContain("見る"); + expect(bases("消えぬ")).toContain("消える"); + expect(bases("せぬ")).toContain("する"); + expect(bases("来ぬ")).toContain("来る"); + }); + + it("labels the reason", () => { + expect(deinflect("畳まぬ").find((c) => c.word === "畳む")?.reasons).toEqual(["negative"]); + }); + + // 死ぬ is itself a ぬ-verb, so the te-form rule んで→ぬ hands these rules a + // word that already ends in ぬ. Without RAW a second pass strips the ending + // the first one produced and たくさんで comes back as 託す. + it("fires on the text as written, never on another rule's output", () => { + // The te-form rule んで→ぬ types these as ぬ-verbs; without RAW the negative + // rules strip the ending it just produced. たくす is the word 託す is + // spelled with, and asserting on 託す instead would be unfailable — + // deinflect only ever returns the script it was handed. + expect(bases("たくさんで")).not.toContain("たくす"); + expect(bases("飲んで")).not.toContain("飲る"); + expect(bases("読んで")).not.toContain("読る"); + expect(bases("言われたんだ")).not.toContain("言われつ"); + // ANY includes RAW, so a rule that emits it used to hand the next one a + // surface that was never written: よぎなくさせる reached 余儀る three rules + // deep, through the suru-noun strip. + expect(bases("よぎなくさせる")).not.toContain("よぎる"); + // 死る is the ichidan rule reading 死 as a stem. No dictionary holds it, so + // it costs a lookup and nothing else; 死ぬ keeps answering as itself. + expect(bases("死ぬ")).toEqual(["死ぬ", "死る"]); + }); + + // These rules are guesses: 弛まぬ, でなく and いななく are themselves entries, + // and a manufactured verb must not be allowed to answer them. + it("marks what it produces as a guess", () => { + const guessed = (word: string, base: string) => + deinflect(word).find((c) => c.word === base)?.guessed; + expect(guessed("畳まぬ", "畳む")).toBe(true); + expect(guessed("誘わなく", "誘う")).toBe(true); + // でる, not 出る: deinflect returns the script it was handed. + expect(guessed("でなく", "でる")).toBe(true); + // The path that does not guess keeps its standing. + expect(guessed("でなく", "でない")).toBe(false); + expect(guessed("せわしなく", "せわしない")).toBe(false); + expect(guessed("畳まぬ", "畳まぬ")).toBe(false); + }); +}); + +// 誘わなくなった had no path at all: the adjective rule く→い reaches 誘わない, +// which nothing spells, and types it ADJ so no verb rule can follow. +describe("negative adverbial ～なく and ～なくて", () => { + it("recovers the verb the negative was built on", () => { + expect(bases("誘わなく")).toContain("誘う"); + expect(bases("出なく")).toContain("出る"); + expect(bases("聞かなく")).toContain("聞く"); + expect(bases("心配しなく")).toContain("心配する"); + expect(bases("来なく")).toContain("来る"); + }); + + it("handles the te-form of the negative", () => { + expect(bases("誘わなくて")).toContain("誘う"); + expect(bases("鳴らなくて")).toContain("鳴る"); + expect(bases("食べなくて")).toContain("食べる"); + }); + + it("labels the reason", () => { + expect(deinflect("誘わなく").find((c) => c.word === "誘う")?.reasons).toEqual(["negative"]); + expect(deinflect("誘わなくて").find((c) => c.word === "誘う")?.reasons).toEqual([ + "negative te-form", + ]); + }); + + // お小遣い**が**なくて is a particle and ない, not a verb: が is あ-row and + // cannot be an ichidan stem, so the bare rule must not reach がる. + it("refuses a stem that cannot be an ichidan one", () => { + expect(bases("お小遣いがなくて")).not.toContain("お小遣いがる"); + expect(bases("がなく")).not.toContain("がる"); + expect(bases("時間がなくて")).not.toContain("時間がる"); + }); +}); diff --git a/packages/japanese-reader/src/deinflect.ts b/packages/japanese-reader/src/deinflect.ts index 073877c..64a2b8b 100644 --- a/packages/japanese-reader/src/deinflect.ts +++ b/packages/japanese-reader/src/deinflect.ts @@ -41,6 +41,16 @@ const NOT_MASU_STEM = ANY & ~V5; * span starting inside よっぽど. */ const RAW = 0x80; +/** + * `ANY` as a rule's OUTPUT. `ANY` includes `RAW`, so a rule that emits it + * hands the next rule a surface that was never written — which is how + * よぎなくさせる reached 余儀る through the suru-noun rule. + */ +const ANY_OUTPUT = ANY & ~RAW; + +/** An ichidan stem's last character: an え/い-row kana, or any kanji. */ +const V1_STEM = + /[えけげせぜてでねへべぺめれいきぎしじちぢにひびぴみり\u3005\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff]/u; // Passive, causative, and potential forms are themselves ichidan verbs. // When they conjugate (past, negative, etc.), the V1 deinflection strips @@ -58,6 +68,18 @@ interface DeinflectRule { reason: string; /** Minimum stem length left after stripping `from`. Guards 1-char inputs. */ minStem?: number; + /** + * Undoing this rule is a guess about what the page meant, so a reading that + * needs no guess wins first: 弛まぬ and でなく are entries, and answering + * them with 緩む and 出る is reaching past a word the dictionary has. + */ + guess?: boolean; + /** + * What the character before the ending has to be. Only an え/い-row kana or a + * kanji can be an ichidan stem, so without it お小遣い**が**なくて deinflects + * to がる and answers a particle with a verb. + */ + stemEnd?: RegExp; } // ─── Deinflection rules ─── @@ -275,7 +297,7 @@ const RULES: DeinflectRule[] = [ { from: "したり", to: "する", typeIn: SURU, typeOut: SURU, reason: "tari" }, { from: "している", to: "する", typeIn: SURU, typeOut: SURU, reason: "te-iru" }, { from: "してる", to: "する", typeIn: SURU, typeOut: SURU, reason: "te-iru (casual)" }, - { from: "する", to: "", typeIn: SURU, typeOut: ANY, reason: SURU_NOUN_REASON }, + { from: "する", to: "", typeIn: SURU, typeOut: ANY_OUTPUT, reason: SURU_NOUN_REASON }, // ── 来る (kuru) irregular ── { from: "きた", to: "くる", typeIn: KURU, typeOut: KURU, reason: "past" }, @@ -341,7 +363,7 @@ const RULES: DeinflectRule[] = [ // volitional 話そう below; both rules are tried and the dictionary decides. { from: "そう", to: "い", typeIn: ADJ, typeOut: ADJ, reason: "appearance" }, // ── Negative ～ず / ～ずに (written negative, the ～ない of literary prose) ── - { from: "ずに", to: "ず", typeIn: ANY, typeOut: ANY, reason: "without doing" }, + { from: "ずに", to: "ず", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "without doing" }, { from: "ず", to: "る", typeIn: V1, typeOut: V1, reason: "negative", minStem: 1 }, { from: "わず", to: "う", typeIn: V5, typeOut: V5, reason: "negative" }, { from: "かず", to: "く", typeIn: V5, typeOut: V5, reason: "negative" }, @@ -355,11 +377,242 @@ const RULES: DeinflectRule[] = [ { from: "せず", to: "する", typeIn: SURU, typeOut: SURU, reason: "negative" }, { from: "こず", to: "くる", typeIn: KURU, typeOut: KURU, reason: "negative" }, + // ── Negative ～ぬ: the ～ない of literary prose, and ～なく / ～なくて ── + // Every rule here fires on the surface as written. ～ぬ is also a verb ending + // — the te-form rule んで→ぬ makes たくさんで a ぬ-verb — so without RAW a + // second pass strips the ending it just produced and 託す answers たくさんで. + { from: "わぬ", to: "う", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "かぬ", to: "く", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "がぬ", to: "ぐ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "さぬ", to: "す", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "たぬ", to: "つ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "なぬ", to: "ぬ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "ばぬ", to: "ぶ", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "まぬ", to: "む", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { from: "らぬ", to: "る", typeIn: RAW, typeOut: V5, reason: "negative", minStem: 1, guess: true }, + { + from: "ぬ", + to: "る", + typeIn: RAW, + typeOut: V1, + reason: "negative", + minStem: 1, + stemEnd: V1_STEM, + guess: true, + }, + { from: "せぬ", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative", guess: true }, + { from: "こぬ", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative", guess: true }, + + // ～なく is the negative's adverbial, the half of 誘わなくなった that is the + // verb. The ADJ rule く→い already reaches 誘わない and stops: nothing is + // spelled that way, and its output is typed ADJ so no verb rule follows it. + { + from: "わなく", + to: "う", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "かなく", + to: "く", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "がなく", + to: "ぐ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "さなく", + to: "す", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "たなく", + to: "つ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "ななく", + to: "ぬ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "ばなく", + to: "ぶ", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "まなく", + to: "む", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "らなく", + to: "る", + typeIn: RAW, + typeOut: V5, + reason: "negative", + minStem: 1, + guess: true, + }, + { + from: "なく", + to: "る", + typeIn: RAW, + typeOut: V1, + reason: "negative", + minStem: 1, + stemEnd: V1_STEM, + guess: true, + }, + { from: "しなく", to: "する", typeIn: RAW, typeOut: SURU, reason: "negative", guess: true }, + { from: "こなく", to: "くる", typeIn: RAW, typeOut: KURU, reason: "negative", guess: true }, + + { + from: "わなくて", + to: "う", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "かなくて", + to: "く", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "がなくて", + to: "ぐ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "さなくて", + to: "す", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "たなくて", + to: "つ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "ななくて", + to: "ぬ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "ばなくて", + to: "ぶ", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "まなくて", + to: "む", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "らなくて", + to: "る", + typeIn: RAW, + typeOut: V5, + reason: "negative te-form", + minStem: 1, + guess: true, + }, + { + from: "なくて", + to: "る", + typeIn: RAW, + typeOut: V1, + reason: "negative te-form", + minStem: 1, + stemEnd: V1_STEM, + guess: true, + }, + { + from: "しなくて", + to: "する", + typeIn: RAW, + typeOut: SURU, + reason: "negative te-form", + guess: true, + }, + { + from: "こなくて", + to: "くる", + typeIn: RAW, + typeOut: KURU, + reason: "negative te-form", + guess: true, + }, + // ── Generic te-iru forms (works across verb types after te-form resolution) ── - { from: "ている", to: "て", typeIn: ANY, typeOut: ANY, reason: "te-iru" }, - { from: "てる", to: "て", typeIn: ANY, typeOut: ANY, reason: "te-iru (casual)" }, - { from: "でいる", to: "で", typeIn: ANY, typeOut: ANY, reason: "te-iru" }, - { from: "でる", to: "で", typeIn: ANY, typeOut: ANY, reason: "te-iru (casual)" }, + { from: "ている", to: "て", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru" }, + { from: "てる", to: "て", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru (casual)" }, + { from: "でいる", to: "で", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru" }, + { from: "でる", to: "で", typeIn: ANY, typeOut: ANY_OUTPUT, reason: "te-iru (casual)" }, // ── ～なさい: the polite imperative, built on the masu-stem ── // Spelled out per row rather than stripping なさい and leaving the stem to the @@ -614,10 +867,18 @@ export interface DeinflectCandidate { */ entryMask: number; reasons: string[]; + /** Some rule on the way here was a guess — see `DeinflectRule.guess`. */ + guessed: boolean; } export function deinflect(word: string): DeinflectCandidate[] { - const first: DeinflectCandidate = { word, typeMask: ANY, entryMask: ANY, reasons: [] }; + const first: DeinflectCandidate = { + word, + typeMask: ANY, + entryMask: ANY, + reasons: [], + guessed: false, + }; const results: DeinflectCandidate[] = [first]; const seen = new Map([[word, first]]); @@ -628,9 +889,15 @@ export function deinflect(word: string): DeinflectCandidate[] { const stemLen = current.word.length - rule.from.length; if (stemLen + rule.to.length <= 0) continue; if (rule.minStem != null && stemLen < rule.minStem) continue; + // By code point: a surrogate pair indexed as a code unit tests half a + // kanji, and the guard would refuse a legitimate stem. + if (rule.stemEnd && !rule.stemEnd.test([...current.word.slice(0, stemLen)].pop() ?? "")) { + continue; + } if (!(current.typeMask & rule.typeIn)) continue; const base = current.word.slice(0, stemLen) + rule.to; + const guessed = current.guessed || !!rule.guess; const already = seen.get(base); if (already) { // A second rule reaching the same word is a second reading of the @@ -638,6 +905,8 @@ export function deinflect(word: string): DeinflectCandidate[] { // and the kuru te-form rule, and 来る is kuru. Keeping only the first // rule's class would refuse the entry that is actually there. already.entryMask |= rule.typeOut; + // Reaching the same word without a guess means it was never one. + if (!guessed) already.guessed = false; continue; } const candidate: DeinflectCandidate = { @@ -645,6 +914,7 @@ export function deinflect(word: string): DeinflectCandidate[] { typeMask: rule.typeOut, entryMask: rule.typeOut, reasons: [...current.reasons, rule.reason], + guessed, }; seen.set(base, candidate); results.push(candidate); diff --git a/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts b/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts new file mode 100644 index 0000000..bdb4d54 --- /dev/null +++ b/packages/japanese-reader/src/furigana.name-trailing-kana.test.ts @@ -0,0 +1,86 @@ +/** + * A kanji followed by a particle is the particle. + * + * 花のつぼみ printed あや over 花 and 水の流れ printed にず over 水: JMnedict + * holds 花の (Ayano) and 水の (Nizuno), the surface was two characters so it + * beat the one-character word, and the ruby that came out covered the kanji + * alone — a reading only the whole name has, over a noun and a particle. + * + * Refusing every kana-tailed name was too wide, and review caught it: it took + * the reading off 十勝ダム and 一戸トンネル entirely (their kanji prefix is a + * counter form, and a settings-rejected counter shadows the span) and garbled + * 三条通り into 三条通【さんじょうどおり】り. Only a particle is the bug. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { resolveFuriganaBatch } from "./furigana"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +async function entry(surface: string): Promise<{ reading?: string; isName?: boolean }> { + const map = await resolveFuriganaBatch([surface], wrap(dict!), wrap(ext!)); + return map[surface] ?? {}; +} + +describe.skipIf(!hasBothDbs)("a name spelled with a trailing particle", () => { + it("is not read as a name", async () => { + expect((await entry("花の")).isName).toBeFalsy(); + expect((await entry("水の")).isName).toBeFalsy(); + expect((await entry("上の")).isName).toBeFalsy(); + expect((await entry("木の")).isName).toBeFalsy(); + }); + + // A guard, not a gate: these read the same before and after the change. + it("leaves the kanji run itself to the word it is", async () => { + expect((await entry("花")).reading).toBe("はな"); + expect((await entry("水")).reading).toBe("みず"); + }); + + it("leaves a name whose tail is not a particle alone", async () => { + // り is the name's own okurigana, and the reading covers 三条通 without it. + expect((await entry("三条通り")).reading).toBe("さんじょうどお"); + expect((await entry("十勝ダム")).isName).toBe(true); + expect((await entry("一戸トンネル")).isName).toBe(true); + expect((await entry("お染め")).reading).toBe("おそ"); + }); + + // Guards, not gates: both read the same before and after the change. + it("keeps the reading on the kanji the ruby actually covered", async () => { + expect((await entry("帝国")).reading).toBe("ていこく"); + }); + + // The same doctrine on the furigana side: 拠なく and 得なく are 拠ない and + // 得ない, entries, and must not be read as 拠る and 得る because those are + // commoner. `scoreFuriganaWordMatch` pays commonness +120 against a + // deinflection penalty of −80, so this has to be a preference, not a weight. + it("prefers a reading that needs no guess", async () => { + expect((await entry("拠なく")).reading).toBe("よんどころ"); + expect((await entry("得なく")).reading).toBe("え"); + expect((await entry("弛まなく")).reading).toBe("たゆ"); + // And a guess still speaks where nothing else does. + expect((await entry("畳まぬ")).reading).toBe("たた"); + expect((await entry("誘わなく")).reading).toBe("さそ"); + }); + + it("leaves a name written in kanji alone", async () => { + expect((await entry("西條")).isName).toBe(true); + expect((await entry("杏子")).reading).toBe("きょうこ"); + expect((await entry("五十嵐")).reading).toBe("いがらし"); + }); +}); diff --git a/packages/japanese-reader/src/furigana.ts b/packages/japanese-reader/src/furigana.ts index 150f600..d18b221 100644 --- a/packages/japanese-reader/src/furigana.ts +++ b/packages/japanese-reader/src/furigana.ts @@ -584,10 +584,39 @@ export function nameReadingDominance( return { share: (best.freq ?? 0) / total, total }; } +/** + * The single-kana particles, the only kana a name's tail is mistaken for. + * ね, よ and さ earn their place: they take 17 more junk name readings off + * kanji (多ね, 晴さ, 嘉よ) and lose none. で is a no-op — no name spelling + * ends in kanji + で. + */ +const PARTICLE_KANA = new Set([ + "の", + "が", + "を", + "に", + "は", + "へ", + "と", + "も", + "や", + "か", + "ね", + "よ", + "さ", +]); + function shouldConsiderNameFuriganaSurface(surface: string): boolean { // 四十三 and 五十八 are given names in JMnedict and numbers everywhere else. // A run written only in numerals is a number. - return hasKanjiText(surface) && !isKanjiNumeralRun(surface); + if (!hasKanjiText(surface) || isKanjiNumeralRun(surface)) return false; + // A kanji followed by a particle is the particle: 花の is the surname Ayano + // and 水の is Nizuno, but in prose they are 花 and 水 and the の, and the ruby + // that came out claimed a reading over the kanji that only the name has. + // Only a particle — the り of 三条通り and the み of 晴み are the name. + const chars = [...surface]; + const last = chars[chars.length - 1]; + return !(PARTICLE_KANA.has(last) && chars.length > 1 && isKanji(chars[chars.length - 2])); } function resolveLexicalSuffixJlpt( @@ -595,7 +624,9 @@ function resolveLexicalSuffixJlpt( lookupMap: Map, ): number | null | undefined { const tryWords = (candidateSurface: string): number | null | undefined => { - const candidateWords = deinflect(candidateSurface).map((c) => c.word); + const candidateWords = deinflect(candidateSurface) + .filter((c) => !c.guessed) + .map((c) => c.word); for (const word of candidateWords) { const match = lookupMap.get(word); if (match?.jlptLevel != null) return match.jlptLevel; @@ -639,22 +670,31 @@ export async function resolveFuriganaBatch( ): Promise> { const { includeNames = true, includeCounters = true } = options; // Deinflect all surfaces, collect unique search words - const surfaceToDeinflected = new Map(); + const surfaceToDeinflected = new Map(); const allSearchWords = new Set(); for (const surface of surfaces) { - const candidates = deinflect(surface); - const words = candidates.map((c) => c.word); + const words: { word: string; guessed: boolean }[] = []; + const at = new Map(); + // A word reachable both ways is not a guess, so the flag is merged rather + // than taken from whichever path was seen first. + const add = (word: string, guessed: boolean) => { + const seen = at.get(word); + if (seen === undefined) { + at.set(word, words.length); + words.push({ word, guessed }); + } else if (!guessed) { + words[seen].guessed = false; + } + }; + for (const c of deinflect(surface)) add(c.word, c.guessed); // Also try digit→kanji normalized forms (e.g. １人 → 一人) const normalized = normalizeDigitsToKanji(surface); if (normalized !== surface) { - const normCandidates = deinflect(normalized); - for (const c of normCandidates) { - if (!words.includes(c.word)) words.push(c.word); - } + for (const c of deinflect(normalized)) add(c.word, c.guessed); } surfaceToDeinflected.set(surface, words); - for (const w of words) allSearchWords.add(w); + for (const w of words) allSearchWords.add(w.word); } // Batch lookup @@ -678,12 +718,19 @@ export async function resolveFuriganaBatch( deinflectedWord: string; } | null = null; - for (const word of deinflected) { - const match = lookupMap.get(word); - if (!match || !match.kanaForm) continue; - const score = scoreFuriganaWordMatch(surface, match, word); - if (!bestWordMatch || score > bestWordMatch.score) { - bestWordMatch = { match, score, deinflectedWord: word }; + // Two passes, not one score: a guess is only allowed to speak when + // reading the surface as written found nothing. 弛まなく is 弛まない, an + // entry, and must not be answered with 弛む because 弛む is commoner. + for (const pass of [false, true]) { + if (bestWordMatch) break; + for (const { word, guessed } of deinflected) { + if (guessed !== pass) continue; + const match = lookupMap.get(word); + if (!match || !match.kanaForm) continue; + const score = scoreFuriganaWordMatch(surface, match, word); + if (!bestWordMatch || score > bestWordMatch.score) { + bestWordMatch = { match, score, deinflectedWord: word }; + } } } diff --git a/packages/japanese-reader/src/lookup.selection.test.ts b/packages/japanese-reader/src/lookup.selection.test.ts new file mode 100644 index 0000000..cec3f50 --- /dev/null +++ b/packages/japanese-reader/src/lookup.selection.test.ts @@ -0,0 +1,95 @@ +/** + * What a drag selection is answered with. + * + * Selecting 展開 in チェーン展開の新古書店 answered チェーン展開: the expansion + * step runs first and returns on its first hit, so a selection that is itself a + * word never got asked about. A drag says where the word ends. + * + * These run the shipping function. The cases in lib/smart-lookup.test.ts + * re-implement the walk to check its shape, which cannot see this. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { autoSelectionLookup as autoSelection, selectionLookup } from "./lookup"; + +async function autoSelectionLookup( + dict: ReturnType, + text: string, + prefix: string, + suffix: string, +) { + return autoSelection(text, dict, wrap(ext!), { prefix, suffix }); +} + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +async function select(text: string, prefix = "", suffix = ""): Promise { + const out: string[] = []; + await selectionLookup(text, wrap(dict!), (result) => out.push(result.matchedText), { + prefix, + suffix, + extendedDb: wrap(ext!), + }); + return out; +} + +describe.skipIf(!hasBothDbs)("a selection that is itself a word", () => { + it("is answered with itself, first", async () => { + expect(await select("展開", "チェーン", "の新古書店")).toEqual(["展開", "チェーン展開"]); + }); + + it("keeps the longer word as the second answer, not the only one", async () => { + expect(await select("べた", "食", "。")).toEqual(["べた", "食べた"]); + }); + + // A one-character drag is not a boundary anybody drew: the reader emits a + // selection for a single character, and the expansion is what it is for. + it("does not apply to a one-character drag", async () => { + expect(await select("意", "注", "にも程がある")).toEqual(["注意"]); + expect(await select("本", "日", "の文化")).toEqual(["日本"]); + }); + + it("does not repeat itself when nothing longer is there", async () => { + expect(await select("散歩", "の", "に誘わ")).toEqual(["散歩"]); + }); +}); + +describe.skipIf(!hasBothDbs)("auto mode, which is what a drag actually runs", () => { + // The reader's default mode calls autoSelectionLookup, not selectionLookup, + // and nothing covered it until review pointed that out. + it("answers the selection first there too", async () => { + const results = await autoSelectionLookup(wrap(dict!), "展開", "チェーン", "の新古書店"); + expect(results.map((r) => r.matchedText)).toEqual(["展開", "チェーン展開"]); + }); +}); + +describe.skipIf(!hasBothDbs)("a selection that is not a word", () => { + it("still reaches past its edges", async () => { + expect(await select("色い", "茶", "シャツ")).toEqual(["茶色い"]); + expect(await select("くもない", "若", "という")).toEqual(["若くもない"]); + }); + + it("still walks a long selection word by word", async () => { + expect(await select("若くもないというのに姿", "", "勢がよく")).toEqual([ + "若くもない", + "というのに", + "姿勢", + ]); + }); +}); diff --git a/packages/japanese-reader/src/lookup.span-starts.test.ts b/packages/japanese-reader/src/lookup.span-starts.test.ts new file mode 100644 index 0000000..0da0141 --- /dev/null +++ b/packages/japanese-reader/src/lookup.span-starts.test.ts @@ -0,0 +1,142 @@ +/** + * Where a span may begin, and what a name has to be to win one. + * + * 赴任したためだった answered したため — the archaic 認む, "to write down" — + * because 赴任した could not prove its own kana tail: the chain runs through + * 赴任する, which no dictionary spells, to 赴任, which every dictionary does, + * and that second step was counted as a second word. + * + * Closing those starts then exposed the other half: the name walk had no such + * guard, so 注文したから came back as the surname たから — a kana run that is a + * name's READING, not a name's spelling. + */ +import Database from "better-sqlite3"; +import { afterAll, describe, expect, it } from "vitest"; + +import { DICT_DB_PATH, EXT_DB_PATH, hasBothDbs } from "../../../test/dictionary-db"; +import { + autoLookupWithOffset, + nameLookupWithOffset, + okuriganaStartsForTap, + smartLookupWithOffset, +} from "./lookup"; + +const dict = hasBothDbs ? new Database(DICT_DB_PATH, { readonly: true }) : null; +const ext = hasBothDbs ? new Database(EXT_DB_PATH, { readonly: true }) : null; +afterAll(() => { + dict?.close(); + ext?.close(); +}); + +const wrap = (db: Database.Database) => ({ + async getAllAsync(sql: string, params?: unknown[]): Promise { + return db.prepare(sql).all(...(params ?? [])) as T[]; + }, + async getFirstAsync(sql: string, params?: unknown[]): Promise { + return (db.prepare(sql).get(...(params ?? [])) as T) ?? null; + }, +}); + +const word = async (text: string, at: number) => + (await smartLookupWithOffset(text, at, wrap(dict!), wrap(ext!)))[0]; +const auto = async (text: string, at: number) => + (await autoLookupWithOffset(text, at, wrap(dict!), wrap(ext!)))[0]; + +describe.skipIf(!hasBothDbs)("a suru-verb noun proves its own kana tail", () => { + it("closes the し and た of 赴任した", async () => { + expect((await word("海外へ赴任したためだった。", 7))?.matchedText).toBe("ため"); + expect((await auto("海外へ赴任したためだった。", 7))?.matchedText).toBe("ため"); + }); + + it("stops a span cutting into 注文した, 勉強して, 合点した", async () => { + expect((await word("切ってみろと注文したから、何だ指ぐらい", 10))?.matchedText).toBe("から"); + expect((await word("学資にして勉強してやろう。六百円を", 9))?.matchedText).toBe("やろう"); + expect((await word("相違ないと合点したものらしい。", 9))?.matchedText).toBe("もの"); + }); + + it("leaves a noun's trailing particle alone", async () => { + // 注意に is not 注意する in any form, so に stays open: the kanji answer the + // noun and the kana answer the particle. Asserting only "not 注意に" passed + // before this change too, which is no gate at all. + expect((await word("注意にも程がある", 0))?.matchedText).toBe("注意"); + expect((await word("注意にも程がある", 2))?.matchedText).toBe("にも"); + }); +}); + +describe.skipIf(!hasBothDbs)("what a guess is allowed to land on", () => { + // The ～なく rules reach する, and the suru-noun strip then hands back + // whatever stood in front of it. A guess claims the surface is an inflected + // word, so the entry it lands on has to be one. + it("refuses a guess that lands on a particle or an adverb", async () => { + expect((await word("る方が楽に極まってる。釣や猟をしなくっちゃ活計が", 14))?.matchedText).toBe( + "を", + ); + expect((await word("と三人で暮していた。おやじは何にもせぬ男で、人の", 14))?.matchedText).toBe( + "何にも", + ); + expect((await word("つは魚に相違ない。生きてるものでなくっちゃ、こう", 13))?.matchedText).toBe( + "もの", + ); + }); + + // Only on a guess: the same filter over every suru-noun strip moves 869 + // corpus taps and takes 直にしていれば from 直に to the broth 煮汁. + it("leaves the suru-noun strip alone where nothing guessed", async () => { + expect((await word("正直にしていれば誰が乗じ", 2))?.matchedText).toBe("直にしていれば"); + expect((await word("勧誘されるのを断った", 0))?.entries[0]?.kanji[0]?.text).toBe("勧誘"); + expect((await word("出たりはいったりして、よう", 5))?.entries[0]?.kanji[0]?.text).toBe("入る"); + }); + + // The gate itself, not the entry filter: でなく and 弛まぬ are entries, and + // the walk must stop asking once a reading that needed no guess matched. + it("stops at a reading that needed no guess", async () => { + expect((await word("問題でなく単なる誤解だ", 2))?.entries[0]?.kanji[0]?.text).toBe("で無い"); + expect((await word("弛まぬ努力を続けた", 0))?.entries[0]?.kanji[0]?.text).toBe("弛まぬ"); + expect((await word("馬がいななく声がした", 2))?.entries[0]?.kanji[0]?.text).toBe("嘶く"); + }); + + it("still lets a guess speak when it lands on a verb", async () => { + expect((await word("心配しなくてもいいと云い", 0))?.matchedText).toBe("心配しなくて"); + expect((await word("散歩に誘わなくなった", 3))?.matchedText).toBe("誘わなく"); + }); +}); + +describe.skipIf(!hasBothDbs)("name mode walks the same spans", () => { + // Name mode has its own walk, and it got the starts only after review: + // without them it answered 注文したから with the surname たから and + // したためだった with 為田, from inside words the word walk had closed. + it("refuses a start inside a word", async () => { + const starts = await okuriganaStartsForTap( + "切ってみろと注文したから、何だ指ぐらい", + 10, + wrap(dict!), + ); + const guarded = await nameLookupWithOffset( + "切ってみろと注文したから、何だ指ぐらい", + 10, + wrap(ext!), + starts, + ); + expect(guarded[0]?.matchedText).not.toBe("たから"); + }); + + it("still answers a name that starts at a word boundary", async () => { + const text = "演じた西條さんがそばに来て"; + const starts = await okuriganaStartsForTap(text, 3, wrap(dict!)); + const hit = await nameLookupWithOffset(text, 3, wrap(ext!), starts); + expect(hit[0]?.matchedText).toBe("西條"); + }); +}); + +describe.skipIf(!hasBothDbs)("what a name has to be to outrank a word", () => { + it("refuses a kana run that is only a name's reading", async () => { + const hit = await auto("切ってみろと注文したから、何だ指ぐらい", 10); + expect(hit?.lookupKind).not.toBe("name"); + expect(hit?.matchedText).toBe("から"); + }); + + it("keeps a name that is spelled the way the page spells it", async () => { + expect((await auto("ひろしに聞いた", 0))?.nameMatches?.[0]?.kana).toBe("ひろし"); + expect((await auto("演じた西條さんがそばに来て", 3))?.matchedText).toBe("西條"); + }); +}); diff --git a/packages/japanese-reader/src/lookup.ts b/packages/japanese-reader/src/lookup.ts index b5dd84f..0fce771 100644 --- a/packages/japanese-reader/src/lookup.ts +++ b/packages/japanese-reader/src/lookup.ts @@ -7,6 +7,7 @@ import { AUTO_NAME_OVERRIDE_MIN_LENGTH, AUTO_NAME_ONLY_WITH_EXACT_WORD_CONFIDENCE, computeAutoNameConfidence, + nameMayOverrideShorterWord, shouldShowBothAutoResults, type AutoNameNameCandidate, type AutoNameWordCandidate, @@ -14,6 +15,7 @@ import { import type { ReaderSqlDb } from "./backend"; import { CONTRACTION_REASONS, + SURU_NOUN_REASON, deinflect, generateSubstrings, kanaSpellings, @@ -41,12 +43,48 @@ interface EntrySortContext { type KanjiReadingRecord = Awaited>; +/** + * What a guess is allowed to land on. Undoing one of the literary negatives + * claims the surface is an inflected word, so the entry has to be one: the + * ～なく rules otherwise reach する, the suru-noun strip hands back whatever + * stood in front of it, and 釣や猟をしなくっちゃ answers the particle を. + * + * Not for a particle swap, which is not an inflection and has its own guard — + * 虫が好かない is `exp` and nothing else. + */ +function entriesAGuessMayLandOn( + candidate: TapCandidate, + entries: ReaderDictEntry[], +): ReaderDictEntry[] { + if (!candidate.guessed || candidate.particleSwapped) return entries; + return entries.filter((entry) => entryInflects(entry) || entryTakesSuru(entry)); +} + +/** + * Undoing a contraction, swapping a particle and the literary negatives are + * all guesses about what the page meant, and a reading that needs no guess + * wins first — so a guess only speaks when the span found nothing else. + */ +function candidateIsGuess(candidate: { + reasons: string[]; + particleSwapped?: boolean; + guessed?: boolean; +}): boolean { + return ( + !!candidate.particleSwapped || + !!candidate.guessed || + candidate.reasons.some((reason) => CONTRACTION_REASONS.has(reason)) + ); +} + /** One spelling the tap walk will ask the dictionary about, and why. */ interface TapCandidate { word: string; reasons: string[]; spelledKanji?: string; particleSwapped?: boolean; + /** A rule on the way here was a guess — see `DeinflectRule.guess`. */ + guessed?: boolean; } const DIGIT_TO_KANJI: Record = { @@ -345,6 +383,19 @@ function topWordExactSurfaceMatch(result: LookupResult): boolean { ); } +/** + * The surface is how a name is WRITTEN, not merely how one is read. ためだった + * holds the reading of 為田 and したから holds 多可良's, and a kana run that + * spells out a name — ひろし, たけし — is a spelling in its own right. + */ +function nameSpellingIsSurface(result: LookupResult): boolean { + return ( + result.nameMatches?.some((name) => + name.kanji ? name.kanji === result.matchedText : name.kana === result.matchedText, + ) ?? false + ); +} + function topNameExactSurfaceMatch(result: LookupResult): boolean { return ( result.nameMatches?.some( @@ -549,6 +600,44 @@ async function lookupNegativeScopeParticleAtTap( * same span could win from one character and lose from the next — which is the * disagreement `yarn check:tap-consistency` measures. */ +/** + * One tap, one lookup per word. The walk asks for the same word from many + * overlapping substrings, and the kana spellings multiply that again. + */ +function makeEntryCache(dictDb: ReaderSqlDb) { + const entryCache = new Map(); + const lookupOnce = async (word: string): Promise => { + const cached = entryCache.get(word); + if (cached) return cached; + const found = await lookupExactJapanese(dictDb, word); + entryCache.set(word, found); + return found; + }; + /** Fill the cache for a whole length's candidates in one round trip. */ + const prefetch = async (words: string[]): Promise => { + const missing = [...new Set(words)].filter((word) => !entryCache.has(word)); + if (missing.length === 0) return; + const found = await lookupExactJapaneseMany(dictDb, missing); + for (const word of missing) entryCache.set(word, found.get(word) ?? []); + }; + return { lookupOnce, prefetch }; +} + +/** + * The positions a span may not begin at, for a tap the caller has not already + * resolved. Auto mode runs the word walk and the name walk side by side and + * both have to refuse the same starts, so it computes this once and hands it + * to each. + */ +export async function okuriganaStartsForTap( + text: string, + tapOffset: number, + dictDb: ReaderSqlDb, +): Promise> { + const { lookupOnce, prefetch } = makeEntryCache(dictDb); + return findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce); +} + /** * Positions where a candidate would begin inside a word rather than at one. * @@ -566,12 +655,15 @@ async function findOkuriganaStarts( lookupOnce: (word: string) => Promise, ): Promise> { const earliest = Math.max(1, tapOffset - 14 - MAX_OKURIGANA); - const tails: { first: number; ends: { end: number; words: string[] }[] }[] = []; + const tails: { + first: number; + ends: { end: number; words: { word: string; viaSuru: boolean }[] }[]; + }[] = []; for (let first = earliest; first <= tapOffset; first++) { if (!isKanaChar(text[first]) || !isKanjiChar(text[first - 1])) continue; let head = first - 1; while (head > 0 && isKanjiChar(text[head - 1])) head--; - const ends: { end: number; words: string[] }[] = []; + const ends: { end: number; words: { word: string; viaSuru: boolean }[] }[] = []; for (let end = first; end < text.length && end < first + MAX_OKURIGANA; end++) { if (!isKanaChar(text[end])) break; ends.push({ @@ -579,24 +671,36 @@ async function findOkuriganaStarts( // One inflection step is the word conjugating itself. Two is a chain // through an auxiliary, which is a second word: 倒して is 倒す in its // te-form, but 倒してや reaches 倒す as well and would swallow やった. + // Stripping する off a suru-verb noun is not a second word: 赴任した + // reaches 赴任する, which no dictionary spells, and then 赴任, which + // every dictionary does. That step does not count towards the bound. words: deinflect(text.slice(head, end + 1)) - .filter((candidate) => candidate.reasons.length <= 1) - .map((candidate) => candidate.word), + .filter( + (candidate) => + candidate.reasons.filter((reason) => reason !== SURU_NOUN_REASON).length <= 1, + ) + .map((candidate) => ({ + word: candidate.word, + viaSuru: candidate.reasons.includes(SURU_NOUN_REASON), + })), }); } if (ends.length > 0) tails.push({ first, ends }); } if (tails.length === 0) return new Set(); - await prefetch(tails.flatMap((tail) => tail.ends.flatMap((option) => option.words))); + await prefetch( + tails.flatMap((tail) => tail.ends.flatMap((option) => option.words.map((w) => w.word))), + ); const starts = new Set(); for (const { first, ends } of tails) { // Longest first: the word decides how far its own tail runs, and taking the // shortest would leave the rest of the tail open to a span that cuts in. for (const { end, words } of [...ends].reverse()) { let inflects = false; - for (const word of words) { - if ((await lookupOnce(word)).some(entryInflects)) { + for (const { word, viaSuru } of words) { + const entries = await lookupOnce(word); + if (entries.some((entry) => entryInflects(entry) || (viaSuru && entryTakesSuru(entry)))) { inflects = true; break; } @@ -612,6 +716,11 @@ async function findOkuriganaStarts( /** How far past a kanji a single word's kana tail is allowed to reach. */ const MAX_OKURIGANA = 4; +/** A noun that takes する: its し, した, して are the verb, not a particle. */ +function entryTakesSuru(entry: ReaderDictEntry): boolean { + return entry.senses.some((sense) => sense.partOfSpeech?.some((pos) => pos.startsWith("vs"))); +} + /** Only an inflecting word has okurigana; a noun's trailing kana is a particle. */ function entryInflects(entry: ReaderDictEntry): boolean { return entry.senses.some((sense) => @@ -657,6 +766,7 @@ function scoreTapCandidate( function chooseAutoLookupVariants( wordResult: LookupResult, nameResults: LookupResult[], + nameCutsAWord = false, ): LookupResult[] { const taggedWord = asWordLookupResult(wordResult); if (nameResults.length === 0) return [taggedWord]; @@ -689,11 +799,17 @@ function chooseAutoLookupVariants( : AUTO_NAME_ONLY_CONFIDENCE; const nameIsLonger = bestName.matchedText.length > taggedWord.matchedText.length; + // The cut test is for the counted two-kanji override only. A longer name + // overlaps an ordinary word all the time — フランクリン contains フランク, + // ゴルキ contains ゴ — and asking it there costs real names. + const cutsAWord = + nameCutsAWord && [...bestName.matchedText].length < AUTO_NAME_OVERRIDE_MIN_LENGTH; if ( nameIsLonger && (nameConfidence >= nameOnlyConfidence || - (topNameExactSurfaceMatch(bestName) && - bestName.matchedText.length >= AUTO_NAME_OVERRIDE_MIN_LENGTH)) + (!cutsAWord && + nameSpellingIsSurface(bestName) && + nameMayOverrideShorterWord(bestName.matchedText, bestName.nameMatches?.[0]?.freq))) ) { return taggedNames; } @@ -755,9 +871,45 @@ function attachAutoSelectionAlternates( }); } +/** + * Whether a span cuts a word at either end — 東京|市 for 京市, 外来|音 for 来音. + * + * The counts cannot tell a straddle from a surname: 京市 and 定家 are both one + * sighting and only one of them is a name. The text can. This is the question + * `findOkuriganaStarts` asks of kana, asked of kanji, and only common entries + * answer it, because a rare one vouching for a boundary is how 滝山 would lose + * to 山城. + */ +async function spanCutsAWord( + text: string, + start: number, + length: number, + lookupMany: (words: string[]) => Promise>, +): Promise { + const end = start + length; + const words: string[] = []; + for (let reach = 1; reach <= 3; reach++) { + for (let into = 1; into <= 3; into++) { + if (start - reach >= 0 && start + into <= text.length) { + words.push(text.slice(start - reach, start + into)); + } + if (end + reach <= text.length && end - into >= 0) { + words.push(text.slice(end - into, end + reach)); + } + } + } + if (words.length === 0) return false; + const found = await lookupMany([...new Set(words)]); + for (const entries of found.values()) { + if (entries.some((entry) => entry.common)) return true; + } + return false; +} + export function chooseAutoLookupResults( wordResults: LookupResult[], nameResults: LookupResult[], + nameCutsAWord = false, ): LookupResult[] { const taggedWordResults = wordResults.map(asWordLookupResult); const taggedNameResults = nameResults.map(asNameLookupResult); @@ -765,7 +917,7 @@ export function chooseAutoLookupResults( if (taggedWordResults.length === 0) return taggedNameResults; if (taggedNameResults.length === 0) return taggedWordResults; - const variants = chooseAutoLookupVariants(taggedWordResults[0], taggedNameResults); + const variants = chooseAutoLookupVariants(taggedWordResults[0], taggedNameResults, nameCutsAWord); if (variants.length <= 1) { return variants[0]?.lookupKind === "name" ? taggedNameResults : taggedWordResults; } @@ -802,6 +954,24 @@ export async function selectionLookup( options?.extendedDb, ); + // A drag says where the word ends. チェーン展開 is the longer word, but a + // selection of 展開 is a question about 展開, so what the selection itself + // spells is answered first and the expansion below lands under it. + // + // Two characters, because one is not a boundary anybody drew on purpose — + // the reader emits a selection for a single character, and 注|意 would then + // answer 意 ahead of 注意, which is what the expansion is for. + const exactSelection = [...trimmed].length >= 2 ? await lookupExactJapanese(dictDb, trimmed) : []; + if (exactSelection.length > 0) { + onResult( + asWordLookupResult({ + matchedText: trimmed, + entries: sortEntriesForMatchedSurface(exactSelection, trimmed, counterHints.get(trimmed)), + deinflectReasons: [], + }), + ); + } + // Expansion step: try substrings of expanded text that fully contain the selection if (prefix.length > 0 || suffix.length > 0) { const expanded = prefix + trimmed + suffix; @@ -819,10 +989,17 @@ export async function selectionLookup( let best: LookupResult | null = null; let bestScore = Number.NEGATIVE_INFINITY; + let literalMatched = false; for (const candidate of candidates) { - const entries = await lookupExactJapanese(dictDb, candidate.word); + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; + const entries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); if (entries.length > 0) { + if (!guessed) literalMatched = true; const sortedEntries = sortEntriesForMatchedSurface( entries, substr, @@ -853,23 +1030,7 @@ export async function selectionLookup( } } - // Try the full selected text as a single lookup first - const fullEntries = await lookupExactJapanese(dictDb, trimmed); - if (fullEntries.length > 0) { - const sortedEntries = sortEntriesForMatchedSurface( - fullEntries, - trimmed, - counterHints.get(trimmed), - ); - onResult( - asWordLookupResult({ - matchedText: trimmed, - entries: sortedEntries, - deinflectReasons: [], - }), - ); - return; - } + if (exactSelection.length > 0) return; // Try deinflecting the full selected text const fullCandidates = deinflect(trimmed); @@ -953,9 +1114,19 @@ async function findFirstWord( // Try all candidates for this substring length, prefer common entries let best: { result: LookupResult; matchLength: number } | null = null; let bestScore = Number.NEGATIVE_INFINITY; + let literalMatched = false; for (const candidate of candidates) { - const allEntries = await lookupExactJapanese(dictDb, candidate.word); + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; + const allEntries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); + // Raised by the literal itself, not by what is left after the entries + // already shown are removed: a literal whose entries were all shown is + // still a reading that needed no guess. + if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { @@ -1024,9 +1195,16 @@ async function findBoundaryWord( let best: LookupResult | null = null; let bestScore = Number.NEGATIVE_INFINITY; + let literalMatched = false; for (const candidate of candidates) { - const allEntries = await lookupExactJapanese(dictDb, candidate.word); + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; + const allEntries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); + if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); if (newEntries.length > 0) { @@ -1084,9 +1262,16 @@ export async function smartLookup( for (const substr of substrings) { const candidates = deinflect(substr); + let literalMatched = false; for (const candidate of candidates) { - const allEntries = await lookupExactJapanese(dictDb, candidate.word); + const guessed = candidateIsGuess(candidate); + if (guessed && literalMatched) continue; + const allEntries = entriesAGuessMayLandOn( + candidate, + await lookupExactJapanese(dictDb, candidate.word), + ); + if (!guessed && allEntries.length > 0) literalMatched = true; const newEntries = allEntries.filter((e) => !seenEntryIds.has(e.id)); @@ -1126,32 +1311,13 @@ export async function smartLookupWithOffset( tapOffset: number, dictDb: ReaderSqlDb, extDb?: ReaderSqlDb | null, + precomputedOkuriganaStarts?: ReadonlySet, ): Promise { const negativeScopeParticle = await lookupNegativeScopeParticleAtTap(text, tapOffset, dictDb); if (negativeScopeParticle) return [asWordLookupResult(negativeScopeParticle)]; const kanjiCache = new Map(); - // The walk asks for the same word from many overlapping substrings, and the - // kana spellings below multiply that again. One tap, one lookup per word. - const entryCache = new Map(); - const lookupOnce = async (word: string): Promise => { - const cached = entryCache.get(word); - if (cached) return cached; - const found = await lookupExactJapanese(dictDb, word); - entryCache.set(word, found); - return found; - }; - /** - * Fill the cache for a whole length's candidates in one round trip. Most - * candidate spellings do not exist, and asking one at a time spent the bulk - * of a tap's queries proving that. - */ - const prefetch = async (words: string[]): Promise => { - const missing = [...new Set(words)].filter((word) => !entryCache.has(word)); - if (missing.length === 0) return; - const found = await lookupExactJapaneseMany(dictDb, missing); - for (const word of missing) entryCache.set(word, found.get(word) ?? []); - }; + const { lookupOnce, prefetch } = makeEntryCache(dictDb); /** * Spellings of `substr` with its one kanji written as a reading instead. @@ -1181,9 +1347,17 @@ export async function smartLookupWithOffset( } } const counterHints = await buildCounterHintMap([...tapSurfaces], extDb); - const okuriganaStarts = await findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce); + const okuriganaStarts = + precomputedOkuriganaStarts ?? + (await findOkuriganaStarts(text, tapOffset, prefetch, lookupOnce)); let bestOverall: { result: LookupResult; score: number; length: number; start: number } | null = null; + // Tap scope, not span scope. Per span, a guess is only ever compared with a + // literal at the SAME characters, and a common guess one character in then + // outscores a rare longer literal — commonness pays 120, the extra character + // 100 — so 要求にこたえられなく answered 耐える off たえられなく. The walk is + // longest-first, so the longer literal is always seen first. + let literalMatched = false; for (let len = Math.min(text.length, 15); len >= 1; len--) { // Valid start positions: substring must contain the tap position @@ -1225,11 +1399,8 @@ export async function smartLookupWithOffset( // span as written found nothing — にしては is an entry in its own right // and must not be answered with にしても, which merely happens to be the // commoner of the two. - let literalMatched = false; for (const candidate of candidates) { - const guessed = - candidate.particleSwapped || - candidate.reasons.some((reason) => CONTRACTION_REASONS.has(reason)); + const guessed = candidateIsGuess(candidate); if (guessed && literalMatched) continue; let entries = await lookupOnce(candidate.word); // Rewriting a kanji as kana claims the entry is that word spelled @@ -1250,6 +1421,17 @@ export async function smartLookupWithOffset( entry.senses.some((sense) => sense.partOfSpeech?.includes("exp")), ); } + // A guess claims the surface is an inflected word, so what it lands on + // has to be one. Without this the ～なく rules reach する and the + // suru-noun strip hands back whatever was in front of it: 釣や猟を + // しなくっちゃ answered the particle を, ものでなくっちゃ answered ので, + // 何にもせぬ swallowed せぬ to answer the adverb 何にも. + // + // Asking this of every deinflected candidate — the obvious + // generalisation — takes 正直にしていれば from 直に to nothing and + // 出たりはいったりして from 入る to 配流, because an adverb reached by + // stripping する is a perfectly good answer. + entries = entriesAGuessMayLandOn(candidate, entries); if (entries.length > 0) { let sortedEntries = sortEntriesForMatchedSurface( entries, @@ -1365,11 +1547,22 @@ export async function autoLookupWithOffset( dictDb: ReaderSqlDb, extDb?: ReaderSqlDb | null, ): Promise { + // Both walks have to refuse the same starts, or the name walk answers a span + // the word walk just refused: 注文したから came back as the surname たから. + const okuriganaStarts = await okuriganaStartsForTap(text, tapOffset, dictDb); const [wordResults, nameResults] = await Promise.all([ - smartLookupWithOffset(text, tapOffset, dictDb, extDb), - extDb ? nameLookupWithOffset(text, tapOffset, extDb) : Promise.resolve([]), + smartLookupWithOffset(text, tapOffset, dictDb, extDb, okuriganaStarts), + extDb ? nameLookupWithOffset(text, tapOffset, extDb, okuriganaStarts) : Promise.resolve([]), ]); - return chooseAutoLookupResults(wordResults, nameResults); + // 東京|市 is not きょういち. Only asked where a name could override, which is + // a minority of taps, and it is one batched lookup. + const topName = nameResults[0]; + const cutsAWord = + topName?.matchStart != null && + (await spanCutsAWord(text, topName.matchStart, topName.matchedText.length, (words) => + lookupExactJapaneseMany(dictDb, words), + )); + return chooseAutoLookupResults(wordResults, nameResults, cutsAWord); } export async function autoSelectionLookup( @@ -1418,12 +1611,14 @@ export async function nameLookupWithOffset( text: string, tapOffset: number, extDb: ReaderSqlDb, + okuriganaStarts?: ReadonlySet, ): Promise { for (let len = Math.min(text.length, 15); len >= 1; len--) { const minStart = Math.max(0, tapOffset - len + 1); const maxStart = Math.min(tapOffset, text.length - len); for (let start = Math.min(tapOffset, maxStart); start >= minStart; start--) { + if (okuriganaStarts?.has(start)) continue; const substr = text.slice(start, start + len); const names = await lookupExactName(extDb, substr); if (names.length > 0) { diff --git a/packages/japanese-reader/src/use-japanese-reader.ts b/packages/japanese-reader/src/use-japanese-reader.ts index 7a97236..338e713 100644 --- a/packages/japanese-reader/src/use-japanese-reader.ts +++ b/packages/japanese-reader/src/use-japanese-reader.ts @@ -42,6 +42,7 @@ import { autoSelectionLookup, nameLookup, nameLookupWithOffset, + okuriganaStartsForTap, selectionLookup, smartLookup, smartLookupWithOffset, @@ -1602,9 +1603,16 @@ export function useJapaneseReader({ } } else { const tapOffset = msg.tapOffset as number | undefined; + // Name mode walks spans too, so it refuses the same starts a word + // walk does: without this, 注文したから answers the surname たから + // from inside 注文した. + const nameModeStarts = + isNameMode && dictDb && tapOffset && tapOffset > 0 + ? await okuriganaStartsForTap(text, tapOffset, dictDb) + : undefined; const results = isNameMode ? tapOffset && tapOffset > 0 - ? await nameLookupWithOffset(text, tapOffset, extendedDb!) + ? await nameLookupWithOffset(text, tapOffset, extendedDb!, nameModeStarts) : await nameLookup(text, extendedDb!) : isAutoMode ? tapOffset && tapOffset > 0 diff --git a/stores/settings.ts b/stores/settings.ts index 967a8c3..a77e1cd 100644 --- a/stores/settings.ts +++ b/stores/settings.ts @@ -68,7 +68,7 @@ export const defaultSettings = Object.freeze({ flashcardSwipeAnimation: true as boolean, flashcardButtonAnimation: true as boolean, dayResetHour: 3 as number, - smartReviewDays: 7 as number, + smartReviewDays: 7 as number | null, }); export type AppSettings = typeof defaultSettings;