Filter final templates in analysis

src/SIL.Machine.Morphology.HermitCrab/AnalysisAffixTemplateRule.cs

@@ -41,6 +41,26 @@ public IEnumerable<Word> Apply(Word input)
             if (_morpher.TraceManager.IsTracing)
                 _morpher.TraceManager.BeginUnapplyTemplate(_template, input);
 
+            // Do not allow a final template to unapply if we don't need to worry about partials
+            // and a non-template was last unapplied.
+            if (
+                (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates)
+                && input.FinalTemplateState == FinalTemplateState.NonTemplate
+                && _template.IsFinal
+            )
+            {
+                if (_morpher.TraceManager.IsTracing)
+                {
+                    _morpher.TraceManager.EndUnapplyTemplate(
+                        _template,
+                        input,
+                        false,
+                        FailureReason.NonPartialRuleProhibitedAfterFinalTemplate
+                    );
+                }
+                return Enumerable.Empty<Word>();
+            }
+
             Word inWord = input.Clone();
             inWord.Freeze();
 
@@ -63,13 +83,13 @@ private void ApplySlots(Word inWord, int index, HashSet<Word> output)
                 if (!_template.Slots[i].Optional)
                 {
                     if (_morpher.TraceManager.IsTracing)
-                        _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, false);
+                        _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, false, FailureReason.None);
                     return;
                 }
             }
 
             if (_morpher.TraceManager.IsTracing)
-                _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true);
+                _morpher.TraceManager.EndUnapplyTemplate(_template, inWord, true, FailureReason.None);
             output.Add(inWord);
         }
 
@@ -101,7 +121,12 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                             if (!_template.Slots[i].Optional)
                             {
                                 if (_morpher.TraceManager.IsTracing)
-                                    _morpher.TraceManager.EndUnapplyTemplate(_template, work.Item1, false);
+                                    _morpher.TraceManager.EndUnapplyTemplate(
+                                        _template,
+                                        work.Item1,
+                                        false,
+                                        FailureReason.None
+                                    );
                                 add = false;
                                 break;
                             }
@@ -110,7 +135,12 @@ private void ParallelApplySlots(Word inWord, HashSet<Word> output)
                         if (add)
                         {
                             if (_morpher.TraceManager.IsTracing)
-                                _morpher.TraceManager.EndUnapplyTemplate(_template, work.Item1, true);
+                                _morpher.TraceManager.EndUnapplyTemplate(
+                                    _template,
+                                    work.Item1,
+                                    true,
+                                    FailureReason.None
+                                );
                             outStack.Push(work.Item1);
                         }
                     }

src/SIL.Machine.Morphology.HermitCrab/AnalysisStateKey.cs

@@ -12,9 +12,11 @@ namespace SIL.Machine.Morphology.HermitCrab
     /// re-run whenever an <c>Analysis*.cs</c> rule changes:
     /// <list type="bullet">
     /// <item><see cref="MorphologicalRules.AnalysisAffixProcessRule"/>: Shape (FST pattern match),
-    /// <see cref="Word.SyntacticFeatureStruct"/> (unifiability gate), per-rule unapplication count.</item>
+    /// <see cref="Word.SyntacticFeatureStruct"/> (unifiability gate), per-rule unapplication count
+    /// and FinalTemplateState.</item>
     /// <item><see cref="MorphologicalRules.AnalysisCompoundingRule"/>: adds <see cref="Word.NonHeadCount"/>
-    /// (<c>MaxStemCount</c> gate) -- never the non-heads' own content, only the count.</item>
+    /// (<c>MaxStemCount</c> gate) -- never the non-heads' own content, only the count
+    /// and FinalTemplateState.</item>
     /// <item><see cref="MorphologicalRules.AnalysisRealizationalAffixProcessRule"/>: adds
     /// <see cref="Word.RealizationalFeatureStruct"/>.</item>
     /// </list>
@@ -34,6 +36,7 @@ namespace SIL.Machine.Morphology.HermitCrab
         private readonly FeatureStruct _realizationalFS;
         private readonly int _nonHeadCount;
         private readonly IReadOnlyDictionary<IMorphologicalRule, int> _ruleCounts;
+        private readonly FinalTemplateState _finalTemplateState;
         private readonly int _hashCode;
 
         /// <summary>
@@ -63,6 +66,7 @@ private AnalysisStateKey(Word word)
             _realizationalFS = word.RealizationalFeatureStruct;
             _nonHeadCount = word.NonHeadCount;
             _ruleCounts = word.UnappliedRuleCounts;
+            _finalTemplateState = word.FinalTemplateState;
 
             // See PinAndKey for why the key pins these rather than just reading them.
             _shape.Freeze();
@@ -75,6 +79,7 @@ private AnalysisStateKey(Word word)
             hash = hash * 31 + _syntacticFS.GetFrozenHashCode();
             hash = hash * 31 + _realizationalFS.GetFrozenHashCode();
             hash = hash * 31 + _nonHeadCount;
+            hash = hash * 31 + _finalTemplateState.GetHashCode();
             if (_ruleCounts != null)
             {
                 // XOR rather than the usual *31 rolling combine: the multiset is unordered, so entries
@@ -101,6 +106,8 @@ public bool Equals(AnalysisStateKey other)
                 return false;
             if (!_syntacticFS.ValueEquals(other._syntacticFS) || !_realizationalFS.ValueEquals(other._realizationalFS))
                 return false;
+            if (_finalTemplateState != other._finalTemplateState)
+                return false;
             return RuleCountsEqual(_ruleCounts, other._ruleCounts);
         }
 

src/SIL.Machine.Morphology.HermitCrab/AnalysisStratumRule.cs

@@ -160,13 +160,22 @@ internal IEnumerable<Word> Apply(Word input, ref int alternativeCount)
                         continue;
                     }
                 }
+                Word newMruleOutWord = mruleOutWord;
+                if (mruleOutWord.FinalTemplateState != FinalTemplateState.None)
+                {
+                    // Clear FinalTemplateState to allow clitics.
+                    newMruleOutWord = mruleOutWord.Clone();
+                    newMruleOutWord.Source = origInput;
+                    newMruleOutWord.FinalTemplateState = FinalTemplateState.None;
+                    newMruleOutWord.Freeze();
+                }
                 // Only cache a canonical that made it into the output. Two words can have different keys yet
                 // be Word.ValueEquals (UnappliedRuleCounts also counts realizational rules, which never enter
                 // _mruleApps), and a rejected canonical would swallow every later word with its key.
-                if (output.Add(mruleOutWord) && mergeEquivalentAnalyses)
-                    wordCache[key] = mruleOutWord;
+                if (output.Add(newMruleOutWord) && mergeEquivalentAnalyses)
+                    wordCache[key] = newMruleOutWord;
                 if (_morpher.TraceManager.IsTracing)
-                    _morpher.TraceManager.EndUnapplyStratum(_stratum, mruleOutWord);
+                    _morpher.TraceManager.EndUnapplyStratum(_stratum, newMruleOutWord);
             }
             return output;
         }

src/SIL.Machine.Morphology.HermitCrab/FinalTemplateState.cs

@@ -0,0 +1,11 @@
+namespace SIL.Machine.Morphology.HermitCrab
+{
+    // Represent the state of a final template in analysis.
+    // This could be extended to represent the state of a final template in synthesis
+    // (replacing or implementing IsLastAppliedRuleFinal using None, FinalTemplate, NonFinalTemplate).
+    enum FinalTemplateState : byte
+    {
+        None,
+        NonTemplate,
+    }
+}

src/SIL.Machine.Morphology.HermitCrab/ITraceManager.cs

@@ -43,7 +43,7 @@ public interface ITraceManager
         void PhonologicalRuleNotUnapplied(IPhonologicalRule rule, int subruleIndex, Word input);
 
         void BeginUnapplyTemplate(AffixTemplate template, Word input);
-        void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied);
+        void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason);
 
         void MorphologicalRuleUnapplied(IMorphologicalRule rule, int subruleIndex, Word input, Word output);
         void MorphologicalRuleNotUnapplied(IMorphologicalRule rule, int subruleIndex, Word input);

src/SIL.Machine.Morphology.HermitCrab/Morpher.cs

@@ -65,6 +65,18 @@ public Morpher(ITraceManager traceManager, Language lang, int maxDegreeOfParalle
             RuleSelector = rule => true;
 
             _morphemes = new ReadOnlyObservableCollection<Morpheme>(morphemes);
+            IsPartial = GetPartialMorphemes().Count() > 0;
+        }
+
+        public IEnumerable<Morpheme> GetPartialMorphemes()
+        {
+            var morphemes = new HashSet<Morpheme>();
+            foreach (Morpheme morpheme in _morphemes)
+            {
+                if (morpheme.IsPartial)
+                    morphemes.Add(morpheme);
+            }
+            return morphemes;
         }
 
         public ITraceManager TraceManager
@@ -88,6 +100,16 @@ public ITraceManager TraceManager
         /// </summary>
         public bool MergeEquivalentAnalyses { get; set; }
 
+        /// <summary>
+        /// A Morpher is partial if any of the elements are partial.
+        /// </summary>
+        public bool IsPartial { get; }
+
+        /// <summary>
+        /// Enforce final templates even if some of the morphemes were partial.
+        /// </summary>
+        public bool AlwaysEnforceFinalTemplates { get; set; }
+
         /// <summary>
         /// Caps the concurrency used within a single parse or generation -- analysis cascade,
         /// affix-template unapplication and synthesis alike. A value of 1 runs the work fully
@@ -490,6 +512,8 @@ private IEnumerable<Word> LexicalGuess(Word input)
                             SyntacticFeatureStruct = input.SyntacticFeatureStruct,
                             Stratum = input.Stratum,
                         };
+                        // Don't update Morpher.IsPartial even if lexEntry.IsPartial is true.
+                        // We don't mind if final templates are filtered during analysis in this case.
                         lexEntry.Allomorphs.Add(root);
                         // Point the root allomorph to the lexical pattern in FieldWorks.
                         if (lexicalPattern.Morpheme != null)

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisAffixProcessRule.cs

@@ -80,6 +80,12 @@ out FeatureStruct syntacticFS
 
                         outWord.SyntacticFeatureStruct = syntacticFS;
                     }
+                    if (!_morpher.IsPartial || _morpher.AlwaysEnforceFinalTemplates)
+                    {
+                        outWord.FinalTemplateState = !_rule.IsTemplateRule
+                            ? FinalTemplateState.NonTemplate
+                            : FinalTemplateState.None;
+                    }
                     outWord.MorphologicalRuleUnapplied(_rule);
                     outWord.Freeze();
                     if (_morpher.TraceManager.IsTracing)

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisCompoundingRule.cs

@@ -148,6 +148,7 @@ out FeatureStruct syntacticFS
                         }
                         outWord.SyntacticFeatureStruct = syntacticFS;
                     }
+                    outWord.FinalTemplateState = FinalTemplateState.NonTemplate;
                     outWord.MorphologicalRuleUnapplied(_rule);
 
                     outWord.Freeze();

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/AnalysisRealizationalAffixProcessRule.cs

@@ -53,6 +53,7 @@ public IEnumerable<Word> Apply(Word input)
                 foreach (Word outWord in _rules[i].Apply(input).RemoveDuplicates())
                 {
                     outWord.RealizationalFeatureStruct = realFS;
+                    outWord.FinalTemplateState = FinalTemplateState.None;
                     outWord.MorphologicalRuleUnapplied(_rule);
 
                     outWord.Freeze();

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisAffixProcessRule.cs

@@ -63,8 +63,7 @@ public IEnumerable<Word> Apply(Word input)
             if (
                 !_rule.IsTemplateRule
                 && (input.IsLastAppliedRuleFinal ?? false)
-                && !input.IsPartial
-                && !_rule.IsPartial
+                && ((!input.IsPartial && !_rule.IsPartial) || _morpher.AlwaysEnforceFinalTemplates)
             )
             {
                 if (_morpher.TraceManager.IsTracing)

src/SIL.Machine.Morphology.HermitCrab/MorphologicalRules/SynthesisCompoundingRule.cs

@@ -62,7 +62,7 @@ public IEnumerable<Word> Apply(Word input)
                 return Enumerable.Empty<Word>();
             }
 
-            if ((input.IsLastAppliedRuleFinal ?? false) && !input.IsPartial)
+            if ((input.IsLastAppliedRuleFinal ?? false) && (!input.IsPartial || _morpher.AlwaysEnforceFinalTemplates))
             {
                 if (_morpher.TraceManager.IsTracing)
                 {

src/SIL.Machine.Morphology.HermitCrab/TraceManager.cs

@@ -58,10 +58,14 @@ public void BeginUnapplyTemplate(AffixTemplate template, Word input)
             );
         }
 
-        public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied)
+        public void EndUnapplyTemplate(AffixTemplate template, Word output, bool unapplied, FailureReason reason)
         {
             ((Trace)output.CurrentTrace).Children.Add(
-                new Trace(TraceType.TemplateAnalysisOutput, template) { Output = unapplied ? output : null }
+                new Trace(TraceType.TemplateAnalysisOutput, template)
+                {
+                    Output = unapplied ? output : null,
+                    FailureReason = reason,
+                }
             );
         }
 

src/SIL.Machine.Morphology.HermitCrab/Word.cs

@@ -28,6 +28,7 @@ public class Word : Freezable<Word>, IAnnotatedData<ShapeNode>, ICloneable<Word>
         private FeatureStruct _realizationalFS;
         private Stratum _stratum;
         private bool? _isLastAppliedRuleFinal;
+        private FinalTemplateState _finalTemplateState;
         private bool _isPartial;
         private readonly Dictionary<string, HashSet<int>> _disjunctiveAllomorphIndices;
         private int _mruleAppCount = 0;
@@ -47,6 +48,7 @@ public Word(RootAllomorph rootAllomorph, FeatureStruct realizationalFS)
             _nonHeadApps = new List<Word>();
             _obligatorySyntacticFeatures = new IDBearerSet<Feature>();
             _isLastAppliedRuleFinal = null;
+            _finalTemplateState = FinalTemplateState.None;
             _disjunctiveAllomorphIndices = new Dictionary<string, HashSet<int>>();
         }
 
@@ -65,6 +67,7 @@ public Word(Stratum stratum, Shape shape)
             _nonHeadApps = new List<Word>();
             _obligatorySyntacticFeatures = new IDBearerSet<Feature>();
             _isLastAppliedRuleFinal = null;
+            _finalTemplateState = FinalTemplateState.None;
             _isPartial = false;
             _disjunctiveAllomorphIndices = new Dictionary<string, HashSet<int>>();
         }
@@ -93,6 +96,7 @@ private Word(Word word, bool cloneNonHeadApps)
             _nonHeadAppIndex = word._nonHeadAppIndex;
             _obligatorySyntacticFeatures = new IDBearerSet<Feature>(word._obligatorySyntacticFeatures);
             _isLastAppliedRuleFinal = word._isLastAppliedRuleFinal;
+            _finalTemplateState = word._finalTemplateState;
             _isPartial = word._isPartial;
             CurrentTrace = word.CurrentTrace;
             AnalysisScope = word.AnalysisScope;
@@ -392,6 +396,16 @@ internal bool? IsLastAppliedRuleFinal
             }
         }
 
+        internal FinalTemplateState FinalTemplateState
+        {
+            get { return _finalTemplateState; }
+            set
+            {
+                CheckFrozen();
+                _finalTemplateState = value;
+            }
+        }
+
         /// <summary>
         /// Gets the number of times the specified morphological rule has been applied.
         /// </summary>
@@ -621,6 +635,7 @@ protected override int FreezeImpl()
             code = code * 31 + _mruleApps.GetSequenceHashCode();
             code = code * 31 + _mruleAppIndex.GetHashCode();
             code = code * 31 + _isLastAppliedRuleFinal.GetHashCode();
+            code = code * 31 + _finalTemplateState.GetHashCode();
             return code;
         }
 
@@ -643,7 +658,8 @@ public override bool ValueEquals(Word other)
                 && _rootAllomorph == other._rootAllomorph
                 && _mruleApps.SequenceEqual(other._mruleApps)
                 && _mruleAppIndex == other._mruleAppIndex
-                && _isLastAppliedRuleFinal == other._isLastAppliedRuleFinal;
+                && _isLastAppliedRuleFinal == other._isLastAppliedRuleFinal
+                && _finalTemplateState == other._finalTemplateState;
         }
 
         public Word Clone()