Fix incremental clearing of Dispatcher state

It wasn't complete. Now `DispatchingFront.forget(occ)`
additionally triggers removal of invalid ReteNodes.

reactor/Core/src/jetbrains/mps/logic/reactor/core/Dispatcher.kt

@@ -113,7 +113,7 @@ class Dispatcher (val ruleIndex: RuleIndex, prevState: DispatchingFrontState = e
          * After that the occurrence can be expanded again without triggering observer reactivation logic.
          * Needed to discern incremental reactivation from observers reactivation.
          * In other words, the state of [DispatchingFront] connected with this occurrence
-         * transitions from "fully-expanded" to "partly-expanded".
+         * transitions from "fully-expanded" to "partially-expanded".
          */
         internal fun forgetExpanded(dropped: Occurrence): DispatchingFront = DispatchingFront(this,
             ruleIndex.forOccurrence(dropped)
@@ -129,7 +129,7 @@ class Dispatcher (val ruleIndex: RuleIndex, prevState: DispatchingFrontState = e
         internal fun forget(dropped: Occurrence): DispatchingFront = DispatchingFront(this,
             ruleIndex.forOccurrence(dropped)
                 .mapNotNull { rule -> ruletag2probe[rule.uniqueTag()] }
-                .map { probe -> probe.contract(dropped).forgetExpanded(dropped).forgetConsumed(dropped) })
+                .map { probe -> probe.forget(dropped) })
 
         /**
          * Serves to indicate that the specified [RuleMatchEx] has been processed (consumed) and has to

reactor/Core/src/jetbrains/mps/logic/reactor/core/RuleMatchingProbe.kt

@@ -56,9 +56,17 @@ interface RuleMatchingProbe {
 
     /**
      * Clears all internal state related to [ruleMatch].
+     * Effect is as if [ruleMatch] was never seen.
      */
     fun forget(ruleMatch: RuleMatchEx): RuleMatchingProbe
 
+    /**
+     * Clears all state related to [Occurrence] [occ].
+     * Same as [contract], but also clears internal state.
+     * Effect is as if [occ] was never seen.
+     */
+    fun forget(occ: Occurrence): RuleMatchingProbe
+
     fun expand(occ: Occurrence): RuleMatchingProbe
 
     fun expand(occ: Occurrence, mask: BitSet, profiler: Profiler? = null): RuleMatchingProbe

reactor/Core/src/jetbrains/mps/logic/reactor/core/internal/ReteRuleMatcherImpl.kt

@@ -355,7 +355,7 @@ internal class ReteRuleMatcherImpl(private var ruleLookup: RuleLookup,
             }
         }
 
-        inner class DropBlock (occurrence: Occurrence) : UpdateBlock(occurrence) {
+        open inner class DropBlock (occurrence: Occurrence) : UpdateBlock(occurrence) {
 
             override fun reset() {
                 // NOP
@@ -368,6 +368,7 @@ internal class ReteRuleMatcherImpl(private var ruleLookup: RuleLookup,
                     }
                 }
                 introTrail.remove(occurrence.identity)
+                // [nodes] are cleared from invalidated nodes later lazily
                 return false
             }
 
@@ -454,9 +455,23 @@ internal class ReteRuleMatcherImpl(private var ruleLookup: RuleLookup,
                 for (layer in layers) {
                     layer.forgetContains(occurrence)
                 }
+                if (lastIntroduced === occurrence) lastIntroduced = null
                 return nextGeneration()
             }
 
+            fun erase(occurrence: Occurrence): Generation {
+                if (lastIntroduced === occurrence) lastIntroduced = null
+                return drop(occurrence).clearInvalidNodes()
+            }
+
+            private fun clearInvalidNodes(): Generation {
+                while (nodesIt.hasNext()) {
+                    val n = nodesIt.next()
+                    if (n.isInvalid()) nodesIt.remove()
+                }
+                return this.reset()
+            }
+
             fun nextGeneration(): Generation = Generation(this)
 
             fun calcMatches(): Collection<RuleMatchImpl> {
@@ -521,6 +536,11 @@ internal class ReteRuleMatcherImpl(private var ruleLookup: RuleLookup,
             return this
         }
 
+        override fun forget(occ: Occurrence): RuleMatchingProbe {
+            this.lastGeneration = lastGeneration.erase(occ)
+            return this
+        }
+
         override fun hasOccurrences(): Boolean {
             return this.lastGeneration.hasOccurrences()
         }

reactor/Core/src/jetbrains/mps/logic/reactor/core/internal/RuleMatcherImpl.kt

@@ -88,6 +88,16 @@ internal class RuleMatcherImpl(private var ruleLookup: RuleLookup,
                             seenOccurrences,
                             consumedSignatures.without(ruleMatch.signatureArray().toSignature()))
 
+        override fun forget(occ: Occurrence): RuleMatchingProbe = with(this.contract(occ)) {
+            val newConsumed = Sets.copyOf(consumedSignatures.filter{ !it.contains(occ.identity) })
+            val newSeen = seenOccurrences.without(occ.identity)
+            RuleMatchFront(trunkNodes,
+                            leafNodes,
+                            leafSignatures,
+                            newSeen,
+                            newConsumed)
+        }
+
         override fun expand(occ: Occurrence): RuleMatchingProbe =
             expand(occ, bitSetOfOnes(head.size))