From f8970f7825ce8b175c1b4503767f8f9be87ecdc4 Mon Sep 17 00:00:00 2001 From: Fedor Isakov Date: Tue, 11 Jul 2017 12:27:45 +0200 Subject: [PATCH] Refactoring and reorganizing the matcher code, preparing for incremental matches search. --- .../mps/logic/reactor/core/MatchTrie.kt | 312 -------------- .../mps/logic/reactor/core/Matcher.kt | 11 +- .../logic/reactor/core/PartialMatchTrie.kt | 384 ++++++++++++++++++ 3 files changed, 388 insertions(+), 319 deletions(-) delete mode 100644 reactor/Core/src/jetbrains/mps/logic/reactor/core/MatchTrie.kt create mode 100644 reactor/Core/src/jetbrains/mps/logic/reactor/core/PartialMatchTrie.kt diff --git a/reactor/Core/src/jetbrains/mps/logic/reactor/core/MatchTrie.kt b/reactor/Core/src/jetbrains/mps/logic/reactor/core/MatchTrie.kt deleted file mode 100644 index bfea0876..00000000 --- a/reactor/Core/src/jetbrains/mps/logic/reactor/core/MatchTrie.kt +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright 2014-2017 JetBrains s.r.o. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package jetbrains.mps.logic.reactor.core - -import jetbrains.mps.logic.reactor.evaluation.ConstraintOccurrence -import jetbrains.mps.logic.reactor.logical.Logical -import jetbrains.mps.logic.reactor.logical.MetaLogical -import jetbrains.mps.logic.reactor.program.Constraint -import jetbrains.mps.logic.reactor.program.Rule -import jetbrains.mps.logic.reactor.util.Profiler -import jetbrains.mps.logic.reactor.util.profile -import jetbrains.mps.unification.Term -import java.util.* -import com.github.andrewoma.dexx.collection.Map as PersMap - -/** - * @author Fedor Isakov - */ - -internal class MatchTrie(val rule: Rule, - val activeOcc: ConstraintOccurrence, - val aux: OccurrenceIndex, - val profiler: Profiler?) : Iterable -{ - - val keptConstraints = rule.headKept().toCollection(ArrayList(4)) - - val discardedConstraints = rule.headReplaced().toCollection(ArrayList(4)) - - val relatedSlots = ArrayList() - - init { - val meta2slots = HashMap, BitSet>() - (keptConstraints + discardedConstraints).forEachIndexed { slot, cst -> - cst.arguments().forEach { arg -> - if (arg is MetaLogical<*>) { - val bs = meta2slots[arg] ?: BitSet().apply { meta2slots[arg] = this } - bs.set(slot) - } - } - } - - (keptConstraints + discardedConstraints).forEachIndexed { slot, cst -> - val bs = BitSet() - cst.arguments().forEach { arg -> - if (arg is MetaLogical<*>) { - bs.or(meta2slots[arg]) - } - } - bs.clear(slot) - relatedSlots.add(bs) - } - } - - private val firstRoot: MatchTrieNode? by lazy(LazyThreadSafetyMode.NONE) { - profiler.profile("initMatchTrie") { - - var first: MatchTrieNode? = null - var prev: MatchTrieNode? = null - for ((idx, cst) in keptConstraints.withIndex()) { - if (cst.symbol() == activeOcc.constraint().symbol()) { - val next = MatchTrieNode(null, true, cst, activeOcc, idx) - if (first == null) { - first = next - } - prev?._nextSibling = next - prev = next - } - } - for ((idx, cst) in discardedConstraints.withIndex()) { - if (cst.symbol() == activeOcc.constraint().symbol()) { - val next = MatchTrieNode(null, false, cst, activeOcc, idx) - if (first == null) { - first = next - } - prev?._nextSibling = next - prev = next - } - } - first - - } - } - - override fun iterator() = object : Iterator { - - var next: PartialMatch? = null - - override fun hasNext(): Boolean { - if (next == null) { - this.next = calcNext() - } - return (next != null) - } - - override fun next(): PartialMatch { - if (next == null) { - this.next = calcNext() - } - if (next != null) { - val tmp: PartialMatch = next!! - this.next = calcNext() - return tmp - } - throw NoSuchElementException() - } - - private fun calcNext(): PartialMatch? { - if (firstRoot == null) { - return null - } - var mtn: MatchTrieNode? = firstRoot - - while (mtn != null) { - if (mtn.isSeen) { - if (mtn.nextSibling == null) { - mtn.parent?.isSeen = true - } - mtn = mtn.nextSibling ?: mtn.parent - } - else if (mtn.isLeaf) { - mtn.isSeen = true - return mtn.match - } - else { - if (mtn.firstChild == null) { - mtn.isSeen = true - } - mtn = mtn.firstChild ?: mtn - } - } - - return null - } - } - - private inner class MatchTrieNode(val parent: MatchTrieNode?, - val keepConstraint: Boolean, - val constraint: Constraint, - val occurrence: ConstraintOccurrence, - val index: Int) - { - - var isSeen: Boolean = false - - var _firstChild: MatchTrieNode? = null - - val firstChild: MatchTrieNode? - get() { - if (_firstChild == null) { - var first: MatchTrieNode? = null - var prev: MatchTrieNode? = null - for (next in calcChildren()) { - if (first == null) { - first = next - } - prev?._nextSibling = next - prev = next - } - this._firstChild = first - } - return _firstChild - } - - var _nextSibling: MatchTrieNode? = null - - val nextSibling: MatchTrieNode? - get() = _nextSibling - - val vacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) { - val result = BitSet() - result.set(0, keptConstraints.size + discardedConstraints.size) - foldNodes(result) { bits, mtn -> - val slot = if (mtn.keepConstraint) mtn.index else mtn.index + keptConstraints.size - bits.clear(slot) - bits - } - result - } - - val relatedFirstVacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) { - val result = vacantSlots.clone() as BitSet - result.and(relatedSlots[if (keepConstraint) index else index + keptConstraints.size]) - result - } - - val isLeaf: Boolean - get() = vacantSlots.isEmpty - - val match: PartialMatch by lazy(LazyThreadSafetyMode.NONE) { - val parentMatch = parent?.match ?: PartialMatch(rule, keptConstraints.size, discardedConstraints.size) - if (keepConstraint) parentMatch.keep(occurrence, index) - else parentMatch.discard(occurrence, index) - } - - private fun calcChildren(): Iterable { - val nextSlots = if (!relatedFirstVacantSlots.isEmpty) relatedFirstVacantSlots else vacantSlots - val nextSlot = nextSlots.nextSetBit(0) - if (nextSlot < 0) { - return emptyList() - } - - val keep = nextSlot < keptConstraints.size - val nextIdx = if (keep) nextSlot else nextSlot - keptConstraints.size - val nextCst = if (keep) keptConstraints[nextSlot] else discardedConstraints[nextIdx] - - val (result, auxOccurrences) = lookupAuxOccurrences(nextCst) - when (result) { - Result.DEFINITIVE -> parent - Result.NONE -> this - Result.INCONCLUSIVE -> null - }?.forEachNode { mtn -> mtn.isSeen = true } - - return auxOccurrences.map { auxOcc -> MatchTrieNode(this, keep, nextCst, auxOcc, nextIdx) } - } - - private fun collectMetaInstances(meta: Collection<*>, list: ArrayList): ArrayList { - constraint.arguments().zip(occurrence.arguments()).forEach { p -> - if (p.first is MetaLogical<*> && meta.contains(p.first) && !list.contains(p.second)) { - list.add(p.second!!) - } - } - return list - } - - private fun metaInstances(meta: Collection<*>): List = - foldNodes(ArrayList(4)) { list, mtn -> mtn.collectMetaInstances(meta, list) } - - // the first component indicates the quality of the result: - // DEFINITIVE indicates the only possible matches were returned - // INCONCLUSIVE may yield some (ir-)relevant results - // NONE means no results could have been found at all, so it's useless to attempt again with the same input - private fun lookupAuxOccurrences(cst: Constraint): Pair> { - return profiler.profile>> ("lookupAux_${cst.symbol()}") { - - val fromArgs = ArrayList(4) - for (arg in metaInstances(cst.arguments()) + cst.arguments().filter { arg -> !(arg is MetaLogical<*>) }) { - when (arg) { - is Logical<*> -> fromArgs.addAll(aux.forLogical(arg)) - - is Term -> fromArgs.addAll(aux.forTermAndConstraint(arg, cst)) - - is Any -> fromArgs.addAll(aux.forValue(arg)) - } - } - if (fromArgs.isNotEmpty()) { - Result.DEFINITIVE.to(fromArgs.filter { occ -> occ.constraint().symbol() == cst.symbol() && !hasOccurrence(occ) }) - - } else { - return profiler.profile>> ("lookupAux_default_${cst.symbol()}") { - - val hasRelated = relatedFirstVacantSlots == vacantSlots - val allArgsMetaLogicals = cst.arguments().all { a -> a is MetaLogical<*> } - val noArgs = cst.arguments().isEmpty() - - if (noArgs || (allArgsMetaLogicals && !hasRelated)) { - Result.INCONCLUSIVE.to(aux.forSymbol(cst.symbol()).filter { occ -> !hasOccurrence(occ) }) - - } - else { - // all candidate occurrences should have been found by this time; if not, then there's none - Result.NONE.to(emptyList()) - } - - } - } - } - } - - private fun hasOccurrence(occ: ConstraintOccurrence): Boolean = - anyNode { mtn -> mtn.occurrence == occ } - - private inline fun anyNode(predicate: (MatchTrieNode) -> Boolean): Boolean = - foldNodes(false) { b, mtn -> b || predicate(mtn) } - - private inline fun forEachNode(proc: (MatchTrieNode) -> Unit) = - foldNodes(Unit) { b, mtn -> proc(mtn) } - - private inline fun foldNodes(start: T, step: (T, MatchTrieNode) -> T): T { - var mtn: MatchTrieNode? = this - var curr = start - while(mtn != null) { - curr = step(curr, mtn) - mtn = mtn.parent - } - return curr - } - - } - - enum class Result { - DEFINITIVE, - INCONCLUSIVE, - NONE - } - -} - diff --git a/reactor/Core/src/jetbrains/mps/logic/reactor/core/Matcher.kt b/reactor/Core/src/jetbrains/mps/logic/reactor/core/Matcher.kt index fb98745e..35bbb1a1 100644 --- a/reactor/Core/src/jetbrains/mps/logic/reactor/core/Matcher.kt +++ b/reactor/Core/src/jetbrains/mps/logic/reactor/core/Matcher.kt @@ -27,12 +27,9 @@ import jetbrains.mps.logic.reactor.program.Predicate import jetbrains.mps.logic.reactor.program.Rule import jetbrains.mps.logic.reactor.util.LazyIterable import jetbrains.mps.logic.reactor.util.Profiler -import jetbrains.mps.logic.reactor.util.profile import jetbrains.mps.unification.Substitution import jetbrains.mps.unification.Term import jetbrains.mps.unification.TermWrapper -import jetbrains.mps.unification.Unification -import org.jetbrains.kotlin.container.topologicalSort import java.util.* import com.github.andrewoma.dexx.collection.List as PersList @@ -46,14 +43,14 @@ class Matcher(val ruleIndex: RuleIndex, val profiler: Profiler? = null) : Iterable { - private val matchTries: LazyIterable by lazy(LazyThreadSafetyMode.NONE) { - LazyIterable (ruleIndex.forOccurrence(activeOcc).toList()) { rule -> - MatchTrie(rule, activeOcc, aux, profiler) + private val matchTries: LazyIterable by lazy(LazyThreadSafetyMode.NONE) { + LazyIterable (ruleIndex.forOccurrence(activeOcc).toList()) { rule -> + PartialMatchTrie(rule, activeOcc, aux, profiler) } } override fun iterator() = matchTries.asSequence().flatMap { matchTrie -> - matchTrie.asSequence() }.map { partialMatch -> + matchTrie.matches().asSequence() }.map { partialMatch -> partialMatch.complete(profiler) }.filter { match -> match.successful }.iterator() diff --git a/reactor/Core/src/jetbrains/mps/logic/reactor/core/PartialMatchTrie.kt b/reactor/Core/src/jetbrains/mps/logic/reactor/core/PartialMatchTrie.kt new file mode 100644 index 00000000..3651a427 --- /dev/null +++ b/reactor/Core/src/jetbrains/mps/logic/reactor/core/PartialMatchTrie.kt @@ -0,0 +1,384 @@ +/* + * Copyright 2014-2017 JetBrains s.r.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package jetbrains.mps.logic.reactor.core + +import jetbrains.mps.logic.reactor.evaluation.ConstraintOccurrence +import jetbrains.mps.logic.reactor.logical.Logical +import jetbrains.mps.logic.reactor.logical.MetaLogical +import jetbrains.mps.logic.reactor.program.Constraint +import jetbrains.mps.logic.reactor.program.Rule +import jetbrains.mps.logic.reactor.util.Profiler +import jetbrains.mps.logic.reactor.util.profile +import jetbrains.mps.unification.Term +import java.util.* +import kotlin.collections.ArrayList +import com.github.andrewoma.dexx.collection.Map as PersMap + +/** + * @author Fedor Isakov + */ + +internal class PartialMatchTrie(val rule: Rule, + val activeOcc: ConstraintOccurrence, + val occIndex: OccurrenceIndex, + val profiler: Profiler?) +{ + + constructor(prototype: PartialMatchTrie, + activeOcc: ConstraintOccurrence, + occurrenceIndex: OccurrenceIndex, + profiler: Profiler?) : + this(prototype.rule, activeOcc, occurrenceIndex, profiler) + { + this._prototype = prototype + } + + private var _prototype: PartialMatchTrie? = null + + private val prototype: PartialMatchTrie? + get() = _prototype + + private val keptConstraints = rule.headKept().toCollection(ArrayList(4)) + + private val discardedConstraints = rule.headReplaced().toCollection(ArrayList(4)) + + private val root: RootMatchTrieNode by lazy(LazyThreadSafetyMode.NONE) { + prototype?.root ?: RootMatchTrieNode() + } + + // for each slot keep a set of "related" slots, that is the slots with matching (meta-)logicals + // this map is built only once per rule + private val relatedSlots: List by lazy(LazyThreadSafetyMode.NONE) { + prototype?.relatedSlots ?: HashMap, BitSet>().let { meta2slots -> + + (keptConstraints + discardedConstraints).forEachIndexed { slot, cst -> + cst.arguments().forEach { arg -> + if (arg is MetaLogical<*>) { + val bs = meta2slots[arg] ?: BitSet().apply { meta2slots[arg] = this } + bs.set(slot) + } + } + } + + ArrayList().apply { + (keptConstraints + discardedConstraints).forEachIndexed { slot, cst -> + val bs = BitSet() + cst.arguments().forEach { arg -> + if (arg is MetaLogical<*>) { + bs.or(meta2slots[arg]) + } + } + bs.clear(slot) + add(bs) + } + } + } + } + + fun matches(): Iterable = object : Iterable { + + override fun iterator() = object : Iterator { + + val firstRoot: MatchTrieNode? = root.firstChild + + var next: PartialMatch? = null + + override fun hasNext(): Boolean { + if (next == null) { + this.next = calcNext() + } + return (next != null) + } + + override fun next(): PartialMatch { + if (next == null) { + this.next = calcNext() + } + if (next != null) { + val tmp: PartialMatch = next!! + this.next = calcNext() + return tmp + } + throw NoSuchElementException() + } + + private fun calcNext(): PartialMatch? { + // the search always starts at the first root + if (firstRoot == null) { + return null + } + var mtn: MatchTrieNode? = firstRoot + + while (mtn != null) { + if (mtn.isSeen) { + if (mtn.nextSibling == null) { + mtn.parent?.setSeen() + } + mtn = mtn.nextSibling ?: mtn.parent + } + else if (mtn.isLeaf) { + mtn.setSeen() + return mtn.match + } + else { + // calculate the children + if (mtn.firstChild == null) { + mtn.setSeen() + } + mtn = mtn.firstChild ?: mtn + } + } + + return null + } + } + + } + + private abstract inner class BaseMatchTrieNode() { + + var _firstChild: MatchTrieNode? = null + + var _nextSibling: MatchTrieNode? = null + + abstract fun reset() + + } + + private inner class RootMatchTrieNode() : BaseMatchTrieNode() { + + val firstChild: MatchTrieNode? + get() { + if (_firstChild == null) { + var first: MatchTrieNode? = null + var prev: MatchTrieNode? = null + for ((idx, cst) in keptConstraints.withIndex()) { + if (cst.symbol() == activeOcc.constraint().symbol()) { + val next = MatchTrieNode(null, true, cst, activeOcc, idx) + if (first == null) { + first = next + } + prev?._nextSibling = next + prev = next + } + } + for ((idx, cst) in discardedConstraints.withIndex()) { + if (cst.symbol() == activeOcc.constraint().symbol()) { + val next = MatchTrieNode(null, false, cst, activeOcc, idx) + if (first == null) { + first = next + } + prev?._nextSibling = next + prev = next + } + } + + this._firstChild = first + } + return _firstChild + } + + override fun reset() { + while (!(_firstChild?.occurrence?.isAlive() ?: true)) { + this._firstChild = _firstChild?._nextSibling + } + _firstChild?.reset() + } + + } + + private inner class MatchTrieNode(val parent: MatchTrieNode?, + val keepConstraint: Boolean, + val constraint: Constraint, + val occurrence: ConstraintOccurrence, + val index: Int) : BaseMatchTrieNode() + { + + private var _isSeen: Boolean = false + + val isSeen: Boolean + get() = _isSeen + + val vacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) { + val result = BitSet() + result.set(0, keptConstraints.size + discardedConstraints.size) + foldPath(result) { bits, mtn -> + val slot = if (mtn.keepConstraint) mtn.index else mtn.index + keptConstraints.size + bits.clear(slot) + bits + } + result + } + + val relatedFirstVacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) { + val result = vacantSlots.clone() as BitSet + result.and(relatedSlots[if (keepConstraint) index else index + keptConstraints.size]) + result + } + + val isLeaf: Boolean + get() = vacantSlots.isEmpty + + val match: PartialMatch by lazy(LazyThreadSafetyMode.NONE) { + val parentMatch = parent?.match ?: PartialMatch(rule, keptConstraints.size, discardedConstraints.size) + if (keepConstraint) parentMatch.keep(occurrence, index) + else parentMatch.discard(occurrence, index) + } + + fun setSeen() { + this._isSeen = true + } + + val firstChild: MatchTrieNode? + get() { + if (_firstChild == null) { + var first: MatchTrieNode? = null + var prev: MatchTrieNode? = null + for (next in calcChildren()) { + if (first == null) { + first = next + } + prev?._nextSibling = next + prev = next + } + + this._firstChild = first + } + return _firstChild + } + + val nextSibling: MatchTrieNode? + get() = _nextSibling + + private fun calcChildren(): Iterable { + val nextSlots = if (!relatedFirstVacantSlots.isEmpty) relatedFirstVacantSlots else vacantSlots + val nextSlot = nextSlots.nextSetBit(0) + if (nextSlot < 0) { + return emptyList() + } + + val keep = nextSlot < keptConstraints.size + val nextIdx = if (keep) nextSlot else nextSlot - keptConstraints.size + val nextCst = if (keep) keptConstraints[nextSlot] else discardedConstraints[nextIdx] + + val (result, auxOccurrences) = lookupAuxOccurrences(nextCst) + when (result) { + Result.DEFINITIVE -> parent + Result.NONE -> this + Result.INCONCLUSIVE -> null + }?.forEachInPath { mtn -> mtn._isSeen = true } + + return auxOccurrences.map { auxOcc -> MatchTrieNode(this, keep, nextCst, auxOcc, nextIdx) } + } + + // the first component indicates the quality of the result: + // DEFINITIVE indicates the only possible matches were returned + // INCONCLUSIVE may yield some (ir-)relevant results + // NONE means no results could have been found at all, so it's useless to attempt again with the same input + private fun lookupAuxOccurrences(cst: Constraint): Pair> { + return profiler.profile>> ("lookupAux_${cst.symbol()}") { + + val fromArgs = ArrayList(4) + for (arg in metaInstances(cst.arguments()) + cst.arguments().filter { arg -> !(arg is MetaLogical<*>) }) { + when (arg) { + is Logical<*> -> fromArgs.addAll(occIndex.forLogical(arg)) + + is Term -> fromArgs.addAll(occIndex.forTermAndConstraint(arg, cst)) + + is Any -> fromArgs.addAll(occIndex.forValue(arg)) + } + } + if (fromArgs.isNotEmpty()) { + Result.DEFINITIVE.to(fromArgs.filter { occ -> occ.constraint().symbol() == cst.symbol() && !hasOccurrence(occ) }) + + } else { + return profiler.profile>> ("lookupAux_default_${cst.symbol()}") { + + val hasRelated = relatedFirstVacantSlots == vacantSlots + val allArgsMetaLogicals = cst.arguments().all { a -> a is MetaLogical<*> } + val noArgs = cst.arguments().isEmpty() + + if (noArgs || (allArgsMetaLogicals && !hasRelated)) { + Result.INCONCLUSIVE.to(occIndex.forSymbol(cst.symbol()).filter { occ -> !hasOccurrence(occ) }) + + } + else { + // all candidate occurrences should have been found by this time; if not, then there's none + Result.NONE.to(emptyList()) + } + + } + } + } + } + + // prepare the node for re-use + override fun reset() { + this._isSeen = false + + while (!(_firstChild?.occurrence?.isAlive() ?: true)) { + this._firstChild = _firstChild?._nextSibling + } + _firstChild?.reset() + + while(!(_nextSibling?.occurrence?.isAlive() ?: true)) { + this._nextSibling = _nextSibling?._nextSibling + } + _nextSibling?.reset() + } + + private fun collectMetaInstances(meta: Collection<*>, list: ArrayList): ArrayList { + constraint.arguments().zip(occurrence.arguments()).forEach { p -> + if (p.first is MetaLogical<*> && meta.contains(p.first) && !list.contains(p.second)) { + list.add(p.second!!) + } + } + return list + } + + private fun metaInstances(meta: Collection<*>): List = + foldPath(ArrayList(4)) { list, mtn -> mtn.collectMetaInstances(meta, list) } + + private fun hasOccurrence(occ: ConstraintOccurrence): Boolean = + anyInPath { mtn -> mtn.occurrence == occ } + + private inline fun anyInPath(predicate: (MatchTrieNode) -> Boolean): Boolean = + foldPath(false) { b, mtn -> b || predicate(mtn) } + + private inline fun forEachInPath(proc: (MatchTrieNode) -> Unit) = + foldPath(Unit) { b, mtn -> proc(mtn) } + + private inline fun foldPath(initVal: T, step: (T, MatchTrieNode) -> T): T { + var mtn: MatchTrieNode? = this + var currVal = initVal + while(mtn != null) { + currVal = step(currVal, mtn) + mtn = mtn.parent + } + return currVal + } + + } + + enum class Result { + DEFINITIVE, + INCONCLUSIVE, + NONE + } + +} +