Refactoring and reorganizing the matcher code, preparing for incremental matches search.

This commit is contained in:
Fedor Isakov 2017-07-11 12:27:45 +02:00
parent f3544af05a
commit f8970f7825
3 changed files with 388 additions and 319 deletions

View File

@ -1,312 +0,0 @@
/*
* Copyright 2014-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package jetbrains.mps.logic.reactor.core
import jetbrains.mps.logic.reactor.evaluation.ConstraintOccurrence
import jetbrains.mps.logic.reactor.logical.Logical
import jetbrains.mps.logic.reactor.logical.MetaLogical
import jetbrains.mps.logic.reactor.program.Constraint
import jetbrains.mps.logic.reactor.program.Rule
import jetbrains.mps.logic.reactor.util.Profiler
import jetbrains.mps.logic.reactor.util.profile
import jetbrains.mps.unification.Term
import java.util.*
import com.github.andrewoma.dexx.collection.Map as PersMap
/**
* @author Fedor Isakov
*/
internal class MatchTrie(val rule: Rule,
val activeOcc: ConstraintOccurrence,
val aux: OccurrenceIndex,
val profiler: Profiler?) : Iterable<PartialMatch>
{
val keptConstraints = rule.headKept().toCollection(ArrayList(4))
val discardedConstraints = rule.headReplaced().toCollection(ArrayList(4))
val relatedSlots = ArrayList<BitSet>()
init {
val meta2slots = HashMap<MetaLogical<*>, BitSet>()
(keptConstraints + discardedConstraints).forEachIndexed { slot, cst ->
cst.arguments().forEach { arg ->
if (arg is MetaLogical<*>) {
val bs = meta2slots[arg] ?: BitSet().apply { meta2slots[arg] = this }
bs.set(slot)
}
}
}
(keptConstraints + discardedConstraints).forEachIndexed { slot, cst ->
val bs = BitSet()
cst.arguments().forEach { arg ->
if (arg is MetaLogical<*>) {
bs.or(meta2slots[arg])
}
}
bs.clear(slot)
relatedSlots.add(bs)
}
}
private val firstRoot: MatchTrieNode? by lazy(LazyThreadSafetyMode.NONE) {
profiler.profile<MatchTrieNode?>("initMatchTrie") {
var first: MatchTrieNode? = null
var prev: MatchTrieNode? = null
for ((idx, cst) in keptConstraints.withIndex()) {
if (cst.symbol() == activeOcc.constraint().symbol()) {
val next = MatchTrieNode(null, true, cst, activeOcc, idx)
if (first == null) {
first = next
}
prev?._nextSibling = next
prev = next
}
}
for ((idx, cst) in discardedConstraints.withIndex()) {
if (cst.symbol() == activeOcc.constraint().symbol()) {
val next = MatchTrieNode(null, false, cst, activeOcc, idx)
if (first == null) {
first = next
}
prev?._nextSibling = next
prev = next
}
}
first
}
}
override fun iterator() = object : Iterator<PartialMatch> {
var next: PartialMatch? = null
override fun hasNext(): Boolean {
if (next == null) {
this.next = calcNext()
}
return (next != null)
}
override fun next(): PartialMatch {
if (next == null) {
this.next = calcNext()
}
if (next != null) {
val tmp: PartialMatch = next!!
this.next = calcNext()
return tmp
}
throw NoSuchElementException()
}
private fun calcNext(): PartialMatch? {
if (firstRoot == null) {
return null
}
var mtn: MatchTrieNode? = firstRoot
while (mtn != null) {
if (mtn.isSeen) {
if (mtn.nextSibling == null) {
mtn.parent?.isSeen = true
}
mtn = mtn.nextSibling ?: mtn.parent
}
else if (mtn.isLeaf) {
mtn.isSeen = true
return mtn.match
}
else {
if (mtn.firstChild == null) {
mtn.isSeen = true
}
mtn = mtn.firstChild ?: mtn
}
}
return null
}
}
private inner class MatchTrieNode(val parent: MatchTrieNode?,
val keepConstraint: Boolean,
val constraint: Constraint,
val occurrence: ConstraintOccurrence,
val index: Int)
{
var isSeen: Boolean = false
var _firstChild: MatchTrieNode? = null
val firstChild: MatchTrieNode?
get() {
if (_firstChild == null) {
var first: MatchTrieNode? = null
var prev: MatchTrieNode? = null
for (next in calcChildren()) {
if (first == null) {
first = next
}
prev?._nextSibling = next
prev = next
}
this._firstChild = first
}
return _firstChild
}
var _nextSibling: MatchTrieNode? = null
val nextSibling: MatchTrieNode?
get() = _nextSibling
val vacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) {
val result = BitSet()
result.set(0, keptConstraints.size + discardedConstraints.size)
foldNodes(result) { bits, mtn ->
val slot = if (mtn.keepConstraint) mtn.index else mtn.index + keptConstraints.size
bits.clear(slot)
bits
}
result
}
val relatedFirstVacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) {
val result = vacantSlots.clone() as BitSet
result.and(relatedSlots[if (keepConstraint) index else index + keptConstraints.size])
result
}
val isLeaf: Boolean
get() = vacantSlots.isEmpty
val match: PartialMatch by lazy(LazyThreadSafetyMode.NONE) {
val parentMatch = parent?.match ?: PartialMatch(rule, keptConstraints.size, discardedConstraints.size)
if (keepConstraint) parentMatch.keep(occurrence, index)
else parentMatch.discard(occurrence, index)
}
private fun calcChildren(): Iterable<MatchTrieNode> {
val nextSlots = if (!relatedFirstVacantSlots.isEmpty) relatedFirstVacantSlots else vacantSlots
val nextSlot = nextSlots.nextSetBit(0)
if (nextSlot < 0) {
return emptyList()
}
val keep = nextSlot < keptConstraints.size
val nextIdx = if (keep) nextSlot else nextSlot - keptConstraints.size
val nextCst = if (keep) keptConstraints[nextSlot] else discardedConstraints[nextIdx]
val (result, auxOccurrences) = lookupAuxOccurrences(nextCst)
when (result) {
Result.DEFINITIVE -> parent
Result.NONE -> this
Result.INCONCLUSIVE -> null
}?.forEachNode { mtn -> mtn.isSeen = true }
return auxOccurrences.map { auxOcc -> MatchTrieNode(this, keep, nextCst, auxOcc, nextIdx) }
}
private fun collectMetaInstances(meta: Collection<*>, list: ArrayList<Any>): ArrayList<Any> {
constraint.arguments().zip(occurrence.arguments()).forEach { p ->
if (p.first is MetaLogical<*> && meta.contains(p.first) && !list.contains(p.second)) {
list.add(p.second!!)
}
}
return list
}
private fun metaInstances(meta: Collection<*>): List<Any> =
foldNodes(ArrayList<Any>(4)) { list, mtn -> mtn.collectMetaInstances(meta, list) }
// the first component indicates the quality of the result:
// DEFINITIVE indicates the only possible matches were returned
// INCONCLUSIVE may yield some (ir-)relevant results
// NONE means no results could have been found at all, so it's useless to attempt again with the same input
private fun lookupAuxOccurrences(cst: Constraint): Pair<Result, Iterable<ConstraintOccurrence>> {
return profiler.profile<Pair<Result, Iterable<ConstraintOccurrence>>> ("lookupAux_${cst.symbol()}") {
val fromArgs = ArrayList<ConstraintOccurrence>(4)
for (arg in metaInstances(cst.arguments()) + cst.arguments().filter { arg -> !(arg is MetaLogical<*>) }) {
when (arg) {
is Logical<*> -> fromArgs.addAll(aux.forLogical(arg))
is Term -> fromArgs.addAll(aux.forTermAndConstraint(arg, cst))
is Any -> fromArgs.addAll(aux.forValue(arg))
}
}
if (fromArgs.isNotEmpty()) {
Result.DEFINITIVE.to(fromArgs.filter { occ -> occ.constraint().symbol() == cst.symbol() && !hasOccurrence(occ) })
} else {
return profiler.profile<Pair<Result, Iterable<ConstraintOccurrence>>> ("lookupAux_default_${cst.symbol()}") {
val hasRelated = relatedFirstVacantSlots == vacantSlots
val allArgsMetaLogicals = cst.arguments().all { a -> a is MetaLogical<*> }
val noArgs = cst.arguments().isEmpty()
if (noArgs || (allArgsMetaLogicals && !hasRelated)) {
Result.INCONCLUSIVE.to(aux.forSymbol(cst.symbol()).filter { occ -> !hasOccurrence(occ) })
}
else {
// all candidate occurrences should have been found by this time; if not, then there's none
Result.NONE.to(emptyList())
}
}
}
}
}
private fun hasOccurrence(occ: ConstraintOccurrence): Boolean =
anyNode { mtn -> mtn.occurrence == occ }
private inline fun anyNode(predicate: (MatchTrieNode) -> Boolean): Boolean =
foldNodes(false) { b, mtn -> b || predicate(mtn) }
private inline fun forEachNode(proc: (MatchTrieNode) -> Unit) =
foldNodes(Unit) { b, mtn -> proc(mtn) }
private inline fun <T> foldNodes(start: T, step: (T, MatchTrieNode) -> T): T {
var mtn: MatchTrieNode? = this
var curr = start
while(mtn != null) {
curr = step(curr, mtn)
mtn = mtn.parent
}
return curr
}
}
enum class Result {
DEFINITIVE,
INCONCLUSIVE,
NONE
}
}

View File

@ -27,12 +27,9 @@ import jetbrains.mps.logic.reactor.program.Predicate
import jetbrains.mps.logic.reactor.program.Rule
import jetbrains.mps.logic.reactor.util.LazyIterable
import jetbrains.mps.logic.reactor.util.Profiler
import jetbrains.mps.logic.reactor.util.profile
import jetbrains.mps.unification.Substitution
import jetbrains.mps.unification.Term
import jetbrains.mps.unification.TermWrapper
import jetbrains.mps.unification.Unification
import org.jetbrains.kotlin.container.topologicalSort
import java.util.*
import com.github.andrewoma.dexx.collection.List as PersList
@ -46,14 +43,14 @@ class Matcher(val ruleIndex: RuleIndex,
val profiler: Profiler? = null) : Iterable<Match>
{
private val matchTries: LazyIterable<Rule, MatchTrie> by lazy(LazyThreadSafetyMode.NONE) {
LazyIterable<Rule, MatchTrie> (ruleIndex.forOccurrence(activeOcc).toList()) { rule ->
MatchTrie(rule, activeOcc, aux, profiler)
private val matchTries: LazyIterable<Rule, PartialMatchTrie> by lazy(LazyThreadSafetyMode.NONE) {
LazyIterable<Rule, PartialMatchTrie> (ruleIndex.forOccurrence(activeOcc).toList()) { rule ->
PartialMatchTrie(rule, activeOcc, aux, profiler)
}
}
override fun iterator() = matchTries.asSequence().flatMap { matchTrie ->
matchTrie.asSequence() }.map { partialMatch ->
matchTrie.matches().asSequence() }.map { partialMatch ->
partialMatch.complete(profiler) }.filter { match ->
match.successful }.iterator()

View File

@ -0,0 +1,384 @@
/*
* Copyright 2014-2017 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package jetbrains.mps.logic.reactor.core
import jetbrains.mps.logic.reactor.evaluation.ConstraintOccurrence
import jetbrains.mps.logic.reactor.logical.Logical
import jetbrains.mps.logic.reactor.logical.MetaLogical
import jetbrains.mps.logic.reactor.program.Constraint
import jetbrains.mps.logic.reactor.program.Rule
import jetbrains.mps.logic.reactor.util.Profiler
import jetbrains.mps.logic.reactor.util.profile
import jetbrains.mps.unification.Term
import java.util.*
import kotlin.collections.ArrayList
import com.github.andrewoma.dexx.collection.Map as PersMap
/**
* @author Fedor Isakov
*/
internal class PartialMatchTrie(val rule: Rule,
val activeOcc: ConstraintOccurrence,
val occIndex: OccurrenceIndex,
val profiler: Profiler?)
{
constructor(prototype: PartialMatchTrie,
activeOcc: ConstraintOccurrence,
occurrenceIndex: OccurrenceIndex,
profiler: Profiler?) :
this(prototype.rule, activeOcc, occurrenceIndex, profiler)
{
this._prototype = prototype
}
private var _prototype: PartialMatchTrie? = null
private val prototype: PartialMatchTrie?
get() = _prototype
private val keptConstraints = rule.headKept().toCollection(ArrayList(4))
private val discardedConstraints = rule.headReplaced().toCollection(ArrayList(4))
private val root: RootMatchTrieNode by lazy(LazyThreadSafetyMode.NONE) {
prototype?.root ?: RootMatchTrieNode()
}
// for each slot keep a set of "related" slots, that is the slots with matching (meta-)logicals
// this map is built only once per rule
private val relatedSlots: List<BitSet> by lazy(LazyThreadSafetyMode.NONE) {
prototype?.relatedSlots ?: HashMap<MetaLogical<*>, BitSet>().let { meta2slots ->
(keptConstraints + discardedConstraints).forEachIndexed { slot, cst ->
cst.arguments().forEach { arg ->
if (arg is MetaLogical<*>) {
val bs = meta2slots[arg] ?: BitSet().apply { meta2slots[arg] = this }
bs.set(slot)
}
}
}
ArrayList<BitSet>().apply {
(keptConstraints + discardedConstraints).forEachIndexed { slot, cst ->
val bs = BitSet()
cst.arguments().forEach { arg ->
if (arg is MetaLogical<*>) {
bs.or(meta2slots[arg])
}
}
bs.clear(slot)
add(bs)
}
}
}
}
fun matches(): Iterable<PartialMatch> = object : Iterable<PartialMatch> {
override fun iterator() = object : Iterator<PartialMatch> {
val firstRoot: MatchTrieNode? = root.firstChild
var next: PartialMatch? = null
override fun hasNext(): Boolean {
if (next == null) {
this.next = calcNext()
}
return (next != null)
}
override fun next(): PartialMatch {
if (next == null) {
this.next = calcNext()
}
if (next != null) {
val tmp: PartialMatch = next!!
this.next = calcNext()
return tmp
}
throw NoSuchElementException()
}
private fun calcNext(): PartialMatch? {
// the search always starts at the first root
if (firstRoot == null) {
return null
}
var mtn: MatchTrieNode? = firstRoot
while (mtn != null) {
if (mtn.isSeen) {
if (mtn.nextSibling == null) {
mtn.parent?.setSeen()
}
mtn = mtn.nextSibling ?: mtn.parent
}
else if (mtn.isLeaf) {
mtn.setSeen()
return mtn.match
}
else {
// calculate the children
if (mtn.firstChild == null) {
mtn.setSeen()
}
mtn = mtn.firstChild ?: mtn
}
}
return null
}
}
}
private abstract inner class BaseMatchTrieNode() {
var _firstChild: MatchTrieNode? = null
var _nextSibling: MatchTrieNode? = null
abstract fun reset()
}
private inner class RootMatchTrieNode() : BaseMatchTrieNode() {
val firstChild: MatchTrieNode?
get() {
if (_firstChild == null) {
var first: MatchTrieNode? = null
var prev: MatchTrieNode? = null
for ((idx, cst) in keptConstraints.withIndex()) {
if (cst.symbol() == activeOcc.constraint().symbol()) {
val next = MatchTrieNode(null, true, cst, activeOcc, idx)
if (first == null) {
first = next
}
prev?._nextSibling = next
prev = next
}
}
for ((idx, cst) in discardedConstraints.withIndex()) {
if (cst.symbol() == activeOcc.constraint().symbol()) {
val next = MatchTrieNode(null, false, cst, activeOcc, idx)
if (first == null) {
first = next
}
prev?._nextSibling = next
prev = next
}
}
this._firstChild = first
}
return _firstChild
}
override fun reset() {
while (!(_firstChild?.occurrence?.isAlive() ?: true)) {
this._firstChild = _firstChild?._nextSibling
}
_firstChild?.reset()
}
}
private inner class MatchTrieNode(val parent: MatchTrieNode?,
val keepConstraint: Boolean,
val constraint: Constraint,
val occurrence: ConstraintOccurrence,
val index: Int) : BaseMatchTrieNode()
{
private var _isSeen: Boolean = false
val isSeen: Boolean
get() = _isSeen
val vacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) {
val result = BitSet()
result.set(0, keptConstraints.size + discardedConstraints.size)
foldPath(result) { bits, mtn ->
val slot = if (mtn.keepConstraint) mtn.index else mtn.index + keptConstraints.size
bits.clear(slot)
bits
}
result
}
val relatedFirstVacantSlots: BitSet by lazy(LazyThreadSafetyMode.NONE) {
val result = vacantSlots.clone() as BitSet
result.and(relatedSlots[if (keepConstraint) index else index + keptConstraints.size])
result
}
val isLeaf: Boolean
get() = vacantSlots.isEmpty
val match: PartialMatch by lazy(LazyThreadSafetyMode.NONE) {
val parentMatch = parent?.match ?: PartialMatch(rule, keptConstraints.size, discardedConstraints.size)
if (keepConstraint) parentMatch.keep(occurrence, index)
else parentMatch.discard(occurrence, index)
}
fun setSeen() {
this._isSeen = true
}
val firstChild: MatchTrieNode?
get() {
if (_firstChild == null) {
var first: MatchTrieNode? = null
var prev: MatchTrieNode? = null
for (next in calcChildren()) {
if (first == null) {
first = next
}
prev?._nextSibling = next
prev = next
}
this._firstChild = first
}
return _firstChild
}
val nextSibling: MatchTrieNode?
get() = _nextSibling
private fun calcChildren(): Iterable<MatchTrieNode> {
val nextSlots = if (!relatedFirstVacantSlots.isEmpty) relatedFirstVacantSlots else vacantSlots
val nextSlot = nextSlots.nextSetBit(0)
if (nextSlot < 0) {
return emptyList()
}
val keep = nextSlot < keptConstraints.size
val nextIdx = if (keep) nextSlot else nextSlot - keptConstraints.size
val nextCst = if (keep) keptConstraints[nextSlot] else discardedConstraints[nextIdx]
val (result, auxOccurrences) = lookupAuxOccurrences(nextCst)
when (result) {
Result.DEFINITIVE -> parent
Result.NONE -> this
Result.INCONCLUSIVE -> null
}?.forEachInPath { mtn -> mtn._isSeen = true }
return auxOccurrences.map { auxOcc -> MatchTrieNode(this, keep, nextCst, auxOcc, nextIdx) }
}
// the first component indicates the quality of the result:
// DEFINITIVE indicates the only possible matches were returned
// INCONCLUSIVE may yield some (ir-)relevant results
// NONE means no results could have been found at all, so it's useless to attempt again with the same input
private fun lookupAuxOccurrences(cst: Constraint): Pair<Result, Iterable<ConstraintOccurrence>> {
return profiler.profile<Pair<Result, Iterable<ConstraintOccurrence>>> ("lookupAux_${cst.symbol()}") {
val fromArgs = ArrayList<ConstraintOccurrence>(4)
for (arg in metaInstances(cst.arguments()) + cst.arguments().filter { arg -> !(arg is MetaLogical<*>) }) {
when (arg) {
is Logical<*> -> fromArgs.addAll(occIndex.forLogical(arg))
is Term -> fromArgs.addAll(occIndex.forTermAndConstraint(arg, cst))
is Any -> fromArgs.addAll(occIndex.forValue(arg))
}
}
if (fromArgs.isNotEmpty()) {
Result.DEFINITIVE.to(fromArgs.filter { occ -> occ.constraint().symbol() == cst.symbol() && !hasOccurrence(occ) })
} else {
return profiler.profile<Pair<Result, Iterable<ConstraintOccurrence>>> ("lookupAux_default_${cst.symbol()}") {
val hasRelated = relatedFirstVacantSlots == vacantSlots
val allArgsMetaLogicals = cst.arguments().all { a -> a is MetaLogical<*> }
val noArgs = cst.arguments().isEmpty()
if (noArgs || (allArgsMetaLogicals && !hasRelated)) {
Result.INCONCLUSIVE.to(occIndex.forSymbol(cst.symbol()).filter { occ -> !hasOccurrence(occ) })
}
else {
// all candidate occurrences should have been found by this time; if not, then there's none
Result.NONE.to(emptyList())
}
}
}
}
}
// prepare the node for re-use
override fun reset() {
this._isSeen = false
while (!(_firstChild?.occurrence?.isAlive() ?: true)) {
this._firstChild = _firstChild?._nextSibling
}
_firstChild?.reset()
while(!(_nextSibling?.occurrence?.isAlive() ?: true)) {
this._nextSibling = _nextSibling?._nextSibling
}
_nextSibling?.reset()
}
private fun collectMetaInstances(meta: Collection<*>, list: ArrayList<Any>): ArrayList<Any> {
constraint.arguments().zip(occurrence.arguments()).forEach { p ->
if (p.first is MetaLogical<*> && meta.contains(p.first) && !list.contains(p.second)) {
list.add(p.second!!)
}
}
return list
}
private fun metaInstances(meta: Collection<*>): List<Any> =
foldPath(ArrayList<Any>(4)) { list, mtn -> mtn.collectMetaInstances(meta, list) }
private fun hasOccurrence(occ: ConstraintOccurrence): Boolean =
anyInPath { mtn -> mtn.occurrence == occ }
private inline fun anyInPath(predicate: (MatchTrieNode) -> Boolean): Boolean =
foldPath(false) { b, mtn -> b || predicate(mtn) }
private inline fun forEachInPath(proc: (MatchTrieNode) -> Unit) =
foldPath(Unit) { b, mtn -> proc(mtn) }
private inline fun <T> foldPath(initVal: T, step: (T, MatchTrieNode) -> T): T {
var mtn: MatchTrieNode? = this
var currVal = initVal
while(mtn != null) {
currVal = step(currVal, mtn)
mtn = mtn.parent
}
return currVal
}
}
enum class Result {
DEFINITIVE,
INCONCLUSIVE,
NONE
}
}