natas_hw
/
mps-coderules
mirror of https://github.com/JetBrains/mps-coderules
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make RuleIndex incrementally updatable. Drop segments feature.

This commit is contained in:
Fedor Isakov 2020-08-04 09:11:30 +02:00
parent a003211a2c
commit eec84330fc
5 changed files with 273 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

165
reactor/Core/src/jetbrains/mps/logic/reactor/core/RuleIndex.kt
View File

@ -16,6 +16,8 @@
package jetbrains.mps.logic.reactor.core
import gnu.trove.TIntObjectHashMap
import gnu.trove.TObjectIntHashMap
import jetbrains.mps.logic.reactor.evaluation.ConstraintOccurrence
import jetbrains.mps.logic.reactor.logical.Logical
import jetbrains.mps.logic.reactor.logical.MetaLogical
@ -33,6 +35,17 @@ import kotlin.collections.HashMap
*/
class RuleIndex(ruleLists: Iterable<RulesList>) : Iterable<Rule>, RuleLookup {
private class IndexedRule {
constructor(idx: Int, rule: Rule) {
this.idx = idx
this.rule = rule
}
var idx: Int
val rule: Rule
}
// Terminology:
// ruleBit - rule's index in the rules list
// headPos - constraint's position in the rule's head (all slots together)
@ -42,21 +55,19 @@ class RuleIndex(ruleLists: Iterable<RulesList>) : Iterable<Rule>, RuleLookup {
private val tag2rule = LinkedHashMap<Any, Rule>()
// TODO replace with a trie?
private val segmentPath2ruleBits = HashMap<List<Any>, BitSet>()
private val tag2bit = TObjectIntHashMap<Any>()
// rule's index is rule's position in this list
private val rulesList = ArrayList<Rule>()
var nextBit: Int = 0
// every rule has a slot mask
// invariant: rulesList.size == slotMasksList.size
private val slotMasksList = ArrayList<SlotMask>()
private val bit2ruleAndMask = TIntObjectHashMap<Pair<IndexedRule, SlotMask>>()
private val allRules = LinkedList<IndexedRule>()
/** the bit set that is going to be reused for all calls to [select] */
private val ruleBits = BitSet(rulesList.size)
private val ruleBits = BitSet(allRules.size)
/** the bit set to be used for temporary processing within [select]*/
private val andRuleIndices = BitSet(rulesList.size)
private val andRuleIndices = BitSet(allRules.size)
init {
buildIndex(ruleLists)
@ -64,31 +75,18 @@ class RuleIndex(ruleLists: Iterable<RulesList>) : Iterable<Rule>, RuleLookup {
override fun lookupRuleByTag(tag: Any): Rule? = tag2rule[tag]
private fun List<Any>.isPrefixOf(that: List<Any>): Boolean {
val thisIt = this@isPrefixOf.iterator()
val thatIt = that.iterator()
while(thisIt.hasNext() && thatIt.hasNext()) {
if (thisIt.next() != thatIt.next()) return false
}
return thisIt.hasNext() == thatIt.hasNext() || thatIt.hasNext()
}
/**
* Returns instances of [Rule] that can potentially match the specified [ConstraintOccurrence].
*/
fun forOccurrence(occ: ConstraintOccurrence): Iterable<Rule> {
val (ruleBits, slotMasks) = symbol2index[occ.constraint().symbol()]?.select(occ) ?: return emptyList()
tag2rule[occ.ruleUniqueTag()]?.segmentPath()?.let {
andRuleIndices.clear()
for ((path, bits) in segmentPath2ruleBits.entries) {
if (!it.isPrefixOf(path)) andRuleIndices.or(bits)
}
ruleBits.andNot(andRuleIndices)
}
val result = ArrayList<Rule>()
val result = ArrayList<IndexedRule>()
val it = ruleBits.allSetBits()
while (it.hasNext()) result.add(rulesList[it.next()])
return result
while (it.hasNext()) bit2ruleAndMask[it.next()]?.let{
result.add(it.first)
}
result.sortBy { it.idx }
return result.map { it.rule }
}
/**
@ -96,56 +94,91 @@ class RuleIndex(ruleLists: Iterable<RulesList>) : Iterable<Rule>, RuleLookup {
*/
fun forOccurrenceWithMask(occ: ConstraintOccurrence): Iterable<Pair<Rule, BitSet>> {
val (ruleBits, argSlotMasks) = symbol2index[occ.constraint().symbol()]?.select(occ) ?: return emptyList()
tag2rule[occ.ruleUniqueTag()]?.segmentPath()?.let {
andRuleIndices.clear()
for ((path, bits) in segmentPath2ruleBits.entries) {
if (!it.isPrefixOf(path)) andRuleIndices.or(bits)
}
ruleBits.andNot(andRuleIndices)
}
val result = ArrayList<Pair<Rule, BitSet>>()
val result = ArrayList<Pair<IndexedRule, BitSet>>()
val it = ruleBits.allSetBits()
while (it.hasNext()) {
val ruleBit = it.next()
slotMasksList[ruleBit][occ]?.let { mask ->
val effMask = argSlotMasks[ruleBit]?.copyApply { and(mask) } ?: mask
result.add(rulesList[ruleBit] to effMask)
bit2ruleAndMask[ruleBit]?.let {(irule, slotMask) ->
slotMask[occ]?.let { mask ->
val effMask = argSlotMasks[ruleBit]?.copyApply { and(mask) } ?: mask
result.add(irule to effMask)
}
}
// slotMasksList[ruleBit][occ]?.let { mask -> result.add(rulesList[ruleBit] to mask) }
}
return result
result.sortBy { it.first.idx }
return result.map { it.first.rule to it.second }
}
override fun iterator(): Iterator<Rule> = rulesList.iterator()
override fun iterator(): Iterator<Rule> = allRules.map { it.rule }.iterator()
fun updateIndex(ruleLists: Iterable<RulesList>, removedTags: Set<Any>) {
val allRulesIt = allRules.listIterator()
var nextIdx = 0
ruleLists.flatMap { it.rules() }.forEach { rule ->
var skip = false
while (allRulesIt.hasNext()) {
val irule = allRulesIt.next()
if (removedTags.contains(irule.rule.uniqueTag())) {
removeRuleFromIndex(irule)
allRulesIt.remove()
} else if (!irule.rule.uniqueTag().equals(rule.uniqueTag())) {
allRulesIt.previous()
break
} else {
irule.idx = nextIdx
skip = true
break
}
}
if (!skip) {
IndexedRule(nextIdx, rule).also {
addRuleToIndex(it)
allRulesIt.add(it)
}
}
nextIdx++
}
while (allRulesIt.hasNext()) {
val irule = allRulesIt.next()
removeRuleFromIndex(irule)
allRulesIt.remove()
}
}
private fun buildIndex(ruleLists: Iterable<RulesList>) {
var ruleBit = 0
for (h in ruleLists) {
for (rule in h.rules()) {
if (tag2rule.containsKey(rule.uniqueTag())) throw IllegalStateException("duplicate rule tag ${rule.uniqueTag()}")
tag2rule[rule.uniqueTag()] = rule
rule.segmentPath()?.let { segmentPath ->
if (segmentPath.isNotEmpty()) {
segmentPath2ruleBits.getOrPut(segmentPath) { BitSet() }.set(ruleBit)
}
}
rulesList.add(rule)
val head = rule.headKept() + rule.headReplaced()
val slotMask = SlotMask()
for ((headPos, cst) in head.withIndex()) {
symbol2index.getOrPut(cst.symbol()) { ArgumentRuleIndex(cst.symbol()) }.update(cst, ruleBit, headPos)
slotMask.update(cst, headPos)
}
slotMasksList.add(slotMask)
ruleBit += 1
}
ruleLists.flatMap { it.rules() }.forEachIndexed { idx, rule ->
val irule = IndexedRule(idx, rule)
addRuleToIndex(irule)
allRules.add(irule)
}
}
private fun removeRuleFromIndex(irule: IndexedRule) {
val uniqueTag = irule.rule.uniqueTag()
tag2rule.remove(uniqueTag)
val bit = tag2bit.remove(uniqueTag)
bit2ruleAndMask.remove(bit)
}
private fun addRuleToIndex(irule: IndexedRule) {
val uniqueTag = irule.rule.uniqueTag()
if (tag2rule.containsKey(uniqueTag)) throw IllegalStateException("duplicate rule tag $uniqueTag")
val head = irule.rule.headKept() + irule.rule.headReplaced()
val slotMask = SlotMask()
for ((headPos, cst) in head.withIndex()) {
symbol2index.getOrPut(cst.symbol()) { ArgumentRuleIndex(cst.symbol()) }.update(cst, nextBit, headPos)
slotMask.update(cst, headPos)
}
tag2rule[uniqueTag] = irule.rule
tag2bit.put(uniqueTag, nextBit)
bit2ruleAndMask.put(nextBit, irule to slotMask)
nextBit += 1
}
/**
* Represents a mask associated with a single rule.
* The mask tells whether or not a particular constraint occurrence can match

8
reactor/Test/test/RulesHelper.kt
View File

@ -131,14 +131,6 @@ fun rule(tag: String, vararg component: RuleBuilder.() -> Unit): () -> Rule = {
rb.toRule()
}
fun rule(tag: String, segmentPath: List<Any>, vararg component: RuleBuilder.() -> Unit): () -> Rule = {
val rb = RuleBuilder(tag, segmentPath)
for (cmp in component) {
rb.cmp()
}
rb.toRule()
}
fun headKept(vararg content: ConjBuilder.() -> Unit): RuleBuilder.() -> Unit = {
appendHeadKept(* buildConjunction(Constraint::class.java, content).toArray())
}

47
reactor/Test/test/TestProgram.kt
View File

@ -67,53 +67,6 @@ class TestProgram {
}
}
@Test
fun segmented() {
programWithRules(
rule("main.foo",
headReplaced(
constraint("main")
),
body(
constraint("foo")
)),
rule("foo.bar", listOf("segment1"),
headKept(
constraint("foo")
),
body(
constraint("bar")
)
),
rule("bar.qux", listOf("segment1"),
headKept(
constraint("bar")
),
body(
constraint("qux")
)
),
rule("bar.dux", listOf("segment2"),
headKept(
constraint("bar")
),
body(
constraint("dux")
)
),
rule("bar.doh",
headKept(
constraint("bar")
),
body(
constraint("doh")
)
)
).session("segmented").run {
constraintSymbols().map { it.id() }.toSet() shouldBe setOf("foo", "bar", "qux", "doh")
}
}
@Test
fun logicalValue() {
val (X, Y, Z) = metaLogical<Int>("X", "Y", "Z")

241
reactor/Test/test/TestRuleIndex.kt
View File

@ -15,16 +15,8 @@
*/
import jetbrains.mps.logic.reactor.core.RuleIndex
import jetbrains.mps.logic.reactor.core.internal.logical
import jetbrains.mps.logic.reactor.util.bitSet
import jetbrains.mps.unification.Term
import jetbrains.mps.unification.test.MockTerm
import jetbrains.mps.unification.test.MockTerm.*
import jetbrains.mps.unification.test.MockTermsParser
import jetbrains.mps.unification.test.MockTermsParser.*
import org.jetbrains.kotlin.js.parser.parse
import jetbrains.mps.unification.test.MockTermsParser.parseTerm
import org.junit.Test
import java.util.*
/*
* Copyright 2014-2019 JetBrains s.r.o.
@ -49,74 +41,189 @@ import java.util.*
class TestRuleIndex {
@Test
fun testProgramWithSegments() {
fun testUpdateRuleIndexReplaceSingle() {
with(programWithRules(
rule("rule0", emptyList(),
headReplaced(
constraint("main")
),
body(
constraint("bar")
)),
rule("rule1", listOf("segment1"),
headReplaced(
constraint("foo", parseTerm("f{g h}"))
),
body(
constraint("qux")
)),
rule("rule2", listOf("segment2"),
headReplaced(
constraint("foo", parseTerm("f{g h}"))
),
body(
constraint("blah")
)),
rule("rule3", listOf("segment1"),
headReplaced(
constraint("blin")
),
body(
constraint("foo", parseTerm("f{g h}"))
)),
rule("rule4", listOf("segment2"),
headReplaced(
constraint("fooblin")
),
body(
constraint("foo", parseTerm("f{g h}"))
))
rule("rule0",
headReplaced(constraint("foo")), body())
) to programWithRules(
rule("rule1",
headReplaced(constraint("bar")), body())
))
{
val ruleIndex = RuleIndex(rulesLists)
// no segment
with (ruleIndex.forOccurrence(occurrence("main"))) {
map { it.tag() }.toSet() shouldBe setOf("rule0")
val ruleIndex = RuleIndex(first.rulesLists)
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule0")
}
with (ruleIndex.forOccurrence(occurrence("foo", parseTerm("f{g h}")))) {
map { it.tag() }.toSet() shouldBe setOf("rule1", "rule2")
with (ruleIndex.forOccurrence(occurrence("bar"))) {
iterator().hasNext() shouldBe false
}
// segment1
with (ruleIndex.forOccurrence(taggedOccurrence("rule0", "foo", parseTerm("f{g h}")))) {
map { it.tag() }.toSet() shouldBe setOf("rule1", "rule2")
ruleIndex.updateIndex(second.rulesLists, setOf("rule0"))
with (ruleIndex.forOccurrence(occurrence("foo"))) {
iterator().hasNext() shouldBe false
}
with (ruleIndex.forOccurrence(taggedOccurrence("rule0", "main"))) {
map { it.tag() }.toSet() shouldBe setOf("rule0")
}
with (ruleIndex.forOccurrence(taggedOccurrence("rule3", "foo", parseTerm("f{g h}")))) {
map { it.tag() }.toSet() shouldBe setOf("rule1")
}
with (ruleIndex.forOccurrence(taggedOccurrence("rule3", "main"))) {
map { it.tag() }.toSet() shouldBe setOf("rule0")
}
// segment2
with (ruleIndex.forOccurrence(taggedOccurrence("rule4", "foo", parseTerm("f{g h}")))) {
map { it.tag() }.toSet() shouldBe setOf("rule2")
with (ruleIndex.forOccurrence(occurrence("bar"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1")
}
}
}
@Test
fun testUpdateRuleIndexAddAtEnds() {
with(programWithRules(
rule("rule0",
headReplaced(constraint("foo")), body())
) to programWithRules(
rule("rule1",
headReplaced(constraint("foo"), constraint("bar")), body()),
rule("rule0",
headReplaced(constraint("foo")), body()),
rule("rule2",
headReplaced(constraint("bar")), body())
))
{
val ruleIndex = RuleIndex(first.rulesLists)
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule0")
}
ruleIndex.updateIndex(second.rulesLists, setOf())
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1","rule0")
}
with (ruleIndex.forOccurrence(occurrence("bar"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1","rule2")
}
}
}
@Test
fun testUpdateRuleIndexRemoveInTheMiddle() {
with(programWithRules(
rule("rule1",
headReplaced(constraint("foo"), constraint("bar")), body()),
rule("rule0",
headReplaced(constraint("foo")), body()),
rule("rule2",
headReplaced(constraint("bar")), body())
) to programWithRules(
rule("rule1",
headReplaced(constraint("foo"), constraint("bar")), body()),
rule("rule2",
headReplaced(constraint("bar")), body())
))
{
val ruleIndex = RuleIndex(first.rulesLists)
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1","rule0")
}
with (ruleIndex.forOccurrence(occurrence("bar"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1","rule2")
}
ruleIndex.updateIndex(second.rulesLists, setOf("rule0"))
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1")
}
}
}
@Test
fun testUpdateRuleIndexRemoveFromEnds() {
with(programWithRules(
rule("rule1",
headReplaced(constraint("foo"), constraint("bar")), body()),
rule("rule0",
headReplaced(constraint("foo")), body()),
rule("rule2",
headReplaced(constraint("bar")), body())
) to programWithRules(
rule("rule0",
headReplaced(constraint("foo")), body())
))
{
val ruleIndex = RuleIndex(first.rulesLists)
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1","rule0")
}
with (ruleIndex.forOccurrence(occurrence("bar"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1","rule2")
}
ruleIndex.updateIndex(second.rulesLists, setOf("rule1", "rule2"))
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule0")
}
with (ruleIndex.forOccurrence(occurrence("bar"))) {
iterator().hasNext() shouldBe false
}
}
}
@Test
fun testUpdateRuleIndex() {
with(programWithRules(
rule("rule0",
headReplaced(constraint("foo")), body()),
rule("rule3",
headReplaced(constraint("foo"), constraint("bar")), body()),
rule("rule1",
headReplaced(constraint("baz")), body()),
rule("rule4",
headReplaced(constraint("qux")), body())
) to programWithRules(
rule("rule5",
headReplaced(constraint("foo"), constraint("foo")), body()),
rule("rule0",
headReplaced(constraint("foo")), body()),
rule("rule2",
headReplaced(constraint("bar")), body()),
rule("rule4",
headReplaced(constraint("qux")), body()),
rule("rule6",
headReplaced(constraint("qux")), body())
))
{
val ruleIndex = RuleIndex(first.rulesLists)
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule0", "rule3")
}
with (ruleIndex.forOccurrence(occurrence("bar"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule3")
}
with (ruleIndex.forOccurrence(occurrence("baz"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule1")
}
with (ruleIndex.forOccurrence(occurrence("qux"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule4")
}
ruleIndex.updateIndex(second.rulesLists, setOf("rule1", "rule3"))
with (ruleIndex.forOccurrence(occurrence("foo"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule5", "rule0")
}
with (ruleIndex.forOccurrence(occurrence("bar"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule2")
}
with (ruleIndex.forOccurrence(occurrence("baz"))) {
iterator().hasNext() shouldBe false
}
with (ruleIndex.forOccurrence(occurrence("qux"))) {
map { it.uniqueTag() }.toList() shouldBe listOf("rule4", "rule6")
}
}
}
}

76
reactor/Test/test/TestRuleMatcher.kt
View File

@ -927,82 +927,6 @@ class TestRuleMatcher {
}
}
@Test
fun testDispatcherWithSegment1() {
with(programWithRules(
rule("rule0", emptyList(),
headReplaced(
constraint("foo")
),
body(
constraint("bar")
)),
rule("rule1", listOf("segment1"),
headReplaced(
constraint("fooblin")
),
body(
constraint("doh")
))
))
{
with(Dispatcher(RuleIndex(rulesLists)).front()) {
expand(occurrence("foo")) }.apply {
matches().count() shouldBe 1 }.run {
}
with(Dispatcher(RuleIndex(rulesLists)).front()) {
expand(taggedOccurrence("rule1", "foo")) }.apply {
matches().count() shouldBe 1 }.run {
}
}
}
@Test
fun testExpandWithSegment2() {
with(programWithRules(
rule("rule1", listOf("segment1"),
headReplaced(
constraint("foo")
),
body(
constraint("bar")
)),
rule("rule2", listOf("segment1"),
headReplaced(
constraint("fooblin")
),
body(
constraint("doh")
)),
rule("rule3", listOf("segment2"),
headReplaced(
constraint("fooblin")
),
body(
constraint("doh")
))
))
{
with(Dispatcher(RuleIndex(rulesLists)).front()) {
expand(occurrence("foo")) }.apply {
matches().count() shouldBe 1 }.run {
}
with(Dispatcher(RuleIndex(rulesLists)).front()) {
expand(taggedOccurrence("rule2", "foo")) }.apply {
matches().count() shouldBe 1 }.run {
}
with(Dispatcher(RuleIndex(rulesLists)).front()) {
expand(taggedOccurrence("rule3", "foo")) }.apply {
matches().count() shouldBe 0 }.run {
}
}
}
@Test
fun testDispatcherIncremental() {
with(programWithRules(