Initial Commit

src/graph_memstore/llrb-iterator.go

@@ -0,0 +1,149 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_memstore
+
+import (
+	"fmt"
+	"github.com/petar/GoLLRB/llrb"
+	"graph"
+	"math"
+	"strings"
+)
+
+type LlrbIterator struct {
+	graph.BaseIterator
+	tree      *llrb.LLRB
+	values    chan llrb.Item
+	another   chan bool
+	data      string
+	isRunning bool
+}
+
+type Int64 int64
+
+func (i Int64) Less(than llrb.Item) bool {
+	return i < than.(Int64)
+}
+
+func IterateAll(tree *llrb.LLRB, c chan llrb.Item, another chan bool) {
+	tree.AscendGreaterOrEqual(Int64(-1), func(i llrb.Item) bool {
+		want_more := <-another
+		if want_more {
+			c <- i
+			return true
+		}
+		return false
+	})
+}
+
+func NewLlrbIterator(tree *llrb.LLRB, data string) *LlrbIterator {
+	var it LlrbIterator
+	graph.BaseIteratorInit(&it.BaseIterator)
+	it.tree = tree
+	it.isRunning = false
+	it.values = make(chan llrb.Item)
+	it.another = make(chan bool, 1)
+	it.data = data
+	return &it
+}
+
+func (it *LlrbIterator) Reset() {
+	if it.another != nil {
+		it.another <- false
+		close(it.another)
+	}
+	it.another = nil
+	if it.values != nil {
+		close(it.values)
+	}
+	it.values = nil
+	it.isRunning = false
+	it.another = make(chan bool)
+	it.values = make(chan llrb.Item)
+}
+
+func (it *LlrbIterator) Clone() graph.Iterator {
+	var new_it = NewLlrbIterator(it.tree, it.data)
+	new_it.CopyTagsFrom(it)
+	return new_it
+}
+
+func (it *LlrbIterator) Close() {
+	if it.another != nil {
+		it.another <- false
+		close(it.another)
+	}
+	it.another = nil
+	if it.values != nil {
+		close(it.values)
+	}
+	it.values = nil
+}
+
+func (it *LlrbIterator) Next() (graph.TSVal, bool) {
+	graph.NextLogIn(it)
+	// Little hack here..
+	if !it.isRunning {
+		go IterateAll(it.tree, it.values, it.another)
+		it.isRunning = true
+	}
+	last := int64(0)
+	if it.Last != nil {
+		last = it.Last.(int64)
+	}
+	if it.tree.Max() == nil || last == int64(it.tree.Max().(Int64)) {
+		return graph.NextLogOut(it, nil, false)
+	}
+	it.another <- true
+	val := <-it.values
+	it.Last = int64(val.(Int64))
+	return graph.NextLogOut(it, it.Last, true)
+}
+
+func (it *LlrbIterator) Size() (int64, bool) {
+	return int64(it.tree.Len()), true
+}
+
+func (it *LlrbIterator) Check(v graph.TSVal) bool {
+	graph.CheckLogIn(it, v)
+	if it.tree.Has(Int64(v.(int64))) {
+		it.Last = v
+		return graph.CheckLogOut(it, v, true)
+	}
+	return graph.CheckLogOut(it, v, false)
+}
+
+func (it *LlrbIterator) DebugString(indent int) string {
+	size, _ := it.Size()
+	return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data)
+}
+
+func (it *LlrbIterator) Type() string {
+	return "llrb"
+}
+func (it *LlrbIterator) Sorted() bool {
+	return true
+}
+func (it *LlrbIterator) Optimize() (graph.Iterator, bool) {
+	return it, false
+}
+
+func (it *LlrbIterator) GetStats() *graph.IteratorStats {
+	return &graph.IteratorStats{
+		CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1,
+		NextCost:  1,
+		Size:      int64(it.tree.Len()),
+	}
+}

src/graph_memstore/memstore-all-iterator.go

@@ -0,0 +1,45 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_memstore
+
+import (
+	"graph"
+)
+
+type MemstoreAllIterator struct {
+	graph.Int64AllIterator
+	ts *MemTripleStore
+}
+
+func NewMemstoreAllIterator(ts *MemTripleStore) *MemstoreAllIterator {
+	var out MemstoreAllIterator
+	out.Int64AllIterator = *graph.NewInt64AllIterator(1, ts.idCounter-1)
+	out.ts = ts
+	return &out
+}
+
+func (memall *MemstoreAllIterator) Next() (graph.TSVal, bool) {
+	next, out := memall.Int64AllIterator.Next()
+	if !out {
+		return next, out
+	}
+	i64 := next.(int64)
+	_, ok := memall.ts.revIdMap[i64]
+	if !ok {
+		return memall.Next()
+	}
+	memall.Last = next
+	return next, out
+}

src/graph_memstore/memtriplestore-iterator-optimize.go

@@ -0,0 +1,53 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_memstore
+
+import (
+	"graph"
+)
+
+func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
+	switch it.Type() {
+	case "linksto":
+		return ts.optimizeLinksTo(it.(*graph.LinksToIterator))
+
+	}
+	return it, false
+}
+
+func (ts *MemTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) {
+	l := it.GetSubIterators()
+	if l.Len() != 1 {
+		return it, false
+	}
+	primaryIt := l.Front().Value.(graph.Iterator)
+	if primaryIt.Type() == "fixed" {
+		size, _ := primaryIt.Size()
+		if size == 1 {
+			val, ok := primaryIt.Next()
+			if !ok {
+				panic("Sizes lie")
+			}
+			newIt := ts.GetTripleIterator(it.Direction(), val)
+			newIt.CopyTagsFrom(it)
+			for _, tag := range primaryIt.Tags() {
+				newIt.AddFixedTag(tag, val)
+			}
+			return newIt, true
+		}
+	}
+	it.Close()
+	return it, false
+}

src/graph_memstore/memtriplestore.go

@@ -0,0 +1,266 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_memstore
+
+import (
+	"fmt"
+	"github.com/barakmich/glog"
+	"github.com/petar/GoLLRB/llrb"
+	"graph"
+)
+
+type TripleDirectionIndex struct {
+	subject    map[int64]*llrb.LLRB
+	predicate  map[int64]*llrb.LLRB
+	object     map[int64]*llrb.LLRB
+	provenance map[int64]*llrb.LLRB
+}
+
+func NewTripleDirectionIndex() *TripleDirectionIndex {
+	var tdi TripleDirectionIndex
+	tdi.subject = make(map[int64]*llrb.LLRB)
+	tdi.predicate = make(map[int64]*llrb.LLRB)
+	tdi.object = make(map[int64]*llrb.LLRB)
+	tdi.provenance = make(map[int64]*llrb.LLRB)
+	return &tdi
+}
+
+func (tdi *TripleDirectionIndex) GetForDir(s string) map[int64]*llrb.LLRB {
+	if s == "s" {
+		return tdi.subject
+	} else if s == "o" {
+		return tdi.object
+	} else if s == "p" {
+		return tdi.predicate
+	} else if s == "c" {
+		return tdi.provenance
+	}
+	panic("Bad direction")
+}
+
+func (tdi *TripleDirectionIndex) GetOrCreate(dir string, id int64) *llrb.LLRB {
+	directionIndex := tdi.GetForDir(dir)
+	if _, ok := directionIndex[id]; !ok {
+		directionIndex[id] = llrb.New()
+	}
+	return directionIndex[id]
+}
+
+func (tdi *TripleDirectionIndex) Get(dir string, id int64) (*llrb.LLRB, bool) {
+	directionIndex := tdi.GetForDir(dir)
+	tree, exists := directionIndex[id]
+	return tree, exists
+}
+
+type MemTripleStore struct {
+	idCounter       int64
+	tripleIdCounter int64
+	idMap           map[string]int64
+	revIdMap        map[int64]string
+	triples         []graph.Triple
+	size            int64
+	index           TripleDirectionIndex
+	// vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree
+}
+
+func NewMemTripleStore() *MemTripleStore {
+	var ts MemTripleStore
+	ts.idMap = make(map[string]int64)
+	ts.revIdMap = make(map[int64]string)
+	ts.triples = make([]graph.Triple, 1, 200)
+
+	// Sentinel null triple so triple indices start at 1
+	ts.triples[0] = graph.Triple{}
+	ts.size = 1
+	ts.index = *NewTripleDirectionIndex()
+	ts.idCounter = 1
+	ts.tripleIdCounter = 1
+	return &ts
+}
+
+func (ts *MemTripleStore) AddTripleSet(triples []*graph.Triple) {
+	for _, t := range triples {
+		ts.AddTriple(t)
+	}
+}
+
+func (ts *MemTripleStore) tripleExists(t *graph.Triple) (bool, int64) {
+	smallest := -1
+	var smallest_tree *llrb.LLRB
+	for _, dir := range graph.TripleDirections {
+		sid := t.Get(dir)
+		if dir == "c" && sid == "" {
+			continue
+		}
+		id, ok := ts.idMap[sid]
+		// If we've never heard about a node, it most not exist
+		if !ok {
+			return false, 0
+		}
+		index, exists := ts.index.Get(dir, id)
+		if !exists {
+			// If it's never been indexed in this direction, it can't exist.
+			return false, 0
+		}
+		if smallest == -1 || index.Len() < smallest {
+			smallest = index.Len()
+			smallest_tree = index
+		}
+	}
+	it := NewLlrbIterator(smallest_tree, "")
+
+	for {
+		val, ok := it.Next()
+		if !ok {
+			break
+		}
+		if t.Equals(&ts.triples[val.(int64)]) {
+			return true, val.(int64)
+		}
+	}
+	return false, 0
+}
+
+func (ts *MemTripleStore) AddTriple(t *graph.Triple) {
+	if exists, _ := ts.tripleExists(t); exists {
+		return
+	}
+	var tripleID int64
+	ts.triples = append(ts.triples, *t)
+	tripleID = ts.tripleIdCounter
+	ts.size++
+	ts.tripleIdCounter++
+
+	for _, dir := range graph.TripleDirections {
+		sid := t.Get(dir)
+		if dir == "c" && sid == "" {
+			continue
+		}
+		if _, ok := ts.idMap[sid]; !ok {
+			ts.idMap[sid] = ts.idCounter
+			ts.revIdMap[ts.idCounter] = sid
+			ts.idCounter++
+		}
+	}
+
+	for _, dir := range graph.TripleDirections {
+		if dir == "c" && t.Get(dir) == "" {
+			continue
+		}
+		id := ts.idMap[t.Get(dir)]
+		tree := ts.index.GetOrCreate(dir, id)
+		tree.ReplaceOrInsert(Int64(tripleID))
+	}
+
+	// TODO(barakmich): Add VIP indexing
+}
+
+func (ts *MemTripleStore) RemoveTriple(t *graph.Triple) {
+	var tripleID int64
+	var exists bool
+	tripleID = 0
+	if exists, tripleID = ts.tripleExists(t); !exists {
+		return
+	}
+
+	ts.triples[tripleID] = graph.Triple{}
+	ts.size--
+
+	for _, dir := range graph.TripleDirections {
+		if dir == "c" && t.Get(dir) == "" {
+			continue
+		}
+		id := ts.idMap[t.Get(dir)]
+		tree := ts.index.GetOrCreate(dir, id)
+		tree.Delete(Int64(tripleID))
+	}
+
+	for _, dir := range graph.TripleDirections {
+		if dir == "c" && t.Get(dir) == "" {
+			continue
+		}
+		id, ok := ts.idMap[t.Get(dir)]
+		if !ok {
+			continue
+		}
+		stillExists := false
+		for _, dir := range graph.TripleDirections {
+			if dir == "c" && t.Get(dir) == "" {
+				continue
+			}
+			nodeTree := ts.index.GetOrCreate(dir, id)
+			if nodeTree.Len() != 0 {
+				stillExists = true
+				break
+			}
+		}
+		if !stillExists {
+			delete(ts.idMap, t.Get(dir))
+			delete(ts.revIdMap, id)
+		}
+	}
+}
+
+func (ts *MemTripleStore) GetTriple(index graph.TSVal) *graph.Triple {
+	return &ts.triples[index.(int64)]
+}
+
+func (ts *MemTripleStore) GetTripleIterator(direction string, value graph.TSVal) graph.Iterator {
+	index, ok := ts.index.Get(direction, value.(int64))
+	data := fmt.Sprintf("dir:%s val:%d", direction, value.(int64))
+	if ok {
+		return NewLlrbIterator(index, data)
+	}
+	return &graph.NullIterator{}
+}
+
+func (ts *MemTripleStore) Size() int64 {
+	return ts.size - 1 // Don't count the sentinel
+}
+
+func (ts *MemTripleStore) DebugPrint() {
+	for i, t := range ts.triples {
+		if i == 0 {
+			continue
+		}
+		glog.V(2).Infoln("%d: %s", i, t.ToString())
+	}
+}
+
+func (ts *MemTripleStore) GetIdFor(name string) graph.TSVal {
+	return ts.idMap[name]
+}
+
+func (ts *MemTripleStore) GetNameFor(id graph.TSVal) string {
+	return ts.revIdMap[id.(int64)]
+}
+
+func (ts *MemTripleStore) GetTriplesAllIterator() graph.Iterator {
+	return graph.NewInt64AllIterator(0, ts.Size())
+}
+
+func (ts *MemTripleStore) MakeFixed() *graph.FixedIterator {
+	return graph.NewFixedIteratorWithCompare(graph.BasicEquality)
+}
+
+func (ts *MemTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal {
+	name := ts.GetTriple(val).Get(direction)
+	return ts.GetIdFor(name)
+}
+
+func (ts *MemTripleStore) GetNodesAllIterator() graph.Iterator {
+	return NewMemstoreAllIterator(ts)
+}
+func (ts *MemTripleStore) Close() {}

src/graph_memstore/memtriplestore_test.go

@@ -0,0 +1,136 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_memstore
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"graph"
+	"sort"
+	"testing"
+)
+
+func TestMemstore(t *testing.T) {
+	Convey("With a simple memstore", t, func() {
+		ts := MakeTestingMemstore()
+		Convey("It should have a reasonable size", func() {
+			So(ts.Size(), ShouldEqual, 11)
+		})
+		Convey("It should have an Id Space that makes sense", func() {
+			v := ts.GetIdFor("C")
+			So(v.(int64), ShouldEqual, 4)
+		})
+	})
+}
+
+func TestIteratorsAndNextResultOrderA(t *testing.T) {
+	ts := MakeTestingMemstore()
+	fixed := ts.MakeFixed()
+	fixed.AddValue(ts.GetIdFor("C"))
+	all := ts.GetNodesAllIterator()
+	lto := graph.NewLinksToIterator(ts, all, "o")
+	innerAnd := graph.NewAndIterator()
+
+	fixed2 := ts.MakeFixed()
+	fixed2.AddValue(ts.GetIdFor("follows"))
+	lto2 := graph.NewLinksToIterator(ts, fixed2, "p")
+	innerAnd.AddSubIterator(lto2)
+	innerAnd.AddSubIterator(lto)
+	hasa := graph.NewHasaIterator(ts, innerAnd, "s")
+	outerAnd := graph.NewAndIterator()
+	outerAnd.AddSubIterator(fixed)
+	outerAnd.AddSubIterator(hasa)
+	val, ok := outerAnd.Next()
+	if !ok {
+		t.Error("Expected one matching subtree")
+	}
+	if ts.GetNameFor(val) != "C" {
+		t.Errorf("Matching subtree should be %s, got %s", "barak", ts.GetNameFor(val))
+	}
+	expected := make([]string, 2)
+	expected[0] = "B"
+	expected[1] = "D"
+	actualOut := make([]string, 2)
+	actualOut[0] = ts.GetNameFor(all.LastResult())
+	nresultOk := outerAnd.NextResult()
+	if !nresultOk {
+		t.Error("Expected two results got one")
+	}
+	actualOut[1] = ts.GetNameFor(all.LastResult())
+	nresultOk = outerAnd.NextResult()
+	if nresultOk {
+		t.Error("Expected two results got three")
+	}
+	CompareStringSlices(t, expected, actualOut)
+	val, ok = outerAnd.Next()
+	if ok {
+		t.Error("More than one possible top level output?")
+	}
+}
+
+func CompareStringSlices(t *testing.T, expected []string, actual []string) {
+	if len(expected) != len(actual) {
+		t.Error("String slices are not the same length")
+	}
+	sort.Strings(expected)
+	sort.Strings(actual)
+	for i := 0; i < len(expected); i++ {
+		if expected[i] != actual[i] {
+			t.Errorf("At index %d, expected \"%s\" and got \"%s\"", i, expected[i], actual[i])
+		}
+	}
+}
+
+func TestLinksToOptimization(t *testing.T) {
+	ts := MakeTestingMemstore()
+	fixed := ts.MakeFixed()
+	fixed.AddValue(ts.GetIdFor("cool"))
+	lto := graph.NewLinksToIterator(ts, fixed, "o")
+	lto.AddTag("foo")
+	newIt, changed := lto.Optimize()
+	if !changed {
+		t.Error("Iterator didn't change")
+	}
+	if newIt.Type() != "llrb" {
+		t.Fatal("Didn't swap out to LLRB")
+	}
+	v := newIt.(*LlrbIterator)
+	v_clone := v.Clone()
+	if v_clone.DebugString(0) != v.DebugString(0) {
+		t.Fatal("Wrong iterator. Got ", v_clone.DebugString(0))
+	}
+	if len(v_clone.Tags()) < 1 || v_clone.Tags()[0] != "foo" {
+		t.Fatal("Tag on LinksTo did not persist")
+	}
+}
+
+func TestRemoveTriple(t *testing.T) {
+	ts := MakeTestingMemstore()
+	ts.RemoveTriple(graph.MakeTriple("E", "follows", "F", ""))
+	fixed := ts.MakeFixed()
+	fixed.AddValue(ts.GetIdFor("E"))
+	lto := graph.NewLinksToIterator(ts, fixed, "s")
+	fixed2 := ts.MakeFixed()
+	fixed2.AddValue(ts.GetIdFor("follows"))
+	lto2 := graph.NewLinksToIterator(ts, fixed2, "p")
+	innerAnd := graph.NewAndIterator()
+	innerAnd.AddSubIterator(lto2)
+	innerAnd.AddSubIterator(lto)
+	hasa := graph.NewHasaIterator(ts, innerAnd, "o")
+	newIt, _ := hasa.Optimize()
+	_, ok := newIt.Next()
+	if ok {
+		t.Error("E should not have any followers.")
+	}
+}

src/graph_memstore/testing_memstore.go

@@ -0,0 +1,45 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_memstore
+
+import "graph"
+
+//    +---+                        +---+
+//    | A |-------               ->| F |<--
+//    +---+       \------>+---+-/  +---+   \--+---+
+//                 ------>|#B#|      |        | E |
+//    +---+-------/      >+---+      |        +---+
+//    | C |             /            v
+//    +---+           -/           +---+
+//      ----    +---+/             |#G#|
+//          \-->|#D#|------------->+---+
+//              +---+
+//
+
+func MakeTestingMemstore() *MemTripleStore {
+	ts := NewMemTripleStore()
+	ts.AddTriple(graph.MakeTriple("A", "follows", "B", ""))
+	ts.AddTriple(graph.MakeTriple("C", "follows", "B", ""))
+	ts.AddTriple(graph.MakeTriple("C", "follows", "D", ""))
+	ts.AddTriple(graph.MakeTriple("D", "follows", "B", ""))
+	ts.AddTriple(graph.MakeTriple("B", "follows", "F", ""))
+	ts.AddTriple(graph.MakeTriple("F", "follows", "G", ""))
+	ts.AddTriple(graph.MakeTriple("D", "follows", "G", ""))
+	ts.AddTriple(graph.MakeTriple("E", "follows", "F", ""))
+	ts.AddTriple(graph.MakeTriple("B", "status", "cool", "status_graph"))
+	ts.AddTriple(graph.MakeTriple("D", "status", "cool", "status_graph"))
+	ts.AddTriple(graph.MakeTriple("G", "status", "cool", "status_graph"))
+	return ts
+}