barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Move iterators into separate package

Browse source 
Also reduce API exposure and use standard library more - and fix bugs I
previously introduces in mongo.
This commit is contained in:
kortschak 2014-06-30 22:22:50 +09:30
parent 88be6bee37
commit 1768e593a8
62 changed files with 3240 additions and 3130 deletions
Download patch file Download diff file Expand all files Collapse all files

9
graph/leveldb/all_iterator.go
View file

@ -19,25 +19,26 @@ import (
"fmt"
"strings"
"github.com/syndtr/goleveldb/leveldb/iterator"
ldbit "github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
)
type AllIterator struct {
graph.BaseIterator
iterator.Base
prefix []byte
dir graph.Direction
open bool
it iterator.Iterator
it ldbit.Iterator
ts *TripleStore
ro *opt.ReadOptions
}
func NewAllIterator(prefix string, d graph.Direction, ts *TripleStore) *AllIterator {
var it AllIterator
graph.BaseIteratorInit(&it.BaseIterator)
iterator.BaseInit(&it.Base)
it.ro = &opt.ReadOptions{}
it.ro.DontFillCache = true
it.it = ts.db.NewIterator(nil, it.ro)

9
graph/leveldb/iterator.go
View file

@ -19,19 +19,20 @@ import (
"fmt"
"strings"
"github.com/syndtr/goleveldb/leveldb/iterator"
ldbit "github.com/syndtr/goleveldb/leveldb/iterator"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
)
type Iterator struct {
graph.BaseIterator
iterator.Base
nextPrefix []byte
checkId []byte
dir graph.Direction
open bool
it iterator.Iterator
it ldbit.Iterator
ts *TripleStore
ro *opt.ReadOptions
originalPrefix string
@ -39,7 +40,7 @@ type Iterator struct {
func NewIterator(prefix string, d graph.Direction, value graph.TSVal, ts *TripleStore) *Iterator {
var it Iterator
graph.BaseIteratorInit(&it.BaseIterator)
iterator.BaseInit(&it.Base)
it.checkId = value.([]byte)
it.dir = d
it.originalPrefix = prefix

75
graph/leveldb/leveldb_test.go
View file

@ -23,21 +23,22 @@ import (
. "github.com/smartystreets/goconvey/convey"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
)
func makeTripleSet() []*graph.Triple {
tripleSet := []*graph.Triple{
graph.MakeTriple("A", "follows", "B", ""),
graph.MakeTriple("C", "follows", "B", ""),
graph.MakeTriple("C", "follows", "D", ""),
graph.MakeTriple("D", "follows", "B", ""),
graph.MakeTriple("B", "follows", "F", ""),
graph.MakeTriple("F", "follows", "G", ""),
graph.MakeTriple("D", "follows", "G", ""),
graph.MakeTriple("E", "follows", "F", ""),
graph.MakeTriple("B", "status", "cool", "status_graph"),
graph.MakeTriple("D", "status", "cool", "status_graph"),
graph.MakeTriple("G", "status", "cool", "status_graph"),
{"A", "follows", "B", ""},
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
{"D", "follows", "B", ""},
{"B", "follows", "F", ""},
{"F", "follows", "G", ""},
{"D", "follows", "G", ""},
{"E", "follows", "F", ""},
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
}
return tripleSet
}
@ -49,7 +50,7 @@ func extractTripleFromIterator(ts graph.TripleStore, it graph.Iterator) []string
if !ok {
break
}
output = append(output, ts.GetTriple(val).ToString())
output = append(output, ts.GetTriple(val).String())
}
return output
}
@ -111,7 +112,7 @@ func TestLoadDatabase(t *testing.T) {
ts = NewTripleStore(tmpDir, nil)
Convey("Can load a single triple", func() {
ts.AddTriple(graph.MakeTriple("Something", "points_to", "Something Else", "context"))
ts.AddTriple(&graph.Triple{"Something", "points_to", "Something Else", "context"})
So(ts.GetNameFor(ts.GetIdFor("Something")), ShouldEqual, "Something")
So(ts.Size(), ShouldEqual, 1)
})
@ -123,7 +124,7 @@ func TestLoadDatabase(t *testing.T) {
So(ts.GetSizeFor(ts.GetIdFor("B")), ShouldEqual, 5)
Convey("Can delete triples", func() {
ts.RemoveTriple(graph.MakeTriple("A", "follows", "B", ""))
ts.RemoveTriple(&graph.Triple{"A", "follows", "B", ""})
So(ts.Size(), ShouldEqual, 10)
So(ts.GetSizeFor(ts.GetIdFor("B")), ShouldEqual, 4)
})
@ -220,9 +221,9 @@ func TestIterator(t *testing.T) {
set := makeTripleSet()
var string_set []string
for _, t := range set {
string_set = append(string_set, t.ToString())
string_set = append(string_set, t.String())
}
So(triple.ToString(), ShouldBeIn, string_set)
So(triple.String(), ShouldBeIn, string_set)
})
Reset(func() {
@ -252,8 +253,8 @@ func TestSetIterator(t *testing.T) {
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("C", "follows", "B", "").ToString(),
graph.MakeTriple("C", "follows", "D", "").ToString(),
(&graph.Triple{"C", "follows", "B", ""}).String(),
(&graph.Triple{"C", "follows", "D", ""}).String(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
@ -262,13 +263,13 @@ func TestSetIterator(t *testing.T) {
})
Convey("And checkable", func() {
and := graph.NewAndIterator()
and := iterator.NewAnd()
and.AddSubIterator(ts.GetTriplesAllIterator())
and.AddSubIterator(it)
expected := []string{
graph.MakeTriple("C", "follows", "B", "").ToString(),
graph.MakeTriple("C", "follows", "D", "").ToString(),
(&graph.Triple{"C", "follows", "B", ""}).String(),
(&graph.Triple{"C", "follows", "D", ""}).String(),
}
actual := extractTripleFromIterator(ts, and)
sort.Strings(actual)
@ -286,8 +287,8 @@ func TestSetIterator(t *testing.T) {
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("B", "follows", "F", "").ToString(),
graph.MakeTriple("E", "follows", "F", "").ToString(),
(&graph.Triple{"B", "follows", "F", ""}).String(),
(&graph.Triple{"E", "follows", "F", ""}).String(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
@ -296,12 +297,12 @@ func TestSetIterator(t *testing.T) {
})
Convey("Mutually and-checkable", func() {
and := graph.NewAndIterator()
and := iterator.NewAnd()
and.AddSubIterator(ts.GetTripleIterator(graph.Subject, ts.GetIdFor("B")))
and.AddSubIterator(it)
expected := []string{
graph.MakeTriple("B", "follows", "F", "").ToString(),
(&graph.Triple{"B", "follows", "F", ""}).String(),
}
actual := extractTripleFromIterator(ts, and)
sort.Strings(actual)
@ -316,9 +317,9 @@ func TestSetIterator(t *testing.T) {
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("D", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("G", "status", "cool", "status_graph").ToString(),
(&graph.Triple{"B", "status", "cool", "status_graph"}).String(),
(&graph.Triple{"D", "status", "cool", "status_graph"}).String(),
(&graph.Triple{"G", "status", "cool", "status_graph"}).String(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
@ -333,9 +334,9 @@ func TestSetIterator(t *testing.T) {
Convey("Containing the right things", func() {
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("D", "status", "cool", "status_graph").ToString(),
graph.MakeTriple("G", "status", "cool", "status_graph").ToString(),
(&graph.Triple{"B", "status", "cool", "status_graph"}).String(),
(&graph.Triple{"D", "status", "cool", "status_graph"}).String(),
(&graph.Triple{"G", "status", "cool", "status_graph"}).String(),
}
actual := extractTripleFromIterator(ts, it)
sort.Strings(actual)
@ -344,26 +345,26 @@ func TestSetIterator(t *testing.T) {
})
Convey("Can be cross-checked", func() {
and := graph.NewAndIterator()
and := iterator.NewAnd()
// Order is important
and.AddSubIterator(ts.GetTripleIterator(graph.Subject, ts.GetIdFor("B")))
and.AddSubIterator(it)
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
(&graph.Triple{"B", "status", "cool", "status_graph"}).String(),
}
actual := extractTripleFromIterator(ts, and)
So(actual, ShouldResemble, expected)
})
Convey("Can check against other iterators", func() {
and := graph.NewAndIterator()
and := iterator.NewAnd()
// Order is important
and.AddSubIterator(it)
and.AddSubIterator(ts.GetTripleIterator(graph.Subject, ts.GetIdFor("B")))
expected := []string{
graph.MakeTriple("B", "status", "cool", "status_graph").ToString(),
(&graph.Triple{"B", "status", "cool", "status_graph"}).String(),
}
actual := extractTripleFromIterator(ts, and)
So(actual, ShouldResemble, expected)
@ -397,10 +398,10 @@ func TestOptimize(t *testing.T) {
ts.AddTripleSet(makeTripleSet())
Convey("With an linksto-fixed pair", func() {
fixed := ts.MakeFixed()
fixed := ts.FixedIterator()
fixed.AddValue(ts.GetIdFor("F"))
fixed.AddTag("internal")
lto = graph.NewLinksToIterator(ts, fixed, graph.Object)
lto = iterator.NewLinksTo(ts, fixed, graph.Object)
Convey("Creates an appropriate iterator", func() {
oldIt := lto.Clone()

11
graph/leveldb/triplestore.go
View file

@ -29,6 +29,7 @@ import (
"github.com/syndtr/goleveldb/leveldb/util"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
)
const (
@ -143,7 +144,7 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
ts.buildWrite(batch, t)
err := ts.db.Write(batch, ts.writeopts)
if err != nil {
glog.Errorf("Couldn't write to DB for triple %s", t.ToString())
glog.Errorf("Couldn't write to DB for triple %s", t)
return
}
ts.size++
@ -180,7 +181,7 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
}
err = ts.db.Write(batch, nil)
if err != nil {
glog.Errorf("Couldn't delete triple %s", t.ToString())
glog.Errorf("Couldn't delete triple %s", t)
return
}
ts.size--
@ -189,7 +190,7 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
func (ts *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) {
bytes, err := json.Marshal(*t)
if err != nil {
glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t.ToString(), err)
glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t, err)
return
}
batch.Put(ts.createKeyFor(spo, t), bytes)
@ -439,6 +440,6 @@ func compareBytes(a, b graph.TSVal) bool {
return bytes.Equal(a.([]uint8), b.([]uint8))
}
func (ts *TripleStore) MakeFixed() *graph.FixedIterator {
return graph.NewFixedIteratorWithCompare(compareBytes)
func (ts *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(compareBytes)
}

5
graph/leveldb/triplestore_iterator_optimize.go
View file

@ -16,18 +16,19 @@ package leveldb
import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
)
func (ts *TripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
switch it.Type() {
case "linksto":
return ts.optimizeLinksTo(it.(*graph.LinksToIterator))
return ts.optimizeLinksTo(it.(*iterator.LinksTo))
}
return it, false
}
func (ts *TripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) {
func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bool) {
subs := it.GetSubIterators()
if len(subs) != 1 {
return it, false