barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Create quads hierarchy

Browse source 
* Move nquads into quad.
* Create cquads simplified parser in quad.
* Move Triple (renamed Quad) to quad.

Also made sure mongo actually implements BulkLoader.
This commit is contained in:
kortschak 2014-07-27 17:42:45 +09:30
parent 01bc63810b
commit 401c58426f
51 changed files with 13400 additions and 5495 deletions
Download patch file Download diff file Expand all files Collapse all files

11
graph/iterator/hasa_iterator.go
View file

@ -40,6 +40,7 @@ import (
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
// A HasA consists of a reference back to the graph.TripleStore that it references,
@ -49,13 +50,13 @@ type HasA struct {
Base
ts graph.TripleStore
primaryIt graph.Iterator
dir graph.Direction
dir quad.Direction
resultIt graph.Iterator
}
// Construct a new HasA iterator, given the triple subiterator, and the triple
// direction for which it stands.
func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d graph.Direction) *HasA {
func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d quad.Direction) *HasA {
var hasa HasA
BaseInit(&hasa.Base)
hasa.ts = ts
@ -83,7 +84,7 @@ func (it *HasA) Clone() graph.Iterator {
}
// Direction accessor.
func (it *HasA) Direction() graph.Direction { return it.dir }
func (it *HasA) Direction() quad.Direction { return it.dir }
// Pass the Optimize() call along to the subiterator. If it becomes Null,
// then the HasA becomes Null (there are no triples that have any directions).
@ -146,7 +147,7 @@ func (it *HasA) GetCheckResult() bool {
break
}
if glog.V(4) {
glog.V(4).Infoln("Triple is", it.ts.Triple(linkVal))
glog.V(4).Infoln("Quad is", it.ts.Quad(linkVal))
}
if it.primaryIt.Check(linkVal) {
it.Last = it.ts.TripleDirection(linkVal, it.dir)
@ -184,7 +185,7 @@ func (it *HasA) Next() (graph.Value, bool) {
if !ok {
return graph.NextLogOut(it, 0, false)
}
name := it.ts.Triple(tID).Get(it.dir)
name := it.ts.Quad(tID).Get(it.dir)
val := it.ts.ValueOf(name)
it.Last = val
return graph.NextLogOut(it, val, true)

7
graph/iterator/linksto_iterator.go
View file

@ -34,6 +34,7 @@ import (
"strings"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
// A LinksTo has a reference back to the graph.TripleStore (to create the iterators
@ -43,13 +44,13 @@ type LinksTo struct {
Base
ts graph.TripleStore
primaryIt graph.Iterator
dir graph.Direction
dir quad.Direction
nextIt graph.Iterator
}
// Construct a new LinksTo iterator around a direction and a subiterator of
// nodes.
func NewLinksTo(ts graph.TripleStore, it graph.Iterator, d graph.Direction) *LinksTo {
func NewLinksTo(ts graph.TripleStore, it graph.Iterator, d quad.Direction) *LinksTo {
var lto LinksTo
BaseInit(&lto.Base)
lto.ts = ts
@ -74,7 +75,7 @@ func (it *LinksTo) Clone() graph.Iterator {
}
// Return the direction under consideration.
func (it *LinksTo) Direction() graph.Direction { return it.dir }
func (it *LinksTo) Direction() quad.Direction { return it.dir }
// Tag these results, and our subiterator's results.
func (it *LinksTo) TagResults(dst map[string]graph.Value) {

6
graph/iterator/linksto_iterator_test.go
View file

@ -17,7 +17,7 @@ package iterator
import (
"testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
func TestLinksTo(t *testing.T) {
@ -32,12 +32,12 @@ func TestLinksTo(t *testing.T) {
t.Fatalf("Failed to return correct value, got:%v expect:1", val)
}
fixed.Add(val)
lto := NewLinksTo(ts, fixed, graph.Object)
lto := NewLinksTo(ts, fixed, quad.Object)
val, ok := lto.Next()
if !ok {
t.Error("At least one triple matches the fixed object")
}
if val != 2 {
t.Errorf("Triple index 2, such as %s, should match %s", ts.Triple(2), ts.Triple(val))
t.Errorf("Quad index 2, such as %s, should match %s", ts.Quad(2), ts.Quad(val))
}
}

43
graph/iterator/mock_ts_test.go
View file

@ -17,15 +17,18 @@ package iterator
// A quickly mocked version of the TripleStore interface, for use in tests.
// Can better used Mock.Called but will fill in as needed.
import "github.com/google/cayley/graph"
import (
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
type store struct {
data []string
iter graph.Iterator
}
func (ts *store) ValueOf(s string) graph.Value {
for i, v := range ts.data {
func (qs *store) ValueOf(s string) graph.Value {
for i, v := range qs.data {
if s == v {
return i
}
@ -33,42 +36,42 @@ func (ts *store) ValueOf(s string) graph.Value {
return nil
}
func (ts *store) AddTriple(*graph.Triple) {}
func (qs *store) AddTriple(*quad.Quad) {}
func (ts *store) AddTripleSet([]*graph.Triple) {}
func (qs *store) AddTripleSet([]*quad.Quad) {}
func (ts *store) Triple(graph.Value) *graph.Triple { return &graph.Triple{} }
func (qs *store) Quad(graph.Value) *quad.Quad { return &quad.Quad{} }
func (ts *store) TripleIterator(d graph.Direction, i graph.Value) graph.Iterator {
return ts.iter
func (qs *store) TripleIterator(d quad.Direction, i graph.Value) graph.Iterator {
return qs.iter
}
func (ts *store) NodesAllIterator() graph.Iterator { return &Null{} }
func (qs *store) NodesAllIterator() graph.Iterator { return &Null{} }
func (ts *store) TriplesAllIterator() graph.Iterator { return &Null{} }
func (qs *store) TriplesAllIterator() graph.Iterator { return &Null{} }
func (ts *store) NameOf(v graph.Value) string {
func (qs *store) NameOf(v graph.Value) string {
i := v.(int)
if i < 0 || i >= len(ts.data) {
if i < 0 || i >= len(qs.data) {
return ""
}
return ts.data[i]
return qs.data[i]
}
func (ts *store) Size() int64 { return 0 }
func (qs *store) Size() int64 { return 0 }
func (ts *store) DebugPrint() {}
func (qs *store) DebugPrint() {}
func (ts *store) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
func (qs *store) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
return &Null{}, false
}
func (ts *store) FixedIterator() graph.FixedIterator {
func (qs *store) FixedIterator() graph.FixedIterator {
return NewFixedIteratorWithCompare(BasicEquality)
}
func (ts *store) Close() {}
func (qs *store) Close() {}
func (ts *store) TripleDirection(graph.Value, graph.Direction) graph.Value { return 0 }
func (qs *store) TripleDirection(graph.Value, quad.Direction) graph.Value { return 0 }
func (ts *store) RemoveTriple(t *graph.Triple) {}
func (qs *store) RemoveTriple(t *quad.Quad) {}

13
graph/iterator/query_shape.go
View file

@ -16,6 +16,7 @@ package iterator
import (
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
type Node struct {
@ -39,7 +40,7 @@ type queryShape struct {
ts graph.TripleStore
nodeId int
hasaIds []int
hasaDirs []graph.Direction
hasaDirs []quad.Direction
}
func OutputQueryShapeForIterator(it graph.Iterator, ts graph.TripleStore, outputMap map[string]interface{}) {
@ -62,11 +63,11 @@ func (qs *queryShape) AddLink(l *Link) {
qs.links = append(qs.links, *l)
}
func (qs *queryShape) LastHasa() (int, graph.Direction) {
func (qs *queryShape) LastHasa() (int, quad.Direction) {
return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1]
}
func (qs *queryShape) PushHasa(i int, d graph.Direction) {
func (qs *queryShape) PushHasa(i int, d quad.Direction) {
qs.hasaIds = append(qs.hasaIds, i)
qs.hasaDirs = append(qs.hasaDirs, d)
}
@ -159,10 +160,10 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
qs.nodeId++
newNode := qs.MakeNode(lto.primaryIt)
hasaID, hasaDir := qs.LastHasa()
if (hasaDir == graph.Subject && lto.dir == graph.Object) ||
(hasaDir == graph.Object && lto.dir == graph.Subject) {
if (hasaDir == quad.Subject && lto.dir == quad.Object) ||
(hasaDir == quad.Object && lto.dir == quad.Subject) {
qs.AddNode(newNode)
if hasaDir == graph.Subject {
if hasaDir == quad.Subject {
qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id})
} else {
qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id})

13
graph/iterator/query_shape_test.go
View file

@ -19,6 +19,7 @@ import (
"testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA {
@ -27,13 +28,13 @@ func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA {
obj := ts.FixedIterator()
obj.Add(ts.ValueOf(target))
obj.AddTag(tag)
and.AddSubIterator(NewLinksTo(ts, obj, graph.Object))
and.AddSubIterator(NewLinksTo(ts, obj, quad.Object))
pred := ts.FixedIterator()
pred.Add(ts.ValueOf("status"))
and.AddSubIterator(NewLinksTo(ts, pred, graph.Predicate))
and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate))
return NewHasA(ts, and, graph.Subject)
return NewHasA(ts, and, quad.Subject)
}
func TestQueryShape(t *testing.T) {
@ -104,11 +105,11 @@ func TestQueryShape(t *testing.T) {
pred.Add(ts.ValueOf("name"))
and := NewAnd()
and.AddSubIterator(NewLinksTo(ts, andInternal, graph.Subject))
and.AddSubIterator(NewLinksTo(ts, pred, graph.Predicate))
and.AddSubIterator(NewLinksTo(ts, andInternal, quad.Subject))
and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate))
shape = make(map[string]interface{})
OutputQueryShapeForIterator(NewHasA(ts, and, graph.Object), ts, shape)
OutputQueryShapeForIterator(NewHasA(ts, and, quad.Object), ts, shape)
links = shape["links"].([]Link)
if len(links) != 3 {

5
graph/leveldb/all_iterator.go
View file

@ -24,19 +24,20 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
type AllIterator struct {
iterator.Base
prefix []byte
dir graph.Direction
dir quad.Direction
open bool
iter ldbit.Iterator
ts *TripleStore
ro *opt.ReadOptions
}
func NewAllIterator(prefix string, d graph.Direction, ts *TripleStore) *AllIterator {
func NewAllIterator(prefix string, d quad.Direction, ts *TripleStore) *AllIterator {
var it AllIterator
iterator.BaseInit(&it.Base)
it.ro = &opt.ReadOptions{}

79
graph/leveldb/iterator.go
View file

@ -24,34 +24,35 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
type Iterator struct {
iterator.Base
nextPrefix []byte
checkId []byte
dir graph.Direction
dir quad.Direction
open bool
iter ldbit.Iterator
ts *TripleStore
qs *TripleStore
ro *opt.ReadOptions
originalPrefix string
}
func NewIterator(prefix string, d graph.Direction, value graph.Value, ts *TripleStore) *Iterator {
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
var it Iterator
iterator.BaseInit(&it.Base)
it.checkId = value.([]byte)
it.dir = d
it.originalPrefix = prefix
it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size())
it.nextPrefix = make([]byte, 0, 2+qs.hasher.Size())
it.nextPrefix = append(it.nextPrefix, []byte(prefix)...)
it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...)
it.ro = &opt.ReadOptions{}
it.ro.DontFillCache = true
it.iter = ts.db.NewIterator(nil, it.ro)
it.iter = qs.db.NewIterator(nil, it.ro)
it.open = true
it.ts = ts
it.qs = qs
ok := it.iter.Seek(it.nextPrefix)
if !ok {
it.open = false
@ -62,7 +63,7 @@ func NewIterator(prefix string, d graph.Direction, value graph.Value, ts *Triple
func (it *Iterator) Reset() {
if !it.open {
it.iter = it.ts.db.NewIterator(nil, it.ro)
it.iter = it.qs.db.NewIterator(nil, it.ro)
it.open = true
}
ok := it.iter.Seek(it.nextPrefix)
@ -73,7 +74,7 @@ func (it *Iterator) Reset() {
}
func (it *Iterator) Clone() graph.Iterator {
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.ts)
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.qs)
out.CopyTagsFrom(it)
return out
}
@ -114,52 +115,52 @@ func (it *Iterator) Next() (graph.Value, bool) {
return nil, false
}
func PositionOf(prefix []byte, d graph.Direction, ts *TripleStore) int {
func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("sp")) {
switch d {
case graph.Subject:
case quad.Subject:
return 2
case graph.Predicate:
return ts.hasher.Size() + 2
case graph.Object:
return 2*ts.hasher.Size() + 2
case graph.Provenance:
case quad.Predicate:
return qs.hasher.Size() + 2
case quad.Object:
return 2*qs.hasher.Size() + 2
case quad.Provenance:
return -1
}
}
if bytes.Equal(prefix, []byte("po")) {
switch d {
case graph.Subject:
return 2*ts.hasher.Size() + 2
case graph.Predicate:
case quad.Subject:
return 2*qs.hasher.Size() + 2
case quad.Predicate:
return 2
case graph.Object:
return ts.hasher.Size() + 2
case graph.Provenance:
case quad.Object:
return qs.hasher.Size() + 2
case quad.Provenance:
return -1
}
}
if bytes.Equal(prefix, []byte("os")) {
switch d {
case graph.Subject:
return ts.hasher.Size() + 2
case graph.Predicate:
return 2*ts.hasher.Size() + 2
case graph.Object:
case quad.Subject:
return qs.hasher.Size() + 2
case quad.Predicate:
return 2*qs.hasher.Size() + 2
case quad.Object:
return 2
case graph.Provenance:
case quad.Provenance:
return -1
}
}
if bytes.Equal(prefix, []byte("cp")) {
switch d {
case graph.Subject:
return 2*ts.hasher.Size() + 2
case graph.Predicate:
return ts.hasher.Size() + 2
case graph.Object:
return 3*ts.hasher.Size() + 2
case graph.Provenance:
case quad.Subject:
return 2*qs.hasher.Size() + 2
case quad.Predicate:
return qs.hasher.Size() + 2
case quad.Object:
return 3*qs.hasher.Size() + 2
case quad.Provenance:
return 2
}
}
@ -171,14 +172,14 @@ func (it *Iterator) Check(v graph.Value) bool {
if val[0] == 'z' {
return false
}
offset := PositionOf(val[0:2], it.dir, it.ts)
offset := PositionOf(val[0:2], it.dir, it.qs)
if offset != -1 {
if bytes.HasPrefix(val[offset:], it.checkId[1:]) {
return true
}
} else {
nameForDir := it.ts.Triple(v).Get(it.dir)
hashForDir := it.ts.ValueOf(nameForDir).([]byte)
nameForDir := it.qs.Quad(v).Get(it.dir)
hashForDir := it.qs.ValueOf(nameForDir).([]byte)
if bytes.Equal(hashForDir, it.checkId) {
return true
}
@ -187,12 +188,12 @@ func (it *Iterator) Check(v graph.Value) bool {
}
func (it *Iterator) Size() (int64, bool) {
return it.ts.SizeOf(it.checkId), true
return it.qs.SizeOf(it.checkId), true
}
func (it *Iterator) DebugString(indent int) string {
size, _ := it.Size()
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.ts.NameOf(it.checkId))
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.qs.NameOf(it.checkId))
}
var levelDBType graph.Type

145
graph/leveldb/leveldb_test.go
View file

@ -23,10 +23,11 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
func makeTripleSet() []*graph.Triple {
tripleSet := []*graph.Triple{
func makeTripleSet() []*quad.Quad {
tripleSet := []*quad.Quad{
{"A", "follows", "B", ""},
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
@ -42,20 +43,20 @@ func makeTripleSet() []*graph.Triple {
return tripleSet
}
func iteratedTriples(ts graph.TripleStore, it graph.Iterator) []*graph.Triple {
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad {
var res ordered
for {
val, ok := it.Next()
if !ok {
break
}
res = append(res, ts.Triple(val))
res = append(res, qs.Quad(val))
}
sort.Sort(res)
return res
}
type ordered []*graph.Triple
type ordered []*quad.Quad
func (o ordered) Len() int { return len(o) }
func (o ordered) Less(i, j int) bool {
@ -82,14 +83,14 @@ func (o ordered) Less(i, j int) bool {
}
func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
func iteratedNames(ts graph.TripleStore, it graph.Iterator) []string {
func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string {
var res []string
for {
val, ok := it.Next()
if !ok {
break
}
res = append(res, ts.NameOf(val))
res = append(res, qs.NameOf(val))
}
sort.Strings(res)
return res
@ -107,14 +108,14 @@ func TestCreateDatabase(t *testing.T) {
t.Fatal("Failed to create LevelDB database.")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
if s := ts.Size(); s != 0 {
if s := qs.Size(); s != 0 {
t.Errorf("Unexpected size, got:%d expected:0", s)
}
ts.Close()
qs.Close()
err = createNewLevelDB("/dev/null/some terrible path", nil)
if err == nil {
@ -137,53 +138,53 @@ func TestLoadDatabase(t *testing.T) {
t.Fatal("Failed to create LevelDB database.")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts.AddTriple(&graph.Triple{"Something", "points_to", "Something Else", "context"})
qs.AddTriple(&quad.Quad{"Something", "points_to", "Something Else", "context"})
for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
if got := ts.NameOf(ts.ValueOf(pq)); got != pq {
if got := qs.NameOf(qs.ValueOf(pq)); got != pq {
t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq)
}
}
if s := ts.Size(); s != 1 {
if s := qs.Size(); s != 1 {
t.Errorf("Unexpected triplestore size, got:%d expect:1", s)
}
ts.Close()
qs.Close()
err = createNewLevelDB(tmpDir, nil)
if err != nil {
t.Fatal("Failed to create LevelDB database.")
}
ts, err = newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err = newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts2, didConvert := ts.(*TripleStore)
ts2, didConvert := qs.(*TripleStore)
if !didConvert {
t.Errorf("Could not convert from generic to LevelDB TripleStore")
}
ts.AddTripleSet(makeTripleSet())
if s := ts.Size(); s != 11 {
qs.AddTripleSet(makeTripleSet())
if s := qs.Size(); s != 11 {
t.Errorf("Unexpected triplestore size, got:%d expect:11", s)
}
if s := ts2.SizeOf(ts.ValueOf("B")); s != 5 {
if s := ts2.SizeOf(qs.ValueOf("B")); s != 5 {
t.Errorf("Unexpected triplestore size, got:%d expect:5", s)
}
ts.RemoveTriple(&graph.Triple{"A", "follows", "B", ""})
if s := ts.Size(); s != 10 {
qs.RemoveTriple(&quad.Quad{"A", "follows", "B", ""})
if s := qs.Size(); s != 10 {
t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s)
}
if s := ts2.SizeOf(ts.ValueOf("B")); s != 4 {
if s := ts2.SizeOf(qs.ValueOf("B")); s != 4 {
t.Errorf("Unexpected triplestore size, got:%d expect:4", s)
}
ts.Close()
qs.Close()
}
func TestIterator(t *testing.T) {
@ -199,14 +200,14 @@ func TestIterator(t *testing.T) {
t.Fatal("Failed to create LevelDB database.")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts.AddTripleSet(makeTripleSet())
qs.AddTripleSet(makeTripleSet())
var it graph.Iterator
it = ts.NodesAllIterator()
it = qs.NodesAllIterator()
if it == nil {
t.Fatal("Got nil iterator.")
}
@ -241,7 +242,7 @@ func TestIterator(t *testing.T) {
}
sort.Strings(expect)
for i := 0; i < 2; i++ {
got := iteratedNames(ts, it)
got := iteratedNames(qs, it)
sort.Strings(got)
if !reflect.DeepEqual(got, expect) {
t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect)
@ -250,23 +251,23 @@ func TestIterator(t *testing.T) {
}
for _, pq := range expect {
if !it.Check(ts.ValueOf(pq)) {
if !it.Check(qs.ValueOf(pq)) {
t.Errorf("Failed to find and check %q correctly", pq)
}
}
// FIXME(kortschak) Why does this fail?
/*
for _, pq := range []string{"baller"} {
if it.Check(ts.ValueOf(pq)) {
if it.Check(qs.ValueOf(pq)) {
t.Errorf("Failed to check %q correctly", pq)
}
}
*/
it.Reset()
it = ts.TriplesAllIterator()
it = qs.TriplesAllIterator()
edge, _ := it.Next()
triple := ts.Triple(edge)
triple := qs.Quad(edge)
set := makeTripleSet()
var ok bool
for _, t := range set {
@ -279,7 +280,7 @@ func TestIterator(t *testing.T) {
t.Errorf("Failed to find %q during iteration, got:%q", triple, set)
}
ts.Close()
qs.Close()
}
func TestSetIterator(t *testing.T) {
@ -292,95 +293,95 @@ func TestSetIterator(t *testing.T) {
t.Fatalf("Failed to create working directory")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
defer ts.Close()
defer qs.Close()
ts.AddTripleSet(makeTripleSet())
qs.AddTripleSet(makeTripleSet())
expect := []*graph.Triple{
expect := []*quad.Quad{
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
}
sort.Sort(ordered(expect))
// Subject iterator.
it := ts.TripleIterator(graph.Subject, ts.ValueOf("C"))
it := qs.TripleIterator(quad.Subject, qs.ValueOf("C"))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
}
it.Reset()
and := iterator.NewAnd()
and.AddSubIterator(ts.TriplesAllIterator())
and.AddSubIterator(qs.TriplesAllIterator())
and.AddSubIterator(it)
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
// Object iterator.
it = ts.TripleIterator(graph.Object, ts.ValueOf("F"))
it = qs.TripleIterator(quad.Object, qs.ValueOf("F"))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "follows", "F", ""},
{"E", "follows", "F", ""},
}
sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
}
and = iterator.NewAnd()
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B")))
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it)
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "follows", "F", ""},
}
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
// Predicate iterator.
it = ts.TripleIterator(graph.Predicate, ts.ValueOf("status"))
it = qs.TripleIterator(quad.Predicate, qs.ValueOf("status"))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
}
sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect)
}
// Provenance iterator.
it = ts.TripleIterator(graph.Provenance, ts.ValueOf("status_graph"))
it = qs.TripleIterator(quad.Provenance, qs.ValueOf("status_graph"))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
}
sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect)
}
it.Reset()
// Order is important
and = iterator.NewAnd()
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B")))
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it)
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
}
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
it.Reset()
@ -388,12 +389,12 @@ func TestSetIterator(t *testing.T) {
// Order is important
and = iterator.NewAnd()
and.AddSubIterator(it)
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B")))
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
}
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
}
@ -406,17 +407,17 @@ func TestOptimize(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create working directory")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts.AddTripleSet(makeTripleSet())
qs.AddTripleSet(makeTripleSet())
// With an linksto-fixed pair
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("F"))
fixed := qs.FixedIterator()
fixed.Add(qs.ValueOf("F"))
fixed.AddTag("internal")
lto := iterator.NewLinksTo(ts, fixed, graph.Object)
lto := iterator.NewLinksTo(qs, fixed, quad.Object)
oldIt := lto.Clone()
newIt, ok := lto.Optimize()
@ -427,8 +428,8 @@ func TestOptimize(t *testing.T) {
t.Errorf("Optimized iterator type does not match original, got:%v expect:%v", newIt.Type(), Type())
}
newTriples := iteratedTriples(ts, newIt)
oldTriples := iteratedTriples(ts, oldIt)
newTriples := iteratedTriples(qs, newIt)
oldTriples := iteratedTriples(qs, oldIt)
if !reflect.DeepEqual(newTriples, oldTriples) {
t.Errorf("Optimized iteration does not match original")
}

253
graph/leveldb/triplestore.go
View file

@ -30,6 +30,7 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
const (
@ -56,110 +57,110 @@ func createNewLevelDB(path string, _ graph.Options) error {
return err
}
defer db.Close()
ts := &TripleStore{}
ts.db = db
ts.writeopts = &opt.WriteOptions{
qs := &TripleStore{}
qs.db = db
qs.writeopts = &opt.WriteOptions{
Sync: true,
}
ts.Close()
qs.Close()
return nil
}
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore
ts.path = path
var qs TripleStore
qs.path = path
cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val
}
ts.dbOpts = &opt.Options{
qs.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
}
ts.dbOpts.ErrorIfMissing = true
qs.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val
}
ts.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
ts.hasher = sha1.New()
ts.writeopts = &opt.WriteOptions{
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.hasher = sha1.New()
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
ts.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(ts.path, ts.dbOpts)
qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil {
panic("Error, couldn't open! " + err.Error())
}
ts.db = db
glog.Infoln(ts.GetStats())
ts.getSize()
return &ts, nil
qs.db = db
glog.Infoln(qs.GetStats())
qs.getSize()
return &qs, nil
}
func (ts *TripleStore) GetStats() string {
func (qs *TripleStore) GetStats() string {
out := ""
stats, err := ts.db.GetProperty("leveldb.stats")
stats, err := qs.db.GetProperty("leveldb.stats")
if err == nil {
out += fmt.Sprintln("Stats: ", stats)
}
out += fmt.Sprintln("Size: ", ts.size)
out += fmt.Sprintln("Size: ", qs.size)
return out
}
func (ts *TripleStore) Size() int64 {
return ts.size
func (qs *TripleStore) Size() int64 {
return qs.size
}
func (ts *TripleStore) createKeyFor(d [3]graph.Direction, triple *graph.Triple) []byte {
key := make([]byte, 0, 2+(ts.hasher.Size()*3))
func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*3))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
return key
}
func (ts *TripleStore) createProvKeyFor(d [3]graph.Direction, triple *graph.Triple) []byte {
key := make([]byte, 0, 2+(ts.hasher.Size()*4))
func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*4))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{graph.Provenance.Prefix(), d[0].Prefix()}...)
key = append(key, ts.convertStringToByteHash(triple.Get(graph.Provenance))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, []byte{quad.Provenance.Prefix(), d[0].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(quad.Provenance))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
return key
}
func (ts *TripleStore) createValueKeyFor(s string) []byte {
key := make([]byte, 0, 1+ts.hasher.Size())
func (qs *TripleStore) createValueKeyFor(s string) []byte {
key := make([]byte, 0, 1+qs.hasher.Size())
key = append(key, []byte("z")...)
key = append(key, ts.convertStringToByteHash(s)...)
key = append(key, qs.convertStringToByteHash(s)...)
return key
}
func (ts *TripleStore) AddTriple(t *graph.Triple) {
func (qs *TripleStore) AddTriple(t *quad.Quad) {
batch := &leveldb.Batch{}
ts.buildWrite(batch, t)
err := ts.db.Write(batch, ts.writeopts)
qs.buildWrite(batch, t)
err := qs.db.Write(batch, qs.writeopts)
if err != nil {
glog.Errorf("Couldn't write to DB for triple %s", t)
return
}
ts.size++
qs.size++
}
// Short hand for direction permutations.
var (
spo = [3]graph.Direction{graph.Subject, graph.Predicate, graph.Object}
osp = [3]graph.Direction{graph.Object, graph.Subject, graph.Predicate}
pos = [3]graph.Direction{graph.Predicate, graph.Object, graph.Subject}
pso = [3]graph.Direction{graph.Predicate, graph.Subject, graph.Object}
spo = [3]quad.Direction{quad.Subject, quad.Predicate, quad.Object}
osp = [3]quad.Direction{quad.Object, quad.Subject, quad.Predicate}
pos = [3]quad.Direction{quad.Predicate, quad.Object, quad.Subject}
pso = [3]quad.Direction{quad.Predicate, quad.Subject, quad.Object}
)
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
_, err := ts.db.Get(ts.createKeyFor(spo, t), ts.readopts)
func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
_, err := qs.db.Get(qs.createKeyFor(spo, t), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorf("Couldn't access DB to confirm deletion")
return
@ -169,45 +170,45 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
return
}
batch := &leveldb.Batch{}
batch.Delete(ts.createKeyFor(spo, t))
batch.Delete(ts.createKeyFor(osp, t))
batch.Delete(ts.createKeyFor(pos, t))
ts.UpdateValueKeyBy(t.Get(graph.Subject), -1, batch)
ts.UpdateValueKeyBy(t.Get(graph.Predicate), -1, batch)
ts.UpdateValueKeyBy(t.Get(graph.Object), -1, batch)
if t.Get(graph.Provenance) != "" {
batch.Delete(ts.createProvKeyFor(pso, t))
ts.UpdateValueKeyBy(t.Get(graph.Provenance), -1, batch)
batch.Delete(qs.createKeyFor(spo, t))
batch.Delete(qs.createKeyFor(osp, t))
batch.Delete(qs.createKeyFor(pos, t))
qs.UpdateValueKeyBy(t.Get(quad.Subject), -1, batch)
qs.UpdateValueKeyBy(t.Get(quad.Predicate), -1, batch)
qs.UpdateValueKeyBy(t.Get(quad.Object), -1, batch)
if t.Get(quad.Provenance) != "" {
batch.Delete(qs.createProvKeyFor(pso, t))
qs.UpdateValueKeyBy(t.Get(quad.Provenance), -1, batch)
}
err = ts.db.Write(batch, nil)
err = qs.db.Write(batch, nil)
if err != nil {
glog.Errorf("Couldn't delete triple %s", t)
return
}
ts.size--
qs.size--
}
func (ts *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) {
func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *quad.Quad) {
bytes, err := json.Marshal(*t)
if err != nil {
glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t, err)
return
}
batch.Put(ts.createKeyFor(spo, t), bytes)
batch.Put(ts.createKeyFor(osp, t), bytes)
batch.Put(ts.createKeyFor(pos, t), bytes)
if t.Get(graph.Provenance) != "" {
batch.Put(ts.createProvKeyFor(pso, t), bytes)
batch.Put(qs.createKeyFor(spo, t), bytes)
batch.Put(qs.createKeyFor(osp, t), bytes)
batch.Put(qs.createKeyFor(pos, t), bytes)
if t.Get(quad.Provenance) != "" {
batch.Put(qs.createProvKeyFor(pso, t), bytes)
}
}
func (ts *TripleStore) buildWrite(batch *leveldb.Batch, t *graph.Triple) {
ts.buildTripleWrite(batch, t)
ts.UpdateValueKeyBy(t.Get(graph.Subject), 1, nil)
ts.UpdateValueKeyBy(t.Get(graph.Predicate), 1, nil)
ts.UpdateValueKeyBy(t.Get(graph.Object), 1, nil)
if t.Get(graph.Provenance) != "" {
ts.UpdateValueKeyBy(t.Get(graph.Provenance), 1, nil)
func (qs *TripleStore) buildWrite(batch *leveldb.Batch, t *quad.Quad) {
qs.buildTripleWrite(batch, t)
qs.UpdateValueKeyBy(t.Get(quad.Subject), 1, nil)
qs.UpdateValueKeyBy(t.Get(quad.Predicate), 1, nil)
qs.UpdateValueKeyBy(t.Get(quad.Object), 1, nil)
if t.Get(quad.Provenance) != "" {
qs.UpdateValueKeyBy(t.Get(quad.Provenance), 1, nil)
}
}
@ -216,10 +217,10 @@ type ValueData struct {
Size int64
}
func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) {
func (qs *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) {
value := &ValueData{name, int64(amount)}
key := ts.createValueKeyFor(name)
b, err := ts.db.Get(key, ts.readopts)
key := qs.createValueKeyFor(name)
b, err := qs.db.Get(key, qs.readopts)
// Error getting the node from the database.
if err != nil && err != leveldb.ErrNotFound {
@ -241,7 +242,7 @@ func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.
if amount < 0 {
if value.Size <= 0 {
if batch == nil {
ts.db.Delete(key, ts.writeopts)
qs.db.Delete(key, qs.writeopts)
} else {
batch.Delete(key)
}
@ -256,18 +257,18 @@ func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.
return
}
if batch == nil {
ts.db.Put(key, bytes, ts.writeopts)
qs.db.Put(key, bytes, qs.writeopts)
} else {
batch.Put(key, bytes)
}
}
func (ts *TripleStore) AddTripleSet(t_s []*graph.Triple) {
func (qs *TripleStore) AddTripleSet(t_s []*quad.Quad) {
batch := &leveldb.Batch{}
newTs := len(t_s)
resizeMap := make(map[string]int)
for _, t := range t_s {
ts.buildTripleWrite(batch, t)
qs.buildTripleWrite(batch, t)
resizeMap[t.Subject]++
resizeMap[t.Predicate]++
resizeMap[t.Object]++
@ -276,68 +277,68 @@ func (ts *TripleStore) AddTripleSet(t_s []*graph.Triple) {
}
}
for k, v := range resizeMap {
ts.UpdateValueKeyBy(k, v, batch)
qs.UpdateValueKeyBy(k, v, batch)
}
err := ts.db.Write(batch, ts.writeopts)
err := qs.db.Write(batch, qs.writeopts)
if err != nil {
glog.Errorf("Couldn't write to DB for tripleset")
return
}
ts.size += int64(newTs)
qs.size += int64(newTs)
}
func (ts *TripleStore) Close() {
func (qs *TripleStore) Close() {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, ts.size)
err := binary.Write(buf, binary.LittleEndian, qs.size)
if err == nil {
werr := ts.db.Put([]byte("__size"), buf.Bytes(), ts.writeopts)
werr := qs.db.Put([]byte("__size"), buf.Bytes(), qs.writeopts)
if werr != nil {
glog.Errorf("Couldn't write size before closing!")
}
} else {
glog.Errorf("Couldn't convert size before closing!")
}
ts.db.Close()
ts.open = false
qs.db.Close()
qs.open = false
}
func (ts *TripleStore) Triple(k graph.Value) *graph.Triple {
var triple graph.Triple
b, err := ts.db.Get(k.([]byte), ts.readopts)
func (qs *TripleStore) Quad(k graph.Value) *quad.Quad {
var triple quad.Quad
b, err := qs.db.Get(k.([]byte), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: couldn't get triple from DB")
return &graph.Triple{}
return &quad.Quad{}
}
if err == leveldb.ErrNotFound {
// No harm, no foul.
return &graph.Triple{}
return &quad.Quad{}
}
err = json.Unmarshal(b, &triple)
if err != nil {
glog.Errorln("Error: couldn't reconstruct triple")
return &graph.Triple{}
return &quad.Quad{}
}
return &triple
}
func (ts *TripleStore) convertStringToByteHash(s string) []byte {
ts.hasher.Reset()
key := make([]byte, 0, ts.hasher.Size())
ts.hasher.Write([]byte(s))
key = ts.hasher.Sum(key)
func (qs *TripleStore) convertStringToByteHash(s string) []byte {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
return key
}
func (ts *TripleStore) ValueOf(s string) graph.Value {
return ts.createValueKeyFor(s)
func (qs *TripleStore) ValueOf(s string) graph.Value {
return qs.createValueKeyFor(s)
}
func (ts *TripleStore) valueData(value_key []byte) ValueData {
func (qs *TripleStore) valueData(value_key []byte) ValueData {
var out ValueData
if glog.V(3) {
glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key)
}
b, err := ts.db.Get(value_key, ts.readopts)
b, err := qs.db.Get(value_key, qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: couldn't get value from DB")
return out
@ -352,30 +353,30 @@ func (ts *TripleStore) valueData(value_key []byte) ValueData {
return out
}
func (ts *TripleStore) NameOf(k graph.Value) string {
func (qs *TripleStore) NameOf(k graph.Value) string {
if k == nil {
glog.V(2).Infoln("k was nil")
return ""
}
return ts.valueData(k.([]byte)).Name
return qs.valueData(k.([]byte)).Name
}
func (ts *TripleStore) SizeOf(k graph.Value) int64 {
func (qs *TripleStore) SizeOf(k graph.Value) int64 {
if k == nil {
return 0
}
return int64(ts.valueData(k.([]byte)).Size)
return int64(qs.valueData(k.([]byte)).Size)
}
func (ts *TripleStore) getSize() {
func (qs *TripleStore) getSize() {
var size int64
b, err := ts.db.Get([]byte("__size"), ts.readopts)
b, err := qs.db.Get([]byte("__size"), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
panic("Couldn't read size " + err.Error())
}
if err == leveldb.ErrNotFound {
// Must be a new database. Cool
ts.size = 0
qs.size = 0
return
}
buf := bytes.NewBuffer(b)
@ -383,10 +384,10 @@ func (ts *TripleStore) getSize() {
if err != nil {
glog.Errorln("Error: couldn't parse size")
}
ts.size = size
qs.size = size
}
func (ts *TripleStore) SizeOfPrefix(pre []byte) (int64, error) {
func (qs *TripleStore) SizeOfPrefix(pre []byte) (int64, error) {
limit := make([]byte, len(pre))
copy(limit, pre)
end := len(limit) - 1
@ -394,45 +395,45 @@ func (ts *TripleStore) SizeOfPrefix(pre []byte) (int64, error) {
ranges := make([]util.Range, 1)
ranges[0].Start = pre
ranges[0].Limit = limit
sizes, err := ts.db.SizeOf(ranges)
sizes, err := qs.db.SizeOf(ranges)
if err == nil {
return (int64(sizes[0]) >> 6) + 1, nil
}
return 0, nil
}
func (ts *TripleStore) TripleIterator(d graph.Direction, val graph.Value) graph.Iterator {
func (qs *TripleStore) TripleIterator(d quad.Direction, val graph.Value) graph.Iterator {
var prefix string
switch d {
case graph.Subject:
case quad.Subject:
prefix = "sp"
case graph.Predicate:
case quad.Predicate:
prefix = "po"
case graph.Object:
case quad.Object:
prefix = "os"
case graph.Provenance:
case quad.Provenance:
prefix = "cp"
default:
panic("unreachable " + d.String())
}
return NewIterator(prefix, d, val, ts)
return NewIterator(prefix, d, val, qs)
}
func (ts *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator("z", graph.Any, ts)
func (qs *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator("z", quad.Any, qs)
}
func (ts *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator("po", graph.Predicate, ts)
func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator("po", quad.Predicate, qs)
}
func (ts *TripleStore) TripleDirection(val graph.Value, d graph.Direction) graph.Value {
func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value {
v := val.([]uint8)
offset := PositionOf(v[0:2], d, ts)
offset := PositionOf(v[0:2], d, qs)
if offset != -1 {
return append([]byte("z"), v[offset:offset+ts.hasher.Size()]...)
return append([]byte("z"), v[offset:offset+qs.hasher.Size()]...)
} else {
return ts.Triple(val).Get(d)
return qs.Quad(val).Get(d)
}
}
@ -440,7 +441,7 @@ func compareBytes(a, b graph.Value) bool {
return bytes.Equal(a.([]uint8), b.([]uint8))
}
func (ts *TripleStore) FixedIterator() graph.FixedIterator {
func (qs *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(compareBytes)
}

63
graph/memstore/triplestore.go
View file

@ -20,6 +20,7 @@ import (
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
"github.com/petar/GoLLRB/llrb"
)
@ -40,21 +41,21 @@ func NewTripleDirectionIndex() *TripleDirectionIndex {
return &tdi
}
func (tdi *TripleDirectionIndex) GetForDir(d graph.Direction) map[int64]*llrb.LLRB {
func (tdi *TripleDirectionIndex) GetForDir(d quad.Direction) map[int64]*llrb.LLRB {
switch d {
case graph.Subject:
case quad.Subject:
return tdi.subject
case graph.Object:
case quad.Object:
return tdi.object
case graph.Predicate:
case quad.Predicate:
return tdi.predicate
case graph.Provenance:
case quad.Provenance:
return tdi.provenance
}
panic("illegal direction")
}
func (tdi *TripleDirectionIndex) GetOrCreate(d graph.Direction, id int64) *llrb.LLRB {
func (tdi *TripleDirectionIndex) GetOrCreate(d quad.Direction, id int64) *llrb.LLRB {
directionIndex := tdi.GetForDir(d)
if _, ok := directionIndex[id]; !ok {
directionIndex[id] = llrb.New()
@ -62,7 +63,7 @@ func (tdi *TripleDirectionIndex) GetOrCreate(d graph.Direction, id int64) *llrb.
return directionIndex[id]
}
func (tdi *TripleDirectionIndex) Get(d graph.Direction, id int64) (*llrb.LLRB, bool) {
func (tdi *TripleDirectionIndex) Get(d quad.Direction, id int64) (*llrb.LLRB, bool) {
directionIndex := tdi.GetForDir(d)
tree, exists := directionIndex[id]
return tree, exists
@ -73,7 +74,7 @@ type TripleStore struct {
tripleIdCounter int64
idMap map[string]int64
revIdMap map[int64]string
triples []graph.Triple
triples []quad.Quad
size int64
index TripleDirectionIndex
// vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree
@ -83,10 +84,10 @@ func newTripleStore() *TripleStore {
var ts TripleStore
ts.idMap = make(map[string]int64)
ts.revIdMap = make(map[int64]string)
ts.triples = make([]graph.Triple, 1, 200)
ts.triples = make([]quad.Quad, 1, 200)
// Sentinel null triple so triple indices start at 1
ts.triples[0] = graph.Triple{}
ts.triples[0] = quad.Quad{}
ts.size = 1
ts.index = *NewTripleDirectionIndex()
ts.idCounter = 1
@ -94,18 +95,18 @@ func newTripleStore() *TripleStore {
return &ts
}
func (ts *TripleStore) AddTripleSet(triples []*graph.Triple) {
func (ts *TripleStore) AddTripleSet(triples []*quad.Quad) {
for _, t := range triples {
ts.AddTriple(t)
}
}
func (ts *TripleStore) tripleExists(t *graph.Triple) (bool, int64) {
func (ts *TripleStore) tripleExists(t *quad.Quad) (bool, int64) {
smallest := -1
var smallest_tree *llrb.LLRB
for d := graph.Subject; d <= graph.Provenance; d++ {
for d := quad.Subject; d <= quad.Provenance; d++ {
sid := t.Get(d)
if d == graph.Provenance && sid == "" {
if d == quad.Provenance && sid == "" {
continue
}
id, ok := ts.idMap[sid]
@ -137,7 +138,7 @@ func (ts *TripleStore) tripleExists(t *graph.Triple) (bool, int64) {
return false, 0
}
func (ts *TripleStore) AddTriple(t *graph.Triple) {
func (ts *TripleStore) AddTriple(t *quad.Quad) {
if exists, _ := ts.tripleExists(t); exists {
return
}
@ -147,9 +148,9 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
ts.size++
ts.tripleIdCounter++
for d := graph.Subject; d <= graph.Provenance; d++ {
for d := quad.Subject; d <= quad.Provenance; d++ {
sid := t.Get(d)
if d == graph.Provenance && sid == "" {
if d == quad.Provenance && sid == "" {
continue
}
if _, ok := ts.idMap[sid]; !ok {
@ -159,8 +160,8 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
}
}
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Provenance; d++ {
if d == quad.Provenance && t.Get(d) == "" {
continue
}
id := ts.idMap[t.Get(d)]
@ -171,7 +172,7 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
// TODO(barakmich): Add VIP indexing
}
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
func (ts *TripleStore) RemoveTriple(t *quad.Quad) {
var tripleID int64
var exists bool
tripleID = 0
@ -179,11 +180,11 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
return
}
ts.triples[tripleID] = graph.Triple{}
ts.triples[tripleID] = quad.Quad{}
ts.size--
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Provenance; d++ {
if d == quad.Provenance && t.Get(d) == "" {
continue
}
id := ts.idMap[t.Get(d)]
@ -191,8 +192,8 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
tree.Delete(Int64(tripleID))
}
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Provenance; d++ {
if d == quad.Provenance && t.Get(d) == "" {
continue
}
id, ok := ts.idMap[t.Get(d)]
@ -200,8 +201,8 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
continue
}
stillExists := false
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Provenance; d++ {
if d == quad.Provenance && t.Get(d) == "" {
continue
}
nodeTree := ts.index.GetOrCreate(d, id)
@ -217,11 +218,11 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
}
}
func (ts *TripleStore) Triple(index graph.Value) *graph.Triple {
func (ts *TripleStore) Quad(index graph.Value) *quad.Quad {
return &ts.triples[index.(int64)]
}
func (ts *TripleStore) TripleIterator(d graph.Direction, value graph.Value) graph.Iterator {
func (ts *TripleStore) TripleIterator(d quad.Direction, value graph.Value) graph.Iterator {
index, ok := ts.index.Get(d, value.(int64))
data := fmt.Sprintf("dir:%s val:%d", d, value.(int64))
if ok {
@ -259,8 +260,8 @@ func (ts *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(iterator.BasicEquality)
}
func (ts *TripleStore) TripleDirection(val graph.Value, d graph.Direction) graph.Value {
name := ts.Triple(val).Get(d)
func (ts *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value {
name := ts.Quad(val).Get(d)
return ts.ValueOf(name)
}

22
graph/memstore/triplestore_test.go
View file

@ -19,8 +19,8 @@ import (
"sort"
"testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
// This is a simple test graph.
@ -36,7 +36,7 @@ import (
// \-->|#D#|------------->+---+
// +---+
//
var simpleGraph = []*graph.Triple{
var simpleGraph = []*quad.Quad{
{"A", "follows", "B", ""},
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
@ -50,7 +50,7 @@ var simpleGraph = []*graph.Triple{
{"G", "status", "cool", "status_graph"},
}
func makeTestStore(data []*graph.Triple) (*TripleStore, []pair) {
func makeTestStore(data []*quad.Quad) (*TripleStore, []pair) {
seen := make(map[string]struct{})
ts := newTripleStore()
var (
@ -105,10 +105,10 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
all := ts.NodesAllIterator()
innerAnd := iterator.NewAnd()
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, graph.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, all, graph.Object))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, quad.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, all, quad.Object))
hasa := iterator.NewHasA(ts, innerAnd, graph.Subject)
hasa := iterator.NewHasA(ts, innerAnd, quad.Subject)
outerAnd := iterator.NewAnd()
outerAnd.AddSubIterator(fixed)
outerAnd.AddSubIterator(hasa)
@ -149,7 +149,7 @@ func TestLinksToOptimization(t *testing.T) {
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("cool"))
lto := iterator.NewLinksTo(ts, fixed, graph.Object)
lto := iterator.NewLinksTo(ts, fixed, quad.Object)
lto.AddTag("foo")
newIt, changed := lto.Optimize()
@ -173,7 +173,7 @@ func TestLinksToOptimization(t *testing.T) {
func TestRemoveTriple(t *testing.T) {
ts, _ := makeTestStore(simpleGraph)
ts.RemoveTriple(&graph.Triple{"E", "follows", "F", ""})
ts.RemoveTriple(&quad.Quad{"E", "follows", "F", ""})
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("E"))
@ -182,10 +182,10 @@ func TestRemoveTriple(t *testing.T) {
fixed2.Add(ts.ValueOf("follows"))
innerAnd := iterator.NewAnd()
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, graph.Subject))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, graph.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, quad.Subject))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, quad.Predicate))
hasa := iterator.NewHasA(ts, innerAnd, graph.Object)
hasa := iterator.NewHasA(ts, innerAnd, quad.Object)
newIt, _ := hasa.Optimize()
_, ok := newIt.Next()

55
graph/mongo/iterator.go
View file

@ -24,12 +24,13 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
type Iterator struct {
iterator.Base
ts *TripleStore
dir graph.Direction
qs *TripleStore
dir quad.Direction
iter *mgo.Iter
hash string
name string
@ -39,27 +40,27 @@ type Iterator struct {
collection string
}
func NewIterator(ts *TripleStore, collection string, d graph.Direction, val graph.Value) *Iterator {
func NewIterator(qs *TripleStore, collection string, d quad.Direction, val graph.Value) *Iterator {
var m Iterator
iterator.BaseInit(&m.Base)
m.name = ts.NameOf(val)
m.name = qs.NameOf(val)
m.collection = collection
switch d {
case graph.Subject:
case quad.Subject:
m.constraint = bson.M{"Subject": m.name}
case graph.Predicate:
case quad.Predicate:
m.constraint = bson.M{"Predicate": m.name}
case graph.Object:
case quad.Object:
m.constraint = bson.M{"Object": m.name}
case graph.Provenance:
case quad.Provenance:
m.constraint = bson.M{"Provenance": m.name}
}
m.ts = ts
m.qs = qs
m.dir = d
m.iter = ts.db.C(collection).Find(m.constraint).Iter()
size, err := ts.db.C(collection).Find(m.constraint).Count()
m.iter = qs.db.C(collection).Find(m.constraint).Iter()
size, err := qs.db.C(collection).Find(m.constraint).Count()
if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
@ -70,14 +71,14 @@ func NewIterator(ts *TripleStore, collection string, d graph.Direction, val grap
return &m
}
func NewAllIterator(ts *TripleStore, collection string) *Iterator {
func NewAllIterator(qs *TripleStore, collection string) *Iterator {
var m Iterator
m.ts = ts
m.dir = graph.Any
m.qs = qs
m.dir = quad.Any
m.constraint = nil
m.collection = collection
m.iter = ts.db.C(collection).Find(nil).Iter()
size, err := ts.db.C(collection).Count()
m.iter = qs.db.C(collection).Find(nil).Iter()
size, err := qs.db.C(collection).Count()
if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
@ -90,7 +91,7 @@ func NewAllIterator(ts *TripleStore, collection string) *Iterator {
func (it *Iterator) Reset() {
it.iter.Close()
it.iter = it.ts.db.C(it.collection).Find(it.constraint).Iter()
it.iter = it.qs.db.C(it.collection).Find(it.constraint).Iter()
}
@ -101,9 +102,9 @@ func (it *Iterator) Close() {
func (it *Iterator) Clone() graph.Iterator {
var newM graph.Iterator
if it.isAll {
newM = NewAllIterator(it.ts, it.collection)
newM = NewAllIterator(it.qs, it.collection)
} else {
newM = NewIterator(it.ts, it.collection, it.dir, it.hash)
newM = NewIterator(it.qs, it.collection, it.dir, it.hash)
}
newM.CopyTagsFrom(it)
return newM
@ -136,16 +137,16 @@ func (it *Iterator) Check(v graph.Value) bool {
}
var offset int
switch it.dir {
case graph.Subject:
case quad.Subject:
offset = 0
case graph.Predicate:
offset = (it.ts.hasher.Size() * 2)
case graph.Object:
offset = (it.ts.hasher.Size() * 2) * 2
case graph.Provenance:
offset = (it.ts.hasher.Size() * 2) * 3
case quad.Predicate:
offset = (it.qs.hasher.Size() * 2)
case quad.Object:
offset = (it.qs.hasher.Size() * 2) * 2
case quad.Provenance:
offset = (it.qs.hasher.Size() * 2) * 3
}
val := v.(string)[offset : it.ts.hasher.Size()*2+offset]
val := v.(string)[offset : it.qs.hasher.Size()*2+offset]
if val == it.hash {
it.Last = v
return graph.CheckLogOut(it, v, true)

180
graph/mongo/triplestore.go
View file

@ -18,6 +18,7 @@ import (
"crypto/sha1"
"encoding/hex"
"hash"
"io"
"log"
"gopkg.in/mgo.v2"
@ -26,8 +27,16 @@ import (
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
func init() {
graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
}
// Guarantee we satisfy graph.Bulkloader.
var _ graph.BulkLoader = (*TripleStore)(nil)
const DefaultDBName = "cayley"
type TripleStore struct {
@ -66,7 +75,7 @@ func createNewMongoGraph(addr string, options graph.Options) error {
}
func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore
var qs TripleStore
conn, err := mgo.Dial(addr)
if err != nil {
return nil, err
@ -76,26 +85,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
if val, ok := options.StringKey("database_name"); ok {
dbName = val
}
ts.db = conn.DB(dbName)
ts.session = conn
ts.hasher = sha1.New()
ts.idCache = NewIDLru(1 << 16)
return &ts, nil
qs.db = conn.DB(dbName)
qs.session = conn
qs.hasher = sha1.New()
qs.idCache = NewIDLru(1 << 16)
return &qs, nil
}
func (ts *TripleStore) getIdForTriple(t *graph.Triple) string {
id := ts.ConvertStringToByteHash(t.Subject)
id += ts.ConvertStringToByteHash(t.Predicate)
id += ts.ConvertStringToByteHash(t.Object)
id += ts.ConvertStringToByteHash(t.Provenance)
func (qs *TripleStore) getIdForTriple(t *quad.Quad) string {
id := qs.ConvertStringToByteHash(t.Subject)
id += qs.ConvertStringToByteHash(t.Predicate)
id += qs.ConvertStringToByteHash(t.Object)
id += qs.ConvertStringToByteHash(t.Provenance)
return id
}
func (ts *TripleStore) ConvertStringToByteHash(s string) string {
ts.hasher.Reset()
key := make([]byte, 0, ts.hasher.Size())
ts.hasher.Write([]byte(s))
key = ts.hasher.Sum(key)
func (qs *TripleStore) ConvertStringToByteHash(s string) string {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
return hex.EncodeToString(key)
}
@ -105,10 +114,10 @@ type MongoNode struct {
Size int "Size"
}
func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
func (qs *TripleStore) updateNodeBy(node_name string, inc int) {
var size MongoNode
node := ts.ValueOf(node_name)
err := ts.db.C("nodes").FindId(node).One(&size)
node := qs.ValueOf(node_name)
err := qs.db.C("nodes").FindId(node).One(&size)
if err != nil {
if err.Error() == "not found" {
// Not found. Okay.
@ -128,7 +137,7 @@ func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
// Removing something...
if inc < 0 {
if size.Size <= 0 {
err := ts.db.C("nodes").RemoveId(node)
err := qs.db.C("nodes").RemoveId(node)
if err != nil {
glog.Error("Error: ", err, " while removing node ", node_name)
return
@ -136,21 +145,21 @@ func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
}
}
_, err2 := ts.db.C("nodes").UpsertId(node, size)
_, err2 := qs.db.C("nodes").UpsertId(node, size)
if err2 != nil {
glog.Error("Error: ", err)
}
}
func (ts *TripleStore) writeTriple(t *graph.Triple) bool {
func (qs *TripleStore) writeTriple(t *quad.Quad) bool {
tripledoc := bson.M{
"_id": ts.getIdForTriple(t),
"_id": qs.getIdForTriple(t),
"Subject": t.Subject,
"Predicate": t.Predicate,
"Object": t.Object,
"Provenance": t.Provenance,
}
err := ts.db.C("triples").Insert(tripledoc)
err := qs.db.C("triples").Insert(tripledoc)
if err != nil {
// Among the reasons I hate MongoDB. "Errors don't happen! Right guys?"
if err.(*mgo.LastError).Code == 11000 {
@ -162,21 +171,21 @@ func (ts *TripleStore) writeTriple(t *graph.Triple) bool {
return true
}
func (ts *TripleStore) AddTriple(t *graph.Triple) {
_ = ts.writeTriple(t)
ts.updateNodeBy(t.Subject, 1)
ts.updateNodeBy(t.Predicate, 1)
ts.updateNodeBy(t.Object, 1)
func (qs *TripleStore) AddTriple(t *quad.Quad) {
_ = qs.writeTriple(t)
qs.updateNodeBy(t.Subject, 1)
qs.updateNodeBy(t.Predicate, 1)
qs.updateNodeBy(t.Object, 1)
if t.Provenance != "" {
ts.updateNodeBy(t.Provenance, 1)
qs.updateNodeBy(t.Provenance, 1)
}
}
func (ts *TripleStore) AddTripleSet(in []*graph.Triple) {
ts.session.SetSafe(nil)
func (qs *TripleStore) AddTripleSet(in []*quad.Quad) {
qs.session.SetSafe(nil)
ids := make(map[string]int)
for _, t := range in {
wrote := ts.writeTriple(t)
wrote := qs.writeTriple(t)
if wrote {
ids[t.Subject]++
ids[t.Object]++
@ -187,34 +196,34 @@ func (ts *TripleStore) AddTripleSet(in []*graph.Triple) {
}
}
for k, v := range ids {
ts.updateNodeBy(k, v)
qs.updateNodeBy(k, v)
}
ts.session.SetSafe(&mgo.Safe{})
qs.session.SetSafe(&mgo.Safe{})
}
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t))
func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
err := qs.db.C("triples").RemoveId(qs.getIdForTriple(t))
if err == mgo.ErrNotFound {
return
} else if err != nil {
log.Println("Error: ", err, " while removing triple ", t)
return
}
ts.updateNodeBy(t.Subject, -1)
ts.updateNodeBy(t.Predicate, -1)
ts.updateNodeBy(t.Object, -1)
qs.updateNodeBy(t.Subject, -1)
qs.updateNodeBy(t.Predicate, -1)
qs.updateNodeBy(t.Object, -1)
if t.Provenance != "" {
ts.updateNodeBy(t.Provenance, -1)
qs.updateNodeBy(t.Provenance, -1)
}
}
func (ts *TripleStore) Triple(val graph.Value) *graph.Triple {
func (qs *TripleStore) Quad(val graph.Value) *quad.Quad {
var bsonDoc bson.M
err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc)
err := qs.db.C("triples").FindId(val.(string)).One(&bsonDoc)
if err != nil {
log.Println("Error: Couldn't retrieve triple", val.(string), err)
}
return &graph.Triple{
return &quad.Quad{
bsonDoc["Subject"].(string),
bsonDoc["Predicate"].(string),
bsonDoc["Object"].(string),
@ -222,38 +231,38 @@ func (ts *TripleStore) Triple(val graph.Value) *graph.Triple {
}
}
func (ts *TripleStore) TripleIterator(d graph.Direction, val graph.Value) graph.Iterator {
return NewIterator(ts, "triples", d, val)
func (qs *TripleStore) TripleIterator(d quad.Direction, val graph.Value) graph.Iterator {
return NewIterator(qs, "triples", d, val)
}
func (ts *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator(ts, "nodes")
func (qs *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator(qs, "nodes")
}
func (ts *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator(ts, "triples")
func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator(qs, "triples")
}
func (ts *TripleStore) ValueOf(s string) graph.Value {
return ts.ConvertStringToByteHash(s)
func (qs *TripleStore) ValueOf(s string) graph.Value {
return qs.ConvertStringToByteHash(s)
}
func (ts *TripleStore) NameOf(v graph.Value) string {
val, ok := ts.idCache.Get(v.(string))
func (qs *TripleStore) NameOf(v graph.Value) string {
val, ok := qs.idCache.Get(v.(string))
if ok {
return val
}
var node MongoNode
err := ts.db.C("nodes").FindId(v.(string)).One(&node)
err := qs.db.C("nodes").FindId(v.(string)).One(&node)
if err != nil {
log.Println("Error: Couldn't retrieve node", v.(string), err)
}
ts.idCache.Put(v.(string), node.Name)
qs.idCache.Put(v.(string), node.Name)
return node.Name
}
func (ts *TripleStore) Size() int64 {
count, err := ts.db.C("triples").Count()
func (qs *TripleStore) Size() int64 {
count, err := qs.db.C("triples").Count()
if err != nil {
glog.Error("Error: ", err)
return 0
@ -265,40 +274,48 @@ func compareStrings(a, b graph.Value) bool {
return a.(string) == b.(string)
}
func (ts *TripleStore) FixedIterator() graph.FixedIterator {
func (qs *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(compareStrings)
}
func (ts *TripleStore) Close() {
ts.db.Session.Close()
func (qs *TripleStore) Close() {
qs.db.Session.Close()
}
func (ts *TripleStore) TripleDirection(in graph.Value, d graph.Direction) graph.Value {
func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.Value {
// Maybe do the trick here
var offset int
switch d {
case graph.Subject:
case quad.Subject:
offset = 0
case graph.Predicate:
offset = (ts.hasher.Size() * 2)
case graph.Object:
offset = (ts.hasher.Size() * 2) * 2
case graph.Provenance:
offset = (ts.hasher.Size() * 2) * 3
case quad.Predicate:
offset = (qs.hasher.Size() * 2)
case quad.Object:
offset = (qs.hasher.Size() * 2) * 2
case quad.Provenance:
offset = (qs.hasher.Size() * 2) * 3
}
val := in.(string)[offset : ts.hasher.Size()*2+offset]
val := in.(string)[offset : qs.hasher.Size()*2+offset]
return val
}
func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool {
if ts.Size() != 0 {
return false
func (qs *TripleStore) BulkLoad(dec quad.Unmarshaler) error {
if qs.Size() != 0 {
return graph.ErrCannotBulkLoad
}
ts.session.SetSafe(nil)
for triple := range t_chan {
ts.writeTriple(triple)
qs.session.SetSafe(nil)
for {
q, err := dec.Unmarshal()
if err != nil {
if err != io.EOF {
return err
}
break
}
qs.writeTriple(q)
}
outputTo := bson.M{"replace": "nodes", "sharded": true}
glog.Infoln("Mapreducing")
job := mgo.MapReduce{
@ -330,16 +347,13 @@ func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool {
`,
Out: outputTo,
}
ts.db.C("triples").Find(nil).MapReduce(&job, nil)
qs.db.C("triples").Find(nil).MapReduce(&job, nil)
glog.Infoln("Fixing")
ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
qs.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
db.nodes.update({"_id": result._id}, result.value)
}) }`}, {"args", bson.D{}}}, nil)
ts.session.SetSafe(&mgo.Safe{})
return true
}
qs.session.SetSafe(&mgo.Safe{})
func init() {
graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
return nil
}

11
graph/sexp/parser.go
View file

@ -19,6 +19,7 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator {
@ -208,7 +209,7 @@ func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Ite
i++
}
it := buildIteratorTree(tree.Children[i], ts)
lto := iterator.NewLinksTo(ts, it, graph.Predicate)
lto := iterator.NewLinksTo(ts, it, quad.Predicate)
return lto
case "RootConstraint":
constraintCount := 0
@ -229,16 +230,16 @@ func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Ite
return and
case "Constraint":
var hasa *iterator.HasA
topLevelDir := graph.Subject
subItDir := graph.Object
topLevelDir := quad.Subject
subItDir := quad.Object
subAnd := iterator.NewAnd()
isOptional := false
for _, c := range tree.Children {
switch c.Name {
case "PredIdentifier":
if c.Children[0].Name == "Reverse" {
topLevelDir = graph.Object
subItDir = graph.Subject
topLevelDir = quad.Object
subItDir = quad.Subject
}
it := buildIteratorTree(c, ts)
subAnd.AddSubIterator(it)

18
graph/sexp/parser_test.go
View file

@ -18,6 +18,8 @@ import (
"testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
_ "github.com/google/cayley/graph/memstore"
)
@ -30,21 +32,21 @@ func TestBadParse(t *testing.T) {
var testQueries = []struct {
message string
add *graph.Triple
add *quad.Quad
query string
typ graph.Type
expect string
}{
{
message: "get a single triple linkage",
add: &graph.Triple{"i", "can", "win", ""},
add: &quad.Quad{"i", "can", "win", ""},
query: "($a (:can \"win\"))",
typ: graph.And,
expect: "i",
},
{
message: "get a single triple linkage",
add: &graph.Triple{"i", "can", "win", ""},
add: &quad.Quad{"i", "can", "win", ""},
query: "(\"i\" (:can $a))",
typ: graph.And,
expect: "i",
@ -77,8 +79,8 @@ func TestMemstoreBackedSexp(t *testing.T) {
func TestTreeConstraintParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
ts.AddTriple(&graph.Triple{"i", "like", "food", ""})
ts.AddTriple(&graph.Triple{"food", "is", "good", ""})
ts.AddTriple(&quad.Quad{"i", "like", "food", ""})
ts.AddTriple(&quad.Quad{"food", "is", "good", ""})
query := "(\"i\"\n" +
"(:like\n" +
"($a (:is :good))))"
@ -97,8 +99,8 @@ func TestTreeConstraintParse(t *testing.T) {
func TestTreeConstraintTagParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
ts.AddTriple(&graph.Triple{"i", "like", "food", ""})
ts.AddTriple(&graph.Triple{"food", "is", "good", ""})
ts.AddTriple(&quad.Quad{"i", "like", "food", ""})
ts.AddTriple(&quad.Quad{"food", "is", "good", ""})
query := "(\"i\"\n" +
"(:like\n" +
"($a (:is :good))))"
@ -117,7 +119,7 @@ func TestTreeConstraintTagParse(t *testing.T) {
func TestMultipleConstraintParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
for _, tv := range []*graph.Triple{
for _, tv := range []*quad.Quad{
{"i", "like", "food", ""},
{"i", "like", "beer", ""},
{"you", "like", "beer", ""},

142
graph/triple.go
View file

@ -1,142 +0,0 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package graph
// Defines the struct which makes the TripleStore possible -- the triple.
//
// At its heart, it consists of three fields -- Subject, Predicate, and Object.
// Three IDs that relate to each other. That's all there is to it. The triples
// are the links in the graph, and the existence of node IDs is defined by the
// fact that some triple in the graph mentions them.
//
// This means that a complete representation of the graph is equivalent to a
// list of triples. The rest is just indexing for speed.
//
// Adding fields to the triple is not to be taken lightly. You'll see I mention
// provenance, but don't as yet use it in any backing store. In general, there
// can be features that can be turned on or off for any store, but I haven't
// decided how to allow/disallow them yet. Another such example would be to add
// a forward and reverse index field -- forward being "order the list of
// objects pointed at by this subject with this predicate" such as first and
// second children, top billing, what have you.
//
// There will never be that much in this file except for the definition, but
// the consequences are not to be taken lightly. But do suggest cool features!
import "fmt"
// TODO(kortschak) Consider providing MashalJSON and UnmarshalJSON
// instead of using struct tags.
// Our triple struct, used throughout.
type Triple struct {
Subject string `json:"subject"`
Predicate string `json:"predicate"`
Object string `json:"object"`
Provenance string `json:"provenance,omitempty"`
}
// Direction specifies an edge's type.
type Direction byte
// List of the valid directions of a triple.
const (
Any Direction = iota
Subject
Predicate
Object
Provenance
)
func (d Direction) Prefix() byte {
switch d {
case Any:
return 'a'
case Subject:
return 's'
case Predicate:
return 'p'
case Provenance:
return 'c'
case Object:
return 'o'
default:
return '\x00'
}
}
func (d Direction) String() string {
switch d {
case Any:
return "any"
case Subject:
return "subject"
case Predicate:
return "predicate"
case Provenance:
return "provenance"
case Object:
return "object"
default:
return fmt.Sprint("illegal direction:", byte(d))
}
}
// TODO(kortschak) Consider writing methods onto the concrete type
// instead of the pointer. This needs benchmarking to make the decision.
// Per-field accessor for triples
func (t *Triple) Get(d Direction) string {
switch d {
case Subject:
return t.Subject
case Predicate:
return t.Predicate
case Provenance:
return t.Provenance
case Object:
return t.Object
default:
panic(d.String())
}
}
func (t *Triple) Equals(o *Triple) bool {
return *t == *o
}
// Pretty-prints a triple.
func (t *Triple) String() string {
// TODO(kortschak) String methods should generally not terminate in '\n'.
return fmt.Sprintf("%s -- %s -> %s\n", t.Subject, t.Predicate, t.Object)
}
func (t *Triple) IsValid() bool {
return t.Subject != "" && t.Predicate != "" && t.Object != ""
}
// TODO(kortschak) NTriple looks like a good candidate for conversion
// to MarshalText() (text []byte, err error) and then move parsing code
// from nquads to here to provide UnmarshalText(text []byte) error.
// Prints a triple in N-Triple format.
func (t *Triple) NTriple() string {
if t.Provenance == "" {
//TODO(barakmich): Proper escaping.
return fmt.Sprintf("%s %s %s .", t.Subject, t.Predicate, t.Object)
} else {
return fmt.Sprintf("%s %s %s %s .", t.Subject, t.Predicate, t.Object, t.Provenance)
}
}

24
graph/triplestore.go
View file

@ -23,7 +23,9 @@ package graph
import (
"errors"
"github.com/barakmich/glog"
"github.com/google/cayley/quad"
)
// Defines an opaque "triple store value" type. However the backend wishes to
@ -38,21 +40,21 @@ type Value interface{}
type TripleStore interface {
// Add a triple to the store.
AddTriple(*Triple)
AddTriple(*quad.Quad)
// Add a set of triples to the store, atomically if possible.
AddTripleSet([]*Triple)
AddTripleSet([]*quad.Quad)
// Removes a triple matching the given one from the database,
// if it exists. Does nothing otherwise.
RemoveTriple(*Triple)
RemoveTriple(*quad.Quad)
// Given an opaque token, returns the triple for that token from the store.
Triple(Value) *Triple
Quad(Value) *quad.Quad
// Given a direction and a token, creates an iterator of links which have
// that node token in that directional field.
TripleIterator(Direction, Value) Iterator
TripleIterator(quad.Direction, Value) Iterator
// Returns an iterator enumerating all nodes in the graph.
NodesAllIterator() Iterator
@ -89,8 +91,8 @@ type TripleStore interface {
// gives the TripleStore the opportunity to make this optimization.
//
// Iterators will call this. At worst, a valid implementation is
// ts.IdFor(ts.Triple(triple_id).Get(dir))
TripleDirection(triple_id Value, d Direction) Value
// ts.IdFor(ts.quad.Quad(id).Get(dir))
TripleDirection(id Value, d quad.Direction) Value
}
type Options map[string]interface{}
@ -122,14 +124,10 @@ func (d Options) StringKey(key string) (string, bool) {
var ErrCannotBulkLoad = errors.New("triplestore: cannot bulk load")
type BulkLoader interface {
// BulkLoad loads Triples from a TripleUnmarshaler in bulk to the TripleStore.
// BulkLoad loads Quads from a quad.Unmarshaler in bulk to the TripleStore.
// It returns ErrCannotBulkLoad if bulk loading is not possible. For example if
// you cannot load in bulk to a non-empty database, and the db is non-empty.
BulkLoad(TripleUnmarshaler) error
}
type TripleUnmarshaler interface {
Unmarshal() (*Triple, error)
BulkLoad(quad.Unmarshaler) error
}
type NewStoreFunc func(string, Options) (TripleStore, error)