barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Create quads hierarchy

Browse source 
* Move nquads into quad.
* Create cquads simplified parser in quad.
* Move Triple (renamed Quad) to quad.

Also made sure mongo actually implements BulkLoader.
This commit is contained in:
kortschak 2014-07-27 17:42:45 +09:30
parent 01bc63810b
commit 401c58426f
51 changed files with 13400 additions and 5495 deletions
Download patch file Download diff file Expand all files Collapse all files

5
db/load.go
View file

@ -25,7 +25,8 @@ import (
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/google/cayley/config" "github.com/google/cayley/config"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/nquads" "github.com/google/cayley/quad"
"github.com/google/cayley/quad/nquads"
) )
func Load(ts graph.TripleStore, cfg *config.Config, path string) error { func Load(ts graph.TripleStore, cfg *config.Config, path string) error {
@ -56,7 +57,7 @@ func Load(ts graph.TripleStore, cfg *config.Config, path string) error {
return err return err
} }
block := make([]*graph.Triple, 0, cfg.LoadSize) block := make([]*quad.Quad, 0, cfg.LoadSize)
for { for {
t, err := dec.Unmarshal() t, err := dec.Unmarshal()
if err != nil { if err != nil {

2
db/repl.go
View file

@ -26,7 +26,7 @@ import (
"github.com/google/cayley/config" "github.com/google/cayley/config"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/sexp" "github.com/google/cayley/graph/sexp"
"github.com/google/cayley/nquads" "github.com/google/cayley/quad/nquads"
"github.com/google/cayley/query/gremlin" "github.com/google/cayley/query/gremlin"
"github.com/google/cayley/query/mql" "github.com/google/cayley/query/mql"
) )

11
graph/iterator/hasa_iterator.go
View file

@ -40,6 +40,7 @@ import (
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/quad"
) )
// A HasA consists of a reference back to the graph.TripleStore that it references, // A HasA consists of a reference back to the graph.TripleStore that it references,
@ -49,13 +50,13 @@ type HasA struct {
Base Base
ts graph.TripleStore ts graph.TripleStore
primaryIt graph.Iterator primaryIt graph.Iterator
dir graph.Direction dir quad.Direction
resultIt graph.Iterator resultIt graph.Iterator
} }
// Construct a new HasA iterator, given the triple subiterator, and the triple // Construct a new HasA iterator, given the triple subiterator, and the triple
// direction for which it stands. // direction for which it stands.
func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d graph.Direction) *HasA { func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d quad.Direction) *HasA {
var hasa HasA var hasa HasA
BaseInit(&hasa.Base) BaseInit(&hasa.Base)
hasa.ts = ts hasa.ts = ts
@ -83,7 +84,7 @@ func (it *HasA) Clone() graph.Iterator {
} }
// Direction accessor. // Direction accessor.
func (it *HasA) Direction() graph.Direction { return it.dir } func (it *HasA) Direction() quad.Direction { return it.dir }
// Pass the Optimize() call along to the subiterator. If it becomes Null, // Pass the Optimize() call along to the subiterator. If it becomes Null,
// then the HasA becomes Null (there are no triples that have any directions). // then the HasA becomes Null (there are no triples that have any directions).
@ -146,7 +147,7 @@ func (it *HasA) GetCheckResult() bool {
break break
} }
if glog.V(4) { if glog.V(4) {
glog.V(4).Infoln("Triple is", it.ts.Triple(linkVal)) glog.V(4).Infoln("Quad is", it.ts.Quad(linkVal))
} }
if it.primaryIt.Check(linkVal) { if it.primaryIt.Check(linkVal) {
it.Last = it.ts.TripleDirection(linkVal, it.dir) it.Last = it.ts.TripleDirection(linkVal, it.dir)
@ -184,7 +185,7 @@ func (it *HasA) Next() (graph.Value, bool) {
if !ok { if !ok {
return graph.NextLogOut(it, 0, false) return graph.NextLogOut(it, 0, false)
} }
name := it.ts.Triple(tID).Get(it.dir) name := it.ts.Quad(tID).Get(it.dir)
val := it.ts.ValueOf(name) val := it.ts.ValueOf(name)
it.Last = val it.Last = val
return graph.NextLogOut(it, val, true) return graph.NextLogOut(it, val, true)

7
graph/iterator/linksto_iterator.go
View file

@ -34,6 +34,7 @@ import (
"strings" "strings"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/quad"
) )
// A LinksTo has a reference back to the graph.TripleStore (to create the iterators // A LinksTo has a reference back to the graph.TripleStore (to create the iterators
@ -43,13 +44,13 @@ type LinksTo struct {
Base Base
ts graph.TripleStore ts graph.TripleStore
primaryIt graph.Iterator primaryIt graph.Iterator
dir graph.Direction dir quad.Direction
nextIt graph.Iterator nextIt graph.Iterator
} }
// Construct a new LinksTo iterator around a direction and a subiterator of // Construct a new LinksTo iterator around a direction and a subiterator of
// nodes. // nodes.
func NewLinksTo(ts graph.TripleStore, it graph.Iterator, d graph.Direction) *LinksTo { func NewLinksTo(ts graph.TripleStore, it graph.Iterator, d quad.Direction) *LinksTo {
var lto LinksTo var lto LinksTo
BaseInit(&lto.Base) BaseInit(&lto.Base)
lto.ts = ts lto.ts = ts
@ -74,7 +75,7 @@ func (it *LinksTo) Clone() graph.Iterator {
} }
// Return the direction under consideration. // Return the direction under consideration.
func (it *LinksTo) Direction() graph.Direction { return it.dir } func (it *LinksTo) Direction() quad.Direction { return it.dir }
// Tag these results, and our subiterator's results. // Tag these results, and our subiterator's results.
func (it *LinksTo) TagResults(dst map[string]graph.Value) { func (it *LinksTo) TagResults(dst map[string]graph.Value) {

6
graph/iterator/linksto_iterator_test.go
View file

@ -17,7 +17,7 @@ package iterator
import ( import (
"testing" "testing"
"github.com/google/cayley/graph" "github.com/google/cayley/quad"
) )
func TestLinksTo(t *testing.T) { func TestLinksTo(t *testing.T) {
@ -32,12 +32,12 @@ func TestLinksTo(t *testing.T) {
t.Fatalf("Failed to return correct value, got:%v expect:1", val) t.Fatalf("Failed to return correct value, got:%v expect:1", val)
} }
fixed.Add(val) fixed.Add(val)
lto := NewLinksTo(ts, fixed, graph.Object) lto := NewLinksTo(ts, fixed, quad.Object)
val, ok := lto.Next() val, ok := lto.Next()
if !ok { if !ok {
t.Error("At least one triple matches the fixed object") t.Error("At least one triple matches the fixed object")
} }
if val != 2 { if val != 2 {
t.Errorf("Triple index 2, such as %s, should match %s", ts.Triple(2), ts.Triple(val)) t.Errorf("Quad index 2, such as %s, should match %s", ts.Quad(2), ts.Quad(val))
} }
} }

43
graph/iterator/mock_ts_test.go
View file

@ -17,15 +17,18 @@ package iterator
// A quickly mocked version of the TripleStore interface, for use in tests. // A quickly mocked version of the TripleStore interface, for use in tests.
// Can better used Mock.Called but will fill in as needed. // Can better used Mock.Called but will fill in as needed.
import "github.com/google/cayley/graph" import (
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
type store struct { type store struct {
data []string data []string
iter graph.Iterator iter graph.Iterator
} }
func (ts *store) ValueOf(s string) graph.Value { func (qs *store) ValueOf(s string) graph.Value {
for i, v := range ts.data { for i, v := range qs.data {
if s == v { if s == v {
return i return i
} }
@ -33,42 +36,42 @@ func (ts *store) ValueOf(s string) graph.Value {
return nil return nil
} }
func (ts *store) AddTriple(*graph.Triple) {} func (qs *store) AddTriple(*quad.Quad) {}
func (ts *store) AddTripleSet([]*graph.Triple) {} func (qs *store) AddTripleSet([]*quad.Quad) {}
func (ts *store) Triple(graph.Value) *graph.Triple { return &graph.Triple{} } func (qs *store) Quad(graph.Value) *quad.Quad { return &quad.Quad{} }
func (ts *store) TripleIterator(d graph.Direction, i graph.Value) graph.Iterator { func (qs *store) TripleIterator(d quad.Direction, i graph.Value) graph.Iterator {
return ts.iter return qs.iter
} }
func (ts *store) NodesAllIterator() graph.Iterator { return &Null{} } func (qs *store) NodesAllIterator() graph.Iterator { return &Null{} }
func (ts *store) TriplesAllIterator() graph.Iterator { return &Null{} } func (qs *store) TriplesAllIterator() graph.Iterator { return &Null{} }
func (ts *store) NameOf(v graph.Value) string { func (qs *store) NameOf(v graph.Value) string {
i := v.(int) i := v.(int)
if i < 0 || i >= len(ts.data) { if i < 0 || i >= len(qs.data) {
return "" return ""
} }
return ts.data[i] return qs.data[i]
} }
func (ts *store) Size() int64 { return 0 } func (qs *store) Size() int64 { return 0 }
func (ts *store) DebugPrint() {} func (qs *store) DebugPrint() {}
func (ts *store) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { func (qs *store) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
return &Null{}, false return &Null{}, false
} }
func (ts *store) FixedIterator() graph.FixedIterator { func (qs *store) FixedIterator() graph.FixedIterator {
return NewFixedIteratorWithCompare(BasicEquality) return NewFixedIteratorWithCompare(BasicEquality)
} }
func (ts *store) Close() {} func (qs *store) Close() {}
func (ts *store) TripleDirection(graph.Value, graph.Direction) graph.Value { return 0 } func (qs *store) TripleDirection(graph.Value, quad.Direction) graph.Value { return 0 }
func (ts *store) RemoveTriple(t *graph.Triple) {} func (qs *store) RemoveTriple(t *quad.Quad) {}

13
graph/iterator/query_shape.go
View file

@ -16,6 +16,7 @@ package iterator
import ( import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/quad"
) )
type Node struct { type Node struct {
@ -39,7 +40,7 @@ type queryShape struct {
ts graph.TripleStore ts graph.TripleStore
nodeId int nodeId int
hasaIds []int hasaIds []int
hasaDirs []graph.Direction hasaDirs []quad.Direction
} }
func OutputQueryShapeForIterator(it graph.Iterator, ts graph.TripleStore, outputMap map[string]interface{}) { func OutputQueryShapeForIterator(it graph.Iterator, ts graph.TripleStore, outputMap map[string]interface{}) {
@ -62,11 +63,11 @@ func (qs *queryShape) AddLink(l *Link) {
qs.links = append(qs.links, *l) qs.links = append(qs.links, *l)
} }
func (qs *queryShape) LastHasa() (int, graph.Direction) { func (qs *queryShape) LastHasa() (int, quad.Direction) {
return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1] return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1]
} }
func (qs *queryShape) PushHasa(i int, d graph.Direction) { func (qs *queryShape) PushHasa(i int, d quad.Direction) {
qs.hasaIds = append(qs.hasaIds, i) qs.hasaIds = append(qs.hasaIds, i)
qs.hasaDirs = append(qs.hasaDirs, d) qs.hasaDirs = append(qs.hasaDirs, d)
} }
@ -159,10 +160,10 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
qs.nodeId++ qs.nodeId++
newNode := qs.MakeNode(lto.primaryIt) newNode := qs.MakeNode(lto.primaryIt)
hasaID, hasaDir := qs.LastHasa() hasaID, hasaDir := qs.LastHasa()
if (hasaDir == graph.Subject && lto.dir == graph.Object) || if (hasaDir == quad.Subject && lto.dir == quad.Object) ||
(hasaDir == graph.Object && lto.dir == graph.Subject) { (hasaDir == quad.Object && lto.dir == quad.Subject) {
qs.AddNode(newNode) qs.AddNode(newNode)
if hasaDir == graph.Subject { if hasaDir == quad.Subject {
qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id}) qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id})
} else { } else {
qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id}) qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id})

13
graph/iterator/query_shape_test.go
View file

@ -19,6 +19,7 @@ import (
"testing" "testing"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/quad"
) )
func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA { func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA {
@ -27,13 +28,13 @@ func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA {
obj := ts.FixedIterator() obj := ts.FixedIterator()
obj.Add(ts.ValueOf(target)) obj.Add(ts.ValueOf(target))
obj.AddTag(tag) obj.AddTag(tag)
and.AddSubIterator(NewLinksTo(ts, obj, graph.Object)) and.AddSubIterator(NewLinksTo(ts, obj, quad.Object))
pred := ts.FixedIterator() pred := ts.FixedIterator()
pred.Add(ts.ValueOf("status")) pred.Add(ts.ValueOf("status"))
and.AddSubIterator(NewLinksTo(ts, pred, graph.Predicate)) and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate))
return NewHasA(ts, and, graph.Subject) return NewHasA(ts, and, quad.Subject)
} }
func TestQueryShape(t *testing.T) { func TestQueryShape(t *testing.T) {
@ -104,11 +105,11 @@ func TestQueryShape(t *testing.T) {
pred.Add(ts.ValueOf("name")) pred.Add(ts.ValueOf("name"))
and := NewAnd() and := NewAnd()
and.AddSubIterator(NewLinksTo(ts, andInternal, graph.Subject)) and.AddSubIterator(NewLinksTo(ts, andInternal, quad.Subject))
and.AddSubIterator(NewLinksTo(ts, pred, graph.Predicate)) and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate))
shape = make(map[string]interface{}) shape = make(map[string]interface{})
OutputQueryShapeForIterator(NewHasA(ts, and, graph.Object), ts, shape) OutputQueryShapeForIterator(NewHasA(ts, and, quad.Object), ts, shape)
links = shape["links"].([]Link) links = shape["links"].([]Link)
if len(links) != 3 { if len(links) != 3 {

5
graph/leveldb/all_iterator.go
View file

@ -24,19 +24,20 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
type AllIterator struct { type AllIterator struct {
iterator.Base iterator.Base
prefix []byte prefix []byte
dir graph.Direction dir quad.Direction
open bool open bool
iter ldbit.Iterator iter ldbit.Iterator
ts *TripleStore ts *TripleStore
ro *opt.ReadOptions ro *opt.ReadOptions
} }
func NewAllIterator(prefix string, d graph.Direction, ts *TripleStore) *AllIterator { func NewAllIterator(prefix string, d quad.Direction, ts *TripleStore) *AllIterator {
var it AllIterator var it AllIterator
iterator.BaseInit(&it.Base) iterator.BaseInit(&it.Base)
it.ro = &opt.ReadOptions{} it.ro = &opt.ReadOptions{}

79
graph/leveldb/iterator.go
View file

@ -24,34 +24,35 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
type Iterator struct { type Iterator struct {
iterator.Base iterator.Base
nextPrefix []byte nextPrefix []byte
checkId []byte checkId []byte
dir graph.Direction dir quad.Direction
open bool open bool
iter ldbit.Iterator iter ldbit.Iterator
ts *TripleStore qs *TripleStore
ro *opt.ReadOptions ro *opt.ReadOptions
originalPrefix string originalPrefix string
} }
func NewIterator(prefix string, d graph.Direction, value graph.Value, ts *TripleStore) *Iterator { func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
var it Iterator var it Iterator
iterator.BaseInit(&it.Base) iterator.BaseInit(&it.Base)
it.checkId = value.([]byte) it.checkId = value.([]byte)
it.dir = d it.dir = d
it.originalPrefix = prefix it.originalPrefix = prefix
it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size()) it.nextPrefix = make([]byte, 0, 2+qs.hasher.Size())
it.nextPrefix = append(it.nextPrefix, []byte(prefix)...) it.nextPrefix = append(it.nextPrefix, []byte(prefix)...)
it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...) it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...)
it.ro = &opt.ReadOptions{} it.ro = &opt.ReadOptions{}
it.ro.DontFillCache = true it.ro.DontFillCache = true
it.iter = ts.db.NewIterator(nil, it.ro) it.iter = qs.db.NewIterator(nil, it.ro)
it.open = true it.open = true
it.ts = ts it.qs = qs
ok := it.iter.Seek(it.nextPrefix) ok := it.iter.Seek(it.nextPrefix)
if !ok { if !ok {
it.open = false it.open = false
@ -62,7 +63,7 @@ func NewIterator(prefix string, d graph.Direction, value graph.Value, ts *Triple
func (it *Iterator) Reset() { func (it *Iterator) Reset() {
if !it.open { if !it.open {
it.iter = it.ts.db.NewIterator(nil, it.ro) it.iter = it.qs.db.NewIterator(nil, it.ro)
it.open = true it.open = true
} }
ok := it.iter.Seek(it.nextPrefix) ok := it.iter.Seek(it.nextPrefix)
@ -73,7 +74,7 @@ func (it *Iterator) Reset() {
} }
func (it *Iterator) Clone() graph.Iterator { func (it *Iterator) Clone() graph.Iterator {
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.ts) out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.qs)
out.CopyTagsFrom(it) out.CopyTagsFrom(it)
return out return out
} }
@ -114,52 +115,52 @@ func (it *Iterator) Next() (graph.Value, bool) {
return nil, false return nil, false
} }
func PositionOf(prefix []byte, d graph.Direction, ts *TripleStore) int { func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("sp")) { if bytes.Equal(prefix, []byte("sp")) {
switch d { switch d {
case graph.Subject: case quad.Subject:
return 2 return 2
case graph.Predicate: case quad.Predicate:
return ts.hasher.Size() + 2 return qs.hasher.Size() + 2
case graph.Object: case quad.Object:
return 2*ts.hasher.Size() + 2 return 2*qs.hasher.Size() + 2
case graph.Provenance: case quad.Provenance:
return -1 return -1
} }
} }
if bytes.Equal(prefix, []byte("po")) { if bytes.Equal(prefix, []byte("po")) {
switch d { switch d {
case graph.Subject: case quad.Subject:
return 2*ts.hasher.Size() + 2 return 2*qs.hasher.Size() + 2
case graph.Predicate: case quad.Predicate:
return 2 return 2
case graph.Object: case quad.Object:
return ts.hasher.Size() + 2 return qs.hasher.Size() + 2
case graph.Provenance: case quad.Provenance:
return -1 return -1
} }
} }
if bytes.Equal(prefix, []byte("os")) { if bytes.Equal(prefix, []byte("os")) {
switch d { switch d {
case graph.Subject: case quad.Subject:
return ts.hasher.Size() + 2 return qs.hasher.Size() + 2
case graph.Predicate: case quad.Predicate:
return 2*ts.hasher.Size() + 2 return 2*qs.hasher.Size() + 2
case graph.Object: case quad.Object:
return 2 return 2
case graph.Provenance: case quad.Provenance:
return -1 return -1
} }
} }
if bytes.Equal(prefix, []byte("cp")) { if bytes.Equal(prefix, []byte("cp")) {
switch d { switch d {
case graph.Subject: case quad.Subject:
return 2*ts.hasher.Size() + 2 return 2*qs.hasher.Size() + 2
case graph.Predicate: case quad.Predicate:
return ts.hasher.Size() + 2 return qs.hasher.Size() + 2
case graph.Object: case quad.Object:
return 3*ts.hasher.Size() + 2 return 3*qs.hasher.Size() + 2
case graph.Provenance: case quad.Provenance:
return 2 return 2
} }
} }
@ -171,14 +172,14 @@ func (it *Iterator) Check(v graph.Value) bool {
if val[0] == 'z' { if val[0] == 'z' {
return false return false
} }
offset := PositionOf(val[0:2], it.dir, it.ts) offset := PositionOf(val[0:2], it.dir, it.qs)
if offset != -1 { if offset != -1 {
if bytes.HasPrefix(val[offset:], it.checkId[1:]) { if bytes.HasPrefix(val[offset:], it.checkId[1:]) {
return true return true
} }
} else { } else {
nameForDir := it.ts.Triple(v).Get(it.dir) nameForDir := it.qs.Quad(v).Get(it.dir)
hashForDir := it.ts.ValueOf(nameForDir).([]byte) hashForDir := it.qs.ValueOf(nameForDir).([]byte)
if bytes.Equal(hashForDir, it.checkId) { if bytes.Equal(hashForDir, it.checkId) {
return true return true
} }
@ -187,12 +188,12 @@ func (it *Iterator) Check(v graph.Value) bool {
} }
func (it *Iterator) Size() (int64, bool) { func (it *Iterator) Size() (int64, bool) {
return it.ts.SizeOf(it.checkId), true return it.qs.SizeOf(it.checkId), true
} }
func (it *Iterator) DebugString(indent int) string { func (it *Iterator) DebugString(indent int) string {
size, _ := it.Size() size, _ := it.Size()
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.ts.NameOf(it.checkId)) return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.qs.NameOf(it.checkId))
} }
var levelDBType graph.Type var levelDBType graph.Type

145
graph/leveldb/leveldb_test.go
View file

@ -23,10 +23,11 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
func makeTripleSet() []*graph.Triple { func makeTripleSet() []*quad.Quad {
tripleSet := []*graph.Triple{ tripleSet := []*quad.Quad{
{"A", "follows", "B", ""}, {"A", "follows", "B", ""},
{"C", "follows", "B", ""}, {"C", "follows", "B", ""},
{"C", "follows", "D", ""}, {"C", "follows", "D", ""},
@ -42,20 +43,20 @@ func makeTripleSet() []*graph.Triple {
return tripleSet return tripleSet
} }
func iteratedTriples(ts graph.TripleStore, it graph.Iterator) []*graph.Triple { func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad {
var res ordered var res ordered
for { for {
val, ok := it.Next() val, ok := it.Next()
if !ok { if !ok {
break break
} }
res = append(res, ts.Triple(val)) res = append(res, qs.Quad(val))
} }
sort.Sort(res) sort.Sort(res)
return res return res
} }
type ordered []*graph.Triple type ordered []*quad.Quad
func (o ordered) Len() int { return len(o) } func (o ordered) Len() int { return len(o) }
func (o ordered) Less(i, j int) bool { func (o ordered) Less(i, j int) bool {
@ -82,14 +83,14 @@ func (o ordered) Less(i, j int) bool {
} }
func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
func iteratedNames(ts graph.TripleStore, it graph.Iterator) []string { func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string {
var res []string var res []string
for { for {
val, ok := it.Next() val, ok := it.Next()
if !ok { if !ok {
break break
} }
res = append(res, ts.NameOf(val)) res = append(res, qs.NameOf(val))
} }
sort.Strings(res) sort.Strings(res)
return res return res
@ -107,14 +108,14 @@ func TestCreateDatabase(t *testing.T) {
t.Fatal("Failed to create LevelDB database.") t.Fatal("Failed to create LevelDB database.")
} }
ts, err := newTripleStore(tmpDir, nil) qs, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil { if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.") t.Error("Failed to create leveldb TripleStore.")
} }
if s := ts.Size(); s != 0 { if s := qs.Size(); s != 0 {
t.Errorf("Unexpected size, got:%d expected:0", s) t.Errorf("Unexpected size, got:%d expected:0", s)
} }
ts.Close() qs.Close()
err = createNewLevelDB("/dev/null/some terrible path", nil) err = createNewLevelDB("/dev/null/some terrible path", nil)
if err == nil { if err == nil {
@ -137,53 +138,53 @@ func TestLoadDatabase(t *testing.T) {
t.Fatal("Failed to create LevelDB database.") t.Fatal("Failed to create LevelDB database.")
} }
ts, err := newTripleStore(tmpDir, nil) qs, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil { if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.") t.Error("Failed to create leveldb TripleStore.")
} }
ts.AddTriple(&graph.Triple{"Something", "points_to", "Something Else", "context"}) qs.AddTriple(&quad.Quad{"Something", "points_to", "Something Else", "context"})
for _, pq := range []string{"Something", "points_to", "Something Else", "context"} { for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
if got := ts.NameOf(ts.ValueOf(pq)); got != pq { if got := qs.NameOf(qs.ValueOf(pq)); got != pq {
t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq) t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq)
} }
} }
if s := ts.Size(); s != 1 { if s := qs.Size(); s != 1 {
t.Errorf("Unexpected triplestore size, got:%d expect:1", s) t.Errorf("Unexpected triplestore size, got:%d expect:1", s)
} }
ts.Close() qs.Close()
err = createNewLevelDB(tmpDir, nil) err = createNewLevelDB(tmpDir, nil)
if err != nil { if err != nil {
t.Fatal("Failed to create LevelDB database.") t.Fatal("Failed to create LevelDB database.")
} }
ts, err = newTripleStore(tmpDir, nil) qs, err = newTripleStore(tmpDir, nil)
if ts == nil || err != nil { if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.") t.Error("Failed to create leveldb TripleStore.")
} }
ts2, didConvert := ts.(*TripleStore) ts2, didConvert := qs.(*TripleStore)
if !didConvert { if !didConvert {
t.Errorf("Could not convert from generic to LevelDB TripleStore") t.Errorf("Could not convert from generic to LevelDB TripleStore")
} }
ts.AddTripleSet(makeTripleSet()) qs.AddTripleSet(makeTripleSet())
if s := ts.Size(); s != 11 { if s := qs.Size(); s != 11 {
t.Errorf("Unexpected triplestore size, got:%d expect:11", s) t.Errorf("Unexpected triplestore size, got:%d expect:11", s)
} }
if s := ts2.SizeOf(ts.ValueOf("B")); s != 5 { if s := ts2.SizeOf(qs.ValueOf("B")); s != 5 {
t.Errorf("Unexpected triplestore size, got:%d expect:5", s) t.Errorf("Unexpected triplestore size, got:%d expect:5", s)
} }
ts.RemoveTriple(&graph.Triple{"A", "follows", "B", ""}) qs.RemoveTriple(&quad.Quad{"A", "follows", "B", ""})
if s := ts.Size(); s != 10 { if s := qs.Size(); s != 10 {
t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s) t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s)
} }
if s := ts2.SizeOf(ts.ValueOf("B")); s != 4 { if s := ts2.SizeOf(qs.ValueOf("B")); s != 4 {
t.Errorf("Unexpected triplestore size, got:%d expect:4", s) t.Errorf("Unexpected triplestore size, got:%d expect:4", s)
} }
ts.Close() qs.Close()
} }
func TestIterator(t *testing.T) { func TestIterator(t *testing.T) {
@ -199,14 +200,14 @@ func TestIterator(t *testing.T) {
t.Fatal("Failed to create LevelDB database.") t.Fatal("Failed to create LevelDB database.")
} }
ts, err := newTripleStore(tmpDir, nil) qs, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil { if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.") t.Error("Failed to create leveldb TripleStore.")
} }
ts.AddTripleSet(makeTripleSet()) qs.AddTripleSet(makeTripleSet())
var it graph.Iterator var it graph.Iterator
it = ts.NodesAllIterator() it = qs.NodesAllIterator()
if it == nil { if it == nil {
t.Fatal("Got nil iterator.") t.Fatal("Got nil iterator.")
} }
@ -241,7 +242,7 @@ func TestIterator(t *testing.T) {
} }
sort.Strings(expect) sort.Strings(expect)
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
got := iteratedNames(ts, it) got := iteratedNames(qs, it)
sort.Strings(got) sort.Strings(got)
if !reflect.DeepEqual(got, expect) { if !reflect.DeepEqual(got, expect) {
t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect) t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect)
@ -250,23 +251,23 @@ func TestIterator(t *testing.T) {
} }
for _, pq := range expect { for _, pq := range expect {
if !it.Check(ts.ValueOf(pq)) { if !it.Check(qs.ValueOf(pq)) {
t.Errorf("Failed to find and check %q correctly", pq) t.Errorf("Failed to find and check %q correctly", pq)
} }
} }
// FIXME(kortschak) Why does this fail? // FIXME(kortschak) Why does this fail?
/* /*
for _, pq := range []string{"baller"} { for _, pq := range []string{"baller"} {
if it.Check(ts.ValueOf(pq)) { if it.Check(qs.ValueOf(pq)) {
t.Errorf("Failed to check %q correctly", pq) t.Errorf("Failed to check %q correctly", pq)
} }
} }
*/ */
it.Reset() it.Reset()
it = ts.TriplesAllIterator() it = qs.TriplesAllIterator()
edge, _ := it.Next() edge, _ := it.Next()
triple := ts.Triple(edge) triple := qs.Quad(edge)
set := makeTripleSet() set := makeTripleSet()
var ok bool var ok bool
for _, t := range set { for _, t := range set {
@ -279,7 +280,7 @@ func TestIterator(t *testing.T) {
t.Errorf("Failed to find %q during iteration, got:%q", triple, set) t.Errorf("Failed to find %q during iteration, got:%q", triple, set)
} }
ts.Close() qs.Close()
} }
func TestSetIterator(t *testing.T) { func TestSetIterator(t *testing.T) {
@ -292,95 +293,95 @@ func TestSetIterator(t *testing.T) {
t.Fatalf("Failed to create working directory") t.Fatalf("Failed to create working directory")
} }
ts, err := newTripleStore(tmpDir, nil) qs, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil { if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.") t.Error("Failed to create leveldb TripleStore.")
} }
defer ts.Close() defer qs.Close()
ts.AddTripleSet(makeTripleSet()) qs.AddTripleSet(makeTripleSet())
expect := []*graph.Triple{ expect := []*quad.Quad{
{"C", "follows", "B", ""}, {"C", "follows", "B", ""},
{"C", "follows", "D", ""}, {"C", "follows", "D", ""},
} }
sort.Sort(ordered(expect)) sort.Sort(ordered(expect))
// Subject iterator. // Subject iterator.
it := ts.TripleIterator(graph.Subject, ts.ValueOf("C")) it := qs.TripleIterator(quad.Subject, qs.ValueOf("C"))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect) t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
} }
it.Reset() it.Reset()
and := iterator.NewAnd() and := iterator.NewAnd()
and.AddSubIterator(ts.TriplesAllIterator()) and.AddSubIterator(qs.TriplesAllIterator())
and.AddSubIterator(it) and.AddSubIterator(it)
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
} }
// Object iterator. // Object iterator.
it = ts.TripleIterator(graph.Object, ts.ValueOf("F")) it = qs.TripleIterator(quad.Object, qs.ValueOf("F"))
expect = []*graph.Triple{ expect = []*quad.Quad{
{"B", "follows", "F", ""}, {"B", "follows", "F", ""},
{"E", "follows", "F", ""}, {"E", "follows", "F", ""},
} }
sort.Sort(ordered(expect)) sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect) t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
} }
and = iterator.NewAnd() and = iterator.NewAnd()
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B"))) and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it) and.AddSubIterator(it)
expect = []*graph.Triple{ expect = []*quad.Quad{
{"B", "follows", "F", ""}, {"B", "follows", "F", ""},
} }
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
} }
// Predicate iterator. // Predicate iterator.
it = ts.TripleIterator(graph.Predicate, ts.ValueOf("status")) it = qs.TripleIterator(quad.Predicate, qs.ValueOf("status"))
expect = []*graph.Triple{ expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"}, {"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"}, {"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"},
} }
sort.Sort(ordered(expect)) sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect) t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect)
} }
// Provenance iterator. // Provenance iterator.
it = ts.TripleIterator(graph.Provenance, ts.ValueOf("status_graph")) it = qs.TripleIterator(quad.Provenance, qs.ValueOf("status_graph"))
expect = []*graph.Triple{ expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"}, {"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"}, {"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"},
} }
sort.Sort(ordered(expect)) sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect) t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect)
} }
it.Reset() it.Reset()
// Order is important // Order is important
and = iterator.NewAnd() and = iterator.NewAnd()
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B"))) and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it) and.AddSubIterator(it)
expect = []*graph.Triple{ expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"}, {"B", "status", "cool", "status_graph"},
} }
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
} }
it.Reset() it.Reset()
@ -388,12 +389,12 @@ func TestSetIterator(t *testing.T) {
// Order is important // Order is important
and = iterator.NewAnd() and = iterator.NewAnd()
and.AddSubIterator(it) and.AddSubIterator(it)
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B"))) and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
expect = []*graph.Triple{ expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"}, {"B", "status", "cool", "status_graph"},
} }
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) { if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect) t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
} }
} }
@ -406,17 +407,17 @@ func TestOptimize(t *testing.T) {
if err != nil { if err != nil {
t.Fatalf("Failed to create working directory") t.Fatalf("Failed to create working directory")
} }
ts, err := newTripleStore(tmpDir, nil) qs, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil { if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.") t.Error("Failed to create leveldb TripleStore.")
} }
ts.AddTripleSet(makeTripleSet()) qs.AddTripleSet(makeTripleSet())
// With an linksto-fixed pair // With an linksto-fixed pair
fixed := ts.FixedIterator() fixed := qs.FixedIterator()
fixed.Add(ts.ValueOf("F")) fixed.Add(qs.ValueOf("F"))
fixed.AddTag("internal") fixed.AddTag("internal")
lto := iterator.NewLinksTo(ts, fixed, graph.Object) lto := iterator.NewLinksTo(qs, fixed, quad.Object)
oldIt := lto.Clone() oldIt := lto.Clone()
newIt, ok := lto.Optimize() newIt, ok := lto.Optimize()
@ -427,8 +428,8 @@ func TestOptimize(t *testing.T) {
t.Errorf("Optimized iterator type does not match original, got:%v expect:%v", newIt.Type(), Type()) t.Errorf("Optimized iterator type does not match original, got:%v expect:%v", newIt.Type(), Type())
} }
newTriples := iteratedTriples(ts, newIt) newTriples := iteratedTriples(qs, newIt)
oldTriples := iteratedTriples(ts, oldIt) oldTriples := iteratedTriples(qs, oldIt)
if !reflect.DeepEqual(newTriples, oldTriples) { if !reflect.DeepEqual(newTriples, oldTriples) {
t.Errorf("Optimized iteration does not match original") t.Errorf("Optimized iteration does not match original")
} }

253
graph/leveldb/triplestore.go
View file

@ -30,6 +30,7 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
const ( const (
@ -56,110 +57,110 @@ func createNewLevelDB(path string, _ graph.Options) error {
return err return err
} }
defer db.Close() defer db.Close()
ts := &TripleStore{} qs := &TripleStore{}
ts.db = db qs.db = db
ts.writeopts = &opt.WriteOptions{ qs.writeopts = &opt.WriteOptions{
Sync: true, Sync: true,
} }
ts.Close() qs.Close()
return nil return nil
} }
func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) { func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore var qs TripleStore
ts.path = path qs.path = path
cache_size := DefaultCacheSize cache_size := DefaultCacheSize
if val, ok := options.IntKey("cache_size_mb"); ok { if val, ok := options.IntKey("cache_size_mb"); ok {
cache_size = val cache_size = val
} }
ts.dbOpts = &opt.Options{ qs.dbOpts = &opt.Options{
BlockCache: cache.NewLRUCache(cache_size * opt.MiB), BlockCache: cache.NewLRUCache(cache_size * opt.MiB),
} }
ts.dbOpts.ErrorIfMissing = true qs.dbOpts.ErrorIfMissing = true
write_buffer_mb := DefaultWriteBufferSize write_buffer_mb := DefaultWriteBufferSize
if val, ok := options.IntKey("write_buffer_mb"); ok { if val, ok := options.IntKey("write_buffer_mb"); ok {
write_buffer_mb = val write_buffer_mb = val
} }
ts.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
ts.hasher = sha1.New() qs.hasher = sha1.New()
ts.writeopts = &opt.WriteOptions{ qs.writeopts = &opt.WriteOptions{
Sync: false, Sync: false,
} }
ts.readopts = &opt.ReadOptions{} qs.readopts = &opt.ReadOptions{}
db, err := leveldb.OpenFile(ts.path, ts.dbOpts) db, err := leveldb.OpenFile(qs.path, qs.dbOpts)
if err != nil { if err != nil {
panic("Error, couldn't open! " + err.Error()) panic("Error, couldn't open! " + err.Error())
} }
ts.db = db qs.db = db
glog.Infoln(ts.GetStats()) glog.Infoln(qs.GetStats())
ts.getSize() qs.getSize()
return &ts, nil return &qs, nil
} }
func (ts *TripleStore) GetStats() string { func (qs *TripleStore) GetStats() string {
out := "" out := ""
stats, err := ts.db.GetProperty("leveldb.stats") stats, err := qs.db.GetProperty("leveldb.stats")
if err == nil { if err == nil {
out += fmt.Sprintln("Stats: ", stats) out += fmt.Sprintln("Stats: ", stats)
} }
out += fmt.Sprintln("Size: ", ts.size) out += fmt.Sprintln("Size: ", qs.size)
return out return out
} }
func (ts *TripleStore) Size() int64 { func (qs *TripleStore) Size() int64 {
return ts.size return qs.size
} }
func (ts *TripleStore) createKeyFor(d [3]graph.Direction, triple *graph.Triple) []byte { func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte {
key := make([]byte, 0, 2+(ts.hasher.Size()*3)) key := make([]byte, 0, 2+(qs.hasher.Size()*3))
// TODO(kortschak) Remove dependence on String() method. // TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...) key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[0]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[1]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[2]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
return key return key
} }
func (ts *TripleStore) createProvKeyFor(d [3]graph.Direction, triple *graph.Triple) []byte { func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte {
key := make([]byte, 0, 2+(ts.hasher.Size()*4)) key := make([]byte, 0, 2+(qs.hasher.Size()*4))
// TODO(kortschak) Remove dependence on String() method. // TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{graph.Provenance.Prefix(), d[0].Prefix()}...) key = append(key, []byte{quad.Provenance.Prefix(), d[0].Prefix()}...)
key = append(key, ts.convertStringToByteHash(triple.Get(graph.Provenance))...) key = append(key, qs.convertStringToByteHash(triple.Get(quad.Provenance))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[0]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[1]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, ts.convertStringToByteHash(triple.Get(d[2]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
return key return key
} }
func (ts *TripleStore) createValueKeyFor(s string) []byte { func (qs *TripleStore) createValueKeyFor(s string) []byte {
key := make([]byte, 0, 1+ts.hasher.Size()) key := make([]byte, 0, 1+qs.hasher.Size())
key = append(key, []byte("z")...) key = append(key, []byte("z")...)
key = append(key, ts.convertStringToByteHash(s)...) key = append(key, qs.convertStringToByteHash(s)...)
return key return key
} }
func (ts *TripleStore) AddTriple(t *graph.Triple) { func (qs *TripleStore) AddTriple(t *quad.Quad) {
batch := &leveldb.Batch{} batch := &leveldb.Batch{}
ts.buildWrite(batch, t) qs.buildWrite(batch, t)
err := ts.db.Write(batch, ts.writeopts) err := qs.db.Write(batch, qs.writeopts)
if err != nil { if err != nil {
glog.Errorf("Couldn't write to DB for triple %s", t) glog.Errorf("Couldn't write to DB for triple %s", t)
return return
} }
ts.size++ qs.size++
} }
// Short hand for direction permutations. // Short hand for direction permutations.
var ( var (
spo = [3]graph.Direction{graph.Subject, graph.Predicate, graph.Object} spo = [3]quad.Direction{quad.Subject, quad.Predicate, quad.Object}
osp = [3]graph.Direction{graph.Object, graph.Subject, graph.Predicate} osp = [3]quad.Direction{quad.Object, quad.Subject, quad.Predicate}
pos = [3]graph.Direction{graph.Predicate, graph.Object, graph.Subject} pos = [3]quad.Direction{quad.Predicate, quad.Object, quad.Subject}
pso = [3]graph.Direction{graph.Predicate, graph.Subject, graph.Object} pso = [3]quad.Direction{quad.Predicate, quad.Subject, quad.Object}
) )
func (ts *TripleStore) RemoveTriple(t *graph.Triple) { func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
_, err := ts.db.Get(ts.createKeyFor(spo, t), ts.readopts) _, err := qs.db.Get(qs.createKeyFor(spo, t), qs.readopts)
if err != nil && err != leveldb.ErrNotFound { if err != nil && err != leveldb.ErrNotFound {
glog.Errorf("Couldn't access DB to confirm deletion") glog.Errorf("Couldn't access DB to confirm deletion")
return return
@ -169,45 +170,45 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
return return
} }
batch := &leveldb.Batch{} batch := &leveldb.Batch{}
batch.Delete(ts.createKeyFor(spo, t)) batch.Delete(qs.createKeyFor(spo, t))
batch.Delete(ts.createKeyFor(osp, t)) batch.Delete(qs.createKeyFor(osp, t))
batch.Delete(ts.createKeyFor(pos, t)) batch.Delete(qs.createKeyFor(pos, t))
ts.UpdateValueKeyBy(t.Get(graph.Subject), -1, batch) qs.UpdateValueKeyBy(t.Get(quad.Subject), -1, batch)
ts.UpdateValueKeyBy(t.Get(graph.Predicate), -1, batch) qs.UpdateValueKeyBy(t.Get(quad.Predicate), -1, batch)
ts.UpdateValueKeyBy(t.Get(graph.Object), -1, batch) qs.UpdateValueKeyBy(t.Get(quad.Object), -1, batch)
if t.Get(graph.Provenance) != "" { if t.Get(quad.Provenance) != "" {
batch.Delete(ts.createProvKeyFor(pso, t)) batch.Delete(qs.createProvKeyFor(pso, t))
ts.UpdateValueKeyBy(t.Get(graph.Provenance), -1, batch) qs.UpdateValueKeyBy(t.Get(quad.Provenance), -1, batch)
} }
err = ts.db.Write(batch, nil) err = qs.db.Write(batch, nil)
if err != nil { if err != nil {
glog.Errorf("Couldn't delete triple %s", t) glog.Errorf("Couldn't delete triple %s", t)
return return
} }
ts.size-- qs.size--
} }
func (ts *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) { func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *quad.Quad) {
bytes, err := json.Marshal(*t) bytes, err := json.Marshal(*t)
if err != nil { if err != nil {
glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t, err) glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t, err)
return return
} }
batch.Put(ts.createKeyFor(spo, t), bytes) batch.Put(qs.createKeyFor(spo, t), bytes)
batch.Put(ts.createKeyFor(osp, t), bytes) batch.Put(qs.createKeyFor(osp, t), bytes)
batch.Put(ts.createKeyFor(pos, t), bytes) batch.Put(qs.createKeyFor(pos, t), bytes)
if t.Get(graph.Provenance) != "" { if t.Get(quad.Provenance) != "" {
batch.Put(ts.createProvKeyFor(pso, t), bytes) batch.Put(qs.createProvKeyFor(pso, t), bytes)
} }
} }
func (ts *TripleStore) buildWrite(batch *leveldb.Batch, t *graph.Triple) { func (qs *TripleStore) buildWrite(batch *leveldb.Batch, t *quad.Quad) {
ts.buildTripleWrite(batch, t) qs.buildTripleWrite(batch, t)
ts.UpdateValueKeyBy(t.Get(graph.Subject), 1, nil) qs.UpdateValueKeyBy(t.Get(quad.Subject), 1, nil)
ts.UpdateValueKeyBy(t.Get(graph.Predicate), 1, nil) qs.UpdateValueKeyBy(t.Get(quad.Predicate), 1, nil)
ts.UpdateValueKeyBy(t.Get(graph.Object), 1, nil) qs.UpdateValueKeyBy(t.Get(quad.Object), 1, nil)
if t.Get(graph.Provenance) != "" { if t.Get(quad.Provenance) != "" {
ts.UpdateValueKeyBy(t.Get(graph.Provenance), 1, nil) qs.UpdateValueKeyBy(t.Get(quad.Provenance), 1, nil)
} }
} }
@ -216,10 +217,10 @@ type ValueData struct {
Size int64 Size int64
} }
func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) { func (qs *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) {
value := &ValueData{name, int64(amount)} value := &ValueData{name, int64(amount)}
key := ts.createValueKeyFor(name) key := qs.createValueKeyFor(name)
b, err := ts.db.Get(key, ts.readopts) b, err := qs.db.Get(key, qs.readopts)
// Error getting the node from the database. // Error getting the node from the database.
if err != nil && err != leveldb.ErrNotFound { if err != nil && err != leveldb.ErrNotFound {
@ -241,7 +242,7 @@ func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.
if amount < 0 { if amount < 0 {
if value.Size <= 0 { if value.Size <= 0 {
if batch == nil { if batch == nil {
ts.db.Delete(key, ts.writeopts) qs.db.Delete(key, qs.writeopts)
} else { } else {
batch.Delete(key) batch.Delete(key)
} }
@ -256,18 +257,18 @@ func (ts *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.
return return
} }
if batch == nil { if batch == nil {
ts.db.Put(key, bytes, ts.writeopts) qs.db.Put(key, bytes, qs.writeopts)
} else { } else {
batch.Put(key, bytes) batch.Put(key, bytes)
} }
} }
func (ts *TripleStore) AddTripleSet(t_s []*graph.Triple) { func (qs *TripleStore) AddTripleSet(t_s []*quad.Quad) {
batch := &leveldb.Batch{} batch := &leveldb.Batch{}
newTs := len(t_s) newTs := len(t_s)
resizeMap := make(map[string]int) resizeMap := make(map[string]int)
for _, t := range t_s { for _, t := range t_s {
ts.buildTripleWrite(batch, t) qs.buildTripleWrite(batch, t)
resizeMap[t.Subject]++ resizeMap[t.Subject]++
resizeMap[t.Predicate]++ resizeMap[t.Predicate]++
resizeMap[t.Object]++ resizeMap[t.Object]++
@ -276,68 +277,68 @@ func (ts *TripleStore) AddTripleSet(t_s []*graph.Triple) {
} }
} }
for k, v := range resizeMap { for k, v := range resizeMap {
ts.UpdateValueKeyBy(k, v, batch) qs.UpdateValueKeyBy(k, v, batch)
} }
err := ts.db.Write(batch, ts.writeopts) err := qs.db.Write(batch, qs.writeopts)
if err != nil { if err != nil {
glog.Errorf("Couldn't write to DB for tripleset") glog.Errorf("Couldn't write to DB for tripleset")
return return
} }
ts.size += int64(newTs) qs.size += int64(newTs)
} }
func (ts *TripleStore) Close() { func (qs *TripleStore) Close() {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
err := binary.Write(buf, binary.LittleEndian, ts.size) err := binary.Write(buf, binary.LittleEndian, qs.size)
if err == nil { if err == nil {
werr := ts.db.Put([]byte("__size"), buf.Bytes(), ts.writeopts) werr := qs.db.Put([]byte("__size"), buf.Bytes(), qs.writeopts)
if werr != nil { if werr != nil {
glog.Errorf("Couldn't write size before closing!") glog.Errorf("Couldn't write size before closing!")
} }
} else { } else {
glog.Errorf("Couldn't convert size before closing!") glog.Errorf("Couldn't convert size before closing!")
} }
ts.db.Close() qs.db.Close()
ts.open = false qs.open = false
} }
func (ts *TripleStore) Triple(k graph.Value) *graph.Triple { func (qs *TripleStore) Quad(k graph.Value) *quad.Quad {
var triple graph.Triple var triple quad.Quad
b, err := ts.db.Get(k.([]byte), ts.readopts) b, err := qs.db.Get(k.([]byte), qs.readopts)
if err != nil && err != leveldb.ErrNotFound { if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: couldn't get triple from DB") glog.Errorln("Error: couldn't get triple from DB")
return &graph.Triple{} return &quad.Quad{}
} }
if err == leveldb.ErrNotFound { if err == leveldb.ErrNotFound {
// No harm, no foul. // No harm, no foul.
return &graph.Triple{} return &quad.Quad{}
} }
err = json.Unmarshal(b, &triple) err = json.Unmarshal(b, &triple)
if err != nil { if err != nil {
glog.Errorln("Error: couldn't reconstruct triple") glog.Errorln("Error: couldn't reconstruct triple")
return &graph.Triple{} return &quad.Quad{}
} }
return &triple return &triple
} }
func (ts *TripleStore) convertStringToByteHash(s string) []byte { func (qs *TripleStore) convertStringToByteHash(s string) []byte {
ts.hasher.Reset() qs.hasher.Reset()
key := make([]byte, 0, ts.hasher.Size()) key := make([]byte, 0, qs.hasher.Size())
ts.hasher.Write([]byte(s)) qs.hasher.Write([]byte(s))
key = ts.hasher.Sum(key) key = qs.hasher.Sum(key)
return key return key
} }
func (ts *TripleStore) ValueOf(s string) graph.Value { func (qs *TripleStore) ValueOf(s string) graph.Value {
return ts.createValueKeyFor(s) return qs.createValueKeyFor(s)
} }
func (ts *TripleStore) valueData(value_key []byte) ValueData { func (qs *TripleStore) valueData(value_key []byte) ValueData {
var out ValueData var out ValueData
if glog.V(3) { if glog.V(3) {
glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key) glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key)
} }
b, err := ts.db.Get(value_key, ts.readopts) b, err := qs.db.Get(value_key, qs.readopts)
if err != nil && err != leveldb.ErrNotFound { if err != nil && err != leveldb.ErrNotFound {
glog.Errorln("Error: couldn't get value from DB") glog.Errorln("Error: couldn't get value from DB")
return out return out
@ -352,30 +353,30 @@ func (ts *TripleStore) valueData(value_key []byte) ValueData {
return out return out
} }
func (ts *TripleStore) NameOf(k graph.Value) string { func (qs *TripleStore) NameOf(k graph.Value) string {
if k == nil { if k == nil {
glog.V(2).Infoln("k was nil") glog.V(2).Infoln("k was nil")
return "" return ""
} }
return ts.valueData(k.([]byte)).Name return qs.valueData(k.([]byte)).Name
} }
func (ts *TripleStore) SizeOf(k graph.Value) int64 { func (qs *TripleStore) SizeOf(k graph.Value) int64 {
if k == nil { if k == nil {
return 0 return 0
} }
return int64(ts.valueData(k.([]byte)).Size) return int64(qs.valueData(k.([]byte)).Size)
} }
func (ts *TripleStore) getSize() { func (qs *TripleStore) getSize() {
var size int64 var size int64
b, err := ts.db.Get([]byte("__size"), ts.readopts) b, err := qs.db.Get([]byte("__size"), qs.readopts)
if err != nil && err != leveldb.ErrNotFound { if err != nil && err != leveldb.ErrNotFound {
panic("Couldn't read size " + err.Error()) panic("Couldn't read size " + err.Error())
} }
if err == leveldb.ErrNotFound { if err == leveldb.ErrNotFound {
// Must be a new database. Cool // Must be a new database. Cool
ts.size = 0 qs.size = 0
return return
} }
buf := bytes.NewBuffer(b) buf := bytes.NewBuffer(b)
@ -383,10 +384,10 @@ func (ts *TripleStore) getSize() {
if err != nil { if err != nil {
glog.Errorln("Error: couldn't parse size") glog.Errorln("Error: couldn't parse size")
} }
ts.size = size qs.size = size
} }
func (ts *TripleStore) SizeOfPrefix(pre []byte) (int64, error) { func (qs *TripleStore) SizeOfPrefix(pre []byte) (int64, error) {
limit := make([]byte, len(pre)) limit := make([]byte, len(pre))
copy(limit, pre) copy(limit, pre)
end := len(limit) - 1 end := len(limit) - 1
@ -394,45 +395,45 @@ func (ts *TripleStore) SizeOfPrefix(pre []byte) (int64, error) {
ranges := make([]util.Range, 1) ranges := make([]util.Range, 1)
ranges[0].Start = pre ranges[0].Start = pre
ranges[0].Limit = limit ranges[0].Limit = limit
sizes, err := ts.db.SizeOf(ranges) sizes, err := qs.db.SizeOf(ranges)
if err == nil { if err == nil {
return (int64(sizes[0]) >> 6) + 1, nil return (int64(sizes[0]) >> 6) + 1, nil
} }
return 0, nil return 0, nil
} }
func (ts *TripleStore) TripleIterator(d graph.Direction, val graph.Value) graph.Iterator { func (qs *TripleStore) TripleIterator(d quad.Direction, val graph.Value) graph.Iterator {
var prefix string var prefix string
switch d { switch d {
case graph.Subject: case quad.Subject:
prefix = "sp" prefix = "sp"
case graph.Predicate: case quad.Predicate:
prefix = "po" prefix = "po"
case graph.Object: case quad.Object:
prefix = "os" prefix = "os"
case graph.Provenance: case quad.Provenance:
prefix = "cp" prefix = "cp"
default: default:
panic("unreachable " + d.String()) panic("unreachable " + d.String())
} }
return NewIterator(prefix, d, val, ts) return NewIterator(prefix, d, val, qs)
} }
func (ts *TripleStore) NodesAllIterator() graph.Iterator { func (qs *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator("z", graph.Any, ts) return NewAllIterator("z", quad.Any, qs)
} }
func (ts *TripleStore) TriplesAllIterator() graph.Iterator { func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator("po", graph.Predicate, ts) return NewAllIterator("po", quad.Predicate, qs)
} }
func (ts *TripleStore) TripleDirection(val graph.Value, d graph.Direction) graph.Value { func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value {
v := val.([]uint8) v := val.([]uint8)
offset := PositionOf(v[0:2], d, ts) offset := PositionOf(v[0:2], d, qs)
if offset != -1 { if offset != -1 {
return append([]byte("z"), v[offset:offset+ts.hasher.Size()]...) return append([]byte("z"), v[offset:offset+qs.hasher.Size()]...)
} else { } else {
return ts.Triple(val).Get(d) return qs.Quad(val).Get(d)
} }
} }
@ -440,7 +441,7 @@ func compareBytes(a, b graph.Value) bool {
return bytes.Equal(a.([]uint8), b.([]uint8)) return bytes.Equal(a.([]uint8), b.([]uint8))
} }
func (ts *TripleStore) FixedIterator() graph.FixedIterator { func (qs *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(compareBytes) return iterator.NewFixedIteratorWithCompare(compareBytes)
} }

63
graph/memstore/triplestore.go
View file

@ -20,6 +20,7 @@ import (
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
"github.com/petar/GoLLRB/llrb" "github.com/petar/GoLLRB/llrb"
) )
@ -40,21 +41,21 @@ func NewTripleDirectionIndex() *TripleDirectionIndex {
return &tdi return &tdi
} }
func (tdi *TripleDirectionIndex) GetForDir(d graph.Direction) map[int64]*llrb.LLRB { func (tdi *TripleDirectionIndex) GetForDir(d quad.Direction) map[int64]*llrb.LLRB {
switch d { switch d {
case graph.Subject: case quad.Subject:
return tdi.subject return tdi.subject
case graph.Object: case quad.Object:
return tdi.object return tdi.object
case graph.Predicate: case quad.Predicate:
return tdi.predicate return tdi.predicate
case graph.Provenance: case quad.Provenance:
return tdi.provenance return tdi.provenance
} }
panic("illegal direction") panic("illegal direction")
} }
func (tdi *TripleDirectionIndex) GetOrCreate(d graph.Direction, id int64) *llrb.LLRB { func (tdi *TripleDirectionIndex) GetOrCreate(d quad.Direction, id int64) *llrb.LLRB {
directionIndex := tdi.GetForDir(d) directionIndex := tdi.GetForDir(d)
if _, ok := directionIndex[id]; !ok { if _, ok := directionIndex[id]; !ok {
directionIndex[id] = llrb.New() directionIndex[id] = llrb.New()
@ -62,7 +63,7 @@ func (tdi *TripleDirectionIndex) GetOrCreate(d graph.Direction, id int64) *llrb.
return directionIndex[id] return directionIndex[id]
} }
func (tdi *TripleDirectionIndex) Get(d graph.Direction, id int64) (*llrb.LLRB, bool) { func (tdi *TripleDirectionIndex) Get(d quad.Direction, id int64) (*llrb.LLRB, bool) {
directionIndex := tdi.GetForDir(d) directionIndex := tdi.GetForDir(d)
tree, exists := directionIndex[id] tree, exists := directionIndex[id]
return tree, exists return tree, exists
@ -73,7 +74,7 @@ type TripleStore struct {
tripleIdCounter int64 tripleIdCounter int64
idMap map[string]int64 idMap map[string]int64
revIdMap map[int64]string revIdMap map[int64]string
triples []graph.Triple triples []quad.Quad
size int64 size int64
index TripleDirectionIndex index TripleDirectionIndex
// vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree // vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree
@ -83,10 +84,10 @@ func newTripleStore() *TripleStore {
var ts TripleStore var ts TripleStore
ts.idMap = make(map[string]int64) ts.idMap = make(map[string]int64)
ts.revIdMap = make(map[int64]string) ts.revIdMap = make(map[int64]string)
ts.triples = make([]graph.Triple, 1, 200) ts.triples = make([]quad.Quad, 1, 200)
// Sentinel null triple so triple indices start at 1 // Sentinel null triple so triple indices start at 1
ts.triples[0] = graph.Triple{} ts.triples[0] = quad.Quad{}
ts.size = 1 ts.size = 1
ts.index = *NewTripleDirectionIndex() ts.index = *NewTripleDirectionIndex()
ts.idCounter = 1 ts.idCounter = 1
@ -94,18 +95,18 @@ func newTripleStore() *TripleStore {
return &ts return &ts
} }
func (ts *TripleStore) AddTripleSet(triples []*graph.Triple) { func (ts *TripleStore) AddTripleSet(triples []*quad.Quad) {
for _, t := range triples { for _, t := range triples {
ts.AddTriple(t) ts.AddTriple(t)
} }
} }
func (ts *TripleStore) tripleExists(t *graph.Triple) (bool, int64) { func (ts *TripleStore) tripleExists(t *quad.Quad) (bool, int64) {
smallest := -1 smallest := -1
var smallest_tree *llrb.LLRB var smallest_tree *llrb.LLRB
for d := graph.Subject; d <= graph.Provenance; d++ { for d := quad.Subject; d <= quad.Provenance; d++ {
sid := t.Get(d) sid := t.Get(d)
if d == graph.Provenance && sid == "" { if d == quad.Provenance && sid == "" {
continue continue
} }
id, ok := ts.idMap[sid] id, ok := ts.idMap[sid]
@ -137,7 +138,7 @@ func (ts *TripleStore) tripleExists(t *graph.Triple) (bool, int64) {
return false, 0 return false, 0
} }
func (ts *TripleStore) AddTriple(t *graph.Triple) { func (ts *TripleStore) AddTriple(t *quad.Quad) {
if exists, _ := ts.tripleExists(t); exists { if exists, _ := ts.tripleExists(t); exists {
return return
} }
@ -147,9 +148,9 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
ts.size++ ts.size++
ts.tripleIdCounter++ ts.tripleIdCounter++
for d := graph.Subject; d <= graph.Provenance; d++ { for d := quad.Subject; d <= quad.Provenance; d++ {
sid := t.Get(d) sid := t.Get(d)
if d == graph.Provenance && sid == "" { if d == quad.Provenance && sid == "" {
continue continue
} }
if _, ok := ts.idMap[sid]; !ok { if _, ok := ts.idMap[sid]; !ok {
@ -159,8 +160,8 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
} }
} }
for d := graph.Subject; d <= graph.Provenance; d++ { for d := quad.Subject; d <= quad.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" { if d == quad.Provenance && t.Get(d) == "" {
continue continue
} }
id := ts.idMap[t.Get(d)] id := ts.idMap[t.Get(d)]
@ -171,7 +172,7 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
// TODO(barakmich): Add VIP indexing // TODO(barakmich): Add VIP indexing
} }
func (ts *TripleStore) RemoveTriple(t *graph.Triple) { func (ts *TripleStore) RemoveTriple(t *quad.Quad) {
var tripleID int64 var tripleID int64
var exists bool var exists bool
tripleID = 0 tripleID = 0
@ -179,11 +180,11 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
return return
} }
ts.triples[tripleID] = graph.Triple{} ts.triples[tripleID] = quad.Quad{}
ts.size-- ts.size--
for d := graph.Subject; d <= graph.Provenance; d++ { for d := quad.Subject; d <= quad.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" { if d == quad.Provenance && t.Get(d) == "" {
continue continue
} }
id := ts.idMap[t.Get(d)] id := ts.idMap[t.Get(d)]
@ -191,8 +192,8 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
tree.Delete(Int64(tripleID)) tree.Delete(Int64(tripleID))
} }
for d := graph.Subject; d <= graph.Provenance; d++ { for d := quad.Subject; d <= quad.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" { if d == quad.Provenance && t.Get(d) == "" {
continue continue
} }
id, ok := ts.idMap[t.Get(d)] id, ok := ts.idMap[t.Get(d)]
@ -200,8 +201,8 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
continue continue
} }
stillExists := false stillExists := false
for d := graph.Subject; d <= graph.Provenance; d++ { for d := quad.Subject; d <= quad.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" { if d == quad.Provenance && t.Get(d) == "" {
continue continue
} }
nodeTree := ts.index.GetOrCreate(d, id) nodeTree := ts.index.GetOrCreate(d, id)
@ -217,11 +218,11 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
} }
} }
func (ts *TripleStore) Triple(index graph.Value) *graph.Triple { func (ts *TripleStore) Quad(index graph.Value) *quad.Quad {
return &ts.triples[index.(int64)] return &ts.triples[index.(int64)]
} }
func (ts *TripleStore) TripleIterator(d graph.Direction, value graph.Value) graph.Iterator { func (ts *TripleStore) TripleIterator(d quad.Direction, value graph.Value) graph.Iterator {
index, ok := ts.index.Get(d, value.(int64)) index, ok := ts.index.Get(d, value.(int64))
data := fmt.Sprintf("dir:%s val:%d", d, value.(int64)) data := fmt.Sprintf("dir:%s val:%d", d, value.(int64))
if ok { if ok {
@ -259,8 +260,8 @@ func (ts *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(iterator.BasicEquality) return iterator.NewFixedIteratorWithCompare(iterator.BasicEquality)
} }
func (ts *TripleStore) TripleDirection(val graph.Value, d graph.Direction) graph.Value { func (ts *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value {
name := ts.Triple(val).Get(d) name := ts.Quad(val).Get(d)
return ts.ValueOf(name) return ts.ValueOf(name)
} }

22
graph/memstore/triplestore_test.go
View file

@ -19,8 +19,8 @@ import (
"sort" "sort"
"testing" "testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
// This is a simple test graph. // This is a simple test graph.
@ -36,7 +36,7 @@ import (
// \-->|#D#|------------->+---+ // \-->|#D#|------------->+---+
// +---+ // +---+
// //
var simpleGraph = []*graph.Triple{ var simpleGraph = []*quad.Quad{
{"A", "follows", "B", ""}, {"A", "follows", "B", ""},
{"C", "follows", "B", ""}, {"C", "follows", "B", ""},
{"C", "follows", "D", ""}, {"C", "follows", "D", ""},
@ -50,7 +50,7 @@ var simpleGraph = []*graph.Triple{
{"G", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"},
} }
func makeTestStore(data []*graph.Triple) (*TripleStore, []pair) { func makeTestStore(data []*quad.Quad) (*TripleStore, []pair) {
seen := make(map[string]struct{}) seen := make(map[string]struct{})
ts := newTripleStore() ts := newTripleStore()
var ( var (
@ -105,10 +105,10 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
all := ts.NodesAllIterator() all := ts.NodesAllIterator()
innerAnd := iterator.NewAnd() innerAnd := iterator.NewAnd()
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, graph.Predicate)) innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, quad.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, all, graph.Object)) innerAnd.AddSubIterator(iterator.NewLinksTo(ts, all, quad.Object))
hasa := iterator.NewHasA(ts, innerAnd, graph.Subject) hasa := iterator.NewHasA(ts, innerAnd, quad.Subject)
outerAnd := iterator.NewAnd() outerAnd := iterator.NewAnd()
outerAnd.AddSubIterator(fixed) outerAnd.AddSubIterator(fixed)
outerAnd.AddSubIterator(hasa) outerAnd.AddSubIterator(hasa)
@ -149,7 +149,7 @@ func TestLinksToOptimization(t *testing.T) {
fixed := ts.FixedIterator() fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("cool")) fixed.Add(ts.ValueOf("cool"))
lto := iterator.NewLinksTo(ts, fixed, graph.Object) lto := iterator.NewLinksTo(ts, fixed, quad.Object)
lto.AddTag("foo") lto.AddTag("foo")
newIt, changed := lto.Optimize() newIt, changed := lto.Optimize()
@ -173,7 +173,7 @@ func TestLinksToOptimization(t *testing.T) {
func TestRemoveTriple(t *testing.T) { func TestRemoveTriple(t *testing.T) {
ts, _ := makeTestStore(simpleGraph) ts, _ := makeTestStore(simpleGraph)
ts.RemoveTriple(&graph.Triple{"E", "follows", "F", ""}) ts.RemoveTriple(&quad.Quad{"E", "follows", "F", ""})
fixed := ts.FixedIterator() fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("E")) fixed.Add(ts.ValueOf("E"))
@ -182,10 +182,10 @@ func TestRemoveTriple(t *testing.T) {
fixed2.Add(ts.ValueOf("follows")) fixed2.Add(ts.ValueOf("follows"))
innerAnd := iterator.NewAnd() innerAnd := iterator.NewAnd()
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, graph.Subject)) innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, quad.Subject))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, graph.Predicate)) innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, quad.Predicate))
hasa := iterator.NewHasA(ts, innerAnd, graph.Object) hasa := iterator.NewHasA(ts, innerAnd, quad.Object)
newIt, _ := hasa.Optimize() newIt, _ := hasa.Optimize()
_, ok := newIt.Next() _, ok := newIt.Next()

55
graph/mongo/iterator.go
View file

@ -24,12 +24,13 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
type Iterator struct { type Iterator struct {
iterator.Base iterator.Base
ts *TripleStore qs *TripleStore
dir graph.Direction dir quad.Direction
iter *mgo.Iter iter *mgo.Iter
hash string hash string
name string name string
@ -39,27 +40,27 @@ type Iterator struct {
collection string collection string
} }
func NewIterator(ts *TripleStore, collection string, d graph.Direction, val graph.Value) *Iterator { func NewIterator(qs *TripleStore, collection string, d quad.Direction, val graph.Value) *Iterator {
var m Iterator var m Iterator
iterator.BaseInit(&m.Base) iterator.BaseInit(&m.Base)
m.name = ts.NameOf(val) m.name = qs.NameOf(val)
m.collection = collection m.collection = collection
switch d { switch d {
case graph.Subject: case quad.Subject:
m.constraint = bson.M{"Subject": m.name} m.constraint = bson.M{"Subject": m.name}
case graph.Predicate: case quad.Predicate:
m.constraint = bson.M{"Predicate": m.name} m.constraint = bson.M{"Predicate": m.name}
case graph.Object: case quad.Object:
m.constraint = bson.M{"Object": m.name} m.constraint = bson.M{"Object": m.name}
case graph.Provenance: case quad.Provenance:
m.constraint = bson.M{"Provenance": m.name} m.constraint = bson.M{"Provenance": m.name}
} }
m.ts = ts m.qs = qs
m.dir = d m.dir = d
m.iter = ts.db.C(collection).Find(m.constraint).Iter() m.iter = qs.db.C(collection).Find(m.constraint).Iter()
size, err := ts.db.C(collection).Find(m.constraint).Count() size, err := qs.db.C(collection).Find(m.constraint).Count()
if err != nil { if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err) glog.Errorln("Trouble getting size for iterator! ", err)
return nil return nil
@ -70,14 +71,14 @@ func NewIterator(ts *TripleStore, collection string, d graph.Direction, val grap
return &m return &m
} }
func NewAllIterator(ts *TripleStore, collection string) *Iterator { func NewAllIterator(qs *TripleStore, collection string) *Iterator {
var m Iterator var m Iterator
m.ts = ts m.qs = qs
m.dir = graph.Any m.dir = quad.Any
m.constraint = nil m.constraint = nil
m.collection = collection m.collection = collection
m.iter = ts.db.C(collection).Find(nil).Iter() m.iter = qs.db.C(collection).Find(nil).Iter()
size, err := ts.db.C(collection).Count() size, err := qs.db.C(collection).Count()
if err != nil { if err != nil {
glog.Errorln("Trouble getting size for iterator! ", err) glog.Errorln("Trouble getting size for iterator! ", err)
return nil return nil
@ -90,7 +91,7 @@ func NewAllIterator(ts *TripleStore, collection string) *Iterator {
func (it *Iterator) Reset() { func (it *Iterator) Reset() {
it.iter.Close() it.iter.Close()
it.iter = it.ts.db.C(it.collection).Find(it.constraint).Iter() it.iter = it.qs.db.C(it.collection).Find(it.constraint).Iter()
} }
@ -101,9 +102,9 @@ func (it *Iterator) Close() {
func (it *Iterator) Clone() graph.Iterator { func (it *Iterator) Clone() graph.Iterator {
var newM graph.Iterator var newM graph.Iterator
if it.isAll { if it.isAll {
newM = NewAllIterator(it.ts, it.collection) newM = NewAllIterator(it.qs, it.collection)
} else { } else {
newM = NewIterator(it.ts, it.collection, it.dir, it.hash) newM = NewIterator(it.qs, it.collection, it.dir, it.hash)
} }
newM.CopyTagsFrom(it) newM.CopyTagsFrom(it)
return newM return newM
@ -136,16 +137,16 @@ func (it *Iterator) Check(v graph.Value) bool {
} }
var offset int var offset int
switch it.dir { switch it.dir {
case graph.Subject: case quad.Subject:
offset = 0 offset = 0
case graph.Predicate: case quad.Predicate:
offset = (it.ts.hasher.Size() * 2) offset = (it.qs.hasher.Size() * 2)
case graph.Object: case quad.Object:
offset = (it.ts.hasher.Size() * 2) * 2 offset = (it.qs.hasher.Size() * 2) * 2
case graph.Provenance: case quad.Provenance:
offset = (it.ts.hasher.Size() * 2) * 3 offset = (it.qs.hasher.Size() * 2) * 3
} }
val := v.(string)[offset : it.ts.hasher.Size()*2+offset] val := v.(string)[offset : it.qs.hasher.Size()*2+offset]
if val == it.hash { if val == it.hash {
it.Last = v it.Last = v
return graph.CheckLogOut(it, v, true) return graph.CheckLogOut(it, v, true)

180
graph/mongo/triplestore.go
View file

@ -18,6 +18,7 @@ import (
"crypto/sha1" "crypto/sha1"
"encoding/hex" "encoding/hex"
"hash" "hash"
"io"
"log" "log"
"gopkg.in/mgo.v2" "gopkg.in/mgo.v2"
@ -26,8 +27,16 @@ import (
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
func init() {
graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
}
// Guarantee we satisfy graph.Bulkloader.
var _ graph.BulkLoader = (*TripleStore)(nil)
const DefaultDBName = "cayley" const DefaultDBName = "cayley"
type TripleStore struct { type TripleStore struct {
@ -66,7 +75,7 @@ func createNewMongoGraph(addr string, options graph.Options) error {
} }
func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) { func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore var qs TripleStore
conn, err := mgo.Dial(addr) conn, err := mgo.Dial(addr)
if err != nil { if err != nil {
return nil, err return nil, err
@ -76,26 +85,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
if val, ok := options.StringKey("database_name"); ok { if val, ok := options.StringKey("database_name"); ok {
dbName = val dbName = val
} }
ts.db = conn.DB(dbName) qs.db = conn.DB(dbName)
ts.session = conn qs.session = conn
ts.hasher = sha1.New() qs.hasher = sha1.New()
ts.idCache = NewIDLru(1 << 16) qs.idCache = NewIDLru(1 << 16)
return &ts, nil return &qs, nil
} }
func (ts *TripleStore) getIdForTriple(t *graph.Triple) string { func (qs *TripleStore) getIdForTriple(t *quad.Quad) string {
id := ts.ConvertStringToByteHash(t.Subject) id := qs.ConvertStringToByteHash(t.Subject)
id += ts.ConvertStringToByteHash(t.Predicate) id += qs.ConvertStringToByteHash(t.Predicate)
id += ts.ConvertStringToByteHash(t.Object) id += qs.ConvertStringToByteHash(t.Object)
id += ts.ConvertStringToByteHash(t.Provenance) id += qs.ConvertStringToByteHash(t.Provenance)
return id return id
} }
func (ts *TripleStore) ConvertStringToByteHash(s string) string { func (qs *TripleStore) ConvertStringToByteHash(s string) string {
ts.hasher.Reset() qs.hasher.Reset()
key := make([]byte, 0, ts.hasher.Size()) key := make([]byte, 0, qs.hasher.Size())
ts.hasher.Write([]byte(s)) qs.hasher.Write([]byte(s))
key = ts.hasher.Sum(key) key = qs.hasher.Sum(key)
return hex.EncodeToString(key) return hex.EncodeToString(key)
} }
@ -105,10 +114,10 @@ type MongoNode struct {
Size int "Size" Size int "Size"
} }
func (ts *TripleStore) updateNodeBy(node_name string, inc int) { func (qs *TripleStore) updateNodeBy(node_name string, inc int) {
var size MongoNode var size MongoNode
node := ts.ValueOf(node_name) node := qs.ValueOf(node_name)
err := ts.db.C("nodes").FindId(node).One(&size) err := qs.db.C("nodes").FindId(node).One(&size)
if err != nil { if err != nil {
if err.Error() == "not found" { if err.Error() == "not found" {
// Not found. Okay. // Not found. Okay.
@ -128,7 +137,7 @@ func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
// Removing something... // Removing something...
if inc < 0 { if inc < 0 {
if size.Size <= 0 { if size.Size <= 0 {
err := ts.db.C("nodes").RemoveId(node) err := qs.db.C("nodes").RemoveId(node)
if err != nil { if err != nil {
glog.Error("Error: ", err, " while removing node ", node_name) glog.Error("Error: ", err, " while removing node ", node_name)
return return
@ -136,21 +145,21 @@ func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
} }
} }
_, err2 := ts.db.C("nodes").UpsertId(node, size) _, err2 := qs.db.C("nodes").UpsertId(node, size)
if err2 != nil { if err2 != nil {
glog.Error("Error: ", err) glog.Error("Error: ", err)
} }
} }
func (ts *TripleStore) writeTriple(t *graph.Triple) bool { func (qs *TripleStore) writeTriple(t *quad.Quad) bool {
tripledoc := bson.M{ tripledoc := bson.M{
"_id": ts.getIdForTriple(t), "_id": qs.getIdForTriple(t),
"Subject": t.Subject, "Subject": t.Subject,
"Predicate": t.Predicate, "Predicate": t.Predicate,
"Object": t.Object, "Object": t.Object,
"Provenance": t.Provenance, "Provenance": t.Provenance,
} }
err := ts.db.C("triples").Insert(tripledoc) err := qs.db.C("triples").Insert(tripledoc)
if err != nil { if err != nil {
// Among the reasons I hate MongoDB. "Errors don't happen! Right guys?" // Among the reasons I hate MongoDB. "Errors don't happen! Right guys?"
if err.(*mgo.LastError).Code == 11000 { if err.(*mgo.LastError).Code == 11000 {
@ -162,21 +171,21 @@ func (ts *TripleStore) writeTriple(t *graph.Triple) bool {
return true return true
} }
func (ts *TripleStore) AddTriple(t *graph.Triple) { func (qs *TripleStore) AddTriple(t *quad.Quad) {
_ = ts.writeTriple(t) _ = qs.writeTriple(t)
ts.updateNodeBy(t.Subject, 1) qs.updateNodeBy(t.Subject, 1)
ts.updateNodeBy(t.Predicate, 1) qs.updateNodeBy(t.Predicate, 1)
ts.updateNodeBy(t.Object, 1) qs.updateNodeBy(t.Object, 1)
if t.Provenance != "" { if t.Provenance != "" {
ts.updateNodeBy(t.Provenance, 1) qs.updateNodeBy(t.Provenance, 1)
} }
} }
func (ts *TripleStore) AddTripleSet(in []*graph.Triple) { func (qs *TripleStore) AddTripleSet(in []*quad.Quad) {
ts.session.SetSafe(nil) qs.session.SetSafe(nil)
ids := make(map[string]int) ids := make(map[string]int)
for _, t := range in { for _, t := range in {
wrote := ts.writeTriple(t) wrote := qs.writeTriple(t)
if wrote { if wrote {
ids[t.Subject]++ ids[t.Subject]++
ids[t.Object]++ ids[t.Object]++
@ -187,34 +196,34 @@ func (ts *TripleStore) AddTripleSet(in []*graph.Triple) {
} }
} }
for k, v := range ids { for k, v := range ids {
ts.updateNodeBy(k, v) qs.updateNodeBy(k, v)
} }
ts.session.SetSafe(&mgo.Safe{}) qs.session.SetSafe(&mgo.Safe{})
} }
func (ts *TripleStore) RemoveTriple(t *graph.Triple) { func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t)) err := qs.db.C("triples").RemoveId(qs.getIdForTriple(t))
if err == mgo.ErrNotFound { if err == mgo.ErrNotFound {
return return
} else if err != nil { } else if err != nil {
log.Println("Error: ", err, " while removing triple ", t) log.Println("Error: ", err, " while removing triple ", t)
return return
} }
ts.updateNodeBy(t.Subject, -1) qs.updateNodeBy(t.Subject, -1)
ts.updateNodeBy(t.Predicate, -1) qs.updateNodeBy(t.Predicate, -1)
ts.updateNodeBy(t.Object, -1) qs.updateNodeBy(t.Object, -1)
if t.Provenance != "" { if t.Provenance != "" {
ts.updateNodeBy(t.Provenance, -1) qs.updateNodeBy(t.Provenance, -1)
} }
} }
func (ts *TripleStore) Triple(val graph.Value) *graph.Triple { func (qs *TripleStore) Quad(val graph.Value) *quad.Quad {
var bsonDoc bson.M var bsonDoc bson.M
err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc) err := qs.db.C("triples").FindId(val.(string)).One(&bsonDoc)
if err != nil { if err != nil {
log.Println("Error: Couldn't retrieve triple", val.(string), err) log.Println("Error: Couldn't retrieve triple", val.(string), err)
} }
return &graph.Triple{ return &quad.Quad{
bsonDoc["Subject"].(string), bsonDoc["Subject"].(string),
bsonDoc["Predicate"].(string), bsonDoc["Predicate"].(string),
bsonDoc["Object"].(string), bsonDoc["Object"].(string),
@ -222,38 +231,38 @@ func (ts *TripleStore) Triple(val graph.Value) *graph.Triple {
} }
} }
func (ts *TripleStore) TripleIterator(d graph.Direction, val graph.Value) graph.Iterator { func (qs *TripleStore) TripleIterator(d quad.Direction, val graph.Value) graph.Iterator {
return NewIterator(ts, "triples", d, val) return NewIterator(qs, "triples", d, val)
} }
func (ts *TripleStore) NodesAllIterator() graph.Iterator { func (qs *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator(ts, "nodes") return NewAllIterator(qs, "nodes")
} }
func (ts *TripleStore) TriplesAllIterator() graph.Iterator { func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator(ts, "triples") return NewAllIterator(qs, "triples")
} }
func (ts *TripleStore) ValueOf(s string) graph.Value { func (qs *TripleStore) ValueOf(s string) graph.Value {
return ts.ConvertStringToByteHash(s) return qs.ConvertStringToByteHash(s)
} }
func (ts *TripleStore) NameOf(v graph.Value) string { func (qs *TripleStore) NameOf(v graph.Value) string {
val, ok := ts.idCache.Get(v.(string)) val, ok := qs.idCache.Get(v.(string))
if ok { if ok {
return val return val
} }
var node MongoNode var node MongoNode
err := ts.db.C("nodes").FindId(v.(string)).One(&node) err := qs.db.C("nodes").FindId(v.(string)).One(&node)
if err != nil { if err != nil {
log.Println("Error: Couldn't retrieve node", v.(string), err) log.Println("Error: Couldn't retrieve node", v.(string), err)
} }
ts.idCache.Put(v.(string), node.Name) qs.idCache.Put(v.(string), node.Name)
return node.Name return node.Name
} }
func (ts *TripleStore) Size() int64 { func (qs *TripleStore) Size() int64 {
count, err := ts.db.C("triples").Count() count, err := qs.db.C("triples").Count()
if err != nil { if err != nil {
glog.Error("Error: ", err) glog.Error("Error: ", err)
return 0 return 0
@ -265,40 +274,48 @@ func compareStrings(a, b graph.Value) bool {
return a.(string) == b.(string) return a.(string) == b.(string)
} }
func (ts *TripleStore) FixedIterator() graph.FixedIterator { func (qs *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(compareStrings) return iterator.NewFixedIteratorWithCompare(compareStrings)
} }
func (ts *TripleStore) Close() { func (qs *TripleStore) Close() {
ts.db.Session.Close() qs.db.Session.Close()
} }
func (ts *TripleStore) TripleDirection(in graph.Value, d graph.Direction) graph.Value { func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.Value {
// Maybe do the trick here // Maybe do the trick here
var offset int var offset int
switch d { switch d {
case graph.Subject: case quad.Subject:
offset = 0 offset = 0
case graph.Predicate: case quad.Predicate:
offset = (ts.hasher.Size() * 2) offset = (qs.hasher.Size() * 2)
case graph.Object: case quad.Object:
offset = (ts.hasher.Size() * 2) * 2 offset = (qs.hasher.Size() * 2) * 2
case graph.Provenance: case quad.Provenance:
offset = (ts.hasher.Size() * 2) * 3 offset = (qs.hasher.Size() * 2) * 3
} }
val := in.(string)[offset : ts.hasher.Size()*2+offset] val := in.(string)[offset : qs.hasher.Size()*2+offset]
return val return val
} }
func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool { func (qs *TripleStore) BulkLoad(dec quad.Unmarshaler) error {
if ts.Size() != 0 { if qs.Size() != 0 {
return false return graph.ErrCannotBulkLoad
} }
ts.session.SetSafe(nil) qs.session.SetSafe(nil)
for triple := range t_chan { for {
ts.writeTriple(triple) q, err := dec.Unmarshal()
if err != nil {
if err != io.EOF {
return err
}
break
}
qs.writeTriple(q)
} }
outputTo := bson.M{"replace": "nodes", "sharded": true} outputTo := bson.M{"replace": "nodes", "sharded": true}
glog.Infoln("Mapreducing") glog.Infoln("Mapreducing")
job := mgo.MapReduce{ job := mgo.MapReduce{
@ -330,16 +347,13 @@ func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool {
`, `,
Out: outputTo, Out: outputTo,
} }
ts.db.C("triples").Find(nil).MapReduce(&job, nil) qs.db.C("triples").Find(nil).MapReduce(&job, nil)
glog.Infoln("Fixing") glog.Infoln("Fixing")
ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) { qs.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
db.nodes.update({"_id": result._id}, result.value) db.nodes.update({"_id": result._id}, result.value)
}) }`}, {"args", bson.D{}}}, nil) }) }`}, {"args", bson.D{}}}, nil)
ts.session.SetSafe(&mgo.Safe{}) qs.session.SetSafe(&mgo.Safe{})
return true
}
func init() { return nil
graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
} }

11
graph/sexp/parser.go
View file

@ -19,6 +19,7 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator { func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator {
@ -208,7 +209,7 @@ func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Ite
i++ i++
} }
it := buildIteratorTree(tree.Children[i], ts) it := buildIteratorTree(tree.Children[i], ts)
lto := iterator.NewLinksTo(ts, it, graph.Predicate) lto := iterator.NewLinksTo(ts, it, quad.Predicate)
return lto return lto
case "RootConstraint": case "RootConstraint":
constraintCount := 0 constraintCount := 0
@ -229,16 +230,16 @@ func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Ite
return and return and
case "Constraint": case "Constraint":
var hasa *iterator.HasA var hasa *iterator.HasA
topLevelDir := graph.Subject topLevelDir := quad.Subject
subItDir := graph.Object subItDir := quad.Object
subAnd := iterator.NewAnd() subAnd := iterator.NewAnd()
isOptional := false isOptional := false
for _, c := range tree.Children { for _, c := range tree.Children {
switch c.Name { switch c.Name {
case "PredIdentifier": case "PredIdentifier":
if c.Children[0].Name == "Reverse" { if c.Children[0].Name == "Reverse" {
topLevelDir = graph.Object topLevelDir = quad.Object
subItDir = graph.Subject subItDir = quad.Subject
} }
it := buildIteratorTree(c, ts) it := buildIteratorTree(c, ts)
subAnd.AddSubIterator(it) subAnd.AddSubIterator(it)

18
graph/sexp/parser_test.go
View file

@ -18,6 +18,8 @@ import (
"testing" "testing"
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/quad"
_ "github.com/google/cayley/graph/memstore" _ "github.com/google/cayley/graph/memstore"
) )
@ -30,21 +32,21 @@ func TestBadParse(t *testing.T) {
var testQueries = []struct { var testQueries = []struct {
message string message string
add *graph.Triple add *quad.Quad
query string query string
typ graph.Type typ graph.Type
expect string expect string
}{ }{
{ {
message: "get a single triple linkage", message: "get a single triple linkage",
add: &graph.Triple{"i", "can", "win", ""}, add: &quad.Quad{"i", "can", "win", ""},
query: "($a (:can \"win\"))", query: "($a (:can \"win\"))",
typ: graph.And, typ: graph.And,
expect: "i", expect: "i",
}, },
{ {
message: "get a single triple linkage", message: "get a single triple linkage",
add: &graph.Triple{"i", "can", "win", ""}, add: &quad.Quad{"i", "can", "win", ""},
query: "(\"i\" (:can $a))", query: "(\"i\" (:can $a))",
typ: graph.And, typ: graph.And,
expect: "i", expect: "i",
@ -77,8 +79,8 @@ func TestMemstoreBackedSexp(t *testing.T) {
func TestTreeConstraintParse(t *testing.T) { func TestTreeConstraintParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil) ts, _ := graph.NewTripleStore("memstore", "", nil)
ts.AddTriple(&graph.Triple{"i", "like", "food", ""}) ts.AddTriple(&quad.Quad{"i", "like", "food", ""})
ts.AddTriple(&graph.Triple{"food", "is", "good", ""}) ts.AddTriple(&quad.Quad{"food", "is", "good", ""})
query := "(\"i\"\n" + query := "(\"i\"\n" +
"(:like\n" + "(:like\n" +
"($a (:is :good))))" "($a (:is :good))))"
@ -97,8 +99,8 @@ func TestTreeConstraintParse(t *testing.T) {
func TestTreeConstraintTagParse(t *testing.T) { func TestTreeConstraintTagParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil) ts, _ := graph.NewTripleStore("memstore", "", nil)
ts.AddTriple(&graph.Triple{"i", "like", "food", ""}) ts.AddTriple(&quad.Quad{"i", "like", "food", ""})
ts.AddTriple(&graph.Triple{"food", "is", "good", ""}) ts.AddTriple(&quad.Quad{"food", "is", "good", ""})
query := "(\"i\"\n" + query := "(\"i\"\n" +
"(:like\n" + "(:like\n" +
"($a (:is :good))))" "($a (:is :good))))"
@ -117,7 +119,7 @@ func TestTreeConstraintTagParse(t *testing.T) {
func TestMultipleConstraintParse(t *testing.T) { func TestMultipleConstraintParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil) ts, _ := graph.NewTripleStore("memstore", "", nil)
for _, tv := range []*graph.Triple{ for _, tv := range []*quad.Quad{
{"i", "like", "food", ""}, {"i", "like", "food", ""},
{"i", "like", "beer", ""}, {"i", "like", "beer", ""},
{"you", "like", "beer", ""}, {"you", "like", "beer", ""},

24
graph/triplestore.go
View file

@ -23,7 +23,9 @@ package graph
import ( import (
"errors" "errors"
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/google/cayley/quad"
) )
// Defines an opaque "triple store value" type. However the backend wishes to // Defines an opaque "triple store value" type. However the backend wishes to
@ -38,21 +40,21 @@ type Value interface{}
type TripleStore interface { type TripleStore interface {
// Add a triple to the store. // Add a triple to the store.
AddTriple(*Triple) AddTriple(*quad.Quad)
// Add a set of triples to the store, atomically if possible. // Add a set of triples to the store, atomically if possible.
AddTripleSet([]*Triple) AddTripleSet([]*quad.Quad)
// Removes a triple matching the given one from the database, // Removes a triple matching the given one from the database,
// if it exists. Does nothing otherwise. // if it exists. Does nothing otherwise.
RemoveTriple(*Triple) RemoveTriple(*quad.Quad)
// Given an opaque token, returns the triple for that token from the store. // Given an opaque token, returns the triple for that token from the store.
Triple(Value) *Triple Quad(Value) *quad.Quad
// Given a direction and a token, creates an iterator of links which have // Given a direction and a token, creates an iterator of links which have
// that node token in that directional field. // that node token in that directional field.
TripleIterator(Direction, Value) Iterator TripleIterator(quad.Direction, Value) Iterator
// Returns an iterator enumerating all nodes in the graph. // Returns an iterator enumerating all nodes in the graph.
NodesAllIterator() Iterator NodesAllIterator() Iterator
@ -89,8 +91,8 @@ type TripleStore interface {
// gives the TripleStore the opportunity to make this optimization. // gives the TripleStore the opportunity to make this optimization.
// //
// Iterators will call this. At worst, a valid implementation is // Iterators will call this. At worst, a valid implementation is
// ts.IdFor(ts.Triple(triple_id).Get(dir)) // ts.IdFor(ts.quad.Quad(id).Get(dir))
TripleDirection(triple_id Value, d Direction) Value TripleDirection(id Value, d quad.Direction) Value
} }
type Options map[string]interface{} type Options map[string]interface{}
@ -122,14 +124,10 @@ func (d Options) StringKey(key string) (string, bool) {
var ErrCannotBulkLoad = errors.New("triplestore: cannot bulk load") var ErrCannotBulkLoad = errors.New("triplestore: cannot bulk load")
type BulkLoader interface { type BulkLoader interface {
// BulkLoad loads Triples from a TripleUnmarshaler in bulk to the TripleStore. // BulkLoad loads Quads from a quad.Unmarshaler in bulk to the TripleStore.
// It returns ErrCannotBulkLoad if bulk loading is not possible. For example if // It returns ErrCannotBulkLoad if bulk loading is not possible. For example if
// you cannot load in bulk to a non-empty database, and the db is non-empty. // you cannot load in bulk to a non-empty database, and the db is non-empty.
BulkLoad(TripleUnmarshaler) error BulkLoad(quad.Unmarshaler) error
}
type TripleUnmarshaler interface {
Unmarshal() (*Triple, error)
} }
type NewStoreFunc func(string, Options) (TripleStore, error) type NewStoreFunc func(string, Options) (TripleStore, error)

10
http/http_test.go
View file

@ -19,13 +19,13 @@ import (
"reflect" "reflect"
"testing" "testing"
"github.com/google/cayley/graph" "github.com/google/cayley/quad"
) )
var parseTests = []struct { var parseTests = []struct {
message string message string
input string input string
expect []*graph.Triple expect []*quad.Quad
err error err error
}{ }{
{ {
@ -34,7 +34,7 @@ var parseTests = []struct {
{"subject": "foo", "predicate": "bar", "object": "baz"}, {"subject": "foo", "predicate": "bar", "object": "baz"},
{"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"} {"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"}
]`, ]`,
expect: []*graph.Triple{ expect: []*quad.Quad{
{"foo", "bar", "baz", ""}, {"foo", "bar", "baz", ""},
{"foo", "bar", "baz", "graph"}, {"foo", "bar", "baz", "graph"},
}, },
@ -45,7 +45,7 @@ var parseTests = []struct {
input: `[ input: `[
{"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"}
]`, ]`,
expect: []*graph.Triple{ expect: []*quad.Quad{
{"foo", "bar", "foo", ""}, {"foo", "bar", "foo", ""},
}, },
err: nil, err: nil,
@ -56,7 +56,7 @@ var parseTests = []struct {
{"subject": "foo", "predicate": "bar"} {"subject": "foo", "predicate": "bar"}
]`, ]`,
expect: nil, expect: nil,
err: fmt.Errorf("Invalid triple at index %d. %v", 0, &graph.Triple{"foo", "bar", "", ""}), err: fmt.Errorf("Invalid triple at index %d. %v", 0, &quad.Quad{"foo", "bar", "", ""}),
}, },
} }

10
http/write.go
View file

@ -25,12 +25,12 @@ import (
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/julienschmidt/httprouter" "github.com/julienschmidt/httprouter"
"github.com/google/cayley/graph" "github.com/google/cayley/quad"
"github.com/google/cayley/nquads" "github.com/google/cayley/quad/nquads"
) )
func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) { func ParseJsonToTripleList(jsonBody []byte) ([]*quad.Quad, error) {
var tripleList []*graph.Triple var tripleList []*quad.Quad
err := json.Unmarshal(jsonBody, &tripleList) err := json.Unmarshal(jsonBody, &tripleList)
if err != nil { if err != nil {
return nil, err return nil, err
@ -83,7 +83,7 @@ func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params
var ( var (
n int n int
block = make([]*graph.Triple, 0, blockSize) block = make([]*quad.Quad, 0, blockSize)
) )
for { for {
t, err := dec.Unmarshal() t, err := dec.Unmarshal()

95
quad/cquads/actions.rl Normal file
View file

@ -0,0 +1,95 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
%%{
machine quads;
action Escape {
isEscaped = true
}
action Quote {
isQuoted = true
}
action StartSubject {
subject = p
}
action StartPredicate {
predicate = p
}
action StartObject {
object = p
}
action StartLabel {
label = p
}
action SetSubject {
if subject < 0 {
panic("unexpected parser state: subject start not set")
}
q.Subject = unEscape(data[subject:p], isQuoted, isEscaped)
isEscaped = false
isQuoted = false
}
action SetPredicate {
if predicate < 0 {
panic("unexpected parser state: predicate start not set")
}
q.Predicate = unEscape(data[predicate:p], isQuoted, isEscaped)
isEscaped = false
isQuoted = false
}
action SetObject {
if object < 0 {
panic("unexpected parser state: object start not set")
}
q.Object = unEscape(data[object:p], isQuoted, isEscaped)
isEscaped = false
isQuoted = false
}
action SetLabel {
if label < 0 {
panic("unexpected parser state: label start not set")
}
q.Provenance = unEscape(data[label:p], isQuoted, isEscaped)
isEscaped = false
isQuoted = false
}
action Return {
return q, nil
}
action Comment {
}
action Error {
if p < len(data) {
if r := data[p]; r < unicode.MaxASCII {
return q, fmt.Errorf("%v: unexpected rune %q at %d", quad.ErrInvalid, data[p], p)
} else {
return q, fmt.Errorf("%v: unexpected rune %q (\\u%04x) at %d", quad.ErrInvalid, data[p], data[p], p)
}
}
return q, quad.ErrIncomplete
}
}%%

141
quad/cquads/cquads.go Normal file
View file

@ -0,0 +1,141 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package cquads implements parsing N-Quads like line-based syntax
// for RDF datasets.
//
// N-Quad parsing is performed as based on a simplified grammar derived from
// the N-Quads grammar defined by http://www.w3.org/TR/n-quads/.
//
// For a complete definition of the grammar, see cquads.rl.
package cquads
import (
"bufio"
"bytes"
"fmt"
"io"
"strconv"
"github.com/google/cayley/quad"
)
// Parse returns a valid quad.Quad or a non-nil error. Parse does
// handle comments except where the comment placement does not prevent
// a complete valid quad.Quad from being defined.
func Parse(str string) (*quad.Quad, error) {
q, err := parse([]rune(str))
return &q, err
}
// Decoder implements simplified N-Quad document parsing.
type Decoder struct {
r *bufio.Reader
line []byte
}
// NewDecoder returns an N-Quad decoder that takes its input from the
// provided io.Reader.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: bufio.NewReader(r)}
}
// Unmarshal returns the next valid N-Quad as a quad.Quad, or an error.
func (dec *Decoder) Unmarshal() (*quad.Quad, error) {
dec.line = dec.line[:0]
var line []byte
for {
for {
l, pre, err := dec.r.ReadLine()
if err != nil {
return nil, err
}
dec.line = append(dec.line, l...)
if !pre {
break
}
}
if line = bytes.TrimSpace(dec.line); len(line) != 0 && line[0] != '#' {
break
}
dec.line = dec.line[:0]
}
triple, err := Parse(string(line))
if err != nil {
return nil, fmt.Errorf("failed to parse %q: %v", dec.line, err)
}
if triple == nil {
return dec.Unmarshal()
}
return triple, nil
}
func unEscape(r []rune, isQuoted, isEscaped bool) string {
if isQuoted {
r = r[1 : len(r)-1]
}
if !isEscaped {
return string(r)
}
buf := bytes.NewBuffer(make([]byte, 0, len(r)))
for i := 0; i < len(r); {
switch r[i] {
case '\\':
i++
var c byte
switch r[i] {
case 't':
c = '\t'
case 'b':
c = '\b'
case 'n':
c = '\n'
case 'r':
c = '\r'
case 'f':
c = '\f'
case '"':
c = '"'
case '\'':
c = '\''
case '\\':
c = '\\'
case 'u':
rc, err := strconv.ParseInt(string(r[i+1:i+5]), 16, 32)
if err != nil {
panic(fmt.Errorf("internal parser error: %v", err))
}
buf.WriteRune(rune(rc))
i += 5
continue
case 'U':
rc, err := strconv.ParseInt(string(r[i+1:i+9]), 16, 32)
if err != nil {
panic(fmt.Errorf("internal parser error: %v", err))
}
buf.WriteRune(rune(rc))
i += 9
continue
}
buf.WriteByte(c)
default:
buf.WriteRune(r[i])
}
i++
}
return buf.String()
}

106
quad/cquads/cquads.rl Normal file
View file

@ -0,0 +1,106 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Ragel gramar definition derived from http://www.w3.org/TR/n-quads/#sec-grammar.
%%{
machine quads;
alphtype rune;
PN_CHARS_BASE = [A-Za-z]
| 0x00c0 .. 0x00d6
| 0x00d8 .. 0x00f6
| 0x00f8 .. 0x02ff
| 0x0370 .. 0x037d
| 0x037f .. 0x1fff
| 0x200c .. 0x200d
| 0x2070 .. 0x218f
| 0x2c00 .. 0x2fef
| 0x3001 .. 0xd7ff
| 0xf900 .. 0xfdcf
| 0xfdf0 .. 0xfffd
| 0x10000 .. 0xeffff
;
PN_CHARS_U = PN_CHARS_BASE | '_' | ':' ;
PN_CHARS = PN_CHARS_U
| '-'
| [0-9]
| 0xb7
| 0x0300 .. 0x036f
| 0x203f .. 0x2040
;
ECHAR = ('\\' [tbnrf"'\\]) %Escape ;
UCHAR = ('\\u' xdigit {4}
| '\\U' xdigit {8}) %Escape
;
BLANK_NODE_LABEL = '_:' (PN_CHARS_U | [0-9]) ((PN_CHARS | '.')* PN_CHARS)? ;
STRING_LITERAL = (
'!'
| '#' .. '['
| ']' .. 0x7e
| 0x80 .. 0x10ffff
| ECHAR
| UCHAR)+ - ('_:' | any* '.' | '#' any*)
;
STRING_LITERAL_QUOTE = '"' (
0x00 .. 0x09
| 0x0b .. 0x0c
| 0x0e .. '!'
| '#' .. '['
| ']' .. 0x10ffff
| ECHAR
| UCHAR)*
'"'
;
IRIREF = '<' (
'!' .. ';'
| '='
| '?' .. '['
| ']'
| '_'
| 'a' .. 'z'
| '~'
| 0x80 .. 0x10ffff
| UCHAR)*
'>'
;
LANGTAG = '@' [a-zA-Z]+ ('-' [a-zA-Z0-9]+)* ;
whitespace = [ \t] ;
literal = STRING_LITERAL | STRING_LITERAL_QUOTE % Quote | STRING_LITERAL_QUOTE ('^^' IRIREF | LANGTAG) ;
subject = (literal | BLANK_NODE_LABEL) ;
predicate = literal ;
object = (literal | BLANK_NODE_LABEL) ;
graphLabel = (literal | BLANK_NODE_LABEL) ;
statement := (
whitespace* subject >StartSubject %SetSubject
whitespace+ predicate >StartPredicate %SetPredicate
whitespace+ object >StartObject %SetObject
(whitespace+ graphLabel >StartLabel %SetLabel)?
whitespace* '.' whitespace* ('#' any*)? >Comment
) %Return @!Error ;
}%%

782
quad/cquads/cquads_test.go Normal file
View file

File diff suppressed because it is too large Load diff

6692
quad/cquads/parse.go Normal file
View file

File diff suppressed because it is too large Load diff

58
quad/cquads/parse.rl Normal file
View file

@ -0,0 +1,58 @@
// GO SOURCE FILE MACHINE GENERATED BY RAGEL; DO NOT EDIT
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cquads
import (
"fmt"
"unicode"
"github.com/google/cayley/quad"
)
%%{
machine quads;
include "actions.rl";
include "cquads.rl";
write data;
}%%
func parse(data []rune) (quad.Quad, error) {
var (
cs, p int
pe = len(data)
eof = pe
subject = -1
predicate = -1
object = -1
label = -1
isEscaped bool
isQuoted bool
q quad.Quad
)
%%write init;
%%write exec;
return quad.Quad{}, quad.ErrInvalid
}

0
nquads/nquad_tests.tar.gz → quad/nquad_tests.tar.gz
View file

16
nquads/actions.rl → quad/nquads/actions.rl
View file

@ -39,7 +39,7 @@
if subject < 0 { if subject < 0 {
panic("unexpected parser state: subject start not set") panic("unexpected parser state: subject start not set")
} }
triple.Subject = unEscape(data[subject:p], isEscaped) q.Subject = unEscape(data[subject:p], isEscaped)
isEscaped = false isEscaped = false
} }
@ -47,7 +47,7 @@
if predicate < 0 { if predicate < 0 {
panic("unexpected parser state: predicate start not set") panic("unexpected parser state: predicate start not set")
} }
triple.Predicate = unEscape(data[predicate:p], isEscaped) q.Predicate = unEscape(data[predicate:p], isEscaped)
isEscaped = false isEscaped = false
} }
@ -55,7 +55,7 @@
if object < 0 { if object < 0 {
panic("unexpected parser state: object start not set") panic("unexpected parser state: object start not set")
} }
triple.Object = unEscape(data[object:p], isEscaped) q.Object = unEscape(data[object:p], isEscaped)
isEscaped = false isEscaped = false
} }
@ -63,12 +63,12 @@
if label < 0 { if label < 0 {
panic("unexpected parser state: label start not set") panic("unexpected parser state: label start not set")
} }
triple.Provenance = unEscape(data[label:p], isEscaped) q.Provenance = unEscape(data[label:p], isEscaped)
isEscaped = false isEscaped = false
} }
action Return { action Return {
return triple, nil return q, nil
} }
action Comment { action Comment {
@ -77,11 +77,11 @@
action Error { action Error {
if p < len(data) { if p < len(data) {
if r := data[p]; r < unicode.MaxASCII { if r := data[p]; r < unicode.MaxASCII {
return triple, fmt.Errorf("%v: unexpected rune %q at %d", ErrInvalid, data[p], p) return q, fmt.Errorf("%v: unexpected rune %q at %d", quad.ErrInvalid, data[p], p)
} else { } else {
return triple, fmt.Errorf("%v: unexpected rune %q (\\u%04x) at %d", ErrInvalid, data[p], data[p], p) return q, fmt.Errorf("%v: unexpected rune %q (\\u%04x) at %d", quad.ErrInvalid, data[p], data[p], p)
} }
} }
return triple, ErrIncomplete return q, quad.ErrIncomplete
} }
}%% }%%

18
nquads/nquads.go → quad/nquads/nquads.go
View file

@ -17,7 +17,7 @@
// //
// N-Quad parsing is performed as defined by http://www.w3.org/TR/n-quads/ // N-Quad parsing is performed as defined by http://www.w3.org/TR/n-quads/
// with the exception that the nquads package will allow relative IRI values, // with the exception that the nquads package will allow relative IRI values,
// which are prohibited by the N-Quads and N-Triples specifications. // which are prohibited by the N-Quads quad-Quads specifications.
package nquads package nquads
import ( import (
@ -27,15 +27,15 @@ import (
"io" "io"
"strconv" "strconv"
"github.com/google/cayley/graph" "github.com/google/cayley/quad"
) )
// Parse returns a valid graph.Triple or a non-nil error. Parse does // Parse returns a valid quad.Quad or a non-nil error. Parse does
// handle comments except where the comment placement does not prevent // handle comments except where the comment placement does not prevent
// a complete valid graph.Triple from being defined. // a complete valid quad.Quad from being defined.
func Parse(str string) (*graph.Triple, error) { func Parse(str string) (*quad.Quad, error) {
t, err := parse([]rune(str)) q, err := parse([]rune(str))
return &t, err return &q, err
} }
// Decoder implements N-Quad document parsing according to the RDF // Decoder implements N-Quad document parsing according to the RDF
@ -51,8 +51,8 @@ func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: bufio.NewReader(r)} return &Decoder{r: bufio.NewReader(r)}
} }
// Unmarshal returns the next valid N-Quad as a graph.Triple, or an error. // Unmarshal returns the next valid N-Quad as a quad.Quad, or an error.
func (dec *Decoder) Unmarshal() (*graph.Triple, error) { func (dec *Decoder) Unmarshal() (*quad.Quad, error) {
dec.line = dec.line[:0] dec.line = dec.line[:0]
var line []byte var line []byte
for { for {

0
nquads/nquads.rl → quad/nquads/nquads.rl
View file

86
nquads/nquads_test.go → quad/nquads/nquads_test.go
View file

@ -25,13 +25,13 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/google/cayley/graph" "github.com/google/cayley/quad"
) )
var testNTriples = []struct { var testNTriples = []struct {
message string message string
input string input string
expect *graph.Triple expect *quad.Quad
err error err error
}{ }{
// Tests taken from http://www.w3.org/TR/n-quads/ and http://www.w3.org/TR/n-triples/. // Tests taken from http://www.w3.org/TR/n-quads/ and http://www.w3.org/TR/n-triples/.
@ -40,7 +40,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with commment", message: "parse triple with commment",
input: `_:100000 </film/performance/actor> </en/larry_fine_1902> . # example from 30movies`, input: `_:100000 </film/performance/actor> </en/larry_fine_1902> . # example from 30movies`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:100000", Subject: "_:100000",
Predicate: "</film/performance/actor>", Predicate: "</film/performance/actor>",
Object: "</en/larry_fine_1902>", Object: "</en/larry_fine_1902>",
@ -52,7 +52,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with commment", message: "parse triple with commment",
input: `_:10011 </film/performance/character> "Tomás de Torquemada" . # example from 30movies with unicode`, input: `_:10011 </film/performance/character> "Tomás de Torquemada" . # example from 30movies with unicode`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:10011", Subject: "_:10011",
Predicate: "</film/performance/character>", Predicate: "</film/performance/character>",
Object: `"Tomás de Torquemada"`, Object: `"Tomás de Torquemada"`,
@ -65,7 +65,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with commment", message: "parse triple with commment",
input: `<http://one.example/subject1> <http://one.example/predicate1> <http://one.example/object1> . # comments here`, input: `<http://one.example/subject1> <http://one.example/predicate1> <http://one.example/object1> . # comments here`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://one.example/subject1>", Subject: "<http://one.example/subject1>",
Predicate: "<http://one.example/predicate1>", Predicate: "<http://one.example/predicate1>",
Object: "<http://one.example/object1>", Object: "<http://one.example/object1>",
@ -76,7 +76,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with blank subject node, literal object and no comment (1)", message: "parse triple with blank subject node, literal object and no comment (1)",
input: `_:subject1 <http://an.example/predicate1> "object1" .`, input: `_:subject1 <http://an.example/predicate1> "object1" .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:subject1", Subject: "_:subject1",
Predicate: "<http://an.example/predicate1>", Predicate: "<http://an.example/predicate1>",
Object: `"object1"`, Object: `"object1"`,
@ -87,7 +87,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with blank subject node, literal object and no comment (2)", message: "parse triple with blank subject node, literal object and no comment (2)",
input: `_:subject2 <http://an.example/predicate2> "object2" .`, input: `_:subject2 <http://an.example/predicate2> "object2" .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:subject2", Subject: "_:subject2",
Predicate: "<http://an.example/predicate2>", Predicate: "<http://an.example/predicate2>",
Object: `"object2"`, Object: `"object2"`,
@ -100,7 +100,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with three IRIREFs", message: "parse triple with three IRIREFs",
input: `<http://example.org/#spiderman> <http://www.perceive.net/schemas/relationship/enemyOf> <http://example.org/#green-goblin> .`, input: `<http://example.org/#spiderman> <http://www.perceive.net/schemas/relationship/enemyOf> <http://example.org/#green-goblin> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/#spiderman>", Subject: "<http://example.org/#spiderman>",
Predicate: "<http://www.perceive.net/schemas/relationship/enemyOf>", Predicate: "<http://www.perceive.net/schemas/relationship/enemyOf>",
Object: "<http://example.org/#green-goblin>", Object: "<http://example.org/#green-goblin>",
@ -113,7 +113,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with blank node labelled subject and object and IRIREF predicate (1)", message: "parse triple with blank node labelled subject and object and IRIREF predicate (1)",
input: `_:alice <http://xmlns.com/foaf/0.1/knows> _:bob .`, input: `_:alice <http://xmlns.com/foaf/0.1/knows> _:bob .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:alice", Subject: "_:alice",
Predicate: "<http://xmlns.com/foaf/0.1/knows>", Predicate: "<http://xmlns.com/foaf/0.1/knows>",
Object: "_:bob", Object: "_:bob",
@ -124,7 +124,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with blank node labelled subject and object and IRIREF predicate (2)", message: "parse triple with blank node labelled subject and object and IRIREF predicate (2)",
input: `_:bob <http://xmlns.com/foaf/0.1/knows> _:alice .`, input: `_:bob <http://xmlns.com/foaf/0.1/knows> _:alice .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:bob", Subject: "_:bob",
Predicate: "<http://xmlns.com/foaf/0.1/knows>", Predicate: "<http://xmlns.com/foaf/0.1/knows>",
Object: "_:alice", Object: "_:alice",
@ -137,7 +137,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with commment", message: "parse quad with commment",
input: `<http://one.example/subject1> <http://one.example/predicate1> <http://one.example/object1> <http://example.org/graph3> . # comments here`, input: `<http://one.example/subject1> <http://one.example/predicate1> <http://one.example/object1> <http://example.org/graph3> . # comments here`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://one.example/subject1>", Subject: "<http://one.example/subject1>",
Predicate: "<http://one.example/predicate1>", Predicate: "<http://one.example/predicate1>",
Object: "<http://one.example/object1>", Object: "<http://one.example/object1>",
@ -148,7 +148,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (1)", message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (1)",
input: `_:subject1 <http://an.example/predicate1> "object1" <http://example.org/graph1> .`, input: `_:subject1 <http://an.example/predicate1> "object1" <http://example.org/graph1> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:subject1", Subject: "_:subject1",
Predicate: "<http://an.example/predicate1>", Predicate: "<http://an.example/predicate1>",
Object: `"object1"`, Object: `"object1"`,
@ -159,7 +159,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (2)", message: "parse quad with blank subject node, literal object, IRIREF predicate and label, and no comment (2)",
input: `_:subject2 <http://an.example/predicate2> "object2" <http://example.org/graph5> .`, input: `_:subject2 <http://an.example/predicate2> "object2" <http://example.org/graph5> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:subject2", Subject: "_:subject2",
Predicate: "<http://an.example/predicate2>", Predicate: "<http://an.example/predicate2>",
Object: `"object2"`, Object: `"object2"`,
@ -172,7 +172,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with all IRIREF parts", message: "parse quad with all IRIREF parts",
input: `<http://example.org/#spiderman> <http://www.perceive.net/schemas/relationship/enemyOf> <http://example.org/#green-goblin> <http://example.org/graphs/spiderman> .`, input: `<http://example.org/#spiderman> <http://www.perceive.net/schemas/relationship/enemyOf> <http://example.org/#green-goblin> <http://example.org/graphs/spiderman> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/#spiderman>", Subject: "<http://example.org/#spiderman>",
Predicate: "<http://www.perceive.net/schemas/relationship/enemyOf>", Predicate: "<http://www.perceive.net/schemas/relationship/enemyOf>",
Object: "<http://example.org/#green-goblin>", Object: "<http://example.org/#green-goblin>",
@ -185,7 +185,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (1)", message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (1)",
input: `_:alice <http://xmlns.com/foaf/0.1/knows> _:bob <http://example.org/graphs/john> .`, input: `_:alice <http://xmlns.com/foaf/0.1/knows> _:bob <http://example.org/graphs/john> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:alice", Subject: "_:alice",
Predicate: "<http://xmlns.com/foaf/0.1/knows>", Predicate: "<http://xmlns.com/foaf/0.1/knows>",
Object: "_:bob", Object: "_:bob",
@ -196,7 +196,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (2)", message: "parse quad with blank node labelled subject and object and IRIREF predicate and label (2)",
input: `_:bob <http://xmlns.com/foaf/0.1/knows> _:alice <http://example.org/graphs/james> .`, input: `_:bob <http://xmlns.com/foaf/0.1/knows> _:alice <http://example.org/graphs/james> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "_:bob", Subject: "_:bob",
Predicate: "<http://xmlns.com/foaf/0.1/knows>", Predicate: "<http://xmlns.com/foaf/0.1/knows>",
Object: "_:alice", Object: "_:alice",
@ -209,7 +209,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with all IRIREF parts", message: "parse triple with all IRIREF parts",
input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Person> .`, input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Person> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>",
Object: "<http://xmlns.com/foaf/0.1/Person>", Object: "<http://xmlns.com/foaf/0.1/Person>",
@ -220,7 +220,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with all IRIREF parts", message: "parse triple with all IRIREF parts",
input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/knows> <http://example.org/alice#me> .`, input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/knows> <http://example.org/alice#me> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://xmlns.com/foaf/0.1/knows>", Predicate: "<http://xmlns.com/foaf/0.1/knows>",
Object: "<http://example.org/alice#me>", Object: "<http://example.org/alice#me>",
@ -231,7 +231,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with IRIREF schema on literal object", message: "parse triple with IRIREF schema on literal object",
input: `<http://example.org/bob#me> <http://schema.org/birthDate> "1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date> .`, input: `<http://example.org/bob#me> <http://schema.org/birthDate> "1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://schema.org/birthDate>", Predicate: "<http://schema.org/birthDate>",
Object: `"1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date>`, Object: `"1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date>`,
@ -242,7 +242,7 @@ var testNTriples = []struct {
{ {
message: "parse commented IRIREF in triple", message: "parse commented IRIREF in triple",
input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/topic_interest> <http://www.wikidata.org/entity/Q12418> .`, input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/topic_interest> <http://www.wikidata.org/entity/Q12418> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://xmlns.com/foaf/0.1/topic_interest>", Predicate: "<http://xmlns.com/foaf/0.1/topic_interest>",
Object: "<http://www.wikidata.org/entity/Q12418>", Object: "<http://www.wikidata.org/entity/Q12418>",
@ -253,7 +253,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with literal subject", message: "parse triple with literal subject",
input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/title> "Mona Lisa" .`, input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/title> "Mona Lisa" .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://www.wikidata.org/entity/Q12418>", Subject: "<http://www.wikidata.org/entity/Q12418>",
Predicate: "<http://purl.org/dc/terms/title>", Predicate: "<http://purl.org/dc/terms/title>",
Object: `"Mona Lisa"`, Object: `"Mona Lisa"`,
@ -264,7 +264,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with all IRIREF parts (1)", message: "parse triple with all IRIREF parts (1)",
input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/creator> <http://dbpedia.org/resource/Leonardo_da_Vinci> .`, input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/creator> <http://dbpedia.org/resource/Leonardo_da_Vinci> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://www.wikidata.org/entity/Q12418>", Subject: "<http://www.wikidata.org/entity/Q12418>",
Predicate: "<http://purl.org/dc/terms/creator>", Predicate: "<http://purl.org/dc/terms/creator>",
Object: "<http://dbpedia.org/resource/Leonardo_da_Vinci>", Object: "<http://dbpedia.org/resource/Leonardo_da_Vinci>",
@ -275,7 +275,7 @@ var testNTriples = []struct {
{ {
message: "parse triple with all IRIREF parts (2)", message: "parse triple with all IRIREF parts (2)",
input: `<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619> <http://purl.org/dc/terms/subject> <http://www.wikidata.org/entity/Q12418> .`, input: `<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619> <http://purl.org/dc/terms/subject> <http://www.wikidata.org/entity/Q12418> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619>", Subject: "<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619>",
Predicate: "<http://purl.org/dc/terms/subject>", Predicate: "<http://purl.org/dc/terms/subject>",
Object: "<http://www.wikidata.org/entity/Q12418>", Object: "<http://www.wikidata.org/entity/Q12418>",
@ -288,7 +288,7 @@ var testNTriples = []struct {
{ {
message: "parse commented IRIREF in quad (1)", message: "parse commented IRIREF in quad (1)",
input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Person> <http://example.org/bob> .`, input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://xmlns.com/foaf/0.1/Person> <http://example.org/bob> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>",
Object: "<http://xmlns.com/foaf/0.1/Person>", Object: "<http://xmlns.com/foaf/0.1/Person>",
@ -299,7 +299,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with all IRIREF parts", message: "parse quad with all IRIREF parts",
input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/knows> <http://example.org/alice#me> <http://example.org/bob> .`, input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/knows> <http://example.org/alice#me> <http://example.org/bob> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://xmlns.com/foaf/0.1/knows>", Predicate: "<http://xmlns.com/foaf/0.1/knows>",
Object: "<http://example.org/alice#me>", Object: "<http://example.org/alice#me>",
@ -310,7 +310,7 @@ var testNTriples = []struct {
{ {
message: "parse quad with IRIREF schema on literal object", message: "parse quad with IRIREF schema on literal object",
input: `<http://example.org/bob#me> <http://schema.org/birthDate> "1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date> <http://example.org/bob> .`, input: `<http://example.org/bob#me> <http://schema.org/birthDate> "1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date> <http://example.org/bob> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://schema.org/birthDate>", Predicate: "<http://schema.org/birthDate>",
Object: `"1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date>`, Object: `"1990-07-04"^^<http://www.w3.org/2001/XMLSchema#date>`,
@ -321,7 +321,7 @@ var testNTriples = []struct {
{ {
message: "parse commented IRIREF in quad (2)", message: "parse commented IRIREF in quad (2)",
input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/topic_interest> <http://www.wikidata.org/entity/Q12418> <http://example.org/bob> .`, input: `<http://example.org/bob#me> <http://xmlns.com/foaf/0.1/topic_interest> <http://www.wikidata.org/entity/Q12418> <http://example.org/bob> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://xmlns.com/foaf/0.1/topic_interest>", Predicate: "<http://xmlns.com/foaf/0.1/topic_interest>",
Object: "<http://www.wikidata.org/entity/Q12418>", Object: "<http://www.wikidata.org/entity/Q12418>",
@ -332,7 +332,7 @@ var testNTriples = []struct {
{ {
message: "parse literal object and colon qualified label in quad", message: "parse literal object and colon qualified label in quad",
input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/title> "Mona Lisa" <https://www.wikidata.org/wiki/Special:EntityData/Q12418> .`, input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/title> "Mona Lisa" <https://www.wikidata.org/wiki/Special:EntityData/Q12418> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://www.wikidata.org/entity/Q12418>", Subject: "<http://www.wikidata.org/entity/Q12418>",
Predicate: "<http://purl.org/dc/terms/title>", Predicate: "<http://purl.org/dc/terms/title>",
Object: `"Mona Lisa"`, Object: `"Mona Lisa"`,
@ -343,7 +343,7 @@ var testNTriples = []struct {
{ {
message: "parse all IRIREF parts with colon qualified label in quad (1)", message: "parse all IRIREF parts with colon qualified label in quad (1)",
input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/creator> <http://dbpedia.org/resource/Leonardo_da_Vinci> <https://www.wikidata.org/wiki/Special:EntityData/Q12418> .`, input: `<http://www.wikidata.org/entity/Q12418> <http://purl.org/dc/terms/creator> <http://dbpedia.org/resource/Leonardo_da_Vinci> <https://www.wikidata.org/wiki/Special:EntityData/Q12418> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://www.wikidata.org/entity/Q12418>", Subject: "<http://www.wikidata.org/entity/Q12418>",
Predicate: "<http://purl.org/dc/terms/creator>", Predicate: "<http://purl.org/dc/terms/creator>",
Object: "<http://dbpedia.org/resource/Leonardo_da_Vinci>", Object: "<http://dbpedia.org/resource/Leonardo_da_Vinci>",
@ -354,7 +354,7 @@ var testNTriples = []struct {
{ {
message: "parse all IRIREF parts with colon qualified label in quad (2)", message: "parse all IRIREF parts with colon qualified label in quad (2)",
input: `<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619> <http://purl.org/dc/terms/subject> <http://www.wikidata.org/entity/Q12418> <https://www.wikidata.org/wiki/Special:EntityData/Q12418> .`, input: `<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619> <http://purl.org/dc/terms/subject> <http://www.wikidata.org/entity/Q12418> <https://www.wikidata.org/wiki/Special:EntityData/Q12418> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619>", Subject: "<http://data.europeana.eu/item/04802/243FA8618938F4117025F17A8B813C5F9AA4D619>",
Predicate: "<http://purl.org/dc/terms/subject>", Predicate: "<http://purl.org/dc/terms/subject>",
Object: "<http://www.wikidata.org/entity/Q12418>", Object: "<http://www.wikidata.org/entity/Q12418>",
@ -365,7 +365,7 @@ var testNTriples = []struct {
{ {
message: "parse all IRIREF parts (quad section - 1)", message: "parse all IRIREF parts (quad section - 1)",
input: `<http://example.org/bob> <http://purl.org/dc/terms/publisher> <http://example.org> .`, input: `<http://example.org/bob> <http://purl.org/dc/terms/publisher> <http://example.org> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob>", Subject: "<http://example.org/bob>",
Predicate: "<http://purl.org/dc/terms/publisher>", Predicate: "<http://purl.org/dc/terms/publisher>",
Object: "<http://example.org>", Object: "<http://example.org>",
@ -376,7 +376,7 @@ var testNTriples = []struct {
{ {
message: "parse all IRIREF parts (quad section - 2)", message: "parse all IRIREF parts (quad section - 2)",
input: `<http://example.org/bob> <http://purl.org/dc/terms/rights> <http://creativecommons.org/licenses/by/3.0/> .`, input: `<http://example.org/bob> <http://purl.org/dc/terms/rights> <http://creativecommons.org/licenses/by/3.0/> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob>", Subject: "<http://example.org/bob>",
Predicate: "<http://purl.org/dc/terms/rights>", Predicate: "<http://purl.org/dc/terms/rights>",
Object: "<http://creativecommons.org/licenses/by/3.0/>", Object: "<http://creativecommons.org/licenses/by/3.0/>",
@ -389,36 +389,36 @@ var testNTriples = []struct {
{ {
message: "parse empty", message: "parse empty",
input: ``, input: ``,
expect: &graph.Triple{}, expect: &quad.Quad{},
err: ErrIncomplete, err: quad.ErrIncomplete,
}, },
{ {
message: "parse commented", message: "parse commented",
input: `# comment`, input: `# comment`,
expect: &graph.Triple{}, expect: &quad.Quad{},
err: fmt.Errorf("%v: unexpected rune '#' at 0", ErrInvalid), err: fmt.Errorf("%v: unexpected rune '#' at 0", quad.ErrInvalid),
}, },
{ {
message: "parse incomplete quad", message: "parse incomplete quad",
input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> .`, input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>",
Object: "", Object: "",
Provenance: "", Provenance: "",
}, },
err: fmt.Errorf("%v: unexpected rune '.' at 78", ErrInvalid), err: fmt.Errorf("%v: unexpected rune '.' at 78", quad.ErrInvalid),
}, },
{ {
message: "parse incomplete quad", message: "parse incomplete quad",
input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> .`, input: `<http://example.org/bob#me> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> .`,
expect: &graph.Triple{ expect: &quad.Quad{
Subject: "<http://example.org/bob#me>", Subject: "<http://example.org/bob#me>",
Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>", Predicate: "<http://www.w3.org/1999/02/22-rdf-syntax-ns#type>",
Object: "", Object: "",
Provenance: "", Provenance: "",
}, },
err: fmt.Errorf("%v: unexpected rune '.' at 78", ErrInvalid), err: fmt.Errorf("%v: unexpected rune '.' at 78", quad.ErrInvalid),
}, },
} }
@ -504,8 +504,8 @@ func TestRDFWorkingGroupSuit(t *testing.T) {
} }
for _, file := range []string{ for _, file := range []string{
"ntriple_tests.tar.gz", filepath.Join("..", "ntriple_tests.tar.gz"),
"nquad_tests.tar.gz", filepath.Join("..", "nquad_tests.tar.gz"),
} { } {
suite, err := os.Open(file) suite, err := os.Open(file)
if err != nil { if err != nil {
@ -580,7 +580,7 @@ func TestUnescape(t *testing.T) {
} }
} }
var result *graph.Triple var result *quad.Quad
func BenchmarkParser(b *testing.B) { func BenchmarkParser(b *testing.B) {
for n := 0; n < b.N; n++ { for n := 0; n < b.N; n++ {

144
nquads/parse.go → quad/nquads/parse.go
View file

File diff suppressed because it is too large Load diff

14
nquads/parse.rl → quad/nquads/parse.rl
View file

@ -17,16 +17,10 @@
package nquads package nquads
import ( import (
"errors"
"fmt" "fmt"
"unicode" "unicode"
"github.com/google/cayley/graph" "github.com/google/cayley/quad"
)
var (
ErrInvalid = errors.New("invalid N-Quad")
ErrIncomplete = errors.New("incomplete N-Quad")
) )
%%{ %%{
@ -39,7 +33,7 @@ var (
write data; write data;
}%% }%%
func parse(data []rune) (graph.Triple, error) { func parse(data []rune) (quad.Quad, error) {
var ( var (
cs, p int cs, p int
pe = len(data) pe = len(data)
@ -52,12 +46,12 @@ func parse(data []rune) (graph.Triple, error) {
isEscaped bool isEscaped bool
triple graph.Triple q quad.Quad
) )
%%write init; %%write init;
%%write exec; %%write exec;
return graph.Triple{}, ErrInvalid return quad.Quad{}, quad.ErrInvalid
} }

0
nquads/quadfix.go → quad/nquads/quadfix.go
View file

0
nquads/ntriple_tests.tar.gz → quad/ntriple_tests.tar.gz
View file

56
graph/triple.go → quad/quad.go
View file

@ -12,7 +12,8 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
package graph // Package quad defines quad and triple handling.
package quad
// Defines the struct which makes the TripleStore possible -- the triple. // Defines the struct which makes the TripleStore possible -- the triple.
// //
@ -25,8 +26,8 @@ package graph
// list of triples. The rest is just indexing for speed. // list of triples. The rest is just indexing for speed.
// //
// Adding fields to the triple is not to be taken lightly. You'll see I mention // Adding fields to the triple is not to be taken lightly. You'll see I mention
// provenance, but don't as yet use it in any backing store. In general, there // provenance, but don'q as yet use it in any backing store. In general, there
// can be features that can be turned on or off for any store, but I haven't // can be features that can be turned on or off for any store, but I haven'q
// decided how to allow/disallow them yet. Another such example would be to add // decided how to allow/disallow them yet. Another such example would be to add
// a forward and reverse index field -- forward being "order the list of // a forward and reverse index field -- forward being "order the list of
// objects pointed at by this subject with this predicate" such as first and // objects pointed at by this subject with this predicate" such as first and
@ -35,13 +36,18 @@ package graph
// There will never be that much in this file except for the definition, but // There will never be that much in this file except for the definition, but
// the consequences are not to be taken lightly. But do suggest cool features! // the consequences are not to be taken lightly. But do suggest cool features!
import "fmt" import (
"errors"
"fmt"
)
// TODO(kortschak) Consider providing MashalJSON and UnmarshalJSON var (
// instead of using struct tags. ErrInvalid = errors.New("invalid N-Quad")
ErrIncomplete = errors.New("incomplete N-Quad")
)
// Our triple struct, used throughout. // Our triple struct, used throughout.
type Triple struct { type Quad struct {
Subject string `json:"subject"` Subject string `json:"subject"`
Predicate string `json:"predicate"` Predicate string `json:"predicate"`
Object string `json:"object"` Object string `json:"object"`
@ -98,45 +104,49 @@ func (d Direction) String() string {
// instead of the pointer. This needs benchmarking to make the decision. // instead of the pointer. This needs benchmarking to make the decision.
// Per-field accessor for triples // Per-field accessor for triples
func (t *Triple) Get(d Direction) string { func (q *Quad) Get(d Direction) string {
switch d { switch d {
case Subject: case Subject:
return t.Subject return q.Subject
case Predicate: case Predicate:
return t.Predicate return q.Predicate
case Provenance: case Provenance:
return t.Provenance return q.Provenance
case Object: case Object:
return t.Object return q.Object
default: default:
panic(d.String()) panic(d.String())
} }
} }
func (t *Triple) Equals(o *Triple) bool { func (q *Quad) Equals(o *Quad) bool {
return *t == *o return *q == *o
} }
// Pretty-prints a triple. // Pretty-prints a triple.
func (t *Triple) String() string { func (q *Quad) String() string {
// TODO(kortschak) String methods should generally not terminate in '\n'. // TODO(kortschak) String methods should generally not terminate in '\n'.
return fmt.Sprintf("%s -- %s -> %s\n", t.Subject, t.Predicate, t.Object) return fmt.Sprintf("%s -- %s -> %s\n", q.Subject, q.Predicate, q.Object)
} }
func (t *Triple) IsValid() bool { func (q *Quad) IsValid() bool {
return t.Subject != "" && t.Predicate != "" && t.Object != "" return q.Subject != "" && q.Predicate != "" && q.Object != ""
} }
// TODO(kortschak) NTriple looks like a good candidate for conversion // TODO(kortschak) NTriple looks like a good candidate for conversion
// to MarshalText() (text []byte, err error) and then move parsing code // to MarshalText() (text []byte, err error) and then move parsing code
// from nquads to here to provide UnmarshalText(text []byte) error. // from nquads to here to provide UnmarshalText(text []byte) error.
// Prints a triple in N-Triple format. // Prints a triple in N-Quad format.
func (t *Triple) NTriple() string { func (q *Quad) NTriple() string {
if t.Provenance == "" { if q.Provenance == "" {
//TODO(barakmich): Proper escaping. //TODO(barakmich): Proper escaping.
return fmt.Sprintf("%s %s %s .", t.Subject, t.Predicate, t.Object) return fmt.Sprintf("%s %s %s .", q.Subject, q.Predicate, q.Object)
} else { } else {
return fmt.Sprintf("%s %s %s %s .", t.Subject, t.Predicate, t.Object, t.Provenance) return fmt.Sprintf("%s %s %s %s .", q.Subject, q.Predicate, q.Object, q.Provenance)
} }
} }
type Unmarshaler interface {
Unmarshal() (*Quad, error)
}

23
query/gremlin/build_iterator.go
View file

@ -22,6 +22,7 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
func getStrings(obj *otto.Object, field string) []string { func getStrings(obj *otto.Object, field string) []string {
@ -139,13 +140,13 @@ func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Itera
} }
} }
in, out := graph.Subject, graph.Object in, out := quad.Subject, quad.Object
if isReverse { if isReverse {
in, out = out, in in, out = out, in
} }
lto := iterator.NewLinksTo(ts, base, in) lto := iterator.NewLinksTo(ts, base, in)
and := iterator.NewAnd() and := iterator.NewAnd()
and.AddSubIterator(iterator.NewLinksTo(ts, predicateNodeIterator, graph.Predicate)) and.AddSubIterator(iterator.NewLinksTo(ts, predicateNodeIterator, quad.Predicate))
and.AddSubIterator(lto) and.AddSubIterator(lto)
return iterator.NewHasA(ts, and, out) return iterator.NewHasA(ts, and, out)
} }
@ -194,9 +195,9 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.
predFixed := ts.FixedIterator() predFixed := ts.FixedIterator()
predFixed.Add(ts.ValueOf(stringArgs[0])) predFixed.Add(ts.ValueOf(stringArgs[0]))
subAnd := iterator.NewAnd() subAnd := iterator.NewAnd()
subAnd.AddSubIterator(iterator.NewLinksTo(ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(iterator.NewLinksTo(ts, predFixed, quad.Predicate))
subAnd.AddSubIterator(iterator.NewLinksTo(ts, all, graph.Object)) subAnd.AddSubIterator(iterator.NewLinksTo(ts, all, quad.Object))
hasa := iterator.NewHasA(ts, subAnd, graph.Subject) hasa := iterator.NewHasA(ts, subAnd, quad.Subject)
and := iterator.NewAnd() and := iterator.NewAnd()
and.AddSubIterator(hasa) and.AddSubIterator(hasa)
and.AddSubIterator(subIt) and.AddSubIterator(subIt)
@ -214,9 +215,9 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.
predFixed := ts.FixedIterator() predFixed := ts.FixedIterator()
predFixed.Add(ts.ValueOf(stringArgs[0])) predFixed.Add(ts.ValueOf(stringArgs[0]))
subAnd := iterator.NewAnd() subAnd := iterator.NewAnd()
subAnd.AddSubIterator(iterator.NewLinksTo(ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(iterator.NewLinksTo(ts, predFixed, quad.Predicate))
subAnd.AddSubIterator(iterator.NewLinksTo(ts, all, graph.Subject)) subAnd.AddSubIterator(iterator.NewLinksTo(ts, all, quad.Subject))
hasa := iterator.NewHasA(ts, subAnd, graph.Object) hasa := iterator.NewHasA(ts, subAnd, quad.Object)
and := iterator.NewAnd() and := iterator.NewAnd()
and.AddSubIterator(hasa) and.AddSubIterator(hasa)
and.AddSubIterator(subIt) and.AddSubIterator(subIt)
@ -232,9 +233,9 @@ func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.
predFixed := ts.FixedIterator() predFixed := ts.FixedIterator()
predFixed.Add(ts.ValueOf(stringArgs[0])) predFixed.Add(ts.ValueOf(stringArgs[0]))
subAnd := iterator.NewAnd() subAnd := iterator.NewAnd()
subAnd.AddSubIterator(iterator.NewLinksTo(ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(iterator.NewLinksTo(ts, predFixed, quad.Predicate))
subAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, graph.Object)) subAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, quad.Object))
hasa := iterator.NewHasA(ts, subAnd, graph.Subject) hasa := iterator.NewHasA(ts, subAnd, quad.Subject)
and := iterator.NewAnd() and := iterator.NewAnd()
and.AddSubIterator(hasa) and.AddSubIterator(hasa)
and.AddSubIterator(subIt) and.AddSubIterator(subIt)

7
query/gremlin/gremlin_test.go
View file

@ -21,6 +21,7 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
_ "github.com/google/cayley/graph/memstore" _ "github.com/google/cayley/graph/memstore"
"github.com/google/cayley/quad"
) )
// This is a simple test graph. // This is a simple test graph.
@ -36,7 +37,7 @@ import (
// \-->|#D#|------------->+---+ // \-->|#D#|------------->+---+
// +---+ // +---+
// //
var simpleGraph = []*graph.Triple{ var simpleGraph = []*quad.Quad{
{"A", "follows", "B", ""}, {"A", "follows", "B", ""},
{"C", "follows", "B", ""}, {"C", "follows", "B", ""},
{"C", "follows", "D", ""}, {"C", "follows", "D", ""},
@ -50,7 +51,7 @@ var simpleGraph = []*graph.Triple{
{"G", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"},
} }
func makeTestSession(data []*graph.Triple) *Session { func makeTestSession(data []*quad.Quad) *Session {
ts, _ := graph.NewTripleStore("memstore", "", nil) ts, _ := graph.NewTripleStore("memstore", "", nil)
for _, t := range data { for _, t := range data {
ts.AddTriple(t) ts.AddTriple(t)
@ -244,7 +245,7 @@ var testQueries = []struct {
}, },
} }
func runQueryGetTag(g []*graph.Triple, query string, tag string) []string { func runQueryGetTag(g []*quad.Quad, query string, tag string) []string {
js := makeTestSession(g) js := makeTestSession(g)
c := make(chan interface{}, 5) c := make(chan interface{}, 5)
js.ExecInput(query, c, -1) js.ExecInput(query, c, -1)

11
query/mql/build_iterator.go
View file

@ -23,6 +23,7 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator" "github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
) )
func (q *Query) buildFixed(s string) graph.Iterator { func (q *Query) buildFixed(s string) graph.Iterator {
@ -139,16 +140,16 @@ func (q *Query) buildIteratorTreeMapInternal(query map[string]interface{}, path
subAnd := iterator.NewAnd() subAnd := iterator.NewAnd()
predFixed := q.ses.ts.FixedIterator() predFixed := q.ses.ts.FixedIterator()
predFixed.Add(q.ses.ts.ValueOf(pred)) predFixed.Add(q.ses.ts.ValueOf(pred))
subAnd.AddSubIterator(iterator.NewLinksTo(q.ses.ts, predFixed, graph.Predicate)) subAnd.AddSubIterator(iterator.NewLinksTo(q.ses.ts, predFixed, quad.Predicate))
if reverse { if reverse {
lto := iterator.NewLinksTo(q.ses.ts, builtIt, graph.Subject) lto := iterator.NewLinksTo(q.ses.ts, builtIt, quad.Subject)
subAnd.AddSubIterator(lto) subAnd.AddSubIterator(lto)
hasa := iterator.NewHasA(q.ses.ts, subAnd, graph.Object) hasa := iterator.NewHasA(q.ses.ts, subAnd, quad.Object)
subit = hasa subit = hasa
} else { } else {
lto := iterator.NewLinksTo(q.ses.ts, builtIt, graph.Object) lto := iterator.NewLinksTo(q.ses.ts, builtIt, quad.Object)
subAnd.AddSubIterator(lto) subAnd.AddSubIterator(lto)
hasa := iterator.NewHasA(q.ses.ts, subAnd, graph.Subject) hasa := iterator.NewHasA(q.ses.ts, subAnd, quad.Subject)
subit = hasa subit = hasa
} }
} }

7
query/mql/mql_test.go
View file

@ -21,6 +21,7 @@ import (
"github.com/google/cayley/graph" "github.com/google/cayley/graph"
_ "github.com/google/cayley/graph/memstore" _ "github.com/google/cayley/graph/memstore"
"github.com/google/cayley/quad"
) )
// This is a simple test graph. // This is a simple test graph.
@ -36,7 +37,7 @@ import (
// \-->|#D#|------------->+---+ // \-->|#D#|------------->+---+
// +---+ // +---+
// //
var simpleGraph = []*graph.Triple{ var simpleGraph = []*quad.Quad{
{"A", "follows", "B", ""}, {"A", "follows", "B", ""},
{"C", "follows", "B", ""}, {"C", "follows", "B", ""},
{"C", "follows", "D", ""}, {"C", "follows", "D", ""},
@ -50,7 +51,7 @@ var simpleGraph = []*graph.Triple{
{"G", "status", "cool", "status_graph"}, {"G", "status", "cool", "status_graph"},
} }
func makeTestSession(data []*graph.Triple) *Session { func makeTestSession(data []*quad.Quad) *Session {
ts, _ := graph.NewTripleStore("memstore", "", nil) ts, _ := graph.NewTripleStore("memstore", "", nil)
for _, t := range data { for _, t := range data {
ts.AddTriple(t) ts.AddTriple(t)
@ -164,7 +165,7 @@ var testQueries = []struct {
}, },
} }
func runQuery(g []*graph.Triple, query string) interface{} { func runQuery(g []*quad.Quad, query string) interface{} {
s := makeTestSession(g) s := makeTestSession(g)
c := make(chan interface{}, 5) c := make(chan interface{}, 5)
go s.ExecInput(query, c, -1) go s.ExecInput(query, c, -1)