barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'master' into nexter

Browse source 
Conflicts:
	graph/leveldb/all_iterator.go
	graph/leveldb/iterator.go
	graph/memstore/triplestore.go
	query/gremlin/finals.go
This commit is contained in:
kortschak 2014-08-07 06:52:36 +09:30
commit 62785d25c2
37 changed files with 882 additions and 467 deletions
Download patch file Download diff file Expand all files Collapse all files

2
graph/leveldb/all_iterator.go
View file

@ -117,7 +117,7 @@ func (it *AllIterator) Next() bool {
it.Close()
return false
}
it.result = out
it.result = Token(out)
return true
}

22
graph/leveldb/iterator.go
View file

@ -42,7 +42,7 @@ type Iterator struct {
}
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
vb := value.([]byte)
vb := value.(Token)
p := make([]byte, 0, 2+qs.hasher.Size())
p = append(p, []byte(prefix)...)
p = append(p, []byte(vb[1:])...)
@ -105,7 +105,7 @@ func (it *Iterator) TagResults(dst map[string]graph.Value) {
}
func (it *Iterator) Clone() graph.Iterator {
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.qs)
out := NewIterator(it.originalPrefix, it.dir, Token(it.checkId), it.qs)
out.tags.CopyFrom(it)
return out
}
@ -134,7 +134,7 @@ func (it *Iterator) Next() bool {
if bytes.HasPrefix(it.iter.Key(), it.nextPrefix) {
out := make([]byte, len(it.iter.Key()))
copy(out, it.iter.Key())
it.result = out
it.result = Token(out)
ok := it.iter.Next()
if !ok {
it.Close()
@ -216,7 +216,7 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
}
func (it *Iterator) Contains(v graph.Value) bool {
val := v.([]byte)
val := v.(Token)
if val[0] == 'z' {
return false
}
@ -227,7 +227,7 @@ func (it *Iterator) Contains(v graph.Value) bool {
}
} else {
nameForDir := it.qs.Quad(v).Get(it.dir)
hashForDir := it.qs.ValueOf(nameForDir).([]byte)
hashForDir := it.qs.ValueOf(nameForDir).(Token)
if bytes.Equal(hashForDir, it.checkId) {
return true
}
@ -236,12 +236,20 @@ func (it *Iterator) Contains(v graph.Value) bool {
}
func (it *Iterator) Size() (int64, bool) {
return it.qs.SizeOf(it.checkId), true
return it.qs.SizeOf(Token(it.checkId)), true
}
func (it *Iterator) DebugString(indent int) string {
size, _ := it.Size()
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.tags.Tags(), it.dir, size, it.qs.NameOf(it.checkId))
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)",
strings.Repeat(" ", indent),
it.Type(),
it.UID(),
it.tags.Tags(),
it.dir,
size,
it.qs.NameOf(Token(it.checkId)),
)
}
var levelDBType graph.Type

26
graph/leveldb/leveldb_test.go
View file

@ -26,8 +26,8 @@ import (
"github.com/google/cayley/quad"
)
func makeTripleSet() []*quad.Quad {
tripleSet := []*quad.Quad{
func makeTripleSet() []quad.Quad {
tripleSet := []quad.Quad{
{"A", "follows", "B", ""},
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
@ -43,7 +43,7 @@ func makeTripleSet() []*quad.Quad {
return tripleSet
}
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad {
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []quad.Quad {
var res ordered
for graph.Next(it) {
res = append(res, qs.Quad(it.Result()))
@ -52,7 +52,7 @@ func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad {
return res
}
type ordered []*quad.Quad
type ordered []quad.Quad
func (o ordered) Len() int { return len(o) }
func (o ordered) Less(i, j int) bool {
@ -135,7 +135,7 @@ func TestLoadDatabase(t *testing.T) {
t.Error("Failed to create leveldb TripleStore.")
}
qs.AddTriple(&quad.Quad{"Something", "points_to", "Something Else", "context"})
qs.AddTriple(quad.Quad{"Something", "points_to", "Something Else", "context"})
for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
if got := qs.NameOf(qs.ValueOf(pq)); got != pq {
t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq)
@ -168,7 +168,7 @@ func TestLoadDatabase(t *testing.T) {
t.Errorf("Unexpected triplestore size, got:%d expect:5", s)
}
qs.RemoveTriple(&quad.Quad{"A", "follows", "B", ""})
qs.RemoveTriple(quad.Quad{"A", "follows", "B", ""})
if s := qs.Size(); s != 10 {
t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s)
}
@ -293,7 +293,7 @@ func TestSetIterator(t *testing.T) {
qs.AddTripleSet(makeTripleSet())
expect := []*quad.Quad{
expect := []quad.Quad{
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
}
@ -318,7 +318,7 @@ func TestSetIterator(t *testing.T) {
// Object iterator.
it = qs.TripleIterator(quad.Object, qs.ValueOf("F"))
expect = []*quad.Quad{
expect = []quad.Quad{
{"B", "follows", "F", ""},
{"E", "follows", "F", ""},
}
@ -331,7 +331,7 @@ func TestSetIterator(t *testing.T) {
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it)
expect = []*quad.Quad{
expect = []quad.Quad{
{"B", "follows", "F", ""},
}
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
@ -341,7 +341,7 @@ func TestSetIterator(t *testing.T) {
// Predicate iterator.
it = qs.TripleIterator(quad.Predicate, qs.ValueOf("status"))
expect = []*quad.Quad{
expect = []quad.Quad{
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
@ -354,7 +354,7 @@ func TestSetIterator(t *testing.T) {
// Label iterator.
it = qs.TripleIterator(quad.Label, qs.ValueOf("status_graph"))
expect = []*quad.Quad{
expect = []quad.Quad{
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
@ -370,7 +370,7 @@ func TestSetIterator(t *testing.T) {
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it)
expect = []*quad.Quad{
expect = []quad.Quad{
{"B", "status", "cool", "status_graph"},
}
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
@ -383,7 +383,7 @@ func TestSetIterator(t *testing.T) {
and.AddSubIterator(it)
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
expect = []*quad.Quad{
expect = []quad.Quad{
{"B", "status", "cool", "status_graph"},
}
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {

48
graph/leveldb/triplestore.go
View file

@ -42,6 +42,12 @@ const (
DefaultWriteBufferSize = 20
)
type Token []byte
func (t Token) Key() interface{} {
return string(t)
}
type TripleStore struct {
dbOpts *opt.Options
db *leveldb.DB
@ -116,7 +122,7 @@ func (qs *TripleStore) Size() int64 {
return qs.size
}
func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte {
func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*3))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
@ -126,7 +132,7 @@ func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple *quad.Quad) []by
return key
}
func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple *quad.Quad) []byte {
func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*4))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{quad.Label.Prefix(), d[0].Prefix()}...)
@ -144,7 +150,7 @@ func (qs *TripleStore) createValueKeyFor(s string) []byte {
return key
}
func (qs *TripleStore) AddTriple(t *quad.Quad) {
func (qs *TripleStore) AddTriple(t quad.Quad) {
batch := &leveldb.Batch{}
qs.buildWrite(batch, t)
err := qs.db.Write(batch, qs.writeopts)
@ -163,7 +169,7 @@ var (
pso = [3]quad.Direction{quad.Predicate, quad.Subject, quad.Object}
)
func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
func (qs *TripleStore) RemoveTriple(t quad.Quad) {
_, err := qs.db.Get(qs.createKeyFor(spo, t), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Error("Couldn't access DB to confirm deletion")
@ -192,8 +198,8 @@ func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
qs.size--
}
func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *quad.Quad) {
bytes, err := json.Marshal(*t)
func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t quad.Quad) {
bytes, err := json.Marshal(t)
if err != nil {
glog.Errorf("Couldn't write to buffer for triple %s: %s", t, err)
return
@ -206,7 +212,7 @@ func (qs *TripleStore) buildTripleWrite(batch *leveldb.Batch, t *quad.Quad) {
}
}
func (qs *TripleStore) buildWrite(batch *leveldb.Batch, t *quad.Quad) {
func (qs *TripleStore) buildWrite(batch *leveldb.Batch, t quad.Quad) {
qs.buildTripleWrite(batch, t)
qs.UpdateValueKeyBy(t.Get(quad.Subject), 1, nil)
qs.UpdateValueKeyBy(t.Get(quad.Predicate), 1, nil)
@ -267,7 +273,7 @@ func (qs *TripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.
}
}
func (qs *TripleStore) AddTripleSet(t_s []*quad.Quad) {
func (qs *TripleStore) AddTripleSet(t_s []quad.Quad) {
batch := &leveldb.Batch{}
newTs := len(t_s)
resizeMap := make(map[string]int)
@ -306,23 +312,23 @@ func (qs *TripleStore) Close() {
qs.open = false
}
func (qs *TripleStore) Quad(k graph.Value) *quad.Quad {
func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
var triple quad.Quad
b, err := qs.db.Get(k.([]byte), qs.readopts)
b, err := qs.db.Get(k.(Token), qs.readopts)
if err != nil && err != leveldb.ErrNotFound {
glog.Error("Error: couldn't get triple from DB.")
return &quad.Quad{}
return quad.Quad{}
}
if err == leveldb.ErrNotFound {
// No harm, no foul.
return &quad.Quad{}
return quad.Quad{}
}
err = json.Unmarshal(b, &triple)
if err != nil {
glog.Error("Error: couldn't reconstruct triple.")
return &quad.Quad{}
return quad.Quad{}
}
return &triple
return triple
}
func (qs *TripleStore) convertStringToByteHash(s string) []byte {
@ -334,7 +340,7 @@ func (qs *TripleStore) convertStringToByteHash(s string) []byte {
}
func (qs *TripleStore) ValueOf(s string) graph.Value {
return qs.createValueKeyFor(s)
return Token(qs.createValueKeyFor(s))
}
func (qs *TripleStore) valueData(value_key []byte) ValueData {
@ -362,14 +368,14 @@ func (qs *TripleStore) NameOf(k graph.Value) string {
glog.V(2).Info("k was nil")
return ""
}
return qs.valueData(k.([]byte)).Name
return qs.valueData(k.(Token)).Name
}
func (qs *TripleStore) SizeOf(k graph.Value) int64 {
if k == nil {
return 0
}
return int64(qs.valueData(k.([]byte)).Size)
return int64(qs.valueData(k.(Token)).Size)
}
func (qs *TripleStore) getSize() {
@ -432,17 +438,17 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
}
func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value {
v := val.([]uint8)
v := val.(Token)
offset := PositionOf(v[0:2], d, qs)
if offset != -1 {
return append([]byte("z"), v[offset:offset+qs.hasher.Size()]...)
return Token(append([]byte("z"), v[offset:offset+qs.hasher.Size()]...))
} else {
return qs.Quad(val).Get(d)
return Token(qs.Quad(val).Get(d))
}
}
func compareBytes(a, b graph.Value) bool {
return bytes.Equal(a.([]uint8), b.([]uint8))
return bytes.Equal(a.(Token), b.(Token))
}
func (qs *TripleStore) FixedIterator() graph.FixedIterator {