barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

fix leveldb (and speed up hasa)

Browse source
This commit is contained in:
Barak Michener 2014-08-13 13:08:50 -04:00
parent ee11a26bc2
commit 104e7d110d
3 changed files with 42 additions and 38 deletions
Download patch file Download diff file Expand all files Collapse all files

3
graph/iterator/hasa_iterator.go
View file

@ -202,8 +202,7 @@ func (it *HasA) Next() bool {
return graph.NextLogOut(it, 0, false) return graph.NextLogOut(it, 0, false)
} }
tID := it.primaryIt.Result() tID := it.primaryIt.Result()
name := it.ts.Quad(tID).Get(it.dir) val := it.ts.TripleDirection(tID, it.dir)
val := it.ts.ValueOf(name)
it.result = val it.result = val
return graph.NextLogOut(it, val, true) return graph.NextLogOut(it, val, true)
} }

20
graph/leveldb/iterator.go
View file

@ -43,7 +43,7 @@ type Iterator struct {
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator { func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
vb := value.(Token) vb := value.(Token)
p := make([]byte, 0, 2+qs.hasher.Size()) p := make([]byte, 0, 2+qs.hasher_size)
p = append(p, []byte(prefix)...) p = append(p, []byte(prefix)...)
p = append(p, []byte(vb[1:])...) p = append(p, []byte(vb[1:])...)
@ -169,9 +169,9 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
case quad.Subject: case quad.Subject:
return 2 return 2
case quad.Predicate: case quad.Predicate:
return qs.hasher.Size() + 2 return qs.hasher_size + 2
case quad.Object: case quad.Object:
return 2*qs.hasher.Size() + 2 return 2*qs.hasher_size + 2
case quad.Label: case quad.Label:
return -1 return -1
} }
@ -179,11 +179,11 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("po")) { if bytes.Equal(prefix, []byte("po")) {
switch d { switch d {
case quad.Subject: case quad.Subject:
return 2*qs.hasher.Size() + 2 return 2*qs.hasher_size + 2
case quad.Predicate: case quad.Predicate:
return 2 return 2
case quad.Object: case quad.Object:
return qs.hasher.Size() + 2 return qs.hasher_size + 2
case quad.Label: case quad.Label:
return -1 return -1
} }
@ -191,9 +191,9 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("os")) { if bytes.Equal(prefix, []byte("os")) {
switch d { switch d {
case quad.Subject: case quad.Subject:
return qs.hasher.Size() + 2 return qs.hasher_size + 2
case quad.Predicate: case quad.Predicate:
return 2*qs.hasher.Size() + 2 return 2*qs.hasher_size + 2
case quad.Object: case quad.Object:
return 2 return 2
case quad.Label: case quad.Label:
@ -203,11 +203,11 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("cp")) { if bytes.Equal(prefix, []byte("cp")) {
switch d { switch d {
case quad.Subject: case quad.Subject:
return 2*qs.hasher.Size() + 2 return 2*qs.hasher_size + 2
case quad.Predicate: case quad.Predicate:
return qs.hasher.Size() + 2 return qs.hasher_size + 2
case quad.Object: case quad.Object:
return 3*qs.hasher.Size() + 2 return 3*qs.hasher_size + 2
case quad.Label: case quad.Label:
return 2 return 2
} }

57
graph/leveldb/triplestore.go
View file

@ -49,14 +49,15 @@ func (t Token) Key() interface{} {
} }
type TripleStore struct { type TripleStore struct {
dbOpts *opt.Options dbOpts *opt.Options
db *leveldb.DB db *leveldb.DB
path string path string
open bool open bool
size int64 size int64
hasher hash.Hash hasher_size int
writeopts *opt.WriteOptions make_hasher func() hash.Hash
readopts *opt.ReadOptions writeopts *opt.WriteOptions
readopts *opt.ReadOptions
} }
func createNewLevelDB(path string, _ graph.Options) error { func createNewLevelDB(path string, _ graph.Options) error {
@ -93,7 +94,8 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro
write_buffer_mb = val write_buffer_mb = val
} }
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.hasher = sha1.New() qs.hasher_size = sha1.Size
qs.make_hasher = func() hash.Hash { return sha1.New() }
qs.writeopts = &opt.WriteOptions{ qs.writeopts = &opt.WriteOptions{
Sync: false, Sync: false,
} }
@ -123,30 +125,33 @@ func (qs *TripleStore) Size() int64 {
} }
func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple quad.Quad) []byte { func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*3)) hasher := qs.make_hasher()
key := make([]byte, 0, 2+(qs.hasher_size*3))
// TODO(kortschak) Remove dependence on String() method. // TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...) key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
return key return key
} }
func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple quad.Quad) []byte { func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*4)) hasher := qs.make_hasher()
key := make([]byte, 0, 2+(qs.hasher_size*4))
// TODO(kortschak) Remove dependence on String() method. // TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{quad.Label.Prefix(), d[0].Prefix()}...) key = append(key, []byte{quad.Label.Prefix(), d[0].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(quad.Label))...) key = append(key, qs.convertStringToByteHash(triple.Get(quad.Label), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...) key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
return key return key
} }
func (qs *TripleStore) createValueKeyFor(s string) []byte { func (qs *TripleStore) createValueKeyFor(s string) []byte {
key := make([]byte, 0, 1+qs.hasher.Size()) hasher := qs.make_hasher()
key := make([]byte, 0, 1+qs.hasher_size)
key = append(key, []byte("z")...) key = append(key, []byte("z")...)
key = append(key, qs.convertStringToByteHash(s)...) key = append(key, qs.convertStringToByteHash(s, hasher)...)
return key return key
} }
@ -331,11 +336,11 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
return triple return triple
} }
func (qs *TripleStore) convertStringToByteHash(s string) []byte { func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte {
qs.hasher.Reset() hasher.Reset()
key := make([]byte, 0, qs.hasher.Size()) key := make([]byte, 0, qs.hasher_size)
qs.hasher.Write([]byte(s)) hasher.Write([]byte(s))
key = qs.hasher.Sum(key) key = hasher.Sum(key)
return key return key
} }
@ -441,7 +446,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
v := val.(Token) v := val.(Token)
offset := PositionOf(v[0:2], d, qs) offset := PositionOf(v[0:2], d, qs)
if offset != -1 { if offset != -1 {
return Token(append([]byte("z"), v[offset:offset+qs.hasher.Size()]...)) return Token(append([]byte("z"), v[offset:offset+qs.hasher_size]...))
} else { } else {
return Token(qs.Quad(val).Get(d)) return Token(qs.Quad(val).Get(d))
} }