barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge pull request #118 from barakmich/issue117

Browse source 
Fixes #117, HasA speedup
This commit is contained in:
Barak Michener 2014-08-13 21:23:01 -04:00
commit 11cfc3dfd8
5 changed files with 69 additions and 61 deletions
Download patch file Download diff file Expand all files Collapse all files

3
graph/iterator/hasa_iterator.go
View file

@ -202,8 +202,7 @@ func (it *HasA) Next() bool {
return graph.NextLogOut(it, 0, false)
}
tID := it.primaryIt.Result()
name := it.ts.Quad(tID).Get(it.dir)
val := it.ts.ValueOf(name)
val := it.ts.TripleDirection(tID, it.dir)
it.result = val
return graph.NextLogOut(it, val, true)
}

20
graph/leveldb/iterator.go
View file

@ -43,7 +43,7 @@ type Iterator struct {
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
vb := value.(Token)
p := make([]byte, 0, 2+qs.hasher.Size())
p := make([]byte, 0, 2+qs.hasherSize)
p = append(p, []byte(prefix)...)
p = append(p, []byte(vb[1:])...)
@ -169,9 +169,9 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
case quad.Subject:
return 2
case quad.Predicate:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Object:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Label:
return -1
}
@ -179,11 +179,11 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("po")) {
switch d {
case quad.Subject:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Predicate:
return 2
case quad.Object:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Label:
return -1
}
@ -191,9 +191,9 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("os")) {
switch d {
case quad.Subject:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Predicate:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Object:
return 2
case quad.Label:
@ -203,11 +203,11 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("cp")) {
switch d {
case quad.Subject:
return 2*qs.hasher.Size() + 2
return 2*qs.hasherSize + 2
case quad.Predicate:
return qs.hasher.Size() + 2
return qs.hasherSize + 2
case quad.Object:
return 3*qs.hasher.Size() + 2
return 3*qs.hasherSize + 2
case quad.Label:
return 2
}

43
graph/leveldb/triplestore.go
View file

@ -54,7 +54,8 @@ type TripleStore struct {
path string
open bool
size int64
hasher hash.Hash
hasherSize int
makeHasher func() hash.Hash
writeopts *opt.WriteOptions
readopts *opt.ReadOptions
}
@ -93,7 +94,8 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro
write_buffer_mb = val
}
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.hasher = sha1.New()
qs.hasherSize = sha1.Size
qs.makeHasher = sha1.New
qs.writeopts = &opt.WriteOptions{
Sync: false,
}
@ -123,30 +125,33 @@ func (qs *TripleStore) Size() int64 {
}
func (qs *TripleStore) createKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*3))
hasher := qs.makeHasher()
key := make([]byte, 0, 2+(qs.hasherSize*3))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
return key
}
func (qs *TripleStore) createProvKeyFor(d [3]quad.Direction, triple quad.Quad) []byte {
key := make([]byte, 0, 2+(qs.hasher.Size()*4))
hasher := qs.makeHasher()
key := make([]byte, 0, 2+(qs.hasherSize*4))
// TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{quad.Label.Prefix(), d[0].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(quad.Label))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(quad.Label), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
return key
}
func (qs *TripleStore) createValueKeyFor(s string) []byte {
key := make([]byte, 0, 1+qs.hasher.Size())
hasher := qs.makeHasher()
key := make([]byte, 0, 1+qs.hasherSize)
key = append(key, []byte("z")...)
key = append(key, qs.convertStringToByteHash(s)...)
key = append(key, qs.convertStringToByteHash(s, hasher)...)
return key
}
@ -331,11 +336,11 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
return triple
}
func (qs *TripleStore) convertStringToByteHash(s string) []byte {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte {
hasher.Reset()
key := make([]byte, 0, qs.hasherSize)
hasher.Write([]byte(s))
key = hasher.Sum(key)
return key
}
@ -441,7 +446,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
v := val.(Token)
offset := PositionOf(v[0:2], d, qs)
if offset != -1 {
return Token(append([]byte("z"), v[offset:offset+qs.hasher.Size()]...))
return Token(append([]byte("z"), v[offset:offset+qs.hasherSize]...))
} else {
return Token(qs.Quad(val).Get(d))
}

8
graph/mongo/iterator.go
View file

@ -185,13 +185,13 @@ func (it *Iterator) Contains(v graph.Value) bool {
case quad.Subject:
offset = 0
case quad.Predicate:
offset = (it.qs.hasher.Size() * 2)
offset = (it.qs.hasherSize * 2)
case quad.Object:
offset = (it.qs.hasher.Size() * 2) * 2
offset = (it.qs.hasherSize * 2) * 2
case quad.Label:
offset = (it.qs.hasher.Size() * 2) * 3
offset = (it.qs.hasherSize * 2) * 3
}
val := v.(string)[offset : it.qs.hasher.Size()*2+offset]
val := v.(string)[offset : it.qs.hasherSize*2+offset]
if val == it.hash {
it.result = v
return graph.ContainsLogOut(it, v, true)

36
graph/mongo/triplestore.go
View file

@ -41,7 +41,8 @@ const DefaultDBName = "cayley"
type TripleStore struct {
session *mgo.Session
db *mgo.Database
hasher hash.Hash
hasherSize int
makeHasher func() hash.Hash
idCache *IDLru
}
@ -86,24 +87,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
}
qs.db = conn.DB(dbName)
qs.session = conn
qs.hasher = sha1.New()
qs.hasherSize = sha1.Size
qs.makeHasher = sha1.New
qs.idCache = NewIDLru(1 << 16)
return &qs, nil
}
func (qs *TripleStore) getIdForTriple(t quad.Quad) string {
id := qs.ConvertStringToByteHash(t.Subject)
id += qs.ConvertStringToByteHash(t.Predicate)
id += qs.ConvertStringToByteHash(t.Object)
id += qs.ConvertStringToByteHash(t.Label)
hasher := qs.makeHasher()
id := qs.convertStringToByteHash(t.Subject, hasher)
id += qs.convertStringToByteHash(t.Predicate, hasher)
id += qs.convertStringToByteHash(t.Object, hasher)
id += qs.convertStringToByteHash(t.Label, hasher)
return id
}
func (qs *TripleStore) ConvertStringToByteHash(s string) string {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) string {
hasher.Reset()
key := make([]byte, 0, qs.hasherSize)
hasher.Write([]byte(s))
key = hasher.Sum(key)
return hex.EncodeToString(key)
}
@ -243,7 +246,8 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
}
func (qs *TripleStore) ValueOf(s string) graph.Value {
return qs.ConvertStringToByteHash(s)
h := qs.makeHasher()
return qs.convertStringToByteHash(s, h)
}
func (qs *TripleStore) NameOf(v graph.Value) string {
@ -288,13 +292,13 @@ func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.V
case quad.Subject:
offset = 0
case quad.Predicate:
offset = (qs.hasher.Size() * 2)
offset = (qs.hasherSize * 2)
case quad.Object:
offset = (qs.hasher.Size() * 2) * 2
offset = (qs.hasherSize * 2) * 2
case quad.Label:
offset = (qs.hasher.Size() * 2) * 3
offset = (qs.hasherSize * 2) * 3
}
val := in.(string)[offset : qs.hasher.Size()*2+offset]
val := in.(string)[offset : qs.hasherSize*2+offset]
return val
}