Merge hash pool in from master
Conflicts: graph/leveldb/triplestore.go graph/mongo/triplestore.go
This commit is contained in:
commit
8720e17d87
4 changed files with 76 additions and 68 deletions
|
|
@ -45,7 +45,7 @@ type Iterator struct {
|
|||
|
||||
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
|
||||
vb := value.(Token)
|
||||
p := make([]byte, 0, 2+qs.hasherSize)
|
||||
p := make([]byte, 0, 2+hashSize)
|
||||
p = append(p, []byte(prefix)...)
|
||||
p = append(p, []byte(vb[1:])...)
|
||||
|
||||
|
|
@ -179,45 +179,45 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
|
|||
case quad.Subject:
|
||||
return 2
|
||||
case quad.Predicate:
|
||||
return qs.hasherSize + 2
|
||||
return hashSize + 2
|
||||
case quad.Object:
|
||||
return 2*qs.hasherSize + 2
|
||||
return 2*hashSize + 2
|
||||
case quad.Label:
|
||||
return 3*qs.hasherSize + 2
|
||||
return 3*hashSize + 2
|
||||
}
|
||||
}
|
||||
if bytes.Equal(prefix, []byte("po")) {
|
||||
switch d {
|
||||
case quad.Subject:
|
||||
return 2*qs.hasherSize + 2
|
||||
return 2*hashSize + 2
|
||||
case quad.Predicate:
|
||||
return 2
|
||||
case quad.Object:
|
||||
return qs.hasherSize + 2
|
||||
return hashSize + 2
|
||||
case quad.Label:
|
||||
return 3*qs.hasherSize + 2
|
||||
return hashSize + 2
|
||||
}
|
||||
}
|
||||
if bytes.Equal(prefix, []byte("os")) {
|
||||
switch d {
|
||||
case quad.Subject:
|
||||
return qs.hasherSize + 2
|
||||
return hashSize + 2
|
||||
case quad.Predicate:
|
||||
return 2*qs.hasherSize + 2
|
||||
return 2*hashSize + 2
|
||||
case quad.Object:
|
||||
return 2
|
||||
case quad.Label:
|
||||
return 3*qs.hasherSize + 2
|
||||
return 3*hashSize + 2
|
||||
}
|
||||
}
|
||||
if bytes.Equal(prefix, []byte("cp")) {
|
||||
switch d {
|
||||
case quad.Subject:
|
||||
return 2*qs.hasherSize + 2
|
||||
return 2*hashSize + 2
|
||||
case quad.Predicate:
|
||||
return qs.hasherSize + 2
|
||||
return hashSize + 2
|
||||
case quad.Object:
|
||||
return 3*qs.hasherSize + 2
|
||||
return 3*hashSize + 2
|
||||
case quad.Label:
|
||||
return 2
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"sync"
|
||||
|
||||
"github.com/barakmich/glog"
|
||||
"github.com/syndtr/goleveldb/leveldb"
|
||||
|
|
@ -43,6 +44,13 @@ const (
|
|||
DefaultWriteBufferSize = 20
|
||||
)
|
||||
|
||||
var (
|
||||
hashPool = sync.Pool{
|
||||
New: func() interface{} { return sha1.New() },
|
||||
}
|
||||
hashSize = sha1.Size
|
||||
)
|
||||
|
||||
type Token []byte
|
||||
|
||||
func (t Token) Key() interface{} {
|
||||
|
|
@ -56,9 +64,6 @@ type TripleStore struct {
|
|||
open bool
|
||||
size int64
|
||||
horizon int64
|
||||
hasher hash.Hash
|
||||
hasherSize int
|
||||
makeHasher func() hash.Hash
|
||||
writeopts *opt.WriteOptions
|
||||
readopts *opt.ReadOptions
|
||||
}
|
||||
|
|
@ -98,8 +103,6 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro
|
|||
write_buffer_mb = val
|
||||
}
|
||||
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
|
||||
qs.hasherSize = sha1.Size
|
||||
qs.makeHasher = sha1.New
|
||||
qs.writeopts = &opt.WriteOptions{
|
||||
Sync: false,
|
||||
}
|
||||
|
|
@ -144,22 +147,20 @@ func (qa *TripleStore) createDeltaKeyFor(d graph.Delta) []byte {
|
|||
}
|
||||
|
||||
func (qs *TripleStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte {
|
||||
hasher := qs.makeHasher()
|
||||
key := make([]byte, 0, 2+(qs.hasherSize*3))
|
||||
key := make([]byte, 0, 2+(hashSize*3))
|
||||
// TODO(kortschak) Remove dependence on String() method.
|
||||
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[3]), hasher)...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
|
||||
key = append(key, qs.convertStringToByteHash(triple.Get(d[3]))...)
|
||||
return key
|
||||
}
|
||||
|
||||
func (qs *TripleStore) createValueKeyFor(s string) []byte {
|
||||
hasher := qs.makeHasher()
|
||||
key := make([]byte, 0, 1+qs.hasherSize)
|
||||
key := make([]byte, 0, 1+hashSize)
|
||||
key = append(key, []byte("z")...)
|
||||
key = append(key, qs.convertStringToByteHash(s, hasher)...)
|
||||
key = append(key, qs.convertStringToByteHash(s)...)
|
||||
return key
|
||||
}
|
||||
|
||||
|
|
@ -346,11 +347,13 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
|
|||
return triple
|
||||
}
|
||||
|
||||
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte {
|
||||
hasher.Reset()
|
||||
key := make([]byte, 0, qs.hasherSize)
|
||||
hasher.Write([]byte(s))
|
||||
key = hasher.Sum(key)
|
||||
func (qs *TripleStore) convertStringToByteHash(s string) []byte {
|
||||
h := hashPool.Get().(hash.Hash)
|
||||
h.Reset()
|
||||
defer hashPool.Put(h)
|
||||
key := make([]byte, 0, hashSize)
|
||||
h.Write([]byte(s))
|
||||
key = h.Sum(key)
|
||||
return key
|
||||
}
|
||||
|
||||
|
|
@ -467,7 +470,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
|
|||
v := val.(Token)
|
||||
offset := PositionOf(v[0:2], d, qs)
|
||||
if offset != -1 {
|
||||
return Token(append([]byte("z"), v[offset:offset+qs.hasherSize]...))
|
||||
return Token(append([]byte("z"), v[offset:offset+hashSize]...))
|
||||
} else {
|
||||
return Token(qs.Quad(val).Get(d))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -177,13 +177,13 @@ func (it *Iterator) Contains(v graph.Value) bool {
|
|||
case quad.Subject:
|
||||
offset = 0
|
||||
case quad.Predicate:
|
||||
offset = (it.qs.hasherSize * 2)
|
||||
offset = (hashSize * 2)
|
||||
case quad.Object:
|
||||
offset = (it.qs.hasherSize * 2) * 2
|
||||
offset = (hashSize * 2) * 2
|
||||
case quad.Label:
|
||||
offset = (it.qs.hasherSize * 2) * 3
|
||||
offset = (hashSize * 2) * 3
|
||||
}
|
||||
val := v.(string)[offset : it.qs.hasherSize*2+offset]
|
||||
val := v.(string)[offset : hashSize*2+offset]
|
||||
if val == it.hash {
|
||||
it.result = v
|
||||
return graph.ContainsLogOut(it, v, true)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import (
|
|||
"crypto/sha1"
|
||||
"encoding/hex"
|
||||
"hash"
|
||||
"sync"
|
||||
|
||||
"gopkg.in/mgo.v2"
|
||||
"gopkg.in/mgo.v2/bson"
|
||||
|
|
@ -34,11 +35,16 @@ func init() {
|
|||
|
||||
const DefaultDBName = "cayley"
|
||||
|
||||
var (
|
||||
hashPool = sync.Pool{
|
||||
New: func() interface{} { return sha1.New() },
|
||||
}
|
||||
hashSize = sha1.Size
|
||||
)
|
||||
|
||||
type TripleStore struct {
|
||||
session *mgo.Session
|
||||
db *mgo.Database
|
||||
hasherSize int
|
||||
makeHasher func() hash.Hash
|
||||
idCache *IDLru
|
||||
}
|
||||
|
||||
|
|
@ -91,26 +97,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
|
|||
}
|
||||
qs.db = conn.DB(dbName)
|
||||
qs.session = conn
|
||||
qs.hasherSize = sha1.Size
|
||||
qs.makeHasher = sha1.New
|
||||
qs.idCache = NewIDLru(1 << 16)
|
||||
return &qs, nil
|
||||
}
|
||||
|
||||
func (qs *TripleStore) getIdForQuad(t quad.Quad) string {
|
||||
hasher := qs.makeHasher()
|
||||
id := qs.convertStringToByteHash(t.Subject, hasher)
|
||||
id += qs.convertStringToByteHash(t.Predicate, hasher)
|
||||
id += qs.convertStringToByteHash(t.Object, hasher)
|
||||
id += qs.convertStringToByteHash(t.Label, hasher)
|
||||
id := qs.convertStringToByteHash(t.Subject)
|
||||
id += qs.convertStringToByteHash(t.Predicate)
|
||||
id += qs.convertStringToByteHash(t.Object)
|
||||
id += qs.convertStringToByteHash(t.Label)
|
||||
return id
|
||||
}
|
||||
|
||||
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) string {
|
||||
hasher.Reset()
|
||||
key := make([]byte, 0, qs.hasherSize)
|
||||
hasher.Write([]byte(s))
|
||||
key = hasher.Sum(key)
|
||||
func (qs *TripleStore) convertStringToByteHash(s string) string {
|
||||
h := hashPool.Get().(hash.Hash)
|
||||
h.Reset()
|
||||
defer hashPool.Put(h)
|
||||
|
||||
key := make([]byte, 0, hashSize)
|
||||
h.Write([]byte(s))
|
||||
key = h.Sum(key)
|
||||
return hex.EncodeToString(key)
|
||||
}
|
||||
|
||||
|
|
@ -282,8 +288,7 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
|
|||
}
|
||||
|
||||
func (qs *TripleStore) ValueOf(s string) graph.Value {
|
||||
h := qs.makeHasher()
|
||||
return qs.convertStringToByteHash(s, h)
|
||||
return qs.convertStringToByteHash(s)
|
||||
}
|
||||
|
||||
func (qs *TripleStore) NameOf(v graph.Value) string {
|
||||
|
|
@ -341,13 +346,13 @@ func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.V
|
|||
case quad.Subject:
|
||||
offset = 0
|
||||
case quad.Predicate:
|
||||
offset = (qs.hasherSize * 2)
|
||||
offset = (hashSize * 2)
|
||||
case quad.Object:
|
||||
offset = (qs.hasherSize * 2) * 2
|
||||
offset = (hashSize * 2) * 2
|
||||
case quad.Label:
|
||||
offset = (qs.hasherSize * 2) * 3
|
||||
offset = (hashSize * 2) * 3
|
||||
}
|
||||
val := in.(string)[offset : qs.hasherSize*2+offset]
|
||||
val := in.(string)[offset : hashSize*2+offset]
|
||||
return val
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue