diff --git a/graph/leveldb/iterator.go b/graph/leveldb/iterator.go index dbc70be..a23d15d 100644 --- a/graph/leveldb/iterator.go +++ b/graph/leveldb/iterator.go @@ -45,7 +45,7 @@ type Iterator struct { func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator { vb := value.(Token) - p := make([]byte, 0, 2+qs.hasherSize) + p := make([]byte, 0, 2+hashSize) p = append(p, []byte(prefix)...) p = append(p, []byte(vb[1:])...) @@ -179,45 +179,45 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int { case quad.Subject: return 2 case quad.Predicate: - return qs.hasherSize + 2 + return hashSize + 2 case quad.Object: - return 2*qs.hasherSize + 2 + return 2*hashSize + 2 case quad.Label: - return 3*qs.hasherSize + 2 + return 3*hashSize + 2 } } if bytes.Equal(prefix, []byte("po")) { switch d { case quad.Subject: - return 2*qs.hasherSize + 2 + return 2*hashSize + 2 case quad.Predicate: return 2 case quad.Object: - return qs.hasherSize + 2 + return hashSize + 2 case quad.Label: - return 3*qs.hasherSize + 2 + return hashSize + 2 } } if bytes.Equal(prefix, []byte("os")) { switch d { case quad.Subject: - return qs.hasherSize + 2 + return hashSize + 2 case quad.Predicate: - return 2*qs.hasherSize + 2 + return 2*hashSize + 2 case quad.Object: return 2 case quad.Label: - return 3*qs.hasherSize + 2 + return 3*hashSize + 2 } } if bytes.Equal(prefix, []byte("cp")) { switch d { case quad.Subject: - return 2*qs.hasherSize + 2 + return 2*hashSize + 2 case quad.Predicate: - return qs.hasherSize + 2 + return hashSize + 2 case quad.Object: - return 3*qs.hasherSize + 2 + return 3*hashSize + 2 case quad.Label: return 2 } diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 8d36a33..dee63f8 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "hash" + "sync" "github.com/barakmich/glog" "github.com/syndtr/goleveldb/leveldb" @@ -43,6 +44,13 @@ const ( DefaultWriteBufferSize = 20 ) +var ( + hashPool = sync.Pool{ + New: func() interface{} { return sha1.New() }, + } + hashSize = sha1.Size +) + type Token []byte func (t Token) Key() interface{} { @@ -50,17 +58,14 @@ func (t Token) Key() interface{} { } type TripleStore struct { - dbOpts *opt.Options - db *leveldb.DB - path string - open bool - size int64 - horizon int64 - hasher hash.Hash - hasherSize int - makeHasher func() hash.Hash - writeopts *opt.WriteOptions - readopts *opt.ReadOptions + dbOpts *opt.Options + db *leveldb.DB + path string + open bool + size int64 + horizon int64 + writeopts *opt.WriteOptions + readopts *opt.ReadOptions } func createNewLevelDB(path string, _ graph.Options) error { @@ -98,8 +103,6 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro write_buffer_mb = val } qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB - qs.hasherSize = sha1.Size - qs.makeHasher = sha1.New qs.writeopts = &opt.WriteOptions{ Sync: false, } @@ -144,22 +147,20 @@ func (qa *TripleStore) createDeltaKeyFor(d graph.Delta) []byte { } func (qs *TripleStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte { - hasher := qs.makeHasher() - key := make([]byte, 0, 2+(qs.hasherSize*3)) + key := make([]byte, 0, 2+(hashSize*3)) // TODO(kortschak) Remove dependence on String() method. key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...) - key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...) - key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...) - key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...) - key = append(key, qs.convertStringToByteHash(triple.Get(d[3]), hasher)...) + key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...) + key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...) + key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...) + key = append(key, qs.convertStringToByteHash(triple.Get(d[3]))...) return key } func (qs *TripleStore) createValueKeyFor(s string) []byte { - hasher := qs.makeHasher() - key := make([]byte, 0, 1+qs.hasherSize) + key := make([]byte, 0, 1+hashSize) key = append(key, []byte("z")...) - key = append(key, qs.convertStringToByteHash(s, hasher)...) + key = append(key, qs.convertStringToByteHash(s)...) return key } @@ -346,11 +347,13 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad { return triple } -func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte { - hasher.Reset() - key := make([]byte, 0, qs.hasherSize) - hasher.Write([]byte(s)) - key = hasher.Sum(key) +func (qs *TripleStore) convertStringToByteHash(s string) []byte { + h := hashPool.Get().(hash.Hash) + h.Reset() + defer hashPool.Put(h) + key := make([]byte, 0, hashSize) + h.Write([]byte(s)) + key = h.Sum(key) return key } @@ -467,7 +470,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph. v := val.(Token) offset := PositionOf(v[0:2], d, qs) if offset != -1 { - return Token(append([]byte("z"), v[offset:offset+qs.hasherSize]...)) + return Token(append([]byte("z"), v[offset:offset+hashSize]...)) } else { return Token(qs.Quad(val).Get(d)) } diff --git a/graph/mongo/iterator.go b/graph/mongo/iterator.go index 3ba695d..3356974 100644 --- a/graph/mongo/iterator.go +++ b/graph/mongo/iterator.go @@ -177,13 +177,13 @@ func (it *Iterator) Contains(v graph.Value) bool { case quad.Subject: offset = 0 case quad.Predicate: - offset = (it.qs.hasherSize * 2) + offset = (hashSize * 2) case quad.Object: - offset = (it.qs.hasherSize * 2) * 2 + offset = (hashSize * 2) * 2 case quad.Label: - offset = (it.qs.hasherSize * 2) * 3 + offset = (hashSize * 2) * 3 } - val := v.(string)[offset : it.qs.hasherSize*2+offset] + val := v.(string)[offset : hashSize*2+offset] if val == it.hash { it.result = v return graph.ContainsLogOut(it, v, true) diff --git a/graph/mongo/triplestore.go b/graph/mongo/triplestore.go index 294ba91..03c4c93 100644 --- a/graph/mongo/triplestore.go +++ b/graph/mongo/triplestore.go @@ -18,6 +18,7 @@ import ( "crypto/sha1" "encoding/hex" "hash" + "sync" "gopkg.in/mgo.v2" "gopkg.in/mgo.v2/bson" @@ -34,12 +35,17 @@ func init() { const DefaultDBName = "cayley" +var ( + hashPool = sync.Pool{ + New: func() interface{} { return sha1.New() }, + } + hashSize = sha1.Size +) + type TripleStore struct { - session *mgo.Session - db *mgo.Database - hasherSize int - makeHasher func() hash.Hash - idCache *IDLru + session *mgo.Session + db *mgo.Database + idCache *IDLru } func createNewMongoGraph(addr string, options graph.Options) error { @@ -91,26 +97,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro } qs.db = conn.DB(dbName) qs.session = conn - qs.hasherSize = sha1.Size - qs.makeHasher = sha1.New qs.idCache = NewIDLru(1 << 16) return &qs, nil } func (qs *TripleStore) getIdForQuad(t quad.Quad) string { - hasher := qs.makeHasher() - id := qs.convertStringToByteHash(t.Subject, hasher) - id += qs.convertStringToByteHash(t.Predicate, hasher) - id += qs.convertStringToByteHash(t.Object, hasher) - id += qs.convertStringToByteHash(t.Label, hasher) + id := qs.convertStringToByteHash(t.Subject) + id += qs.convertStringToByteHash(t.Predicate) + id += qs.convertStringToByteHash(t.Object) + id += qs.convertStringToByteHash(t.Label) return id } -func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) string { - hasher.Reset() - key := make([]byte, 0, qs.hasherSize) - hasher.Write([]byte(s)) - key = hasher.Sum(key) +func (qs *TripleStore) convertStringToByteHash(s string) string { + h := hashPool.Get().(hash.Hash) + h.Reset() + defer hashPool.Put(h) + + key := make([]byte, 0, hashSize) + h.Write([]byte(s)) + key = h.Sum(key) return hex.EncodeToString(key) } @@ -282,8 +288,7 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator { } func (qs *TripleStore) ValueOf(s string) graph.Value { - h := qs.makeHasher() - return qs.convertStringToByteHash(s, h) + return qs.convertStringToByteHash(s) } func (qs *TripleStore) NameOf(v graph.Value) string { @@ -341,13 +346,13 @@ func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.V case quad.Subject: offset = 0 case quad.Predicate: - offset = (qs.hasherSize * 2) + offset = (hashSize * 2) case quad.Object: - offset = (qs.hasherSize * 2) * 2 + offset = (hashSize * 2) * 2 case quad.Label: - offset = (qs.hasherSize * 2) * 3 + offset = (hashSize * 2) * 3 } - val := in.(string)[offset : qs.hasherSize*2+offset] + val := in.(string)[offset : hashSize*2+offset] return val }