barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge hash pool in from master

Browse source 
Conflicts:
	graph/leveldb/triplestore.go
	graph/mongo/triplestore.go
This commit is contained in:
Barak Michener 2014-08-14 21:03:36 -04:00
commit 8720e17d87
4 changed files with 76 additions and 68 deletions
Download patch file Download diff file Expand all files Collapse all files

26
graph/leveldb/iterator.go
View file

@ -45,7 +45,7 @@ type Iterator struct {
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator { func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
vb := value.(Token) vb := value.(Token)
p := make([]byte, 0, 2+qs.hasherSize) p := make([]byte, 0, 2+hashSize)
p = append(p, []byte(prefix)...) p = append(p, []byte(prefix)...)
p = append(p, []byte(vb[1:])...) p = append(p, []byte(vb[1:])...)
@ -179,45 +179,45 @@ func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
case quad.Subject: case quad.Subject:
return 2 return 2
case quad.Predicate: case quad.Predicate:
return qs.hasherSize + 2 return hashSize + 2
case quad.Object: case quad.Object:
return 2*qs.hasherSize + 2 return 2*hashSize + 2
case quad.Label: case quad.Label:
return 3*qs.hasherSize + 2 return 3*hashSize + 2
} }
} }
if bytes.Equal(prefix, []byte("po")) { if bytes.Equal(prefix, []byte("po")) {
switch d { switch d {
case quad.Subject: case quad.Subject:
return 2*qs.hasherSize + 2 return 2*hashSize + 2
case quad.Predicate: case quad.Predicate:
return 2 return 2
case quad.Object: case quad.Object:
return qs.hasherSize + 2 return hashSize + 2
case quad.Label: case quad.Label:
return 3*qs.hasherSize + 2 return hashSize + 2
} }
} }
if bytes.Equal(prefix, []byte("os")) { if bytes.Equal(prefix, []byte("os")) {
switch d { switch d {
case quad.Subject: case quad.Subject:
return qs.hasherSize + 2 return hashSize + 2
case quad.Predicate: case quad.Predicate:
return 2*qs.hasherSize + 2 return 2*hashSize + 2
case quad.Object: case quad.Object:
return 2 return 2
case quad.Label: case quad.Label:
return 3*qs.hasherSize + 2 return 3*hashSize + 2
} }
} }
if bytes.Equal(prefix, []byte("cp")) { if bytes.Equal(prefix, []byte("cp")) {
switch d { switch d {
case quad.Subject: case quad.Subject:
return 2*qs.hasherSize + 2 return 2*hashSize + 2
case quad.Predicate: case quad.Predicate:
return qs.hasherSize + 2 return hashSize + 2
case quad.Object: case quad.Object:
return 3*qs.hasherSize + 2 return 3*hashSize + 2
case quad.Label: case quad.Label:
return 2 return 2
} }

59
graph/leveldb/triplestore.go
View file

@ -22,6 +22,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"hash" "hash"
"sync"
"github.com/barakmich/glog" "github.com/barakmich/glog"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
@ -43,6 +44,13 @@ const (
DefaultWriteBufferSize = 20 DefaultWriteBufferSize = 20
) )
var (
hashPool = sync.Pool{
New: func() interface{} { return sha1.New() },
}
hashSize = sha1.Size
)
type Token []byte type Token []byte
func (t Token) Key() interface{} { func (t Token) Key() interface{} {
@ -50,17 +58,14 @@ func (t Token) Key() interface{} {
} }
type TripleStore struct { type TripleStore struct {
dbOpts *opt.Options dbOpts *opt.Options
db *leveldb.DB db *leveldb.DB
path string path string
open bool open bool
size int64 size int64
horizon int64 horizon int64
hasher hash.Hash writeopts *opt.WriteOptions
hasherSize int readopts *opt.ReadOptions
makeHasher func() hash.Hash
writeopts *opt.WriteOptions
readopts *opt.ReadOptions
} }
func createNewLevelDB(path string, _ graph.Options) error { func createNewLevelDB(path string, _ graph.Options) error {
@ -98,8 +103,6 @@ func newTripleStore(path string, options graph.Options) (graph.TripleStore, erro
write_buffer_mb = val write_buffer_mb = val
} }
qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB qs.dbOpts.WriteBuffer = write_buffer_mb * opt.MiB
qs.hasherSize = sha1.Size
qs.makeHasher = sha1.New
qs.writeopts = &opt.WriteOptions{ qs.writeopts = &opt.WriteOptions{
Sync: false, Sync: false,
} }
@ -144,22 +147,20 @@ func (qa *TripleStore) createDeltaKeyFor(d graph.Delta) []byte {
} }
func (qs *TripleStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte { func (qs *TripleStore) createKeyFor(d [4]quad.Direction, triple quad.Quad) []byte {
hasher := qs.makeHasher() key := make([]byte, 0, 2+(hashSize*3))
key := make([]byte, 0, 2+(qs.hasherSize*3))
// TODO(kortschak) Remove dependence on String() method. // TODO(kortschak) Remove dependence on String() method.
key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...) key = append(key, []byte{d[0].Prefix(), d[1].Prefix()}...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[0]), hasher)...) key = append(key, qs.convertStringToByteHash(triple.Get(d[0]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[1]), hasher)...) key = append(key, qs.convertStringToByteHash(triple.Get(d[1]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[2]), hasher)...) key = append(key, qs.convertStringToByteHash(triple.Get(d[2]))...)
key = append(key, qs.convertStringToByteHash(triple.Get(d[3]), hasher)...) key = append(key, qs.convertStringToByteHash(triple.Get(d[3]))...)
return key return key
} }
func (qs *TripleStore) createValueKeyFor(s string) []byte { func (qs *TripleStore) createValueKeyFor(s string) []byte {
hasher := qs.makeHasher() key := make([]byte, 0, 1+hashSize)
key := make([]byte, 0, 1+qs.hasherSize)
key = append(key, []byte("z")...) key = append(key, []byte("z")...)
key = append(key, qs.convertStringToByteHash(s, hasher)...) key = append(key, qs.convertStringToByteHash(s)...)
return key return key
} }
@ -346,11 +347,13 @@ func (qs *TripleStore) Quad(k graph.Value) quad.Quad {
return triple return triple
} }
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) []byte { func (qs *TripleStore) convertStringToByteHash(s string) []byte {
hasher.Reset() h := hashPool.Get().(hash.Hash)
key := make([]byte, 0, qs.hasherSize) h.Reset()
hasher.Write([]byte(s)) defer hashPool.Put(h)
key = hasher.Sum(key) key := make([]byte, 0, hashSize)
h.Write([]byte(s))
key = h.Sum(key)
return key return key
} }
@ -467,7 +470,7 @@ func (qs *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.
v := val.(Token) v := val.(Token)
offset := PositionOf(v[0:2], d, qs) offset := PositionOf(v[0:2], d, qs)
if offset != -1 { if offset != -1 {
return Token(append([]byte("z"), v[offset:offset+qs.hasherSize]...)) return Token(append([]byte("z"), v[offset:offset+hashSize]...))
} else { } else {
return Token(qs.Quad(val).Get(d)) return Token(qs.Quad(val).Get(d))
} }

8
graph/mongo/iterator.go
View file

@ -177,13 +177,13 @@ func (it *Iterator) Contains(v graph.Value) bool {
case quad.Subject: case quad.Subject:
offset = 0 offset = 0
case quad.Predicate: case quad.Predicate:
offset = (it.qs.hasherSize * 2) offset = (hashSize * 2)
case quad.Object: case quad.Object:
offset = (it.qs.hasherSize * 2) * 2 offset = (hashSize * 2) * 2
case quad.Label: case quad.Label:
offset = (it.qs.hasherSize * 2) * 3 offset = (hashSize * 2) * 3
} }
val := v.(string)[offset : it.qs.hasherSize*2+offset] val := v.(string)[offset : hashSize*2+offset]
if val == it.hash { if val == it.hash {
it.result = v it.result = v
return graph.ContainsLogOut(it, v, true) return graph.ContainsLogOut(it, v, true)

51
graph/mongo/triplestore.go
View file

@ -18,6 +18,7 @@ import (
"crypto/sha1" "crypto/sha1"
"encoding/hex" "encoding/hex"
"hash" "hash"
"sync"
"gopkg.in/mgo.v2" "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson" "gopkg.in/mgo.v2/bson"
@ -34,12 +35,17 @@ func init() {
const DefaultDBName = "cayley" const DefaultDBName = "cayley"
var (
hashPool = sync.Pool{
New: func() interface{} { return sha1.New() },
}
hashSize = sha1.Size
)
type TripleStore struct { type TripleStore struct {
session *mgo.Session session *mgo.Session
db *mgo.Database db *mgo.Database
hasherSize int idCache *IDLru
makeHasher func() hash.Hash
idCache *IDLru
} }
func createNewMongoGraph(addr string, options graph.Options) error { func createNewMongoGraph(addr string, options graph.Options) error {
@ -91,26 +97,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
} }
qs.db = conn.DB(dbName) qs.db = conn.DB(dbName)
qs.session = conn qs.session = conn
qs.hasherSize = sha1.Size
qs.makeHasher = sha1.New
qs.idCache = NewIDLru(1 << 16) qs.idCache = NewIDLru(1 << 16)
return &qs, nil return &qs, nil
} }
func (qs *TripleStore) getIdForQuad(t quad.Quad) string { func (qs *TripleStore) getIdForQuad(t quad.Quad) string {
hasher := qs.makeHasher() id := qs.convertStringToByteHash(t.Subject)
id := qs.convertStringToByteHash(t.Subject, hasher) id += qs.convertStringToByteHash(t.Predicate)
id += qs.convertStringToByteHash(t.Predicate, hasher) id += qs.convertStringToByteHash(t.Object)
id += qs.convertStringToByteHash(t.Object, hasher) id += qs.convertStringToByteHash(t.Label)
id += qs.convertStringToByteHash(t.Label, hasher)
return id return id
} }
func (qs *TripleStore) convertStringToByteHash(s string, hasher hash.Hash) string { func (qs *TripleStore) convertStringToByteHash(s string) string {
hasher.Reset() h := hashPool.Get().(hash.Hash)
key := make([]byte, 0, qs.hasherSize) h.Reset()
hasher.Write([]byte(s)) defer hashPool.Put(h)
key = hasher.Sum(key)
key := make([]byte, 0, hashSize)
h.Write([]byte(s))
key = h.Sum(key)
return hex.EncodeToString(key) return hex.EncodeToString(key)
} }
@ -282,8 +288,7 @@ func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
} }
func (qs *TripleStore) ValueOf(s string) graph.Value { func (qs *TripleStore) ValueOf(s string) graph.Value {
h := qs.makeHasher() return qs.convertStringToByteHash(s)
return qs.convertStringToByteHash(s, h)
} }
func (qs *TripleStore) NameOf(v graph.Value) string { func (qs *TripleStore) NameOf(v graph.Value) string {
@ -341,13 +346,13 @@ func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.V
case quad.Subject: case quad.Subject:
offset = 0 offset = 0
case quad.Predicate: case quad.Predicate:
offset = (qs.hasherSize * 2) offset = (hashSize * 2)
case quad.Object: case quad.Object:
offset = (qs.hasherSize * 2) * 2 offset = (hashSize * 2) * 2
case quad.Label: case quad.Label:
offset = (qs.hasherSize * 2) * 3 offset = (hashSize * 2) * 3
} }
val := in.(string)[offset : qs.hasherSize*2+offset] val := in.(string)[offset : hashSize*2+offset]
return val return val
} }