From e780c1ceb99cd1484b362db9af1881f07142eb82 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Wed, 16 Jul 2014 13:49:39 -0400 Subject: [PATCH 1/8] Register should be RegisterIterator --- graph/iterator.go | 4 ++-- graph/leveldb/iterator.go | 2 +- graph/memstore/iterator.go | 2 +- graph/mongo/iterator.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/graph/iterator.go b/graph/iterator.go index 3ad28de..27a6f02 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -167,12 +167,12 @@ var ( } ) -// Register adds a new iterator type to the set of acceptable types, returning +// RegisterIterator adds a new iterator type to the set of acceptable types, returning // the registered Type. // Calls to Register are idempotent and must be made prior to use of the iterator. // The conventional approach for use is to include a call to Register in a package // init() function, saving the Type to a private package var. -func Register(name string) Type { +func RegisterIterator(name string) Type { lock.Lock() defer lock.Unlock() for i, t := range types { diff --git a/graph/leveldb/iterator.go b/graph/leveldb/iterator.go index be044e9..0e3f6e0 100644 --- a/graph/leveldb/iterator.go +++ b/graph/leveldb/iterator.go @@ -198,7 +198,7 @@ func (it *Iterator) DebugString(indent int) string { var levelDBType graph.Type func init() { - levelDBType = graph.Register("leveldb") + levelDBType = graph.RegisterIterator("leveldb") } func Type() graph.Type { return levelDBType } diff --git a/graph/memstore/iterator.go b/graph/memstore/iterator.go index b3bf9f4..3b7bfa8 100644 --- a/graph/memstore/iterator.go +++ b/graph/memstore/iterator.go @@ -104,7 +104,7 @@ func (it *Iterator) DebugString(indent int) string { var memType graph.Type func init() { - memType = graph.Register("llrb") + memType = graph.RegisterIterator("llrb") } func Type() graph.Type { return memType } diff --git a/graph/mongo/iterator.go b/graph/mongo/iterator.go index 4587b3c..58c50e4 100644 --- a/graph/mongo/iterator.go +++ b/graph/mongo/iterator.go @@ -160,7 +160,7 @@ func (it *Iterator) Size() (int64, bool) { var mongoType graph.Type func init() { - mongoType = graph.Register("mongo") + mongoType = graph.RegisterIterator("mongo") } func Type() graph.Type { return mongoType } From d808d9347cef2b368f6378a5d9a36a147ea44787 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Wed, 16 Jul 2014 16:49:55 -0400 Subject: [PATCH 2/8] move to registry interface for backends --- cayley.go | 7 ++++++- config/config.go | 2 +- db/init.go | 22 ++++++++-------------- db/load.go | 36 +++++++++++++++--------------------- db/open.go | 24 +++++++++++------------- graph/leveldb/triplestore.go | 16 +++++++++++----- graph/memstore/triplestore.go | 8 +++++++- graph/mongo/triplestore.go | 26 ++++++++++++++++++-------- graph/triplestore.go | 40 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 117 insertions(+), 64 deletions(-) diff --git a/cayley.go b/cayley.go index a848931..7ff1fd8 100644 --- a/cayley.go +++ b/cayley.go @@ -26,6 +26,11 @@ import ( "github.com/google/cayley/db" "github.com/google/cayley/graph" "github.com/google/cayley/http" + + // load all supported backends + _ "github.com/google/cayley/graph/leveldb" + _ "github.com/google/cayley/graph/memstore" + _ "github.com/google/cayley/graph/mongo" ) var tripleFile = flag.String("triples", "", "Triple File to load before going to REPL.") @@ -72,7 +77,7 @@ func main() { db.Init(cfg, *tripleFile) case "load": ts = db.Open(cfg) - db.Load(ts, cfg, *tripleFile, false) + db.Load(ts, cfg, *tripleFile) ts.Close() case "repl": ts = db.Open(cfg) diff --git a/config/config.go b/config/config.go index 1c577fa..d521043 100644 --- a/config/config.go +++ b/config/config.go @@ -34,7 +34,7 @@ type Config struct { } var databasePath = flag.String("dbpath", "/tmp/testdb", "Path to the database.") -var databaseBackend = flag.String("db", "mem", "Database Backend.") +var databaseBackend = flag.String("db", "memstore", "Database Backend.") var host = flag.String("host", "0.0.0.0", "Host to listen on (defaults to all).") var loadSize = flag.Int("load_size", 10000, "Size of triplesets to load") var port = flag.String("port", "64210", "Port to listen on.") diff --git a/db/init.go b/db/init.go index 8efb863..a8b3897 100644 --- a/db/init.go +++ b/db/init.go @@ -16,25 +16,19 @@ package db import ( "github.com/google/cayley/config" - "github.com/google/cayley/graph/leveldb" - "github.com/google/cayley/graph/mongo" + "github.com/google/cayley/graph" ) func Init(cfg *config.Config, triplePath string) bool { - created := false - dbpath := cfg.DatabasePath - switch cfg.DatabaseType { - case "mongo", "mongodb": - created = mongo.CreateNewMongoGraph(dbpath, cfg.DatabaseOptions) - case "leveldb": - created = leveldb.CreateNewLevelDB(dbpath) - case "mem": - return true + err := graph.InitTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) + if err != nil { + return false } - if created && triplePath != "" { + + if triplePath != "" { ts := Open(cfg) - Load(ts, cfg, triplePath, true) + Load(ts, cfg, triplePath) ts.Close() } - return created + return true } diff --git a/db/load.go b/db/load.go index f09962e..d4d2bd4 100644 --- a/db/load.go +++ b/db/load.go @@ -21,30 +21,26 @@ import ( "github.com/google/cayley/config" "github.com/google/cayley/graph" - "github.com/google/cayley/graph/mongo" "github.com/google/cayley/nquads" ) -func Load(ts graph.TripleStore, cfg *config.Config, triplePath string, firstTime bool) { - switch cfg.DatabaseType { - case "mongo", "mongodb": - if firstTime { - loadMongo(ts.(*mongo.TripleStore), triplePath) - } else { - LoadTriplesFromFileInto(ts, triplePath, cfg.LoadSize) - } - case "leveldb": - LoadTriplesFromFileInto(ts, triplePath, cfg.LoadSize) - case "mem": - LoadTriplesFromFileInto(ts, triplePath, cfg.LoadSize) - } - +type bulkLoadable interface { + // BulkLoad loads Triples from a channel in bulk to the TripleStore. It + // returns false if bulk loading is not possible (i.e. if you cannot load + // in bulk to a non-empty database, and the current database is non-empty) + BulkLoad(chan *graph.Triple) bool } -func loadMongo(ts *mongo.TripleStore, path string) { +func Load(ts graph.TripleStore, cfg *config.Config, triplePath string) { tChan := make(chan *graph.Triple) - go ReadTriplesFromFile(tChan, path) - ts.BulkLoad(tChan) + go ReadTriplesFromFile(tChan, triplePath) + + bulker, canBulk := ts.(bulkLoadable) + if canBulk && bulker.BulkLoad(tChan) { + return + } + + LoadTriplesInto(tChan, ts, cfg.LoadSize) } func ReadTriplesFromFile(c chan *graph.Triple, tripleFile string) { @@ -62,9 +58,7 @@ func ReadTriplesFromFile(c chan *graph.Triple, tripleFile string) { nquads.ReadNQuadsFromReader(c, f) } -func LoadTriplesFromFileInto(ts graph.TripleStore, filename string, loadSize int) { - tChan := make(chan *graph.Triple) - go ReadTriplesFromFile(tChan, filename) +func LoadTriplesInto(tChan chan *graph.Triple, ts graph.TripleStore, loadSize int) { tripleblock := make([]*graph.Triple, loadSize) i := 0 for t := range tChan { diff --git a/db/open.go b/db/open.go index 6997ba9..b0973d7 100644 --- a/db/open.go +++ b/db/open.go @@ -19,22 +19,20 @@ import ( "github.com/google/cayley/config" "github.com/google/cayley/graph" - "github.com/google/cayley/graph/leveldb" - "github.com/google/cayley/graph/memstore" - "github.com/google/cayley/graph/mongo" ) func Open(cfg *config.Config) graph.TripleStore { glog.Infof("Opening database \"%s\" at %s", cfg.DatabaseType, cfg.DatabasePath) - switch cfg.DatabaseType { - case "mongo", "mongodb": - return mongo.NewTripleStore(cfg.DatabasePath, cfg.DatabaseOptions) - case "leveldb": - return leveldb.NewTripleStore(cfg.DatabasePath, cfg.DatabaseOptions) - case "mem": - ts := memstore.NewTripleStore() - Load(ts, cfg, cfg.DatabasePath, true) - return ts + ts, err := graph.NewTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) + if err != nil { + glog.Fatalln(err.Error()) + return nil } - panic("Unsupported database backend " + cfg.DatabaseType) + + // memstore is not persistent, so MUST be loaded + if cfg.DatabaseType == "memstore" { + Load(ts, cfg, cfg.DatabasePath) + } + + return ts } diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 7880165..798895a 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -48,12 +48,12 @@ type TripleStore struct { readopts *opt.ReadOptions } -func CreateNewLevelDB(path string) bool { +func createNewLevelDB(path string, _ graph.Options) error { opts := &opt.Options{} db, err := leveldb.OpenFile(path, opts) if err != nil { glog.Errorln("Error: couldn't create database: ", err) - return false + return err } defer db.Close() ts := &TripleStore{} @@ -62,10 +62,10 @@ func CreateNewLevelDB(path string) bool { Sync: true, } ts.Close() - return true + return nil } -func NewTripleStore(path string, options graph.Options) *TripleStore { +func newTripleStore(path string, options graph.Options) (graph.TripleStore, error) { var ts TripleStore ts.path = path cache_size := DefaultCacheSize @@ -94,7 +94,7 @@ func NewTripleStore(path string, options graph.Options) *TripleStore { ts.db = db glog.Infoln(ts.GetStats()) ts.getSize() - return &ts + return &ts, nil } func (ts *TripleStore) GetStats() string { @@ -443,3 +443,9 @@ func compareBytes(a, b graph.Value) bool { func (ts *TripleStore) FixedIterator() graph.FixedIterator { return iterator.NewFixedIteratorWithCompare(compareBytes) } + +func init() { + graph.RegisterTripleStore("leveldb", + graph.TripleStoreGetter(newTripleStore), + graph.TripleStoreInit(createNewLevelDB)) +} diff --git a/graph/memstore/triplestore.go b/graph/memstore/triplestore.go index a53422c..29fe94c 100644 --- a/graph/memstore/triplestore.go +++ b/graph/memstore/triplestore.go @@ -79,7 +79,7 @@ type TripleStore struct { // vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree } -func NewTripleStore() *TripleStore { +func newTripleStore() *TripleStore { var ts TripleStore ts.idMap = make(map[string]int64) ts.revIdMap = make(map[int64]string) @@ -268,3 +268,9 @@ func (ts *TripleStore) NodesAllIterator() graph.Iterator { return NewMemstoreAllIterator(ts) } func (ts *TripleStore) Close() {} + +func init() { + graph.RegisterTripleStore("memstore", func(string, graph.Options) (graph.TripleStore, error) { + return newTripleStore(), nil + }) +} diff --git a/graph/mongo/triplestore.go b/graph/mongo/triplestore.go index 001c539..7ff220f 100644 --- a/graph/mongo/triplestore.go +++ b/graph/mongo/triplestore.go @@ -37,11 +37,10 @@ type TripleStore struct { idCache *IDLru } -func CreateNewMongoGraph(addr string, options graph.Options) bool { +func createNewMongoGraph(addr string, options graph.Options) error { conn, err := mgo.Dial(addr) if err != nil { - glog.Fatal("Error connecting: ", err) - return false + return err } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName @@ -63,14 +62,14 @@ func CreateNewMongoGraph(addr string, options graph.Options) bool { db.C("triples").EnsureIndex(indexOpts) indexOpts.Key = []string{"Provenance"} db.C("triples").EnsureIndex(indexOpts) - return true + return nil } -func NewTripleStore(addr string, options graph.Options) *TripleStore { +func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) { var ts TripleStore conn, err := mgo.Dial(addr) if err != nil { - glog.Fatal("Error connecting: ", err) + return nil, err } conn.SetSafe(&mgo.Safe{}) dbName := DefaultDBName @@ -81,7 +80,7 @@ func NewTripleStore(addr string, options graph.Options) *TripleStore { ts.session = conn ts.hasher = sha1.New() ts.idCache = NewIDLru(1 << 16) - return &ts + return &ts, nil } func (ts *TripleStore) getIdForTriple(t *graph.Triple) string { @@ -291,7 +290,11 @@ func (ts *TripleStore) TripleDirection(in graph.Value, d graph.Direction) graph. return val } -func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) { +func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool { + if ts.Size() != 0 { + return false + } + ts.session.SetSafe(nil) for triple := range t_chan { ts.writeTriple(triple) @@ -334,4 +337,11 @@ func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) { }) }`}, {"args", bson.D{}}}, nil) ts.session.SetSafe(&mgo.Safe{}) + return true +} + +func init() { + graph.RegisterTripleStore("mongo", + graph.TripleStoreGetter(newTripleStore), + graph.TripleStoreInit(createNewMongoGraph)) } diff --git a/graph/triplestore.go b/graph/triplestore.go index 92ed9e9..f8ddef9 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -22,6 +22,7 @@ package graph // triple backing store we prefer. import ( + "fmt" "github.com/barakmich/glog" ) @@ -117,3 +118,42 @@ func (d Options) StringKey(key string) (string, bool) { } return "", false } + +type TripleStoreGetter func(string, Options) (TripleStore, error) +type TripleStoreInit func(string, Options) error + +var storeRegistry = make(map[string]TripleStoreGetter) +var storeInitRegistry = make(map[string]TripleStoreInit) + +func RegisterTripleStore(name string, getter TripleStoreGetter, initer ...TripleStoreInit) { + if _, found := storeRegistry[name]; found { + panic("already registered TripleStore " + name) + } + storeRegistry[name] = getter + if len(initer) > 0 { + storeInitRegistry[name] = initer[0] + } +} + +func NewTripleStore(name, dbpath string, opts Options) (TripleStore, error) { + getter, hasGetter := storeRegistry[name] + if !hasGetter { + return nil, fmt.Errorf("unknown triplestore '%s'", name) + } + return getter(dbpath, opts) +} + +func InitTripleStore(name, dbpath string, opts Options) error { + initer, hasInit := storeInitRegistry[name] + if hasInit { + return initer(dbpath, opts) + } + return fmt.Errorf("unknown triplestore '%s'", name) +} + +func TripleStores() (t []string) { + for n := range storeRegistry { + t = append(t, n) + } + return +} From f9c60a5f30331ef1ef765e185a46810381e36a90 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Fri, 18 Jul 2014 11:07:19 -0400 Subject: [PATCH 3/8] update names per discussion at google/cayley#38 --- db/load.go | 19 +++++++++---------- graph/leveldb/triplestore.go | 4 +--- graph/memstore/triplestore.go | 2 +- graph/mongo/triplestore.go | 4 +--- graph/triplestore.go | 38 +++++++++++++++++++++++++------------- 5 files changed, 37 insertions(+), 30 deletions(-) diff --git a/db/load.go b/db/load.go index d4d2bd4..d81ee35 100644 --- a/db/load.go +++ b/db/load.go @@ -24,20 +24,19 @@ import ( "github.com/google/cayley/nquads" ) -type bulkLoadable interface { - // BulkLoad loads Triples from a channel in bulk to the TripleStore. It - // returns false if bulk loading is not possible (i.e. if you cannot load - // in bulk to a non-empty database, and the current database is non-empty) - BulkLoad(chan *graph.Triple) bool -} - func Load(ts graph.TripleStore, cfg *config.Config, triplePath string) { tChan := make(chan *graph.Triple) go ReadTriplesFromFile(tChan, triplePath) - bulker, canBulk := ts.(bulkLoadable) - if canBulk && bulker.BulkLoad(tChan) { - return + bulker, canBulk := ts.(graph.BulkLoader) + if canBulk { + err := bulker.BulkLoad(tChan) + if err == nil { + return + } + if err != graph.ErrCannotBulkLoad { + glog.Errorln("Error attempting to bulk load: ", err) + } } LoadTriplesInto(tChan, ts, cfg.LoadSize) diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 798895a..75c995c 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -445,7 +445,5 @@ func (ts *TripleStore) FixedIterator() graph.FixedIterator { } func init() { - graph.RegisterTripleStore("leveldb", - graph.TripleStoreGetter(newTripleStore), - graph.TripleStoreInit(createNewLevelDB)) + graph.RegisterTripleStore("leveldb", newTripleStore, createNewLevelDB) } diff --git a/graph/memstore/triplestore.go b/graph/memstore/triplestore.go index 29fe94c..5b3413e 100644 --- a/graph/memstore/triplestore.go +++ b/graph/memstore/triplestore.go @@ -272,5 +272,5 @@ func (ts *TripleStore) Close() {} func init() { graph.RegisterTripleStore("memstore", func(string, graph.Options) (graph.TripleStore, error) { return newTripleStore(), nil - }) + }, nil) } diff --git a/graph/mongo/triplestore.go b/graph/mongo/triplestore.go index 7ff220f..81b1ceb 100644 --- a/graph/mongo/triplestore.go +++ b/graph/mongo/triplestore.go @@ -341,7 +341,5 @@ func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool { } func init() { - graph.RegisterTripleStore("mongo", - graph.TripleStoreGetter(newTripleStore), - graph.TripleStoreInit(createNewMongoGraph)) + graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph) } diff --git a/graph/triplestore.go b/graph/triplestore.go index f8ddef9..9b73735 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -119,34 +119,46 @@ func (d Options) StringKey(key string) (string, bool) { return "", false } -type TripleStoreGetter func(string, Options) (TripleStore, error) -type TripleStoreInit func(string, Options) error +var ErrCannotBulkLoad = fmt.Errorf("cannot bulk load") -var storeRegistry = make(map[string]TripleStoreGetter) -var storeInitRegistry = make(map[string]TripleStoreInit) +type BulkLoader interface { + // BulkLoad loads Triples from a channel in bulk to the TripleStore. It + // returns ErrCannotBulkLoad if bulk loading is not possible (i.e. if you + // cannot load in bulk to a non-empty database, and the db is non-empty) + BulkLoad(chan *Triple) error +} -func RegisterTripleStore(name string, getter TripleStoreGetter, initer ...TripleStoreInit) { +type NewStoreFunc func(string, Options) (TripleStore, error) +type InitStoreFunc func(string, Options) error + +var storeRegistry = make(map[string]NewStoreFunc) +var storeInitRegistry = make(map[string]InitStoreFunc) + +func RegisterTripleStore(name string, newFunc NewStoreFunc, initFunc InitStoreFunc) { if _, found := storeRegistry[name]; found { panic("already registered TripleStore " + name) } - storeRegistry[name] = getter - if len(initer) > 0 { - storeInitRegistry[name] = initer[0] + storeRegistry[name] = newFunc + if initFunc != nil { + storeInitRegistry[name] = initFunc } } func NewTripleStore(name, dbpath string, opts Options) (TripleStore, error) { - getter, hasGetter := storeRegistry[name] - if !hasGetter { + newFunc, hasNew := storeRegistry[name] + if !hasNew { return nil, fmt.Errorf("unknown triplestore '%s'", name) } - return getter(dbpath, opts) + return newFunc(dbpath, opts) } func InitTripleStore(name, dbpath string, opts Options) error { - initer, hasInit := storeInitRegistry[name] + initFunc, hasInit := storeInitRegistry[name] if hasInit { - return initer(dbpath, opts) + return initFunc(dbpath, opts) + } + if _, isRegistered := storeRegistry[name]; isRegistered { + return nil } return fmt.Errorf("unknown triplestore '%s'", name) } From 0641309a8fc121785fc879b776f03244db742e51 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Fri, 18 Jul 2014 11:46:36 -0400 Subject: [PATCH 4/8] adding to A+C --- AUTHORS | 1 + CONTRIBUTORS | 1 + 2 files changed, 2 insertions(+) diff --git a/AUTHORS b/AUTHORS index e769a7d..7035a5b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -11,6 +11,7 @@ Alexander Peters Google Inc. Jay Graves +Jeremy Jay Pius Uzamere Robert Daniel Kortschak Timothy Armstrong diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 7d4f7a5..10ae14f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -14,6 +14,7 @@ Alexander Peters Barak Michener Jay Graves +Jeremy Jay Pius Uzamere Robert Daniel Kortschak Timothy Armstrong From a6dc5c953273609facfb9674a0fd121451d55cc9 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Fri, 18 Jul 2014 14:09:57 -0400 Subject: [PATCH 5/8] fix tests for new method names --- graph/leveldb/leveldb_test.go | 74 ++++++++++++++++++++------------------ graph/memstore/triplestore_test.go | 2 +- graph/sexp/parser_test.go | 10 +++--- query/gremlin/gremlin_test.go | 4 +-- query/mql/mql_test.go | 4 +-- 5 files changed, 49 insertions(+), 45 deletions(-) diff --git a/graph/leveldb/leveldb_test.go b/graph/leveldb/leveldb_test.go index 3e50122..59b32b0 100644 --- a/graph/leveldb/leveldb_test.go +++ b/graph/leveldb/leveldb_test.go @@ -102,12 +102,13 @@ func TestCreateDatabase(t *testing.T) { } t.Log(tmpDir) - if created := CreateNewLevelDB(tmpDir); !created { + err = createNewLevelDB(tmpDir, nil) + if err != nil { t.Fatal("Failed to create LevelDB database.") } - ts := NewTripleStore(tmpDir, nil) - if ts == nil { + ts, err := newTripleStore(tmpDir, nil) + if ts == nil || err != nil { t.Error("Failed to create leveldb TripleStore.") } if s := ts.Size(); s != 0 { @@ -115,21 +116,10 @@ func TestCreateDatabase(t *testing.T) { } ts.Close() - if created := CreateNewLevelDB("/dev/null/some terrible path"); created { + err = createNewLevelDB("/dev/null/some terrible path", nil) + if err == nil { t.Errorf("Created LevelDB database for bad path.") } - // TODO(kortschak) Invalidate this test by using error returns rather than panics. - var panicked bool - func() { - defer func() { - r := recover() - panicked = r != nil - }() - NewTripleStore("/dev/null/some terrible path", nil) - }() - if !panicked { - t.Error("NewTripleStore failed to panic with bad path.") - } os.RemoveAll(tmpDir) } @@ -142,14 +132,13 @@ func TestLoadDatabase(t *testing.T) { defer os.RemoveAll(tmpDir) t.Log(tmpDir) - if created := CreateNewLevelDB(tmpDir); !created { + err = createNewLevelDB(tmpDir, nil) + if err != nil { t.Fatal("Failed to create LevelDB database.") } - var ts *TripleStore - - ts = NewTripleStore(tmpDir, nil) - if ts == nil { + ts, err := newTripleStore(tmpDir, nil) + if ts == nil || err != nil { t.Error("Failed to create leveldb TripleStore.") } @@ -164,19 +153,25 @@ func TestLoadDatabase(t *testing.T) { } ts.Close() - if created := CreateNewLevelDB(tmpDir); !created { + err = createNewLevelDB(tmpDir, nil) + if err != nil { t.Fatal("Failed to create LevelDB database.") } - ts = NewTripleStore(tmpDir, nil) - if ts == nil { + ts, err = newTripleStore(tmpDir, nil) + if ts == nil || err != nil { t.Error("Failed to create leveldb TripleStore.") } + ts2, didConvert := ts.(*TripleStore) + if !didConvert { + t.Errorf("Could not convert from generic to LevelDB TripleStore") + } + ts.AddTripleSet(makeTripleSet()) if s := ts.Size(); s != 11 { t.Errorf("Unexpected triplestore size, got:%d expect:11", s) } - if s := ts.SizeOf(ts.ValueOf("B")); s != 5 { + if s := ts2.SizeOf(ts.ValueOf("B")); s != 5 { t.Errorf("Unexpected triplestore size, got:%d expect:5", s) } @@ -184,7 +179,7 @@ func TestLoadDatabase(t *testing.T) { if s := ts.Size(); s != 10 { t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s) } - if s := ts.SizeOf(ts.ValueOf("B")); s != 4 { + if s := ts2.SizeOf(ts.ValueOf("B")); s != 4 { t.Errorf("Unexpected triplestore size, got:%d expect:4", s) } @@ -199,12 +194,15 @@ func TestIterator(t *testing.T) { defer os.RemoveAll(tmpDir) t.Log(tmpDir) - if created := CreateNewLevelDB(tmpDir); !created { + err = createNewLevelDB(tmpDir, nil) + if err != nil { t.Fatal("Failed to create LevelDB database.") } - var ts *TripleStore - ts = NewTripleStore(tmpDir, nil) + ts, err := newTripleStore(tmpDir, nil) + if ts == nil || err != nil { + t.Error("Failed to create leveldb TripleStore.") + } ts.AddTripleSet(makeTripleSet()) var it graph.Iterator @@ -289,12 +287,15 @@ func TestSetIterator(t *testing.T) { tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test") t.Log(tmpDir) defer os.RemoveAll(tmpDir) - ok := CreateNewLevelDB(tmpDir) - if !ok { + err := createNewLevelDB(tmpDir, nil) + if err != nil { t.Fatalf("Failed to create working directory") } - ts := NewTripleStore(tmpDir, nil) + ts, err := newTripleStore(tmpDir, nil) + if ts == nil || err != nil { + t.Error("Failed to create leveldb TripleStore.") + } defer ts.Close() ts.AddTripleSet(makeTripleSet()) @@ -401,11 +402,14 @@ func TestOptimize(t *testing.T) { tmpDir, _ := ioutil.TempDir(os.TempDir(), "cayley_test") t.Log(tmpDir) defer os.RemoveAll(tmpDir) - ok := CreateNewLevelDB(tmpDir) - if !ok { + err := createNewLevelDB(tmpDir, nil) + if err != nil { t.Fatalf("Failed to create working directory") } - ts := NewTripleStore(tmpDir, nil) + ts, err := newTripleStore(tmpDir, nil) + if ts == nil || err != nil { + t.Error("Failed to create leveldb TripleStore.") + } ts.AddTripleSet(makeTripleSet()) // With an linksto-fixed pair diff --git a/graph/memstore/triplestore_test.go b/graph/memstore/triplestore_test.go index 1372ef2..8b911f1 100644 --- a/graph/memstore/triplestore_test.go +++ b/graph/memstore/triplestore_test.go @@ -52,7 +52,7 @@ var simpleGraph = []*graph.Triple{ func makeTestStore(data []*graph.Triple) (*TripleStore, []pair) { seen := make(map[string]struct{}) - ts := NewTripleStore() + ts := newTripleStore() var ( val int64 ind []pair diff --git a/graph/sexp/parser_test.go b/graph/sexp/parser_test.go index 78f7886..bd9c13b 100644 --- a/graph/sexp/parser_test.go +++ b/graph/sexp/parser_test.go @@ -18,7 +18,7 @@ import ( "testing" "github.com/google/cayley/graph" - "github.com/google/cayley/graph/memstore" + _ "github.com/google/cayley/graph/memstore" ) func TestBadParse(t *testing.T) { @@ -52,7 +52,7 @@ var testQueries = []struct { } func TestMemstoreBackedSexp(t *testing.T) { - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) it := BuildIteratorTreeForQuery(ts, "()") if it.Type() != graph.Null { t.Errorf(`Incorrect type for empty query, got:%q expect: "null"`, it.Type()) @@ -76,7 +76,7 @@ func TestMemstoreBackedSexp(t *testing.T) { } func TestTreeConstraintParse(t *testing.T) { - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) ts.AddTriple(&graph.Triple{"i", "like", "food", ""}) ts.AddTriple(&graph.Triple{"food", "is", "good", ""}) query := "(\"i\"\n" + @@ -96,7 +96,7 @@ func TestTreeConstraintParse(t *testing.T) { } func TestTreeConstraintTagParse(t *testing.T) { - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) ts.AddTriple(&graph.Triple{"i", "like", "food", ""}) ts.AddTriple(&graph.Triple{"food", "is", "good", ""}) query := "(\"i\"\n" + @@ -116,7 +116,7 @@ func TestTreeConstraintTagParse(t *testing.T) { } func TestMultipleConstraintParse(t *testing.T) { - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) for _, tv := range []*graph.Triple{ {"i", "like", "food", ""}, {"i", "like", "beer", ""}, diff --git a/query/gremlin/gremlin_test.go b/query/gremlin/gremlin_test.go index 416c651..3b19885 100644 --- a/query/gremlin/gremlin_test.go +++ b/query/gremlin/gremlin_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/google/cayley/graph" - "github.com/google/cayley/graph/memstore" + _ "github.com/google/cayley/graph/memstore" ) // This is a simple test graph. @@ -51,7 +51,7 @@ var simpleGraph = []*graph.Triple{ } func makeTestSession(data []*graph.Triple) *Session { - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) for _, t := range data { ts.AddTriple(t) } diff --git a/query/mql/mql_test.go b/query/mql/mql_test.go index a1afdb8..9129898 100644 --- a/query/mql/mql_test.go +++ b/query/mql/mql_test.go @@ -20,7 +20,7 @@ import ( "testing" "github.com/google/cayley/graph" - "github.com/google/cayley/graph/memstore" + _ "github.com/google/cayley/graph/memstore" ) // This is a simple test graph. @@ -51,7 +51,7 @@ var simpleGraph = []*graph.Triple{ } func makeTestSession(data []*graph.Triple) *Session { - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) for _, t := range data { ts.AddTriple(t) } From 5731ca7b42779ca4c35bd10ba024ef584b892d36 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Fri, 18 Jul 2014 14:19:02 -0400 Subject: [PATCH 6/8] 6 of one, half-dozen of the other --- graph/triplestore.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/graph/triplestore.go b/graph/triplestore.go index 9b73735..76a18cc 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -22,7 +22,7 @@ package graph // triple backing store we prefer. import ( - "fmt" + "errors" "github.com/barakmich/glog" ) @@ -119,7 +119,7 @@ func (d Options) StringKey(key string) (string, bool) { return "", false } -var ErrCannotBulkLoad = fmt.Errorf("cannot bulk load") +var ErrCannotBulkLoad = errors.New("triplestore: cannot bulk load") type BulkLoader interface { // BulkLoad loads Triples from a channel in bulk to the TripleStore. It @@ -147,7 +147,7 @@ func RegisterTripleStore(name string, newFunc NewStoreFunc, initFunc InitStoreFu func NewTripleStore(name, dbpath string, opts Options) (TripleStore, error) { newFunc, hasNew := storeRegistry[name] if !hasNew { - return nil, fmt.Errorf("unknown triplestore '%s'", name) + return nil, errors.New("triplestore: name '" + name + "' is not registered") } return newFunc(dbpath, opts) } @@ -160,7 +160,7 @@ func InitTripleStore(name, dbpath string, opts Options) error { if _, isRegistered := storeRegistry[name]; isRegistered { return nil } - return fmt.Errorf("unknown triplestore '%s'", name) + return errors.New("triplestore: name '" + name + "' is not registered") } func TripleStores() (t []string) { From fb3e181a62e8eb8aec5aab0ad05c8105a8fa76f1 Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Fri, 18 Jul 2014 14:29:23 -0400 Subject: [PATCH 7/8] fix failing build --- appengine/appengine.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appengine/appengine.go b/appengine/appengine.go index 64071fb..c594af8 100644 --- a/appengine/appengine.go +++ b/appengine/appengine.go @@ -21,15 +21,16 @@ import ( "github.com/google/cayley/config" "github.com/google/cayley/graph" - "github.com/google/cayley/graph/memstore" "github.com/google/cayley/http" "github.com/google/cayley/nquads" + + _ "github.com/google/cayley/graph/memstore" ) func init() { glog.SetToStderr(true) cfg := config.ParseConfigFromFile("cayley_appengine.cfg") - ts := memstore.NewTripleStore() + ts, _ := graph.NewTripleStore("memstore", "", nil) glog.Errorln(cfg) LoadTriplesFromFileInto(ts, cfg.DatabasePath, cfg.LoadSize) http.SetupRoutes(ts, cfg) From 923679b36c9d47deefcf079dfaf165de5f6c085d Mon Sep 17 00:00:00 2001 From: Jeremy Jay Date: Fri, 18 Jul 2014 14:54:29 -0400 Subject: [PATCH 8/8] comment tweaks --- cayley.go | 2 +- db/open.go | 3 +-- graph/triplestore.go | 5 +++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cayley.go b/cayley.go index 7ff1fd8..d5dea02 100644 --- a/cayley.go +++ b/cayley.go @@ -27,7 +27,7 @@ import ( "github.com/google/cayley/graph" "github.com/google/cayley/http" - // load all supported backends + // Load all supported backends. _ "github.com/google/cayley/graph/leveldb" _ "github.com/google/cayley/graph/memstore" _ "github.com/google/cayley/graph/mongo" diff --git a/db/open.go b/db/open.go index b0973d7..6b78e52 100644 --- a/db/open.go +++ b/db/open.go @@ -26,10 +26,9 @@ func Open(cfg *config.Config) graph.TripleStore { ts, err := graph.NewTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) if err != nil { glog.Fatalln(err.Error()) - return nil } - // memstore is not persistent, so MUST be loaded + // Memstore is not persistent, so it MUST be loaded. if cfg.DatabaseType == "memstore" { Load(ts, cfg, cfg.DatabasePath) } diff --git a/graph/triplestore.go b/graph/triplestore.go index 76a18cc..9690bdb 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -163,9 +163,10 @@ func InitTripleStore(name, dbpath string, opts Options) error { return errors.New("triplestore: name '" + name + "' is not registered") } -func TripleStores() (t []string) { +func TripleStores() []string { + t := make([]string, 0, len(storeRegistry)) for n := range storeRegistry { t = append(t, n) } - return + return t }