From 35ccfe767747c80ab4c22be8edf34577015f41fd Mon Sep 17 00:00:00 2001 From: = <=> Date: Thu, 27 Nov 2014 15:32:46 +0100 Subject: [PATCH] Added functionality so quadstore is generated per request (if needed) for the new appengine backend \n CR : nobody \n Tests run: unit tests --- config/config.go | 63 ++++++++++++++++++++++++--------------------- graph/bolt/quadstore.go | 10 ++++++- graph/leveldb/quadstore.go | 7 ++++- graph/memstore/quadstore.go | 10 +++++-- graph/mongo/quadstore.go | 13 +++++++--- graph/quadstore.go | 29 ++++++++++++++++----- http/http.go | 24 +++++++++++++++++ http/write.go | 25 +++++++++++++----- 8 files changed, 130 insertions(+), 51 deletions(-) diff --git a/config/config.go b/config/config.go index 90f7741..fa18b57 100644 --- a/config/config.go +++ b/config/config.go @@ -24,29 +24,31 @@ import ( // Config defines the behavior of cayley database instances. type Config struct { - DatabaseType string - DatabasePath string - DatabaseOptions map[string]interface{} - ReplicationType string - ReplicationOptions map[string]interface{} - ListenHost string - ListenPort string - ReadOnly bool - Timeout time.Duration - LoadSize int + DatabaseType string + DatabasePath string + DatabaseOptions map[string]interface{} + ReplicationType string + ReplicationOptions map[string]interface{} + ListenHost string + ListenPort string + ReadOnly bool + Timeout time.Duration + LoadSize int + RequiresHTTPRequestContext bool } type config struct { - DatabaseType string `json:"database"` - DatabasePath string `json:"db_path"` - DatabaseOptions map[string]interface{} `json:"db_options"` - ReplicationType string `json:"replication"` - ReplicationOptions map[string]interface{} `json:"replication_options"` - ListenHost string `json:"listen_host"` - ListenPort string `json:"listen_port"` - ReadOnly bool `json:"read_only"` - Timeout duration `json:"timeout"` - LoadSize int `json:"load_size"` + DatabaseType string `json:"database"` + DatabasePath string `json:"db_path"` + DatabaseOptions map[string]interface{} `json:"db_options"` + ReplicationType string `json:"replication"` + ReplicationOptions map[string]interface{} `json:"replication_options"` + ListenHost string `json:"listen_host"` + ListenPort string `json:"listen_port"` + ReadOnly bool `json:"read_only"` + Timeout duration `json:"timeout"` + LoadSize int `json:"load_size"` + RequiresHTTPRequestContext bool `json:"http_request_context"` } func (c *Config) UnmarshalJSON(data []byte) error { @@ -56,16 +58,17 @@ func (c *Config) UnmarshalJSON(data []byte) error { return err } *c = Config{ - DatabaseType: t.DatabaseType, - DatabasePath: t.DatabasePath, - DatabaseOptions: t.DatabaseOptions, - ReplicationType: t.ReplicationType, - ReplicationOptions: t.ReplicationOptions, - ListenHost: t.ListenHost, - ListenPort: t.ListenPort, - ReadOnly: t.ReadOnly, - Timeout: time.Duration(t.Timeout), - LoadSize: t.LoadSize, + DatabaseType: t.DatabaseType, + DatabasePath: t.DatabasePath, + DatabaseOptions: t.DatabaseOptions, + ReplicationType: t.ReplicationType, + ReplicationOptions: t.ReplicationOptions, + ListenHost: t.ListenHost, + ListenPort: t.ListenPort, + ReadOnly: t.ReadOnly, + Timeout: time.Duration(t.Timeout), + LoadSize: t.LoadSize, + RequiresHTTPRequestContext: t.RequiresHTTPRequestContext, } return nil } diff --git a/graph/bolt/quadstore.go b/graph/bolt/quadstore.go index 0cbcb7c..761cd5b 100644 --- a/graph/bolt/quadstore.go +++ b/graph/bolt/quadstore.go @@ -33,7 +33,7 @@ import ( ) func init() { - graph.RegisterQuadStore("bolt", true, newQuadStore, createNewBolt) + graph.RegisterQuadStore("bolt", true, newQuadStore, createNewBolt, nil) } var ( @@ -44,6 +44,10 @@ var ( localFillPercent = 0.7 ) +const ( + QuadStoreType = "bolt" +) + type Token struct { bucket []byte key []byte @@ -515,3 +519,7 @@ func compareTokens(a, b graph.Value) bool { func (qs *QuadStore) FixedIterator() graph.FixedIterator { return iterator.NewFixed(compareTokens) } + +func (qs *QuadStore) GetType() string { + return QuadStoreType +} diff --git a/graph/leveldb/quadstore.go b/graph/leveldb/quadstore.go index 9f04b3d..aca9833 100644 --- a/graph/leveldb/quadstore.go +++ b/graph/leveldb/quadstore.go @@ -35,12 +35,13 @@ import ( ) func init() { - graph.RegisterQuadStore("leveldb", true, newQuadStore, createNewLevelDB) + graph.RegisterQuadStore(QuadStoreType, true, newQuadStore, createNewLevelDB, nil) } const ( DefaultCacheSize = 2 DefaultWriteBufferSize = 20 + QuadStoreType = "leveldb" ) var ( @@ -495,3 +496,7 @@ func compareBytes(a, b graph.Value) bool { func (qs *QuadStore) FixedIterator() graph.FixedIterator { return iterator.NewFixed(compareBytes) } + +func (qs *QuadStore) GetType() string { + return QuadStoreType +} diff --git a/graph/memstore/quadstore.go b/graph/memstore/quadstore.go index 8e8f8c4..d9203e5 100644 --- a/graph/memstore/quadstore.go +++ b/graph/memstore/quadstore.go @@ -27,10 +27,12 @@ import ( "github.com/google/cayley/quad" ) +const QuadStoreType = "memstore" + func init() { - graph.RegisterQuadStore("memstore", false, func(string, graph.Options) (graph.QuadStore, error) { + graph.RegisterQuadStore(QuadStoreType, false, func(string, graph.Options) (graph.QuadStore, error) { return newQuadStore(), nil - }, nil) + }, nil, nil) } type QuadDirectionIndex struct { @@ -270,3 +272,7 @@ func (qs *QuadStore) NodesAllIterator() graph.Iterator { } func (qs *QuadStore) Close() {} + +func (qs *QuadStore) GetType() string { + return QuadStoreType +} diff --git a/graph/mongo/quadstore.go b/graph/mongo/quadstore.go index 636e1d1..36458f1 100644 --- a/graph/mongo/quadstore.go +++ b/graph/mongo/quadstore.go @@ -30,11 +30,12 @@ import ( "github.com/google/cayley/quad" ) -func init() { - graph.RegisterQuadStore("mongo", true, newQuadStore, createNewMongoGraph) -} - const DefaultDBName = "cayley" +const QuadStoreType = "mongo" + +func init() { + graph.RegisterQuadStore(QuadStoreType, true, newQuadStore, createNewMongoGraph, nil) +} var ( hashPool = sync.Pool{ @@ -366,3 +367,7 @@ func (qs *QuadStore) QuadDirection(in graph.Value, d quad.Direction) graph.Value } // TODO(barakmich): Rewrite bulk loader. For now, iterating around blocks is the way we'll go about it. + +func (qs *QuadStore) GetType() string { + return QuadStoreType +} diff --git a/graph/quadstore.go b/graph/quadstore.go index cb7eca0..7a7f2b6 100644 --- a/graph/quadstore.go +++ b/graph/quadstore.go @@ -95,6 +95,10 @@ type QuadStore interface { // qs.ValueOf(qs.Quad(id).Get(dir)) // QuadDirection(id Value, d quad.Direction) Value + + // Get the type of QuadStore + //TODO replace this using reflection + GetType() string } type Options map[string]interface{} @@ -146,23 +150,26 @@ type BulkLoader interface { type NewStoreFunc func(string, Options) (QuadStore, error) type InitStoreFunc func(string, Options) error +type NewStoreForRequestFunc func(QuadStore, Options) (QuadStore, error) type register struct { - newFunc NewStoreFunc - initFunc InitStoreFunc - isPersistent bool + newFunc NewStoreFunc + newForRequestFunc NewStoreForRequestFunc + initFunc InitStoreFunc + isPersistent bool } var storeRegistry = make(map[string]register) -func RegisterQuadStore(name string, persists bool, newFunc NewStoreFunc, initFunc InitStoreFunc) { +func RegisterQuadStore(name string, persists bool, newFunc NewStoreFunc, initFunc InitStoreFunc, newForRequestFunc NewStoreForRequestFunc) { if _, found := storeRegistry[name]; found { panic("already registered QuadStore " + name) } storeRegistry[name] = register{ - newFunc: newFunc, - initFunc: initFunc, - isPersistent: persists, + newFunc: newFunc, + initFunc: initFunc, + newForRequestFunc: newForRequestFunc, + isPersistent: persists, } } @@ -182,6 +189,14 @@ func InitQuadStore(name, dbpath string, opts Options) error { return errors.New("quadstore: name '" + name + "' is not registered") } +func NewQuadStoreForRequest(qs QuadStore, opts Options) (QuadStore, error) { + r, registered := storeRegistry[qs.GetType()] + if registered { + return r.newForRequestFunc(qs, opts) + } + return nil, errors.New("QuadStore does not support Per Request construction, check config") +} + func IsPersistent(name string) bool { return storeRegistry[name].isPersistent } diff --git a/http/http.go b/http/http.go index 8b0b648..abe2e27 100644 --- a/http/http.go +++ b/http/http.go @@ -26,6 +26,7 @@ import ( "github.com/julienschmidt/httprouter" "github.com/google/cayley/config" + "github.com/google/cayley/db" "github.com/google/cayley/graph" ) @@ -55,6 +56,10 @@ func findAssetsPath() string { return "." } + if hasAssets("..") { + return ".." + } + gopathPath := os.ExpandEnv("$GOPATH/src/github.com/google/cayley") if hasAssets(gopathPath) { return gopathPath @@ -105,6 +110,25 @@ type API struct { handle *graph.Handle } +func (api *API) GetHandleForRequest(r *http.Request) (*graph.Handle, error) { + if !api.config.RequiresHTTPRequestContext { + return api.handle, nil + } + + opts := make(graph.Options) + opts["HTTPRequest"] = r + + qs, err := graph.NewQuadStoreForRequest(api.handle.QuadStore, opts) + if err != nil { + return nil, err + } + qw, err := db.OpenQuadWriter(qs, api.config) + if err != nil { + return nil, err + } + return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil +} + func (api *API) APIv1(r *httprouter.Router) { r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) diff --git a/http/write.go b/http/write.go index 9ba037b..78743f4 100644 --- a/http/write.go +++ b/http/write.go @@ -55,7 +55,12 @@ func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httproute if err != nil { return jsonResponse(w, 400, err) } - api.handle.QuadWriter.AddQuadSet(quads) + h, err := api.GetHandleForRequest(r) + if err != nil { + return jsonResponse(w, 400, err) + } + + h.QuadWriter.AddQuadSet(quads) fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", len(quads)) return 200 } @@ -81,9 +86,13 @@ func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params // TODO(kortschak) Make this configurable from the web UI. dec := cquads.NewDecoder(formFile) - var ( - n int + h, err := api.GetHandleForRequest(r) + if err != nil { + return jsonResponse(w, 400, err) + } + var ( + n int block = make([]quad.Quad, 0, blockSize) ) for { @@ -97,11 +106,11 @@ func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params block = append(block, t) n++ if len(block) == cap(block) { - api.handle.QuadWriter.AddQuadSet(block) + h.QuadWriter.AddQuadSet(block) block = block[:0] } } - api.handle.QuadWriter.AddQuadSet(block) + h.QuadWriter.AddQuadSet(block) fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n) @@ -120,9 +129,13 @@ func (api *API) ServeV1Delete(w http.ResponseWriter, r *http.Request, params htt if err != nil { return jsonResponse(w, 400, err) } + h, err := api.GetHandleForRequest(r) + if err != nil { + return jsonResponse(w, 400, err) + } count := 0 for _, q := range quads { - api.handle.QuadWriter.RemoveQuad(q) + h.QuadWriter.RemoveQuad(q) count++ } fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d quads.\"}", count)