From a8b3a04eda0ce6f18da12d2adb62853199a8d26f Mon Sep 17 00:00:00 2001 From: kortschak Date: Fri, 22 May 2015 14:21:20 +0930 Subject: [PATCH] internal: move {config,db,http} into internal --- appengine/appengine.go | 9 +- cmd/cayley/cayley.go | 6 +- config/config.go | 145 ----------------------- db/db.go | 105 ----------------- db/repl.go | 251 ---------------------------------------- db/repl_test.go | 66 ----------- http/docs.go | 74 ------------ http/http.go | 171 --------------------------- http/http_test.go | 73 ------------ http/query.go | 146 ----------------------- http/write.go | 144 ----------------------- integration/integration_test.go | 4 +- internal/config/config.go | 145 +++++++++++++++++++++++ internal/db/db.go | 105 +++++++++++++++++ internal/db/repl.go | 251 ++++++++++++++++++++++++++++++++++++++++ internal/db/repl_test.go | 66 +++++++++++ internal/http/docs.go | 74 ++++++++++++ internal/http/http.go | 171 +++++++++++++++++++++++++++ internal/http/http_test.go | 73 ++++++++++++ internal/http/query.go | 146 +++++++++++++++++++++++ internal/http/write.go | 144 +++++++++++++++++++++++ internal/load.go | 4 +- 22 files changed, 1187 insertions(+), 1186 deletions(-) delete mode 100644 config/config.go delete mode 100644 db/db.go delete mode 100644 db/repl.go delete mode 100644 db/repl_test.go delete mode 100644 http/docs.go delete mode 100644 http/http.go delete mode 100644 http/http_test.go delete mode 100644 http/query.go delete mode 100644 http/write.go create mode 100644 internal/config/config.go create mode 100644 internal/db/db.go create mode 100644 internal/db/repl.go create mode 100644 internal/db/repl_test.go create mode 100644 internal/http/docs.go create mode 100644 internal/http/http.go create mode 100644 internal/http/http_test.go create mode 100644 internal/http/query.go create mode 100644 internal/http/write.go diff --git a/appengine/appengine.go b/appengine/appengine.go index 1f8706e..bb972c1 100644 --- a/appengine/appengine.go +++ b/appengine/appengine.go @@ -18,13 +18,14 @@ package main import ( "fmt" - "github.com/barakmich/glog" "os" "time" - "github.com/google/cayley/config" - "github.com/google/cayley/db" - "github.com/google/cayley/http" + "github.com/barakmich/glog" + + "github.com/google/cayley/internal/config" + "github.com/google/cayley/internal/db" + "github.com/google/cayley/internal/http" _ "github.com/google/cayley/graph/gaedatastore" _ "github.com/google/cayley/writer" diff --git a/cmd/cayley/cayley.go b/cmd/cayley/cayley.go index 77c6cde..302d9e7 100644 --- a/cmd/cayley/cayley.go +++ b/cmd/cayley/cayley.go @@ -25,11 +25,11 @@ import ( "github.com/barakmich/glog" - "github.com/google/cayley/config" - "github.com/google/cayley/db" "github.com/google/cayley/graph" - "github.com/google/cayley/http" "github.com/google/cayley/internal" + "github.com/google/cayley/internal/config" + "github.com/google/cayley/internal/db" + "github.com/google/cayley/internal/http" // Load all supported backends. _ "github.com/google/cayley/graph/bolt" diff --git a/config/config.go b/config/config.go deleted file mode 100644 index fa18b57..0000000 --- a/config/config.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package config - -import ( - "encoding/json" - "fmt" - "os" - "strconv" - "time" -) - -// Config defines the behavior of cayley database instances. -type Config struct { - DatabaseType string - DatabasePath string - DatabaseOptions map[string]interface{} - ReplicationType string - ReplicationOptions map[string]interface{} - ListenHost string - ListenPort string - ReadOnly bool - Timeout time.Duration - LoadSize int - RequiresHTTPRequestContext bool -} - -type config struct { - DatabaseType string `json:"database"` - DatabasePath string `json:"db_path"` - DatabaseOptions map[string]interface{} `json:"db_options"` - ReplicationType string `json:"replication"` - ReplicationOptions map[string]interface{} `json:"replication_options"` - ListenHost string `json:"listen_host"` - ListenPort string `json:"listen_port"` - ReadOnly bool `json:"read_only"` - Timeout duration `json:"timeout"` - LoadSize int `json:"load_size"` - RequiresHTTPRequestContext bool `json:"http_request_context"` -} - -func (c *Config) UnmarshalJSON(data []byte) error { - var t config - err := json.Unmarshal(data, &t) - if err != nil { - return err - } - *c = Config{ - DatabaseType: t.DatabaseType, - DatabasePath: t.DatabasePath, - DatabaseOptions: t.DatabaseOptions, - ReplicationType: t.ReplicationType, - ReplicationOptions: t.ReplicationOptions, - ListenHost: t.ListenHost, - ListenPort: t.ListenPort, - ReadOnly: t.ReadOnly, - Timeout: time.Duration(t.Timeout), - LoadSize: t.LoadSize, - RequiresHTTPRequestContext: t.RequiresHTTPRequestContext, - } - return nil -} - -func (c *Config) MarshalJSON() ([]byte, error) { - return json.Marshal(config{ - DatabaseType: c.DatabaseType, - DatabasePath: c.DatabasePath, - DatabaseOptions: c.DatabaseOptions, - ReplicationType: c.ReplicationType, - ReplicationOptions: c.ReplicationOptions, - ListenHost: c.ListenHost, - ListenPort: c.ListenPort, - ReadOnly: c.ReadOnly, - Timeout: duration(c.Timeout), - LoadSize: c.LoadSize, - }) -} - -// duration is a time.Duration that satisfies the -// json.UnMarshaler and json.Marshaler interfaces. -type duration time.Duration - -// UnmarshalJSON unmarshals a duration according to the following scheme: -// * If the element is absent the duration is zero. -// * If the element is parsable as a time.Duration, the parsed value is kept. -// * If the element is parsable as a number, that number of seconds is kept. -func (d *duration) UnmarshalJSON(data []byte) error { - if len(data) == 0 { - *d = 0 - return nil - } - text := string(data) - t, err := time.ParseDuration(text) - if err == nil { - *d = duration(t) - return nil - } - i, err := strconv.ParseInt(text, 10, 64) - if err == nil { - *d = duration(time.Duration(i) * time.Second) - return nil - } - // This hack is to get around strconv.ParseFloat - // not handling e-notation for integers. - f, err := strconv.ParseFloat(text, 64) - *d = duration(time.Duration(f) * time.Second) - return err -} - -func (d *duration) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf("%q", *d)), nil -} - -// Load reads a JSON-encoded config contained in the given file. A zero value -// config is returned if the filename is empty. -func Load(file string) (*Config, error) { - config := &Config{} - if file == "" { - return config, nil - } - f, err := os.Open(file) - if err != nil { - return nil, fmt.Errorf("could not open config file %q: %v", file, err) - } - defer f.Close() - - dec := json.NewDecoder(f) - err = dec.Decode(config) - if err != nil { - return nil, fmt.Errorf("could not parse config file %q: %v", file, err) - } - return config, nil -} diff --git a/db/db.go b/db/db.go deleted file mode 100644 index a01429e..0000000 --- a/db/db.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db - -import ( - "errors" - "fmt" - "io" - - "github.com/barakmich/glog" - - "github.com/google/cayley/config" - "github.com/google/cayley/graph" - "github.com/google/cayley/quad" -) - -var ErrNotPersistent = errors.New("database type is not persistent") - -func Init(cfg *config.Config) error { - if !graph.IsPersistent(cfg.DatabaseType) { - return fmt.Errorf("ignoring unproductive database initialization request: %v", ErrNotPersistent) - } - - return graph.InitQuadStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) -} - -func Open(cfg *config.Config) (*graph.Handle, error) { - qs, err := OpenQuadStore(cfg) - if err != nil { - return nil, err - } - qw, err := OpenQuadWriter(qs, cfg) - if err != nil { - return nil, err - } - return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil -} - -func OpenQuadStore(cfg *config.Config) (graph.QuadStore, error) { - glog.Infof("Opening quad store %q at %s", cfg.DatabaseType, cfg.DatabasePath) - qs, err := graph.NewQuadStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) - if err != nil { - return nil, err - } - - return qs, nil -} - -func OpenQuadWriter(qs graph.QuadStore, cfg *config.Config) (graph.QuadWriter, error) { - glog.Infof("Opening replication method %q", cfg.ReplicationType) - w, err := graph.NewQuadWriter(cfg.ReplicationType, qs, cfg.ReplicationOptions) - if err != nil { - return nil, err - } - - return w, nil -} - -func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error { - block := make([]quad.Quad, 0, cfg.LoadSize) - count := 0 - for { - t, err := dec.Unmarshal() - if err != nil { - if err == io.EOF { - break - } - return err - } - block = append(block, t) - if len(block) == cap(block) { - count += len(block) - err := qw.AddQuadSet(block) - if err != nil { - return fmt.Errorf("db: failed to load data: %v", err) - } - block = block[:0] - if glog.V(2) { - glog.V(2).Infof("Wrote %d quads.", count) - } - } - } - count += len(block) - err := qw.AddQuadSet(block) - if err != nil { - return fmt.Errorf("db: failed to load data: %v", err) - } - if glog.V(2) { - glog.V(2).Infof("Wrote %d quads.", count) - } - - return nil -} diff --git a/db/repl.go b/db/repl.go deleted file mode 100644 index 9af2a6f..0000000 --- a/db/repl.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db - -import ( - "fmt" - "io" - "os" - "os/signal" - "strconv" - "strings" - "time" - - "github.com/peterh/liner" - - "github.com/google/cayley/config" - "github.com/google/cayley/graph" - "github.com/google/cayley/quad/cquads" - "github.com/google/cayley/query" - "github.com/google/cayley/query/gremlin" - "github.com/google/cayley/query/mql" - "github.com/google/cayley/query/sexp" -) - -func trace(s string) (string, time.Time) { - return s, time.Now() -} - -func un(s string, startTime time.Time) { - endTime := time.Now() - - fmt.Printf(s, float64(endTime.UnixNano()-startTime.UnixNano())/float64(1E6)) -} - -func Run(query string, ses query.Session) { - nResults := 0 - startTrace, startTime := trace("Elapsed time: %g ms\n\n") - defer func() { - if nResults > 0 { - un(startTrace, startTime) - } - }() - fmt.Printf("\n") - c := make(chan interface{}, 5) - go ses.Execute(query, c, 100) - for res := range c { - fmt.Print(ses.Format(res)) - nResults++ - } - if nResults > 0 { - results := "Result" - if nResults > 1 { - results += "s" - } - fmt.Printf("-----------\n%d %s\n", nResults, results) - } -} - -const ( - ps1 = "cayley> " - ps2 = "... " - - history = ".cayley_history" -) - -func Repl(h *graph.Handle, queryLanguage string, cfg *config.Config) error { - var ses query.Session - switch queryLanguage { - case "sexp": - ses = sexp.NewSession(h.QuadStore) - case "mql": - ses = mql.NewSession(h.QuadStore) - case "gremlin": - fallthrough - default: - ses = gremlin.NewSession(h.QuadStore, cfg.Timeout, true) - } - - term, err := terminal(history) - if os.IsNotExist(err) { - fmt.Printf("creating new history file: %q\n", history) - } - defer persist(term, history) - - var ( - prompt = ps1 - - code string - ) - - for { - if len(code) == 0 { - prompt = ps1 - } else { - prompt = ps2 - } - line, err := term.Prompt(prompt) - if err != nil { - if err == io.EOF { - fmt.Println() - return nil - } - return err - } - - term.AppendHistory(line) - - line = strings.TrimSpace(line) - if len(line) == 0 || line[0] == '#' { - continue - } - - if code == "" { - cmd, args := splitLine(line) - - switch cmd { - case ":debug": - args = strings.TrimSpace(args) - var debug bool - switch args { - case "t": - debug = true - case "f": - // Do nothing. - default: - debug, err = strconv.ParseBool(args) - if err != nil { - fmt.Printf("Error: cannot parse %q as a valid boolean - acceptable values: 't'|'true' or 'f'|'false'\n", args) - continue - } - } - ses.Debug(debug) - fmt.Printf("Debug set to %t\n", debug) - continue - - case ":a": - quad, err := cquads.Parse(args) - if err != nil { - fmt.Printf("Error: not a valid quad: %v\n", err) - continue - } - - h.QuadWriter.AddQuad(quad) - continue - - case ":d": - quad, err := cquads.Parse(args) - if err != nil { - fmt.Printf("Error: not a valid quad: %v\n", err) - continue - } - h.QuadWriter.RemoveQuad(quad) - continue - - case "exit": - term.Close() - os.Exit(0) - - default: - if cmd[0] == ':' { - fmt.Printf("Unknown command: %q\n", cmd) - continue - } - } - } - - code += line - - result, err := ses.Parse(code) - switch result { - case query.Parsed: - Run(code, ses) - code = "" - case query.ParseFail: - fmt.Println("Error: ", err) - code = "" - case query.ParseMore: - } - } -} - -// Splits a line into a command and its arguments -// e.g. ":a b c d ." will be split into ":a" and " b c d ." -func splitLine(line string) (string, string) { - var command, arguments string - - line = strings.TrimSpace(line) - - // An empty line/a line consisting of whitespace contains neither command nor arguments - if len(line) > 0 { - command = strings.Fields(line)[0] - - // A line containing only a command has no arguments - if len(line) > len(command) { - arguments = line[len(command):] - } - } - - return command, arguments -} - -func terminal(path string) (*liner.State, error) { - term := liner.NewLiner() - - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, os.Kill) - <-c - - err := persist(term, history) - if err != nil { - fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err) - os.Exit(1) - } - - os.Exit(0) - }() - - f, err := os.Open(path) - if err != nil { - return term, err - } - defer f.Close() - _, err = term.ReadHistory(f) - return term, err -} - -func persist(term *liner.State, path string) error { - f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) - if err != nil { - return fmt.Errorf("could not open %q to append history: %v", path, err) - } - defer f.Close() - _, err = term.WriteHistory(f) - if err != nil { - return fmt.Errorf("could not write history to %q: %v", path, err) - } - return term.Close() -} diff --git a/db/repl_test.go b/db/repl_test.go deleted file mode 100644 index 0a48433..0000000 --- a/db/repl_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db - -import ( - "testing" -) - -var testSplitLines = []struct { - line string - expectedCommand string - expectedArguments string - err error -}{ - { - line: ":a arg1 arg2 arg3 .", - expectedCommand: ":a", - expectedArguments: " arg1 arg2 arg3 .", - }, - { - line: ":debug t", - expectedCommand: ":debug", - expectedArguments: " t", - }, - { - line: "", - // expectedCommand is nil - // expectedArguments is nil - }, - { - line: `:d . # comments here`, - expectedCommand: ":d", - expectedArguments: ` . # comments here`, - }, - { - line: ` :a subject "predicate with spaces" object . `, - expectedCommand: ":a", - expectedArguments: ` subject "predicate with spaces" object .`, - }, -} - -func TestSplitLines(t *testing.T) { - for _, testcase := range testSplitLines { - command, arguments := splitLine(testcase.line) - - if testcase.expectedCommand != command { - t.Errorf("Error splitting lines: got: %v expected: %v", command, testcase.expectedCommand) - } - - if testcase.expectedArguments != arguments { - t.Errorf("Error splitting lines: got: %v expected: %v", arguments, testcase.expectedArguments) - } - } -} diff --git a/http/docs.go b/http/docs.go deleted file mode 100644 index 0d1d234..0000000 --- a/http/docs.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "fmt" - "io/ioutil" - "net/http" - "os" - - "github.com/julienschmidt/httprouter" - "github.com/russross/blackfriday" -) - -type DocRequestHandler struct { - assets string -} - -func MarkdownWithCSS(input []byte, title string) []byte { - // set up the HTML renderer - htmlFlags := 0 - htmlFlags |= blackfriday.HTML_USE_XHTML - htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS - htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS - htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES - htmlFlags |= blackfriday.HTML_COMPLETE_PAGE - renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS) - - // set up the parser - extensions := 0 - //extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS - extensions |= blackfriday.EXTENSION_TABLES - extensions |= blackfriday.EXTENSION_FENCED_CODE - extensions |= blackfriday.EXTENSION_AUTOLINK - extensions |= blackfriday.EXTENSION_STRIKETHROUGH - //extensions |= blackfriday.EXTENSION_SPACE_HEADERS - extensions |= blackfriday.EXTENSION_HEADER_IDS - extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS - - return blackfriday.Markdown(input, renderer, extensions) -} - -func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { - docpage := params.ByName("docpage") - if docpage == "" { - docpage = "Index" - } - file, err := os.Open(fmt.Sprintf("%s/docs/%s.md", h.assets, docpage)) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - data, err := ioutil.ReadAll(file) - if err != nil { - http.Error(w, err.Error(), http.StatusNoContent) - return - } - output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage)) - fmt.Fprint(w, string(output)) -} - -var markdownCSS = "/static/css/docs.css" diff --git a/http/http.go b/http/http.go deleted file mode 100644 index abe2e27..0000000 --- a/http/http.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "flag" - "fmt" - "html/template" - "net/http" - "os" - "time" - - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/config" - "github.com/google/cayley/db" - "github.com/google/cayley/graph" -) - -type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int - -var assetsPath = flag.String("assets", "", "Explicit path to the HTTP assets.") -var assetsDirs = []string{"templates", "static", "docs"} - -func hasAssets(path string) bool { - for _, dir := range assetsDirs { - if _, err := os.Stat(fmt.Sprint(path, "/", dir)); os.IsNotExist(err) { - return false - } - } - return true -} - -func findAssetsPath() string { - if *assetsPath != "" { - if hasAssets(*assetsPath) { - return *assetsPath - } - glog.Fatalln("Cannot find assets at", *assetsPath, ".") - } - - if hasAssets(".") { - return "." - } - - if hasAssets("..") { - return ".." - } - - gopathPath := os.ExpandEnv("$GOPATH/src/github.com/google/cayley") - if hasAssets(gopathPath) { - return gopathPath - } - glog.Fatalln("Cannot find assets in any of the default search paths. Please run in the same directory, in a Go workspace, or set --assets .") - panic("cannot reach") -} - -func LogRequest(handler ResponseHandler) httprouter.Handle { - return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { - start := time.Now() - addr := req.Header.Get("X-Real-IP") - if addr == "" { - addr = req.Header.Get("X-Forwarded-For") - if addr == "" { - addr = req.RemoteAddr - } - } - glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr) - code := handler(w, req, params) - glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start)) - - } -} - -func jsonResponse(w http.ResponseWriter, code int, err interface{}) int { - http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code) - return code -} - -type TemplateRequestHandler struct { - templates *template.Template -} - -func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { - uiType := params.ByName("ui_type") - if r.URL.Path == "/" { - uiType = "query" - } - err := h.templates.ExecuteTemplate(w, uiType+".html", h) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -type API struct { - config *config.Config - handle *graph.Handle -} - -func (api *API) GetHandleForRequest(r *http.Request) (*graph.Handle, error) { - if !api.config.RequiresHTTPRequestContext { - return api.handle, nil - } - - opts := make(graph.Options) - opts["HTTPRequest"] = r - - qs, err := graph.NewQuadStoreForRequest(api.handle.QuadStore, opts) - if err != nil { - return nil, err - } - qw, err := db.OpenQuadWriter(qs, api.config) - if err != nil { - return nil, err - } - return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil -} - -func (api *API) APIv1(r *httprouter.Router) { - r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) - r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) - r.POST("/api/v1/write", LogRequest(api.ServeV1Write)) - r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad)) - //TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form? - r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete)) -} - -func SetupRoutes(handle *graph.Handle, cfg *config.Config) { - r := httprouter.New() - assets := findAssetsPath() - if glog.V(2) { - glog.V(2).Infoln("Found assets at", assets) - } - var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl"))) - templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html")) - root := &TemplateRequestHandler{templates: templates} - docs := &DocRequestHandler{assets: assets} - api := &API{config: cfg, handle: handle} - api.APIv1(r) - - //m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true})) - //r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) - r.GET("/docs/:docpage", docs.ServeHTTP) - r.GET("/ui/:ui_type", root.ServeHTTP) - r.GET("/", root.ServeHTTP) - http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/"))))) - http.Handle("/", r) -} - -func Serve(handle *graph.Handle, cfg *config.Config) { - SetupRoutes(handle, cfg) - glog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) - fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) - err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil) - if err != nil { - glog.Fatal("ListenAndServe: ", err) - } -} diff --git a/http/http_test.go b/http/http_test.go deleted file mode 100644 index 1b1490b..0000000 --- a/http/http_test.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "fmt" - "reflect" - "testing" - - "github.com/google/cayley/quad" -) - -var parseTests = []struct { - message string - input string - expect []quad.Quad - err error -}{ - { - message: "parse correct JSON", - input: `[ - {"subject": "foo", "predicate": "bar", "object": "baz"}, - {"subject": "foo", "predicate": "bar", "object": "baz", "label": "graph"} - ]`, - expect: []quad.Quad{ - {"foo", "bar", "baz", ""}, - {"foo", "bar", "baz", "graph"}, - }, - err: nil, - }, - { - message: "parse correct JSON with extra field", - input: `[ - {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} - ]`, - expect: []quad.Quad{ - {"foo", "bar", "foo", ""}, - }, - err: nil, - }, - { - message: "reject incorrect JSON", - input: `[ - {"subject": "foo", "predicate": "bar"} - ]`, - expect: nil, - err: fmt.Errorf("invalid quad at index %d. %v", 0, quad.Quad{"foo", "bar", "", ""}), - }, -} - -func TestParseJSON(t *testing.T) { - for _, test := range parseTests { - got, err := ParseJSONToQuadList([]byte(test.input)) - if fmt.Sprint(err) != fmt.Sprint(test.err) { - t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err) - } - if !reflect.DeepEqual(got, test.expect) { - t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect) - } - } -} diff --git a/http/query.go b/http/query.go deleted file mode 100644 index 44d5fb4..0000000 --- a/http/query.go +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/query" - "github.com/google/cayley/query/gremlin" - "github.com/google/cayley/query/mql" -) - -type SuccessQueryWrapper struct { - Result interface{} `json:"result"` -} - -type ErrorQueryWrapper struct { - Error string `json:"error"` -} - -func WrapErrResult(err error) ([]byte, error) { - var wrap ErrorQueryWrapper - wrap.Error = err.Error() - return json.MarshalIndent(wrap, "", " ") -} - -func WrapResult(result interface{}) ([]byte, error) { - var wrap SuccessQueryWrapper - wrap.Result = result - return json.MarshalIndent(wrap, "", " ") -} - -func Run(q string, ses query.HTTP) (interface{}, error) { - c := make(chan interface{}, 5) - go ses.Execute(q, c, 100) - for res := range c { - ses.Collate(res) - } - return ses.Results() -} - -func GetQueryShape(q string, ses query.HTTP) ([]byte, error) { - s, err := ses.ShapeOf(q) - if err != nil { - return nil, err - } - return json.Marshal(s) -} - -// TODO(barakmich): Turn this into proper middleware. -func (api *API) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - h, err := api.GetHandleForRequest(r) - var ses query.HTTP - switch params.ByName("query_lang") { - case "gremlin": - ses = gremlin.NewSession(h.QuadStore, api.config.Timeout, false) - case "mql": - ses = mql.NewSession(h.QuadStore) - default: - return jsonResponse(w, 400, "Need a query language.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return jsonResponse(w, 400, err) - } - code := string(bodyBytes) - result, err := ses.Parse(code) - switch result { - case query.Parsed: - var output interface{} - var bytes []byte - var err error - output, err = Run(code, ses) - if err != nil { - bytes, err = WrapErrResult(err) - http.Error(w, string(bytes), 400) - ses = nil - return 400 - } - bytes, err = WrapResult(output) - if err != nil { - ses = nil - return jsonResponse(w, 400, err) - } - fmt.Fprint(w, string(bytes)) - ses = nil - return 200 - case query.ParseFail: - ses = nil - return jsonResponse(w, 400, err) - default: - ses = nil - return jsonResponse(w, 500, "Incomplete data?") - } -} - -func (api *API) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - h, err := api.GetHandleForRequest(r) - var ses query.HTTP - switch params.ByName("query_lang") { - case "gremlin": - ses = gremlin.NewSession(h.QuadStore, api.config.Timeout, false) - case "mql": - ses = mql.NewSession(h.QuadStore) - default: - return jsonResponse(w, 400, "Need a query language.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return jsonResponse(w, 400, err) - } - code := string(bodyBytes) - result, err := ses.Parse(code) - switch result { - case query.Parsed: - var output []byte - var err error - output, err = GetQueryShape(code, ses) - if err != nil { - return jsonResponse(w, 400, err) - } - fmt.Fprint(w, string(output)) - return 200 - case query.ParseFail: - return jsonResponse(w, 400, err) - default: - return jsonResponse(w, 500, "Incomplete data?") - } -} diff --git a/http/write.go b/http/write.go deleted file mode 100644 index de2c24b..0000000 --- a/http/write.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "strconv" - - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/internal" - "github.com/google/cayley/quad" - "github.com/google/cayley/quad/cquads" -) - -func ParseJSONToQuadList(jsonBody []byte) ([]quad.Quad, error) { - var quads []quad.Quad - err := json.Unmarshal(jsonBody, &quads) - if err != nil { - return nil, err - } - for i, q := range quads { - if !q.IsValid() { - return nil, fmt.Errorf("invalid quad at index %d. %s", i, q) - } - } - return quads, nil -} - -func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int { - if api.config.ReadOnly { - return jsonResponse(w, 400, "Database is read-only.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return jsonResponse(w, 400, err) - } - quads, err := ParseJSONToQuadList(bodyBytes) - if err != nil { - return jsonResponse(w, 400, err) - } - h, err := api.GetHandleForRequest(r) - if err != nil { - return jsonResponse(w, 400, err) - } - - h.QuadWriter.AddQuadSet(quads) - fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", len(quads)) - return 200 -} - -func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - if api.config.ReadOnly { - return jsonResponse(w, 400, "Database is read-only.") - } - - formFile, _, err := r.FormFile("NQuadFile") - if err != nil { - glog.Errorln(err) - return jsonResponse(w, 500, "Couldn't read file: "+err.Error()) - } - defer formFile.Close() - - blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) - if blockErr != nil { - blockSize = int64(api.config.LoadSize) - } - - quadReader, err := internal.Decompressor(formFile) - // TODO(kortschak) Make this configurable from the web UI. - dec := cquads.NewDecoder(quadReader) - - h, err := api.GetHandleForRequest(r) - if err != nil { - return jsonResponse(w, 400, err) - } - - var ( - n int - block = make([]quad.Quad, 0, blockSize) - ) - for { - t, err := dec.Unmarshal() - if err != nil { - if err == io.EOF { - break - } - glog.Fatalln("what can do this here?", err) // FIXME(kortschak) - } - block = append(block, t) - n++ - if len(block) == cap(block) { - h.QuadWriter.AddQuadSet(block) - block = block[:0] - } - } - h.QuadWriter.AddQuadSet(block) - - fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n) - - return 200 -} - -func (api *API) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - if api.config.ReadOnly { - return jsonResponse(w, 400, "Database is read-only.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return jsonResponse(w, 400, err) - } - quads, err := ParseJSONToQuadList(bodyBytes) - if err != nil { - return jsonResponse(w, 400, err) - } - h, err := api.GetHandleForRequest(r) - if err != nil { - return jsonResponse(w, 400, err) - } - count := 0 - for _, q := range quads { - h.QuadWriter.RemoveQuad(q) - count++ - } - fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d quads.\"}", count) - return 200 -} diff --git a/integration/integration_test.go b/integration/integration_test.go index c6a39ed..4b5b726 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -25,10 +25,10 @@ import ( "testing" "time" - "github.com/google/cayley/config" - "github.com/google/cayley/db" "github.com/google/cayley/graph" "github.com/google/cayley/internal" + "github.com/google/cayley/internal/config" + "github.com/google/cayley/internal/db" "github.com/google/cayley/quad" "github.com/google/cayley/query/gremlin" diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..fa18b57 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,145 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "encoding/json" + "fmt" + "os" + "strconv" + "time" +) + +// Config defines the behavior of cayley database instances. +type Config struct { + DatabaseType string + DatabasePath string + DatabaseOptions map[string]interface{} + ReplicationType string + ReplicationOptions map[string]interface{} + ListenHost string + ListenPort string + ReadOnly bool + Timeout time.Duration + LoadSize int + RequiresHTTPRequestContext bool +} + +type config struct { + DatabaseType string `json:"database"` + DatabasePath string `json:"db_path"` + DatabaseOptions map[string]interface{} `json:"db_options"` + ReplicationType string `json:"replication"` + ReplicationOptions map[string]interface{} `json:"replication_options"` + ListenHost string `json:"listen_host"` + ListenPort string `json:"listen_port"` + ReadOnly bool `json:"read_only"` + Timeout duration `json:"timeout"` + LoadSize int `json:"load_size"` + RequiresHTTPRequestContext bool `json:"http_request_context"` +} + +func (c *Config) UnmarshalJSON(data []byte) error { + var t config + err := json.Unmarshal(data, &t) + if err != nil { + return err + } + *c = Config{ + DatabaseType: t.DatabaseType, + DatabasePath: t.DatabasePath, + DatabaseOptions: t.DatabaseOptions, + ReplicationType: t.ReplicationType, + ReplicationOptions: t.ReplicationOptions, + ListenHost: t.ListenHost, + ListenPort: t.ListenPort, + ReadOnly: t.ReadOnly, + Timeout: time.Duration(t.Timeout), + LoadSize: t.LoadSize, + RequiresHTTPRequestContext: t.RequiresHTTPRequestContext, + } + return nil +} + +func (c *Config) MarshalJSON() ([]byte, error) { + return json.Marshal(config{ + DatabaseType: c.DatabaseType, + DatabasePath: c.DatabasePath, + DatabaseOptions: c.DatabaseOptions, + ReplicationType: c.ReplicationType, + ReplicationOptions: c.ReplicationOptions, + ListenHost: c.ListenHost, + ListenPort: c.ListenPort, + ReadOnly: c.ReadOnly, + Timeout: duration(c.Timeout), + LoadSize: c.LoadSize, + }) +} + +// duration is a time.Duration that satisfies the +// json.UnMarshaler and json.Marshaler interfaces. +type duration time.Duration + +// UnmarshalJSON unmarshals a duration according to the following scheme: +// * If the element is absent the duration is zero. +// * If the element is parsable as a time.Duration, the parsed value is kept. +// * If the element is parsable as a number, that number of seconds is kept. +func (d *duration) UnmarshalJSON(data []byte) error { + if len(data) == 0 { + *d = 0 + return nil + } + text := string(data) + t, err := time.ParseDuration(text) + if err == nil { + *d = duration(t) + return nil + } + i, err := strconv.ParseInt(text, 10, 64) + if err == nil { + *d = duration(time.Duration(i) * time.Second) + return nil + } + // This hack is to get around strconv.ParseFloat + // not handling e-notation for integers. + f, err := strconv.ParseFloat(text, 64) + *d = duration(time.Duration(f) * time.Second) + return err +} + +func (d *duration) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("%q", *d)), nil +} + +// Load reads a JSON-encoded config contained in the given file. A zero value +// config is returned if the filename is empty. +func Load(file string) (*Config, error) { + config := &Config{} + if file == "" { + return config, nil + } + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("could not open config file %q: %v", file, err) + } + defer f.Close() + + dec := json.NewDecoder(f) + err = dec.Decode(config) + if err != nil { + return nil, fmt.Errorf("could not parse config file %q: %v", file, err) + } + return config, nil +} diff --git a/internal/db/db.go b/internal/db/db.go new file mode 100644 index 0000000..c139b1e --- /dev/null +++ b/internal/db/db.go @@ -0,0 +1,105 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "errors" + "fmt" + "io" + + "github.com/barakmich/glog" + + "github.com/google/cayley/graph" + "github.com/google/cayley/internal/config" + "github.com/google/cayley/quad" +) + +var ErrNotPersistent = errors.New("database type is not persistent") + +func Init(cfg *config.Config) error { + if !graph.IsPersistent(cfg.DatabaseType) { + return fmt.Errorf("ignoring unproductive database initialization request: %v", ErrNotPersistent) + } + + return graph.InitQuadStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) +} + +func Open(cfg *config.Config) (*graph.Handle, error) { + qs, err := OpenQuadStore(cfg) + if err != nil { + return nil, err + } + qw, err := OpenQuadWriter(qs, cfg) + if err != nil { + return nil, err + } + return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil +} + +func OpenQuadStore(cfg *config.Config) (graph.QuadStore, error) { + glog.Infof("Opening quad store %q at %s", cfg.DatabaseType, cfg.DatabasePath) + qs, err := graph.NewQuadStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) + if err != nil { + return nil, err + } + + return qs, nil +} + +func OpenQuadWriter(qs graph.QuadStore, cfg *config.Config) (graph.QuadWriter, error) { + glog.Infof("Opening replication method %q", cfg.ReplicationType) + w, err := graph.NewQuadWriter(cfg.ReplicationType, qs, cfg.ReplicationOptions) + if err != nil { + return nil, err + } + + return w, nil +} + +func Load(qw graph.QuadWriter, cfg *config.Config, dec quad.Unmarshaler) error { + block := make([]quad.Quad, 0, cfg.LoadSize) + count := 0 + for { + t, err := dec.Unmarshal() + if err != nil { + if err == io.EOF { + break + } + return err + } + block = append(block, t) + if len(block) == cap(block) { + count += len(block) + err := qw.AddQuadSet(block) + if err != nil { + return fmt.Errorf("db: failed to load data: %v", err) + } + block = block[:0] + if glog.V(2) { + glog.V(2).Infof("Wrote %d quads.", count) + } + } + } + count += len(block) + err := qw.AddQuadSet(block) + if err != nil { + return fmt.Errorf("db: failed to load data: %v", err) + } + if glog.V(2) { + glog.V(2).Infof("Wrote %d quads.", count) + } + + return nil +} diff --git a/internal/db/repl.go b/internal/db/repl.go new file mode 100644 index 0000000..20ec017 --- /dev/null +++ b/internal/db/repl.go @@ -0,0 +1,251 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "fmt" + "io" + "os" + "os/signal" + "strconv" + "strings" + "time" + + "github.com/peterh/liner" + + "github.com/google/cayley/graph" + "github.com/google/cayley/internal/config" + "github.com/google/cayley/quad/cquads" + "github.com/google/cayley/query" + "github.com/google/cayley/query/gremlin" + "github.com/google/cayley/query/mql" + "github.com/google/cayley/query/sexp" +) + +func trace(s string) (string, time.Time) { + return s, time.Now() +} + +func un(s string, startTime time.Time) { + endTime := time.Now() + + fmt.Printf(s, float64(endTime.UnixNano()-startTime.UnixNano())/float64(1E6)) +} + +func Run(query string, ses query.Session) { + nResults := 0 + startTrace, startTime := trace("Elapsed time: %g ms\n\n") + defer func() { + if nResults > 0 { + un(startTrace, startTime) + } + }() + fmt.Printf("\n") + c := make(chan interface{}, 5) + go ses.Execute(query, c, 100) + for res := range c { + fmt.Print(ses.Format(res)) + nResults++ + } + if nResults > 0 { + results := "Result" + if nResults > 1 { + results += "s" + } + fmt.Printf("-----------\n%d %s\n", nResults, results) + } +} + +const ( + ps1 = "cayley> " + ps2 = "... " + + history = ".cayley_history" +) + +func Repl(h *graph.Handle, queryLanguage string, cfg *config.Config) error { + var ses query.Session + switch queryLanguage { + case "sexp": + ses = sexp.NewSession(h.QuadStore) + case "mql": + ses = mql.NewSession(h.QuadStore) + case "gremlin": + fallthrough + default: + ses = gremlin.NewSession(h.QuadStore, cfg.Timeout, true) + } + + term, err := terminal(history) + if os.IsNotExist(err) { + fmt.Printf("creating new history file: %q\n", history) + } + defer persist(term, history) + + var ( + prompt = ps1 + + code string + ) + + for { + if len(code) == 0 { + prompt = ps1 + } else { + prompt = ps2 + } + line, err := term.Prompt(prompt) + if err != nil { + if err == io.EOF { + fmt.Println() + return nil + } + return err + } + + term.AppendHistory(line) + + line = strings.TrimSpace(line) + if len(line) == 0 || line[0] == '#' { + continue + } + + if code == "" { + cmd, args := splitLine(line) + + switch cmd { + case ":debug": + args = strings.TrimSpace(args) + var debug bool + switch args { + case "t": + debug = true + case "f": + // Do nothing. + default: + debug, err = strconv.ParseBool(args) + if err != nil { + fmt.Printf("Error: cannot parse %q as a valid boolean - acceptable values: 't'|'true' or 'f'|'false'\n", args) + continue + } + } + ses.Debug(debug) + fmt.Printf("Debug set to %t\n", debug) + continue + + case ":a": + quad, err := cquads.Parse(args) + if err != nil { + fmt.Printf("Error: not a valid quad: %v\n", err) + continue + } + + h.QuadWriter.AddQuad(quad) + continue + + case ":d": + quad, err := cquads.Parse(args) + if err != nil { + fmt.Printf("Error: not a valid quad: %v\n", err) + continue + } + h.QuadWriter.RemoveQuad(quad) + continue + + case "exit": + term.Close() + os.Exit(0) + + default: + if cmd[0] == ':' { + fmt.Printf("Unknown command: %q\n", cmd) + continue + } + } + } + + code += line + + result, err := ses.Parse(code) + switch result { + case query.Parsed: + Run(code, ses) + code = "" + case query.ParseFail: + fmt.Println("Error: ", err) + code = "" + case query.ParseMore: + } + } +} + +// Splits a line into a command and its arguments +// e.g. ":a b c d ." will be split into ":a" and " b c d ." +func splitLine(line string) (string, string) { + var command, arguments string + + line = strings.TrimSpace(line) + + // An empty line/a line consisting of whitespace contains neither command nor arguments + if len(line) > 0 { + command = strings.Fields(line)[0] + + // A line containing only a command has no arguments + if len(line) > len(command) { + arguments = line[len(command):] + } + } + + return command, arguments +} + +func terminal(path string) (*liner.State, error) { + term := liner.NewLiner() + + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, os.Kill) + <-c + + err := persist(term, history) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err) + os.Exit(1) + } + + os.Exit(0) + }() + + f, err := os.Open(path) + if err != nil { + return term, err + } + defer f.Close() + _, err = term.ReadHistory(f) + return term, err +} + +func persist(term *liner.State, path string) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("could not open %q to append history: %v", path, err) + } + defer f.Close() + _, err = term.WriteHistory(f) + if err != nil { + return fmt.Errorf("could not write history to %q: %v", path, err) + } + return term.Close() +} diff --git a/internal/db/repl_test.go b/internal/db/repl_test.go new file mode 100644 index 0000000..0a48433 --- /dev/null +++ b/internal/db/repl_test.go @@ -0,0 +1,66 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package db + +import ( + "testing" +) + +var testSplitLines = []struct { + line string + expectedCommand string + expectedArguments string + err error +}{ + { + line: ":a arg1 arg2 arg3 .", + expectedCommand: ":a", + expectedArguments: " arg1 arg2 arg3 .", + }, + { + line: ":debug t", + expectedCommand: ":debug", + expectedArguments: " t", + }, + { + line: "", + // expectedCommand is nil + // expectedArguments is nil + }, + { + line: `:d . # comments here`, + expectedCommand: ":d", + expectedArguments: ` . # comments here`, + }, + { + line: ` :a subject "predicate with spaces" object . `, + expectedCommand: ":a", + expectedArguments: ` subject "predicate with spaces" object .`, + }, +} + +func TestSplitLines(t *testing.T) { + for _, testcase := range testSplitLines { + command, arguments := splitLine(testcase.line) + + if testcase.expectedCommand != command { + t.Errorf("Error splitting lines: got: %v expected: %v", command, testcase.expectedCommand) + } + + if testcase.expectedArguments != arguments { + t.Errorf("Error splitting lines: got: %v expected: %v", arguments, testcase.expectedArguments) + } + } +} diff --git a/internal/http/docs.go b/internal/http/docs.go new file mode 100644 index 0000000..0d1d234 --- /dev/null +++ b/internal/http/docs.go @@ -0,0 +1,74 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + + "github.com/julienschmidt/httprouter" + "github.com/russross/blackfriday" +) + +type DocRequestHandler struct { + assets string +} + +func MarkdownWithCSS(input []byte, title string) []byte { + // set up the HTML renderer + htmlFlags := 0 + htmlFlags |= blackfriday.HTML_USE_XHTML + htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS + htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS + htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES + htmlFlags |= blackfriday.HTML_COMPLETE_PAGE + renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS) + + // set up the parser + extensions := 0 + //extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS + extensions |= blackfriday.EXTENSION_TABLES + extensions |= blackfriday.EXTENSION_FENCED_CODE + extensions |= blackfriday.EXTENSION_AUTOLINK + extensions |= blackfriday.EXTENSION_STRIKETHROUGH + //extensions |= blackfriday.EXTENSION_SPACE_HEADERS + extensions |= blackfriday.EXTENSION_HEADER_IDS + extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS + + return blackfriday.Markdown(input, renderer, extensions) +} + +func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + docpage := params.ByName("docpage") + if docpage == "" { + docpage = "Index" + } + file, err := os.Open(fmt.Sprintf("%s/docs/%s.md", h.assets, docpage)) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + data, err := ioutil.ReadAll(file) + if err != nil { + http.Error(w, err.Error(), http.StatusNoContent) + return + } + output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage)) + fmt.Fprint(w, string(output)) +} + +var markdownCSS = "/static/css/docs.css" diff --git a/internal/http/http.go b/internal/http/http.go new file mode 100644 index 0000000..90691b1 --- /dev/null +++ b/internal/http/http.go @@ -0,0 +1,171 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "flag" + "fmt" + "html/template" + "net/http" + "os" + "time" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/internal/config" + "github.com/google/cayley/internal/db" +) + +type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int + +var assetsPath = flag.String("assets", "", "Explicit path to the HTTP assets.") +var assetsDirs = []string{"templates", "static", "docs"} + +func hasAssets(path string) bool { + for _, dir := range assetsDirs { + if _, err := os.Stat(fmt.Sprint(path, "/", dir)); os.IsNotExist(err) { + return false + } + } + return true +} + +func findAssetsPath() string { + if *assetsPath != "" { + if hasAssets(*assetsPath) { + return *assetsPath + } + glog.Fatalln("Cannot find assets at", *assetsPath, ".") + } + + if hasAssets(".") { + return "." + } + + if hasAssets("..") { + return ".." + } + + gopathPath := os.ExpandEnv("$GOPATH/src/github.com/google/cayley") + if hasAssets(gopathPath) { + return gopathPath + } + glog.Fatalln("Cannot find assets in any of the default search paths. Please run in the same directory, in a Go workspace, or set --assets .") + panic("cannot reach") +} + +func LogRequest(handler ResponseHandler) httprouter.Handle { + return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + start := time.Now() + addr := req.Header.Get("X-Real-IP") + if addr == "" { + addr = req.Header.Get("X-Forwarded-For") + if addr == "" { + addr = req.RemoteAddr + } + } + glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr) + code := handler(w, req, params) + glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start)) + + } +} + +func jsonResponse(w http.ResponseWriter, code int, err interface{}) int { + http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code) + return code +} + +type TemplateRequestHandler struct { + templates *template.Template +} + +func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + uiType := params.ByName("ui_type") + if r.URL.Path == "/" { + uiType = "query" + } + err := h.templates.ExecuteTemplate(w, uiType+".html", h) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +type API struct { + config *config.Config + handle *graph.Handle +} + +func (api *API) GetHandleForRequest(r *http.Request) (*graph.Handle, error) { + if !api.config.RequiresHTTPRequestContext { + return api.handle, nil + } + + opts := make(graph.Options) + opts["HTTPRequest"] = r + + qs, err := graph.NewQuadStoreForRequest(api.handle.QuadStore, opts) + if err != nil { + return nil, err + } + qw, err := db.OpenQuadWriter(qs, api.config) + if err != nil { + return nil, err + } + return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil +} + +func (api *API) APIv1(r *httprouter.Router) { + r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) + r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) + r.POST("/api/v1/write", LogRequest(api.ServeV1Write)) + r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad)) + //TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form? + r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete)) +} + +func SetupRoutes(handle *graph.Handle, cfg *config.Config) { + r := httprouter.New() + assets := findAssetsPath() + if glog.V(2) { + glog.V(2).Infoln("Found assets at", assets) + } + var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl"))) + templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html")) + root := &TemplateRequestHandler{templates: templates} + docs := &DocRequestHandler{assets: assets} + api := &API{config: cfg, handle: handle} + api.APIv1(r) + + //m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true})) + //r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) + r.GET("/docs/:docpage", docs.ServeHTTP) + r.GET("/ui/:ui_type", root.ServeHTTP) + r.GET("/", root.ServeHTTP) + http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/"))))) + http.Handle("/", r) +} + +func Serve(handle *graph.Handle, cfg *config.Config) { + SetupRoutes(handle, cfg) + glog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) + fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) + err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil) + if err != nil { + glog.Fatal("ListenAndServe: ", err) + } +} diff --git a/internal/http/http_test.go b/internal/http/http_test.go new file mode 100644 index 0000000..1b1490b --- /dev/null +++ b/internal/http/http_test.go @@ -0,0 +1,73 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "reflect" + "testing" + + "github.com/google/cayley/quad" +) + +var parseTests = []struct { + message string + input string + expect []quad.Quad + err error +}{ + { + message: "parse correct JSON", + input: `[ + {"subject": "foo", "predicate": "bar", "object": "baz"}, + {"subject": "foo", "predicate": "bar", "object": "baz", "label": "graph"} + ]`, + expect: []quad.Quad{ + {"foo", "bar", "baz", ""}, + {"foo", "bar", "baz", "graph"}, + }, + err: nil, + }, + { + message: "parse correct JSON with extra field", + input: `[ + {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} + ]`, + expect: []quad.Quad{ + {"foo", "bar", "foo", ""}, + }, + err: nil, + }, + { + message: "reject incorrect JSON", + input: `[ + {"subject": "foo", "predicate": "bar"} + ]`, + expect: nil, + err: fmt.Errorf("invalid quad at index %d. %v", 0, quad.Quad{"foo", "bar", "", ""}), + }, +} + +func TestParseJSON(t *testing.T) { + for _, test := range parseTests { + got, err := ParseJSONToQuadList([]byte(test.input)) + if fmt.Sprint(err) != fmt.Sprint(test.err) { + t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err) + } + if !reflect.DeepEqual(got, test.expect) { + t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect) + } + } +} diff --git a/internal/http/query.go b/internal/http/query.go new file mode 100644 index 0000000..44d5fb4 --- /dev/null +++ b/internal/http/query.go @@ -0,0 +1,146 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/query" + "github.com/google/cayley/query/gremlin" + "github.com/google/cayley/query/mql" +) + +type SuccessQueryWrapper struct { + Result interface{} `json:"result"` +} + +type ErrorQueryWrapper struct { + Error string `json:"error"` +} + +func WrapErrResult(err error) ([]byte, error) { + var wrap ErrorQueryWrapper + wrap.Error = err.Error() + return json.MarshalIndent(wrap, "", " ") +} + +func WrapResult(result interface{}) ([]byte, error) { + var wrap SuccessQueryWrapper + wrap.Result = result + return json.MarshalIndent(wrap, "", " ") +} + +func Run(q string, ses query.HTTP) (interface{}, error) { + c := make(chan interface{}, 5) + go ses.Execute(q, c, 100) + for res := range c { + ses.Collate(res) + } + return ses.Results() +} + +func GetQueryShape(q string, ses query.HTTP) ([]byte, error) { + s, err := ses.ShapeOf(q) + if err != nil { + return nil, err + } + return json.Marshal(s) +} + +// TODO(barakmich): Turn this into proper middleware. +func (api *API) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + h, err := api.GetHandleForRequest(r) + var ses query.HTTP + switch params.ByName("query_lang") { + case "gremlin": + ses = gremlin.NewSession(h.QuadStore, api.config.Timeout, false) + case "mql": + ses = mql.NewSession(h.QuadStore) + default: + return jsonResponse(w, 400, "Need a query language.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return jsonResponse(w, 400, err) + } + code := string(bodyBytes) + result, err := ses.Parse(code) + switch result { + case query.Parsed: + var output interface{} + var bytes []byte + var err error + output, err = Run(code, ses) + if err != nil { + bytes, err = WrapErrResult(err) + http.Error(w, string(bytes), 400) + ses = nil + return 400 + } + bytes, err = WrapResult(output) + if err != nil { + ses = nil + return jsonResponse(w, 400, err) + } + fmt.Fprint(w, string(bytes)) + ses = nil + return 200 + case query.ParseFail: + ses = nil + return jsonResponse(w, 400, err) + default: + ses = nil + return jsonResponse(w, 500, "Incomplete data?") + } +} + +func (api *API) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + h, err := api.GetHandleForRequest(r) + var ses query.HTTP + switch params.ByName("query_lang") { + case "gremlin": + ses = gremlin.NewSession(h.QuadStore, api.config.Timeout, false) + case "mql": + ses = mql.NewSession(h.QuadStore) + default: + return jsonResponse(w, 400, "Need a query language.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return jsonResponse(w, 400, err) + } + code := string(bodyBytes) + result, err := ses.Parse(code) + switch result { + case query.Parsed: + var output []byte + var err error + output, err = GetQueryShape(code, ses) + if err != nil { + return jsonResponse(w, 400, err) + } + fmt.Fprint(w, string(output)) + return 200 + case query.ParseFail: + return jsonResponse(w, 400, err) + default: + return jsonResponse(w, 500, "Incomplete data?") + } +} diff --git a/internal/http/write.go b/internal/http/write.go new file mode 100644 index 0000000..de2c24b --- /dev/null +++ b/internal/http/write.go @@ -0,0 +1,144 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "strconv" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/internal" + "github.com/google/cayley/quad" + "github.com/google/cayley/quad/cquads" +) + +func ParseJSONToQuadList(jsonBody []byte) ([]quad.Quad, error) { + var quads []quad.Quad + err := json.Unmarshal(jsonBody, &quads) + if err != nil { + return nil, err + } + for i, q := range quads { + if !q.IsValid() { + return nil, fmt.Errorf("invalid quad at index %d. %s", i, q) + } + } + return quads, nil +} + +func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int { + if api.config.ReadOnly { + return jsonResponse(w, 400, "Database is read-only.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return jsonResponse(w, 400, err) + } + quads, err := ParseJSONToQuadList(bodyBytes) + if err != nil { + return jsonResponse(w, 400, err) + } + h, err := api.GetHandleForRequest(r) + if err != nil { + return jsonResponse(w, 400, err) + } + + h.QuadWriter.AddQuadSet(quads) + fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", len(quads)) + return 200 +} + +func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + if api.config.ReadOnly { + return jsonResponse(w, 400, "Database is read-only.") + } + + formFile, _, err := r.FormFile("NQuadFile") + if err != nil { + glog.Errorln(err) + return jsonResponse(w, 500, "Couldn't read file: "+err.Error()) + } + defer formFile.Close() + + blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) + if blockErr != nil { + blockSize = int64(api.config.LoadSize) + } + + quadReader, err := internal.Decompressor(formFile) + // TODO(kortschak) Make this configurable from the web UI. + dec := cquads.NewDecoder(quadReader) + + h, err := api.GetHandleForRequest(r) + if err != nil { + return jsonResponse(w, 400, err) + } + + var ( + n int + block = make([]quad.Quad, 0, blockSize) + ) + for { + t, err := dec.Unmarshal() + if err != nil { + if err == io.EOF { + break + } + glog.Fatalln("what can do this here?", err) // FIXME(kortschak) + } + block = append(block, t) + n++ + if len(block) == cap(block) { + h.QuadWriter.AddQuadSet(block) + block = block[:0] + } + } + h.QuadWriter.AddQuadSet(block) + + fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n) + + return 200 +} + +func (api *API) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + if api.config.ReadOnly { + return jsonResponse(w, 400, "Database is read-only.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return jsonResponse(w, 400, err) + } + quads, err := ParseJSONToQuadList(bodyBytes) + if err != nil { + return jsonResponse(w, 400, err) + } + h, err := api.GetHandleForRequest(r) + if err != nil { + return jsonResponse(w, 400, err) + } + count := 0 + for _, q := range quads { + h.QuadWriter.RemoveQuad(q) + count++ + } + fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d quads.\"}", count) + return 200 +} diff --git a/internal/load.go b/internal/load.go index 57b46da..eb5020a 100644 --- a/internal/load.go +++ b/internal/load.go @@ -8,9 +8,9 @@ import ( "os" "path/filepath" - "github.com/google/cayley/config" - "github.com/google/cayley/db" "github.com/google/cayley/graph" + "github.com/google/cayley/internal/config" + "github.com/google/cayley/internal/db" "github.com/google/cayley/quad" "github.com/google/cayley/quad/cquads" "github.com/google/cayley/quad/nquads"