diff --git a/.gitignore b/.gitignore index 17c43e6..206a77a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ main *.test *.peg.go cayley.cfg +.cayley_history diff --git a/.travis.yml b/.travis.yml index 66af39d..bd0558f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ install: - go get github.com/barakmich/glog - go get github.com/julienschmidt/httprouter - go get github.com/petar/GoLLRB/llrb + - go get github.com/peterh/liner - go get github.com/robertkrimen/otto - go get github.com/russross/blackfriday - go get github.com/syndtr/goleveldb/leveldb diff --git a/README.md b/README.md index 0e616a8..65c0faa 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,13 @@ Its goal is to be a part of the developer's toolbox where [Linked Data](http://l [![Build Status](https://travis-ci.org/google/cayley.png?branch=master)](https://travis-ci.org/google/cayley) ## What's new? -* 2014-07-12: - * Massive cleanup and restructuring is largely done, it should be even easier to add to Cayley. (thanks @kortschak) - * A couple new backends are in progress, namely Postgres and Cassandra -- PRs when they come around. - * Cayley is [now in Homebrew](https://github.com/Homebrew/homebrew/commit/1bd2fb2a61c7101a8c79c05afc90eeb02e9aa240), thanks to @whitlockjc - * Our first client API (for Clojure, thanks to @wjb) -- list is now started on the [Client API wiki page](https://github.com/google/cayley/wiki/Client-APIs) +* 2014-08-06: + * 0.3.1 Binary Release including: + * New Quad Parser (more strictly passing the [W3C spec](http://www.w3.org/TR/n-quads) and test suite) + * Automatic decompression of quad files + * Ruby and a Node.JS [client libraries](https://github.com/google/cayley/wiki/Client-APIs) from the community. + * Benchmarks + * [Large speedups on HEAD](https://github.com/google/cayley/pull/101) (in for the next binary release) ## Features diff --git a/TODO.md b/TODO.md index 92c0118..74973c9 100644 --- a/TODO.md +++ b/TODO.md @@ -52,7 +52,7 @@ An important failure of MQL before was that it was never well-specified. Let's n ### New Iterators #### Limit Iterator -The necessary component to make mid-query limit work. Acts as a limit on Next(), a passthrough on Contains(), and a limit on NextResult() +The necessary component to make mid-query limit work. Acts as a limit on Next(), a passthrough on Contains(), and a limit on NextPath() ## Medium Term diff --git a/cayley.go b/cayley.go index c68d487..7e58749 100644 --- a/cayley.go +++ b/cayley.go @@ -17,9 +17,17 @@ package main import ( + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" "flag" "fmt" + "io" + client "net/http" + "net/url" "os" + "path/filepath" "runtime" "github.com/barakmich/glog" @@ -28,6 +36,9 @@ import ( "github.com/google/cayley/db" "github.com/google/cayley/graph" "github.com/google/cayley/http" + "github.com/google/cayley/quad" + "github.com/google/cayley/quad/cquads" + "github.com/google/cayley/quad/nquads" // Load all supported backends. _ "github.com/google/cayley/graph/leveldb" @@ -35,14 +46,19 @@ import ( _ "github.com/google/cayley/graph/mongo" ) -var tripleFile = flag.String("triples", "", "Triple File to load before going to REPL.") -var cpuprofile = flag.String("prof", "", "Output profiling file.") -var queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.") -var configFile = flag.String("config", "", "Path to an explicit configuration file.") +var ( + tripleFile = flag.String("triples", "", "Triple File to load before going to REPL.") + tripleType = flag.String("format", "cquad", `Triple format to use for loading ("cquad" or "nquad").`) + cpuprofile = flag.String("prof", "", "Output profiling file.") + queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.") + configFile = flag.String("config", "", "Path to an explicit configuration file.") +) // Filled in by `go build ldflags="-X main.VERSION `ver`"`. -var BUILD_DATE string -var VERSION string +var ( + BUILD_DATE string + VERSION string +) func Usage() { fmt.Println("Cayley is a graph store and graph query layer.") @@ -100,40 +116,140 @@ func main() { fmt.Println("Cayley snapshot") } os.Exit(0) + case "init": - err = db.Init(cfg, *tripleFile) + err = db.Init(cfg) + if err != nil { + break + } + if *tripleFile != "" { + ts, err = db.Open(cfg) + if err != nil { + break + } + err = load(ts, cfg, *tripleFile, *tripleType) + if err != nil { + break + } + ts.Close() + } + case "load": ts, err = db.Open(cfg) if err != nil { break } - err = db.Load(ts, cfg, *tripleFile) + err = load(ts, cfg, *tripleFile, *tripleType) if err != nil { break } + ts.Close() + case "repl": ts, err = db.Open(cfg) if err != nil { break } - err = db.Repl(ts, *queryLanguage, cfg) - if err != nil { - break + if !graph.IsPersistent(cfg.DatabaseType) { + err = load(ts, cfg, "", *tripleType) + if err != nil { + break + } } + + err = db.Repl(ts, *queryLanguage, cfg) + ts.Close() + case "http": ts, err = db.Open(cfg) if err != nil { break } + if !graph.IsPersistent(cfg.DatabaseType) { + err = load(ts, cfg, "", *tripleType) + if err != nil { + break + } + } + http.Serve(ts, cfg) + ts.Close() + default: fmt.Println("No command", cmd) flag.Usage() } if err != nil { - glog.Fatalln(err) + glog.Errorln(err) + } +} + +func load(ts graph.TripleStore, cfg *config.Config, path, typ string) error { + var r io.Reader + + if path == "" { + path = cfg.DatabasePath + } + u, err := url.Parse(path) + if err != nil || u.Scheme == "file" || u.Scheme == "" { + // Don't alter relative URL path or non-URL path parameter. + if u.Scheme != "" && err == nil { + // Recovery heuristic for mistyping "file://path/to/file". + path = filepath.Join(u.Host, u.Path) + } + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("could not open file %q: %v", path, err) + } + defer f.Close() + r = f + } else { + res, err := client.Get(path) + if err != nil { + return fmt.Errorf("could not get resource <%s>: %v", u, err) + } + defer res.Body.Close() + r = res.Body + } + + r, err = decompressor(r) + if err != nil { + return err + } + + var dec quad.Unmarshaler + switch typ { + case "cquad": + dec = cquads.NewDecoder(r) + case "nquad": + dec = nquads.NewDecoder(r) + default: + return fmt.Errorf("unknown quad format %q", typ) + } + + return db.Load(ts, cfg, dec) +} + +const ( + gzipMagic = "\x1f\x8b" + b2zipMagic = "BZh" +) + +func decompressor(r io.Reader) (io.Reader, error) { + br := bufio.NewReader(r) + buf, err := br.Peek(3) + if err != nil { + return nil, err + } + switch { + case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0: + return gzip.NewReader(br) + case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0: + return bzip2.NewReader(br), nil + default: + return br, nil } } diff --git a/cayley_test.go b/cayley_test.go index 55ef3c2..d108f85 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -15,6 +15,11 @@ package main import ( + "bytes" + "compress/bzip2" + "compress/gzip" + "io" + "strings" "sync" "testing" "time" @@ -308,6 +313,13 @@ func prepare(t testing.TB) { if err != nil { t.Fatalf("Failed to open %q: %v", cfg.DatabasePath, err) } + + if !graph.IsPersistent(cfg.DatabaseType) { + err = load(ts, cfg, "", "cquad") + if err != nil { + t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err) + } + } }) } @@ -408,3 +420,84 @@ func BenchmarkKeanuOther(b *testing.B) { func BenchmarkKeanuBullockOther(b *testing.B) { runBench(8, b) } + +// reader is a test helper to filter non-io.Reader methods from the contained io.Reader. +type reader struct { + r io.Reader +} + +func (r reader) Read(p []byte) (int, error) { + return r.r.Read(p) +} + +var testDecompressor = []struct { + message string + input io.Reader + expect []byte + err error + readErr error +}{ + { + message: "text input", + input: strings.NewReader("cayley data\n"), + err: nil, + expect: []byte("cayley data\n"), + readErr: nil, + }, + { + message: "gzip input", + input: bytes.NewReader([]byte{ + 0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad, + 0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00, + }), + err: nil, + expect: []byte("cayley data\n"), + readErr: nil, + }, + { + message: "bzip2 input", + input: bytes.NewReader([]byte{ + 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00, + 0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c, + 0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16, + 0xa9, 0x7c, 0x78, 0x80, + }), + err: nil, + expect: []byte("cayley data\n"), + readErr: nil, + }, + { + message: "bad gzip input", + input: strings.NewReader("\x1f\x8bcayley data\n"), + err: gzip.ErrHeader, + expect: nil, + readErr: nil, + }, + { + message: "bad bzip2 input", + input: strings.NewReader("\x42\x5a\x68cayley data\n"), + err: nil, + expect: nil, + readErr: bzip2.StructuralError("invalid compression level"), + }, +} + +func TestDecompressor(t *testing.T) { + for _, test := range testDecompressor { + r, err := decompressor(test.input) + if err != test.err { + t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err) + } + if err != nil { + continue + } + p := make([]byte, len(test.expect)*2) + n, err := r.Read(p) + if err != test.readErr { + t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err) + } + if bytes.Compare(p[:n], test.expect) != 0 { + t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect) + } + } +} diff --git a/db/load.go b/db/db.go similarity index 56% rename from db/load.go rename to db/db.go index 9a3b069..8ea30db 100644 --- a/db/load.go +++ b/db/db.go @@ -15,46 +15,48 @@ package db import ( - "bytes" - "compress/bzip2" - "compress/gzip" + "errors" "fmt" "io" - "os" "github.com/barakmich/glog" + "github.com/google/cayley/config" "github.com/google/cayley/graph" "github.com/google/cayley/quad" - "github.com/google/cayley/quad/cquads" ) -func Load(ts graph.TripleStore, cfg *config.Config, path string) error { - f, err := os.Open(path) - if err != nil { - return fmt.Errorf("could not open file %q: %v", path, err) - } - defer f.Close() +var ErrNotPersistent = errors.New("database type is not persistent") - r, err := decompressor(f) - if err != nil { - glog.Fatalln(err) +func Init(cfg *config.Config) error { + if !graph.IsPersistent(cfg.DatabaseType) { + return fmt.Errorf("ignoring unproductive database initialization request: %v", ErrNotPersistent) } - dec := cquads.NewDecoder(r) + return graph.InitTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) +} +func Open(cfg *config.Config) (graph.TripleStore, error) { + glog.Infof("Opening database %q at %s", cfg.DatabaseType, cfg.DatabasePath) + ts, err := graph.NewTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) + if err != nil { + return nil, err + } + + return ts, nil +} + +func Load(ts graph.TripleStore, cfg *config.Config, dec quad.Unmarshaler) error { bulker, canBulk := ts.(graph.BulkLoader) if canBulk { - err = bulker.BulkLoad(dec) - if err == nil { + switch err := bulker.BulkLoad(dec); err { + case nil: return nil + case graph.ErrCannotBulkLoad: + // Try individual loading. + default: + return err } - if err == graph.ErrCannotBulkLoad { - err = nil - } - } - if err != nil { - return err } block := make([]quad.Quad, 0, cfg.LoadSize) @@ -76,29 +78,3 @@ func Load(ts graph.TripleStore, cfg *config.Config, path string) error { return nil } - -const ( - gzipMagic = "\x1f\x8b" - b2zipMagic = "BZh" -) - -type readAtReader interface { - io.Reader - io.ReaderAt -} - -func decompressor(r readAtReader) (io.Reader, error) { - var buf [3]byte - _, err := r.ReadAt(buf[:], 0) - if err != nil { - return nil, err - } - switch { - case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0: - return gzip.NewReader(r) - case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0: - return bzip2.NewReader(r), nil - default: - return r, nil - } -} diff --git a/db/init.go b/db/init.go deleted file mode 100644 index a791a8f..0000000 --- a/db/init.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db - -import ( - "github.com/google/cayley/config" - "github.com/google/cayley/graph" -) - -func Init(cfg *config.Config, triplePath string) error { - err := graph.InitTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) - if err != nil { - return err - } - if triplePath != "" { - ts, err := Open(cfg) - if err != nil { - return err - } - err = Load(ts, cfg, triplePath) - if err != nil { - return err - } - ts.Close() - } - return err -} diff --git a/db/load_test.go b/db/load_test.go deleted file mode 100644 index 17ed6c5..0000000 --- a/db/load_test.go +++ /dev/null @@ -1,81 +0,0 @@ -package db - -import ( - "bytes" - "compress/bzip2" - "compress/gzip" - "testing" -) - -var testDecompressor = []struct { - message string - input []byte - expect []byte - err error - readErr error -}{ - { - message: "text input", - input: []byte("cayley data\n"), - err: nil, - expect: []byte("cayley data\n"), - readErr: nil, - }, - { - message: "gzip input", - input: []byte{ - 0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad, - 0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00, - }, - err: nil, - expect: []byte("cayley data\n"), - readErr: nil, - }, - { - message: "bzip2 input", - input: []byte{ - 0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00, - 0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c, - 0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16, - 0xa9, 0x7c, 0x78, 0x80, - }, - err: nil, - expect: []byte("cayley data\n"), - readErr: nil, - }, - { - message: "bad gzip input", - input: []byte{0x1f, 0x8b, 'c', 'a', 'y', 'l', 'e', 'y', ' ', 'd', 'a', 't', 'a', '\n'}, - err: gzip.ErrHeader, - expect: nil, - readErr: nil, - }, - { - message: "bad bzip2 input", - input: []byte{0x42, 0x5a, 0x68, 'c', 'a', 'y', 'l', 'e', 'y', ' ', 'd', 'a', 't', 'a', '\n'}, - err: nil, - expect: nil, - readErr: bzip2.StructuralError("invalid compression level"), - }, -} - -func TestDecompressor(t *testing.T) { - for _, test := range testDecompressor { - buf := bytes.NewReader(test.input) - r, err := decompressor(buf) - if err != test.err { - t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err) - } - if err != nil { - continue - } - p := make([]byte, len(test.expect)*2) - n, err := r.Read(p) - if err != test.readErr { - t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err) - } - if bytes.Compare(p[:n], test.expect) != 0 { - t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect) - } - } -} diff --git a/db/open.go b/db/open.go deleted file mode 100644 index 3e1ee24..0000000 --- a/db/open.go +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package db - -import ( - "github.com/barakmich/glog" - - "github.com/google/cayley/config" - "github.com/google/cayley/graph" -) - -func Open(cfg *config.Config) (graph.TripleStore, error) { - glog.Infof("Opening database %q at %s", cfg.DatabaseType, cfg.DatabasePath) - ts, err := graph.NewTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions) - if err != nil { - return nil, err - } - - // Memstore is not persistent, so it MUST be loaded. - if cfg.DatabaseType == "memstore" { - err = Load(ts, cfg, cfg.DatabasePath) - if err != nil { - return nil, err - } - } - - return ts, nil -} diff --git a/db/repl.go b/db/repl.go index 730d59d..11ecfea 100644 --- a/db/repl.go +++ b/db/repl.go @@ -15,14 +15,15 @@ package db import ( - "bufio" - "bytes" - "errors" "fmt" "io" "os" + "os/signal" + "strings" "time" + "github.com/peterh/liner" + "github.com/google/cayley/config" "github.com/google/cayley/graph" "github.com/google/cayley/quad/cquads" @@ -62,6 +63,13 @@ func Run(query string, ses query.Session) { } } +const ( + ps1 = "cayley> " + ps2 = "... " + + history = ".cayley_history" +) + func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error { var ses query.Session switch queryLanguage { @@ -74,80 +82,123 @@ func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error default: ses = gremlin.NewSession(ts, cfg.Timeout, true) } - buf := bufio.NewReader(os.Stdin) - var line []byte + + term, err := terminal(history) + if os.IsNotExist(err) { + fmt.Printf("creating new history file: %q\n", history) + } + defer persist(term, history) + + var ( + prompt = ps1 + + code string + ) + for { - if len(line) == 0 { - fmt.Print("cayley> ") + if len(code) == 0 { + prompt = ps1 } else { - fmt.Print("... ") + prompt = ps2 } - l, prefix, err := buf.ReadLine() - if err == io.EOF { - if len(line) != 0 { - line = line[:0] - } else { + line, err := term.Prompt(prompt) + if err != nil { + if err == io.EOF { return nil } + return err } - if err != nil { - line = line[:0] - } - if prefix { - return errors.New("line too long") - } - line = append(line, l...) - if len(line) == 0 { - continue - } - line = bytes.TrimSpace(line) + + term.AppendHistory(line) + + line = strings.TrimSpace(line) if len(line) == 0 || line[0] == '#' { - line = line[:0] continue } - if bytes.HasPrefix(line, []byte(":debug")) { - ses.ToggleDebug() - fmt.Println("Debug Toggled") - line = line[:0] - continue - } - if bytes.HasPrefix(line, []byte(":a")) { - var tripleStmt = line[3:] - triple, err := cquads.Parse(string(tripleStmt)) - if !triple.IsValid() { - if err != nil { - fmt.Printf("not a valid triple: %v\n", err) + + if code == "" { + switch { + case strings.HasPrefix(line, ":debug"): + ses.ToggleDebug() + fmt.Println("Debug Toggled") + continue + + case strings.HasPrefix(line, ":a"): + triple, err := cquads.Parse(line[3:]) + if !triple.IsValid() { + if err != nil { + fmt.Printf("not a valid triple: %v\n", err) + } + continue } - line = line[:0] + ts.AddTriple(triple) + continue + + case strings.HasPrefix(line, ":d"): + triple, err := cquads.Parse(line[3:]) + if !triple.IsValid() { + if err != nil { + fmt.Printf("not a valid triple: %v\n", err) + } + continue + } + ts.RemoveTriple(triple) continue } - ts.AddTriple(triple) - line = line[:0] - continue } - if bytes.HasPrefix(line, []byte(":d")) { - var tripleStmt = line[3:] - triple, err := cquads.Parse(string(tripleStmt)) - if !triple.IsValid() { - if err != nil { - fmt.Printf("not a valid triple: %v\n", err) - } - line = line[:0] - continue - } - ts.RemoveTriple(triple) - line = line[:0] - continue - } - result, err := ses.InputParses(string(line)) + + code += line + + result, err := ses.InputParses(code) switch result { case query.Parsed: - Run(string(line), ses) - line = line[:0] + Run(code, ses) + code = "" case query.ParseFail: fmt.Println("Error: ", err) - line = line[:0] + code = "" case query.ParseMore: } } } + +func terminal(path string) (*liner.State, error) { + term := liner.NewLiner() + + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt, os.Kill) + <-c + + persist(term, history) + + err := term.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err) + os.Exit(1) + } + + os.Exit(0) + }() + + f, err := os.Open(path) + if err != nil { + return term, err + } + defer f.Close() + _, err = term.ReadHistory(f) + return term, err +} + +func persist(term *liner.State, path string) error { + f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666) + if err != nil { + return fmt.Errorf("could not open %q to append history: %v", path, err) + } + defer f.Close() + _, err = term.WriteHistory(f) + if err != nil { + return fmt.Errorf("could not write history to %q: %v", path, err) + } + return nil +} diff --git a/graph/iterator.go b/graph/iterator.go index 3880b80..96f1587 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -77,17 +77,21 @@ type Iterator interface { // the iteration interface. // // To get the full results of iteration, do the following: - // while (!Next()): - // emit result - // while (!NextResult()): - // emit result + // + // for graph.Next(it) { + // val := it.Result() + // ... do things with val. + // for it.NextPath() { + // ... find other paths to iterate + // } + // } // // All of them should set iterator.Last to be the last returned value, to // make results work. // - // NextResult() advances iterators that may have more than one valid result, + // NextPath() advances iterators that may have more than one valid result, // from the bottom up. - NextResult() bool + NextPath() bool // Contains returns whether the value is within the set held by the iterator. Contains(Value) bool @@ -135,22 +139,22 @@ type Iterator interface { } type Nexter interface { - // Next() advances the iterator and returns the next valid result. Returns - // (, true) or (nil, false) - Next() (Value, bool) + // Next advances the iterator to the next value, which will then be available through + // the Result method. It returns false if no further advancement is possible. + Next() bool Iterator } // Next is a convenience function that conditionally calls the Next method // of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next -// return a nil Value and false. -func Next(it Iterator) (Value, bool) { +// returns false. +func Next(it Iterator) bool { if n, ok := it.(Nexter); ok { return n.Next() } glog.Errorln("Nexting an un-nextable iterator") - return nil, false + return false } // Height is a convienence function to measure the height of an iterator tree. @@ -271,7 +275,7 @@ func NextLogIn(it Iterator) { } } -func NextLogOut(it Iterator, val Value, ok bool) (Value, bool) { +func NextLogOut(it Iterator, val Value, ok bool) bool { if glog.V(4) { if ok { glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type().String()), it.UID(), val) @@ -279,5 +283,5 @@ func NextLogOut(it Iterator, val Value, ok bool) (Value, bool) { glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID()) } } - return val, ok + return ok } diff --git a/graph/iterator/all_iterator.go b/graph/iterator/all_iterator.go index a3e1174..80e471c 100644 --- a/graph/iterator/all_iterator.go +++ b/graph/iterator/all_iterator.go @@ -87,7 +87,7 @@ func (it *Int64) DebugString(indent int) string { // Next() on an Int64 all iterator is a simple incrementing counter. // Return the next integer, and mark it as the result. -func (it *Int64) Next() (graph.Value, bool) { +func (it *Int64) Next() bool { graph.NextLogIn(it) if it.at == -1 { return graph.NextLogOut(it, nil, false) @@ -110,7 +110,7 @@ func (it *Int64) Result() graph.Value { return it.result } -func (it *Int64) NextResult() bool { +func (it *Int64) NextPath() bool { return false } diff --git a/graph/iterator/and_iterator.go b/graph/iterator/and_iterator.go index 654f7c6..7cb7577 100644 --- a/graph/iterator/and_iterator.go +++ b/graph/iterator/and_iterator.go @@ -151,25 +151,20 @@ func (it *And) AddSubIterator(sub graph.Iterator) { it.itCount++ } -// Returns the Next value from the And iterator. Because the And is the -// intersection of its subiterators, it must choose one subiterator to produce a -// candidate, and check this value against the subiterators. A productive choice -// of primary iterator is therefore very important. -func (it *And) Next() (graph.Value, bool) { +// Returns advances the And iterator. Because the And is the intersection of its +// subiterators, it must choose one subiterator to produce a candidate, and check +// this value against the subiterators. A productive choice of primary iterator +// is therefore very important. +func (it *And) Next() bool { graph.NextLogIn(it) - var curr graph.Value - var exists bool - for { - curr, exists = graph.Next(it.primaryIt) - if !exists { - return graph.NextLogOut(it, nil, false) - } + for graph.Next(it.primaryIt) { + curr := it.primaryIt.Result() if it.subItsContain(curr) { it.result = curr return graph.NextLogOut(it, curr, true) } } - panic("unreachable") + return graph.NextLogOut(it, nil, false) } func (it *And) Result() graph.Value { @@ -236,15 +231,15 @@ func (it *And) Size() (int64, bool) { return val, b } -// An And has no NextResult of its own -- that is, there are no other values +// An And has no NextPath of its own -- that is, there are no other values // which satisfy our previous result that are not the result itself. Our // subiterators might, however, so just pass the call recursively. -func (it *And) NextResult() bool { - if it.primaryIt.NextResult() { +func (it *And) NextPath() bool { + if it.primaryIt.NextPath() { return true } for _, sub := range it.internalIterators { - if sub.NextResult() { + if sub.NextPath() { return true } } diff --git a/graph/iterator/and_iterator_test.go b/graph/iterator/and_iterator_test.go index 82d290a..3023bfd 100644 --- a/graph/iterator/and_iterator_test.go +++ b/graph/iterator/and_iterator_test.go @@ -36,10 +36,10 @@ func TestTag(t *testing.T) { t.Errorf("Cannot get tag back, got %s", out[0]) } - val, ok := and.Next() - if !ok { + if !and.Next() { t.Errorf("And did not next") } + val := and.Result() if val != 234 { t.Errorf("Unexpected value") } @@ -76,18 +76,15 @@ func TestAndAndFixedIterators(t *testing.T) { t.Error("not accurate") } - val, ok := and.Next() - if val != 3 || ok == false { + if !and.Next() || and.Result() != 3 { t.Error("Incorrect first value") } - val, ok = and.Next() - if val != 4 || ok == false { + if !and.Next() || and.Result() != 4 { t.Error("Incorrect second value") } - val, ok = and.Next() - if ok { + if and.Next() { t.Error("Too many values") } @@ -117,8 +114,7 @@ func TestNonOverlappingFixedIterators(t *testing.T) { t.Error("not accurate") } - _, ok := and.Next() - if ok { + if and.Next() { t.Error("Too many values") } @@ -131,18 +127,15 @@ func TestAllIterators(t *testing.T) { and.AddSubIterator(all2) and.AddSubIterator(all1) - val, ok := and.Next() - if val.(int64) != 4 || ok == false { + if !and.Next() || and.Result() != int64(4) { t.Error("Incorrect first value") } - val, ok = and.Next() - if val.(int64) != 5 || ok == false { + if !and.Next() || and.Result() != int64(5) { t.Error("Incorrect second value") } - val, ok = and.Next() - if ok { + if and.Next() { t.Error("Too many values") } diff --git a/graph/iterator/fixed_iterator.go b/graph/iterator/fixed_iterator.go index 7efee2d..a2d57b1 100644 --- a/graph/iterator/fixed_iterator.go +++ b/graph/iterator/fixed_iterator.go @@ -135,8 +135,8 @@ func (it *Fixed) Contains(v graph.Value) bool { return graph.ContainsLogOut(it, v, false) } -// Return the next stored value from the iterator. -func (it *Fixed) Next() (graph.Value, bool) { +// Next advances the iterator. +func (it *Fixed) Next() bool { graph.NextLogIn(it) if it.lastIndex == len(it.values) { return graph.NextLogOut(it, nil, false) @@ -156,7 +156,7 @@ func (it *Fixed) Result() graph.Value { return it.result } -func (it *Fixed) NextResult() bool { +func (it *Fixed) NextPath() bool { return false } diff --git a/graph/iterator/hasa_iterator.go b/graph/iterator/hasa_iterator.go index 954102a..93e224a 100644 --- a/graph/iterator/hasa_iterator.go +++ b/graph/iterator/hasa_iterator.go @@ -27,9 +27,9 @@ package iterator // value to check, it means "Check all predicates that have this value for your // direction against the subiterator." This would imply that there's more than // one possibility for the same Contains()ed value. While we could return the -// number of options, it's simpler to return one, and then call NextResult() +// number of options, it's simpler to return one, and then call NextPath() // enough times to enumerate the options. (In fact, one could argue that the -// raison d'etre for NextResult() is this iterator). +// raison d'etre for NextPath() is this iterator). // // Alternatively, can be seen as the dual of the LinksTo iterator. @@ -158,16 +158,13 @@ func (it *HasA) Contains(val graph.Value) bool { // result iterator (a triple iterator based on the last checked value) and returns true if // another match is made. func (it *HasA) NextContains() bool { - for { - linkVal, ok := graph.Next(it.resultIt) - if !ok { - break - } + for graph.Next(it.resultIt) { + link := it.resultIt.Result() if glog.V(4) { - glog.V(4).Infoln("Quad is", it.ts.Quad(linkVal)) + glog.V(4).Infoln("Quad is", it.ts.Quad(link)) } - if it.primaryIt.Contains(linkVal) { - it.result = it.ts.TripleDirection(linkVal, it.dir) + if it.primaryIt.Contains(link) { + it.result = it.ts.TripleDirection(link, it.dir) return true } } @@ -175,33 +172,33 @@ func (it *HasA) NextContains() bool { } // Get the next result that matches this branch. -func (it *HasA) NextResult() bool { - // Order here is important. If the subiterator has a NextResult, then we +func (it *HasA) NextPath() bool { + // Order here is important. If the subiterator has a NextPath, then we // need do nothing -- there is a next result, and we shouldn't move forward. // However, we then need to get the next result from our last Contains(). // - // The upshot is, the end of NextResult() bubbles up from the bottom of the + // The upshot is, the end of NextPath() bubbles up from the bottom of the // iterator tree up, and we need to respect that. - if it.primaryIt.NextResult() { + if it.primaryIt.NextPath() { return true } return it.NextContains() } -// Get the next result from this iterator. This is simpler than Contains. We have a +// Next advances the iterator. This is simpler than Contains. We have a // subiterator we can get a value from, and we can take that resultant triple, // pull our direction out of it, and return that. -func (it *HasA) Next() (graph.Value, bool) { +func (it *HasA) Next() bool { graph.NextLogIn(it) if it.resultIt != nil { it.resultIt.Close() } it.resultIt = &Null{} - tID, ok := graph.Next(it.primaryIt) - if !ok { + if !graph.Next(it.primaryIt) { return graph.NextLogOut(it, 0, false) } + tID := it.primaryIt.Result() name := it.ts.Quad(tID).Get(it.dir) val := it.ts.ValueOf(name) it.result = val diff --git a/graph/iterator/iterator.go b/graph/iterator/iterator.go index 69b26c5..67d8a80 100644 --- a/graph/iterator/iterator.go +++ b/graph/iterator/iterator.go @@ -79,8 +79,8 @@ func (it *Null) DebugString(indent int) string { return strings.Repeat(" ", indent) + "(null)" } -func (it *Null) Next() (graph.Value, bool) { - return nil, false +func (it *Null) Next() bool { + return false } func (it *Null) Result() graph.Value { @@ -95,7 +95,7 @@ func (it *Null) SubIterators() []graph.Iterator { return nil } -func (it *Null) NextResult() bool { +func (it *Null) NextPath() bool { return false } diff --git a/graph/iterator/linksto_iterator.go b/graph/iterator/linksto_iterator.go index a79b34b..517fc11 100644 --- a/graph/iterator/linksto_iterator.go +++ b/graph/iterator/linksto_iterator.go @@ -153,23 +153,23 @@ func (it *LinksTo) Optimize() (graph.Iterator, bool) { } // Next()ing a LinksTo operates as described above. -func (it *LinksTo) Next() (graph.Value, bool) { +func (it *LinksTo) Next() bool { graph.NextLogIn(it) - val, ok := graph.Next(it.nextIt) - if !ok { - // Subiterator is empty, get another one - candidate, ok := graph.Next(it.primaryIt) - if !ok { - // We're out of nodes in our subiterator, so we're done as well. - return graph.NextLogOut(it, 0, false) - } - it.nextIt.Close() - it.nextIt = it.ts.TripleIterator(it.dir, candidate) - // Recurse -- return the first in the next set. - return it.Next() + if graph.Next(it.nextIt) { + it.result = it.nextIt.Result() + return graph.NextLogOut(it, it.nextIt, true) } - it.result = val - return graph.NextLogOut(it, val, ok) + + // Subiterator is empty, get another one + if !graph.Next(it.primaryIt) { + // We're out of nodes in our subiterator, so we're done as well. + return graph.NextLogOut(it, 0, false) + } + it.nextIt.Close() + it.nextIt = it.ts.TripleIterator(it.dir, it.primaryIt.Result()) + + // Recurse -- return the first in the next set. + return it.Next() } func (it *LinksTo) Result() graph.Value { @@ -183,8 +183,8 @@ func (it *LinksTo) Close() { } // We won't ever have a new result, but our subiterators might. -func (it *LinksTo) NextResult() bool { - return it.primaryIt.NextResult() +func (it *LinksTo) NextPath() bool { + return it.primaryIt.NextPath() } // Register the LinksTo. diff --git a/graph/iterator/linksto_iterator_test.go b/graph/iterator/linksto_iterator_test.go index 797ce6e..b1eb1fa 100644 --- a/graph/iterator/linksto_iterator_test.go +++ b/graph/iterator/linksto_iterator_test.go @@ -33,10 +33,10 @@ func TestLinksTo(t *testing.T) { } fixed.Add(val) lto := NewLinksTo(ts, fixed, quad.Object) - val, ok := lto.Next() - if !ok { + if !lto.Next() { t.Error("At least one triple matches the fixed object") } + val = lto.Result() if val != 2 { t.Errorf("Quad index 2, such as %s, should match %s", ts.Quad(2), ts.Quad(val)) } diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 46baf0c..ac6a1bc 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -32,8 +32,12 @@ type result struct { tags map[string]graph.Value } -type hasher interface { - Hasher() interface{} +// Keyer provides a method for comparing types that are not otherwise comparable. +// The Key method must return a dynamic type that is comparable according to the +// Go language specification. The returned value must be unique for each receiver +// value. +type Keyer interface { + Key() interface{} } type Materialize struct { @@ -179,7 +183,7 @@ func (it *Materialize) Stats() graph.IteratorStats { } } -func (it *Materialize) Next() (graph.Value, bool) { +func (it *Materialize) Next() bool { graph.NextLogIn(it) if !it.hasRun { it.materializeSet() @@ -205,8 +209,8 @@ func (it *Materialize) Contains(v graph.Value) bool { return it.subIt.Contains(v) } key := v - if h, ok := v.(hasher); ok { - key = h.Hasher() + if h, ok := v.(Keyer); ok { + key = h.Key() } if i, ok := it.containsMap[key]; ok { it.index = i @@ -216,12 +220,12 @@ func (it *Materialize) Contains(v graph.Value) bool { return graph.ContainsLogOut(it, v, false) } -func (it *Materialize) NextResult() bool { +func (it *Materialize) NextPath() bool { if !it.hasRun { it.materializeSet() } if it.aborted { - return it.subIt.NextResult() + return it.subIt.NextPath() } it.subindex++ @@ -235,19 +239,16 @@ func (it *Materialize) NextResult() bool { func (it *Materialize) materializeSet() { i := 0 - for { - id, ok := graph.Next(it.subIt) - if !ok { - break - } - i += 1 + for graph.Next(it.subIt) { + i++ if i > abortMaterializeAt { it.aborted = true break } + id := it.subIt.Result() val := id - if h, ok := id.(hasher); ok { - val = h.Hasher() + if h, ok := id.(Keyer); ok { + val = h.Key() } if _, ok := it.containsMap[val]; !ok { it.containsMap[val] = len(it.values) @@ -257,7 +258,7 @@ func (it *Materialize) materializeSet() { tags := make(map[string]graph.Value) it.subIt.TagResults(tags) it.values[index] = append(it.values[index], result{id: id, tags: tags}) - for it.subIt.NextResult() == true { + for it.subIt.NextPath() { tags := make(map[string]graph.Value) it.subIt.TagResults(tags) it.values[index] = append(it.values[index], result{id: id, tags: tags}) diff --git a/graph/iterator/optional_iterator.go b/graph/iterator/optional_iterator.go index 646bc7f..2432390 100644 --- a/graph/iterator/optional_iterator.go +++ b/graph/iterator/optional_iterator.go @@ -51,8 +51,6 @@ func NewOptional(it graph.Iterator) *Optional { } } -func (it *Optional) CanNext() bool { return false } - func (it *Optional) UID() uint64 { return it.uid } @@ -88,9 +86,9 @@ func (it *Optional) Result() graph.Value { // An optional iterator only has a next result if, (a) last time we checked // we had any results whatsoever, and (b) there was another subresult in our // optional subbranch. -func (it *Optional) NextResult() bool { +func (it *Optional) NextPath() bool { if it.lastCheck { - return it.subIt.NextResult() + return it.subIt.NextPath() } return false } diff --git a/graph/iterator/or_iterator.go b/graph/iterator/or_iterator.go index b7e765a..ddff768 100644 --- a/graph/iterator/or_iterator.go +++ b/graph/iterator/or_iterator.go @@ -141,35 +141,34 @@ func (it *Or) AddSubIterator(sub graph.Iterator) { it.itCount++ } -// Returns the Next value from the Or graph.iterator. Because the Or is the -// union of its subiterators, it must produce from all subiterators -- unless -// it's shortcircuiting, in which case, it's the first one that returns anything. -func (it *Or) Next() (graph.Value, bool) { +// Next advances the Or graph.iterator. Because the Or is the union of its +// subiterators, it must produce from all subiterators -- unless it it +// shortcircuiting, in which case, it is the first one that returns anything. +func (it *Or) Next() bool { graph.NextLogIn(it) - var curr graph.Value - var exists bool - firstTime := false + var first bool for { if it.currentIterator == -1 { it.currentIterator = 0 - firstTime = true + first = true } curIt := it.internalIterators[it.currentIterator] - curr, exists = graph.Next(curIt) - if !exists { - if it.isShortCircuiting && !firstTime { - return graph.NextLogOut(it, nil, false) - } - it.currentIterator++ - if it.currentIterator == it.itCount { - return graph.NextLogOut(it, nil, false) - } - } else { - it.result = curr - return graph.NextLogOut(it, curr, true) + + if graph.Next(curIt) { + it.result = curIt.Result() + return graph.NextLogOut(it, it.result, true) + } + + if it.isShortCircuiting && !first { + break + } + it.currentIterator++ + if it.currentIterator == it.itCount { + break } } - panic("unreachable") + + return graph.NextLogOut(it, nil, false) } func (it *Or) Result() graph.Value { @@ -228,13 +227,13 @@ func (it *Or) Size() (int64, bool) { return val, b } -// An Or has no NextResult of its own -- that is, there are no other values +// An Or has no NextPath of its own -- that is, there are no other values // which satisfy our previous result that are not the result itself. Our // subiterators might, however, so just pass the call recursively. In the case of // shortcircuiting, only allow new results from the currently checked graph.iterator -func (it *Or) NextResult() bool { +func (it *Or) NextPath() bool { if it.currentIterator != -1 { - return it.internalIterators[it.currentIterator].NextResult() + return it.internalIterators[it.currentIterator].NextPath() } return false } diff --git a/graph/iterator/or_iterator_test.go b/graph/iterator/or_iterator_test.go index bbd5de6..b147d56 100644 --- a/graph/iterator/or_iterator_test.go +++ b/graph/iterator/or_iterator_test.go @@ -23,12 +23,8 @@ import ( func iterated(it graph.Iterator) []int { var res []int - for { - val, ok := graph.Next(it) - if !ok { - break - } - res = append(res, val.(int)) + for graph.Next(it) { + res = append(res, it.Result().(int)) } return res } diff --git a/graph/iterator/query_shape.go b/graph/iterator/query_shape.go index 973f6db..77e10eb 100644 --- a/graph/iterator/query_shape.go +++ b/graph/iterator/query_shape.go @@ -129,12 +129,8 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node { } case graph.Fixed: n.IsFixed = true - for { - val, more := graph.Next(it) - if !more { - break - } - n.Values = append(n.Values, qs.ts.NameOf(val)) + for graph.Next(it) { + n.Values = append(n.Values, qs.ts.NameOf(it.Result())) } case graph.HasA: hasa := it.(*HasA) diff --git a/graph/iterator/value_comparison_iterator.go b/graph/iterator/value_comparison_iterator.go index 7c2eb58..91b1985 100644 --- a/graph/iterator/value_comparison_iterator.go +++ b/graph/iterator/value_comparison_iterator.go @@ -127,20 +127,15 @@ func (it *Comparison) Clone() graph.Iterator { return out } -func (it *Comparison) Next() (graph.Value, bool) { - var val graph.Value - var ok bool - for { - val, ok = graph.Next(it.subIt) - if !ok { - return nil, false - } +func (it *Comparison) Next() bool { + for graph.Next(it.subIt) { + val := it.subIt.Result() if it.doComparison(val) { - break + it.result = val + return true } } - it.result = val - return val, ok + return false } // DEPRECATED @@ -152,9 +147,9 @@ func (it *Comparison) Result() graph.Value { return it.result } -func (it *Comparison) NextResult() bool { +func (it *Comparison) NextPath() bool { for { - hasNext := it.subIt.NextResult() + hasNext := it.subIt.NextPath() if !hasNext { return false } diff --git a/graph/iterator/value_comparison_iterator_test.go b/graph/iterator/value_comparison_iterator_test.go index e7482e8..a2bb108 100644 --- a/graph/iterator/value_comparison_iterator_test.go +++ b/graph/iterator/value_comparison_iterator_test.go @@ -69,12 +69,8 @@ func TestValueComparison(t *testing.T) { vc := NewComparison(simpleFixedIterator(), test.operator, test.operand, ts) var got []string - for { - val, ok := vc.Next() - if !ok { - break - } - got = append(got, ts.NameOf(val)) + for vc.Next() { + got = append(got, ts.NameOf(vc.Result())) } if !reflect.DeepEqual(got, test.expect) { t.Errorf("Failed to show %s, got:%q expect:%q", test.message, got, test.expect) diff --git a/graph/leveldb/all_iterator.go b/graph/leveldb/all_iterator.go index a810792..ecfaf02 100644 --- a/graph/leveldb/all_iterator.go +++ b/graph/leveldb/all_iterator.go @@ -101,10 +101,10 @@ func (it *AllIterator) Clone() graph.Iterator { return out } -func (it *AllIterator) Next() (graph.Value, bool) { +func (it *AllIterator) Next() bool { if !it.open { it.result = nil - return nil, false + return false } var out []byte out = make([]byte, len(it.iter.Key())) @@ -115,10 +115,10 @@ func (it *AllIterator) Next() (graph.Value, bool) { } if !bytes.HasPrefix(out, it.prefix) { it.Close() - return nil, false + return false } it.result = Token(out) - return it.result, true + return true } func (it *AllIterator) ResultTree() *graph.ResultTree { @@ -129,7 +129,7 @@ func (it *AllIterator) Result() graph.Value { return it.result } -func (it *AllIterator) NextResult() bool { +func (it *AllIterator) NextPath() bool { return false } diff --git a/graph/leveldb/iterator.go b/graph/leveldb/iterator.go index a2342bc..d3c5ea5 100644 --- a/graph/leveldb/iterator.go +++ b/graph/leveldb/iterator.go @@ -125,19 +125,19 @@ func (it *Iterator) isLiveValue(val []byte) bool { return len(entry.History)%2 != 0 } -func (it *Iterator) Next() (graph.Value, bool) { +func (it *Iterator) Next() bool { if it.iter == nil { it.result = nil - return nil, false + return false } if !it.open { it.result = nil - return nil, false + return false } if !it.iter.Valid() { it.result = nil it.Close() - return nil, false + return false } if bytes.HasPrefix(it.iter.Key(), it.nextPrefix) { if !it.isLiveValue(it.iter.Value()) { @@ -150,11 +150,11 @@ func (it *Iterator) Next() (graph.Value, bool) { if !ok { it.Close() } - return Token(out), true + return true } it.Close() it.result = nil - return nil, false + return false } func (it *Iterator) ResultTree() *graph.ResultTree { @@ -165,7 +165,7 @@ func (it *Iterator) Result() graph.Value { return it.result } -func (it *Iterator) NextResult() bool { +func (it *Iterator) NextPath() bool { return false } diff --git a/graph/leveldb/leveldb_test.go b/graph/leveldb/leveldb_test.go index ee4131d..3c3a057 100644 --- a/graph/leveldb/leveldb_test.go +++ b/graph/leveldb/leveldb_test.go @@ -46,12 +46,8 @@ func makeTripleSet() []quad.Quad { func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []quad.Quad { var res ordered - for { - val, ok := graph.Next(it) - if !ok { - break - } - res = append(res, qs.Quad(val)) + for graph.Next(it) { + res = append(res, qs.Quad(it.Result())) } sort.Sort(res) return res @@ -86,12 +82,8 @@ func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] } func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string { var res []string - for { - val, ok := graph.Next(it) - if !ok { - break - } - res = append(res, qs.NameOf(val)) + for graph.Next(it) { + res = append(res, qs.NameOf(it.Result())) } sort.Strings(res) return res @@ -271,8 +263,8 @@ func TestIterator(t *testing.T) { it.Reset() it = qs.TriplesAllIterator() - edge, _ := graph.Next(it) - triple := qs.Quad(edge) + graph.Next(it) + triple := qs.Quad(it.Result()) set := makeTripleSet() var ok bool for _, t := range set { diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go index 2f32a83..9648d1c 100644 --- a/graph/leveldb/triplestore.go +++ b/graph/leveldb/triplestore.go @@ -35,7 +35,7 @@ import ( ) func init() { - graph.RegisterTripleStore("leveldb", newTripleStore, createNewLevelDB) + graph.RegisterTripleStore("leveldb", true, newTripleStore, createNewLevelDB) } const ( @@ -45,7 +45,7 @@ const ( type Token []byte -func (t Token) Hasher() interface{} { +func (t Token) Key() interface{} { return string(t) } diff --git a/graph/leveldb/triplestore_iterator_optimize.go b/graph/leveldb/triplestore_iterator_optimize.go index 9aab0f2..31b7f7d 100644 --- a/graph/leveldb/triplestore_iterator_optimize.go +++ b/graph/leveldb/triplestore_iterator_optimize.go @@ -37,10 +37,10 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo if primary.Type() == graph.Fixed { size, _ := primary.Size() if size == 1 { - val, ok := graph.Next(primary) - if !ok { - panic("Sizes lie") + if !graph.Next(primary) { + panic("unexpected size during optimize") } + val := primary.Result() newIt := ts.TripleIterator(it.Direction(), val) nt := newIt.Tagger() nt.CopyFrom(it) diff --git a/graph/memstore/all_iterator.go b/graph/memstore/all_iterator.go index 8e173a5..eb01f78 100644 --- a/graph/memstore/all_iterator.go +++ b/graph/memstore/all_iterator.go @@ -35,21 +35,19 @@ func NewMemstoreNodesAllIterator(ts *TripleStore) *NodesAllIterator { } // No subiterators. -func (it *AllIterator) SubIterators() []graph.Iterator { +func (nit *NodesAllIterator) SubIterators() []graph.Iterator { return nil } -func (nit *NodesAllIterator) Next() (graph.Value, bool) { - next, out := nit.Int64.Next() - if !out { - return next, out +func (nit *NodesAllIterator) Next() bool { + if !nit.Int64.Next() { + return false } - i64 := next.(int64) - _, ok := nit.ts.revIdMap[i64] + _, ok := nit.ts.revIdMap[nit.Int64.Result().(int64)] if !ok { return nit.Next() } - return next, out + return true } func NewMemstoreQuadsAllIterator(ts *TripleStore) *QuadsAllIterator { @@ -59,13 +57,13 @@ func NewMemstoreQuadsAllIterator(ts *TripleStore) *QuadsAllIterator { return &out } -func (qit *QuadsAllIterator) Next() (graph.Value, bool) { - next, out := qit.Int64.Next() +func (qit *QuadsAllIterator) Next() bool { + out := qit.Int64.Next() if out { - i64 := next.(int64) + i64 := qit.Int64.Result().(int64) if qit.ts.log[i64].DeletedBy != 0 || qit.ts.log[i64].Action == graph.Delete { return qit.Next() } } - return next, out + return out } diff --git a/graph/memstore/iterator.go b/graph/memstore/iterator.go index 3ab9d34..277f86a 100644 --- a/graph/memstore/iterator.go +++ b/graph/memstore/iterator.go @@ -99,7 +99,7 @@ func (it *Iterator) checkValid(index int64) bool { return it.ts.log[index].DeletedBy == 0 } -func (it *Iterator) Next() (graph.Value, bool) { +func (it *Iterator) Next() bool { graph.NextLogIn(it) if it.tree.Max() == nil || it.iterLast == it.tree.Max().(Int64) { return graph.NextLogOut(it, nil, false) @@ -120,7 +120,7 @@ func (it *Iterator) Result() graph.Value { return it.result } -func (it *Iterator) NextResult() bool { +func (it *Iterator) NextPath() bool { return false } diff --git a/graph/memstore/triplestore.go b/graph/memstore/triplestore.go index edba01f..babae77 100644 --- a/graph/memstore/triplestore.go +++ b/graph/memstore/triplestore.go @@ -26,7 +26,7 @@ import ( ) func init() { - graph.RegisterTripleStore("memstore", func(string, graph.Options) (graph.TripleStore, error) { + graph.RegisterTripleStore("memstore", false, func(string, graph.Options) (graph.TripleStore, error) { return newTripleStore(), nil }, nil) } @@ -145,14 +145,10 @@ func (ts *TripleStore) quadExists(t quad.Quad) (bool, int64) { } it := NewLlrbIterator(smallest_tree, "", ts) - for { - val, ok := it.Next() - if !ok { - break - } - ival := val.(int64) - if t == ts.log[ival].Quad { - return true, ival + for it.Next() { + val := it.Result() + if t == ts.log[val.(int64)].Quad { + return true, val.(int64) } } return false, 0 diff --git a/graph/memstore/triplestore_iterator_optimize.go b/graph/memstore/triplestore_iterator_optimize.go index 3dc2c2c..ae5628f 100644 --- a/graph/memstore/triplestore_iterator_optimize.go +++ b/graph/memstore/triplestore_iterator_optimize.go @@ -37,10 +37,10 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo if primary.Type() == graph.Fixed { size, _ := primary.Size() if size == 1 { - val, ok := graph.Next(primary) - if !ok { - panic("Sizes lie") + if !graph.Next(primary) { + panic("unexpected size during optimize") } + val := primary.Result() newIt := ts.TripleIterator(it.Direction(), val) nt := newIt.Tagger() nt.CopyFrom(it) diff --git a/graph/memstore/triplestore_test.go b/graph/memstore/triplestore_test.go index 38f9780..6f1959f 100644 --- a/graph/memstore/triplestore_test.go +++ b/graph/memstore/triplestore_test.go @@ -117,10 +117,10 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) { outerAnd.AddSubIterator(fixed) outerAnd.AddSubIterator(hasa) - val, ok := outerAnd.Next() - if !ok { + if !outerAnd.Next() { t.Error("Expected one matching subtree") } + val := outerAnd.Result() if ts.NameOf(val) != "C" { t.Errorf("Matching subtree should be %s, got %s", "barak", ts.NameOf(val)) } @@ -131,7 +131,7 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) { ) for { got = append(got, ts.NameOf(all.Result())) - if !outerAnd.NextResult() { + if !outerAnd.NextPath() { break } } @@ -141,8 +141,7 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) { t.Errorf("Unexpected result, got:%q expect:%q", got, expect) } - val, ok = outerAnd.Next() - if ok { + if outerAnd.Next() { t.Error("More than one possible top level output?") } } @@ -193,8 +192,7 @@ func TestRemoveTriple(t *testing.T) { hasa := iterator.NewHasA(ts, innerAnd, quad.Object) newIt, _ := hasa.Optimize() - _, ok := graph.Next(newIt) - if ok { + if graph.Next(newIt) { t.Error("E should not have any followers.") } } diff --git a/graph/mongo/iterator.go b/graph/mongo/iterator.go index f9cea5b..9e37089 100644 --- a/graph/mongo/iterator.go +++ b/graph/mongo/iterator.go @@ -138,7 +138,7 @@ func (it *Iterator) Clone() graph.Iterator { return m } -func (it *Iterator) Next() (graph.Value, bool) { +func (it *Iterator) Next() bool { var result struct { Id string "_id" //Sub string "Sub" @@ -151,10 +151,10 @@ func (it *Iterator) Next() (graph.Value, bool) { if err != nil { glog.Errorln("Error Nexting Iterator: ", err) } - return nil, false + return false } it.result = result.Id - return result.Id, true + return true } func (it *Iterator) ResultTree() *graph.ResultTree { @@ -165,7 +165,7 @@ func (it *Iterator) Result() graph.Value { return it.result } -func (it *Iterator) NextResult() bool { +func (it *Iterator) NextPath() bool { return false } diff --git a/graph/mongo/triplestore.go b/graph/mongo/triplestore.go index 20fea2f..364d195 100644 --- a/graph/mongo/triplestore.go +++ b/graph/mongo/triplestore.go @@ -30,7 +30,7 @@ import ( ) func init() { - graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph) + graph.RegisterTripleStore("mongo", true, newTripleStore, createNewMongoGraph) } // Guarantee we satisfy graph.Bulkloader. diff --git a/graph/mongo/triplestore_iterator_optimize.go b/graph/mongo/triplestore_iterator_optimize.go index d2d1a05..99fb25b 100644 --- a/graph/mongo/triplestore_iterator_optimize.go +++ b/graph/mongo/triplestore_iterator_optimize.go @@ -37,10 +37,10 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo if primary.Type() == graph.Fixed { size, _ := primary.Size() if size == 1 { - val, ok := graph.Next(primary) - if !ok { - panic("Sizes lie") + if !graph.Next(primary) { + panic("unexpected size during optimize") } + val := primary.Result() newIt := ts.TripleIterator(it.Direction(), val) nt := newIt.Tagger() nt.CopyFrom(it) diff --git a/graph/result_tree_evaluator.go b/graph/result_tree_evaluator.go index e2feb33..56947d4 100644 --- a/graph/result_tree_evaluator.go +++ b/graph/result_tree_evaluator.go @@ -14,7 +14,10 @@ package graph -import "fmt" +import ( + "bytes" + "fmt" +) type ResultTree struct { result Value @@ -26,14 +29,13 @@ func NewResultTree(result Value) *ResultTree { } func (t *ResultTree) String() string { - base := fmt.Sprintf("(%d", t.result) - if len(t.subtrees) != 0 { - for _, sub := range t.subtrees { - base += fmt.Sprintf(" %s", sub) - } + var buf bytes.Buffer + fmt.Fprintf(&buf, "(%d", t.result) + for _, sub := range t.subtrees { + fmt.Fprintf(&buf, " %s", sub) } - base += ")" - return base + buf.WriteByte(')') + return buf.String() } func (t *ResultTree) AddSubtree(sub *ResultTree) { @@ -41,22 +43,15 @@ func (t *ResultTree) AddSubtree(sub *ResultTree) { } func StringResultTreeEvaluator(it Nexter) string { - ok := true - out := "" - for { - _, ok = it.Next() - if !ok { - break - } - out += it.ResultTree().String() - out += "\n" - for it.NextResult() == true { - out += " " - out += it.ResultTree().String() - out += "\n" + var buf bytes.Buffer + for it.Next() { + fmt.Fprintln(&buf, it.ResultTree()) + for it.NextPath() { + buf.WriteByte(' ') + fmt.Fprintln(&buf, it.ResultTree()) } } - return out + return buf.String() } func PrintResultTreeEvaluator(it Nexter) { diff --git a/graph/triplestore.go b/graph/triplestore.go index 7bfe38e..3f884ff 100644 --- a/graph/triplestore.go +++ b/graph/triplestore.go @@ -37,7 +37,7 @@ import ( // pointers to structs, or merely triples, or whatever works best for the // backing store. // -// These must be comparable, or implement a `Hasher() interface{}` function +// These must be comparable, or implement a `Key() interface{}` function // so that they may be stored in maps. type Value interface{} @@ -133,38 +133,45 @@ type BulkLoader interface { type NewStoreFunc func(string, Options) (TripleStore, error) type InitStoreFunc func(string, Options) error -var storeRegistry = make(map[string]NewStoreFunc) -var storeInitRegistry = make(map[string]InitStoreFunc) +type register struct { + newFunc NewStoreFunc + initFunc InitStoreFunc + isPersistent bool +} -func RegisterTripleStore(name string, newFunc NewStoreFunc, initFunc InitStoreFunc) { +var storeRegistry = make(map[string]register) + +func RegisterTripleStore(name string, persists bool, newFunc NewStoreFunc, initFunc InitStoreFunc) { if _, found := storeRegistry[name]; found { panic("already registered TripleStore " + name) } - storeRegistry[name] = newFunc - if initFunc != nil { - storeInitRegistry[name] = initFunc + storeRegistry[name] = register{ + newFunc: newFunc, + initFunc: initFunc, + isPersistent: persists, } } func NewTripleStore(name, dbpath string, opts Options) (TripleStore, error) { - newFunc, hasNew := storeRegistry[name] - if !hasNew { + r, registered := storeRegistry[name] + if !registered { return nil, errors.New("triplestore: name '" + name + "' is not registered") } - return newFunc(dbpath, opts) + return r.newFunc(dbpath, opts) } func InitTripleStore(name, dbpath string, opts Options) error { - initFunc, hasInit := storeInitRegistry[name] - if hasInit { - return initFunc(dbpath, opts) - } - if _, isRegistered := storeRegistry[name]; isRegistered { - return nil + r, registered := storeRegistry[name] + if registered { + return r.initFunc(dbpath, opts) } return errors.New("triplestore: name '" + name + "' is not registered") } +func IsPersistent(name string) bool { + return storeRegistry[name].isPersistent +} + func TripleStores() []string { t := make([]string, 0, len(storeRegistry)) for n := range storeRegistry { diff --git a/http/write.go b/http/write.go index f810941..54e5537 100644 --- a/http/write.go +++ b/http/write.go @@ -26,7 +26,7 @@ import ( "github.com/julienschmidt/httprouter" "github.com/google/cayley/quad" - "github.com/google/cayley/quad/nquads" + "github.com/google/cayley/quad/cquads" ) func ParseJsonToTripleList(jsonBody []byte) ([]quad.Quad, error) { @@ -78,7 +78,8 @@ func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params blockSize = int64(api.config.LoadSize) } - dec := nquads.NewDecoder(formFile) + // TODO(kortschak) Make this configurable from the web UI. + dec := cquads.NewDecoder(formFile) var ( n int diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index 916cb26..bdd7de2 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -153,8 +153,7 @@ func runIteratorToArray(it graph.Iterator, ses *Session, limit int) []map[string return nil default: } - _, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { break } tags := make(map[string]graph.Value) @@ -164,7 +163,7 @@ func runIteratorToArray(it graph.Iterator, ses *Session, limit int) []map[string if limit >= 0 && count >= limit { break } - for it.NextResult() == true { + for it.NextPath() { select { case <-ses.kill: return nil @@ -193,11 +192,10 @@ func runIteratorToArrayNoTags(it graph.Iterator, ses *Session, limit int) []stri return nil default: } - val, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { break } - output = append(output, ses.ts.NameOf(val)) + output = append(output, ses.ts.NameOf(it.Result())) count++ if limit >= 0 && count >= limit { break @@ -216,8 +214,7 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu return default: } - _, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { break } tags := make(map[string]graph.Value) @@ -228,7 +225,7 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu if limit >= 0 && count >= limit { break } - for it.NextResult() == true { + for it.NextPath() { select { case <-ses.kill: return @@ -260,8 +257,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { return default: } - _, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { break } tags := make(map[string]graph.Value) @@ -269,7 +265,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { if !ses.SendResult(&Result{actualResults: &tags}) { break } - for it.NextResult() == true { + for it.NextPath() { select { case <-ses.kill: return diff --git a/query/mql/session.go b/query/mql/session.go index c272005..63aa6ba 100644 --- a/query/mql/session.go +++ b/query/mql/session.go @@ -88,15 +88,11 @@ func (s *Session) ExecInput(input string, c chan interface{}, limit int) { if glog.V(2) { glog.V(2).Infoln(it.DebugString(0)) } - for { - _, ok := graph.Next(it) - if !ok { - break - } + for graph.Next(it) { tags := make(map[string]graph.Value) it.TagResults(tags) c <- tags - for it.NextResult() == true { + for it.NextPath() == true { tags := make(map[string]graph.Value) it.TagResults(tags) c <- tags diff --git a/query/sexp/parser_test.go b/query/sexp/parser_test.go index ed4776a..d4d16af 100644 --- a/query/sexp/parser_test.go +++ b/query/sexp/parser_test.go @@ -67,10 +67,10 @@ func TestMemstoreBackedSexp(t *testing.T) { if it.Type() != test.typ { t.Errorf("Incorrect type for %s, got:%q expect %q", test.message, it.Type(), test.expect) } - got, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { t.Errorf("Failed to %s", test.message) } + got := it.Result() if expect := ts.ValueOf(test.expect); got != expect { t.Errorf("Incorrect result for %s, got:%v expect %v", test.message, got, expect) } @@ -88,10 +88,10 @@ func TestTreeConstraintParse(t *testing.T) { if it.Type() != graph.And { t.Error("Odd iterator tree. Got: %s", it.DebugString(0)) } - out, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { t.Error("Got no results") } + out := it.Result() if out != ts.ValueOf("i") { t.Errorf("Got %d, expected %d", out, ts.ValueOf("i")) } @@ -105,8 +105,7 @@ func TestTreeConstraintTagParse(t *testing.T) { "(:like\n" + "($a (:is :good))))" it := BuildIteratorTreeForQuery(ts, query) - _, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { t.Error("Got no results") } tags := make(map[string]graph.Value) @@ -135,15 +134,14 @@ func TestMultipleConstraintParse(t *testing.T) { if it.Type() != graph.And { t.Error("Odd iterator tree. Got: %s", it.DebugString(0)) } - out, ok := graph.Next(it) - if !ok { + if !graph.Next(it) { t.Error("Got no results") } + out := it.Result() if out != ts.ValueOf("i") { t.Errorf("Got %d, expected %d", out, ts.ValueOf("i")) } - _, ok = graph.Next(it) - if ok { + if graph.Next(it) { t.Error("Too many results") } } diff --git a/query/sexp/session.go b/query/sexp/session.go index c1a227b..eea974f 100644 --- a/query/sexp/session.go +++ b/query/sexp/session.go @@ -77,11 +77,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { fmt.Println(it.DebugString(0)) } nResults := 0 - for { - _, ok := graph.Next(it) - if !ok { - break - } + for graph.Next(it) { tags := make(map[string]graph.Value) it.TagResults(tags) out <- &tags @@ -89,7 +85,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) { if nResults > limit && limit != -1 { break } - for it.NextResult() == true { + for it.NextPath() == true { tags := make(map[string]graph.Value) it.TagResults(tags) out <- &tags