Merge with new Next() interface

.gitignore

@@ -3,3 +3,4 @@ main
 *.test
 *.peg.go
 cayley.cfg
+.cayley_history

.travis.yml

@@ -10,6 +10,7 @@ install:
   - go get github.com/barakmich/glog
   - go get github.com/julienschmidt/httprouter
   - go get github.com/petar/GoLLRB/llrb
+  - go get github.com/peterh/liner
   - go get github.com/robertkrimen/otto
   - go get github.com/russross/blackfriday
   - go get github.com/syndtr/goleveldb/leveldb

README.md

@@ -8,11 +8,13 @@ Its goal is to be a part of the developer's toolbox where [Linked Data](http://l
 [![Build Status](https://travis-ci.org/google/cayley.png?branch=master)](https://travis-ci.org/google/cayley)
 
 ## What's new?
-* 2014-07-12:
+* 2014-08-06:
-  * Massive cleanup and restructuring is largely done, it should be even easier to add to Cayley. (thanks @kortschak)
+  * 0.3.1 Binary Release including:
-  * A couple new backends are in progress, namely Postgres and Cassandra -- PRs when they come around.
+    * New Quad Parser (more strictly passing the [W3C spec](http://www.w3.org/TR/n-quads) and test suite)
-  * Cayley is [now in Homebrew](https://github.com/Homebrew/homebrew/commit/1bd2fb2a61c7101a8c79c05afc90eeb02e9aa240), thanks to @whitlockjc
+    * Automatic decompression of quad files
-  * Our first client API (for Clojure, thanks to @wjb) -- list is now started on the [Client API wiki page](https://github.com/google/cayley/wiki/Client-APIs)
+  * Ruby and a Node.JS [client libraries](https://github.com/google/cayley/wiki/Client-APIs) from the community.
+  * Benchmarks
+  * [Large speedups on HEAD](https://github.com/google/cayley/pull/101) (in for the next binary release)
 
 ## Features
 

TODO.md

@@ -52,7 +52,7 @@ An important failure of MQL before was that it was never well-specified. Let's n
 ### New Iterators
 
 #### Limit Iterator
-The necessary component to make mid-query limit work. Acts as a limit on Next(), a passthrough on Contains(), and a limit on NextResult()
+The necessary component to make mid-query limit work. Acts as a limit on Next(), a passthrough on Contains(), and a limit on NextPath()
 
 ## Medium Term
 

cayley.go

@@ -17,9 +17,17 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
 	"flag"
 	"fmt"
+	"io"
+	client "net/http"
+	"net/url"
 	"os"
+	"path/filepath"
 	"runtime"
 
 	"github.com/barakmich/glog"
@@ -28,6 +36,9 @@ import (
 	"github.com/google/cayley/db"
 	"github.com/google/cayley/graph"
 	"github.com/google/cayley/http"
+	"github.com/google/cayley/quad"
+	"github.com/google/cayley/quad/cquads"
+	"github.com/google/cayley/quad/nquads"
 
 	// Load all supported backends.
 	_ "github.com/google/cayley/graph/leveldb"
@@ -35,14 +46,19 @@ import (
 	_ "github.com/google/cayley/graph/mongo"
 )
 
-var tripleFile = flag.String("triples", "", "Triple File to load before going to REPL.")
+var (
-var cpuprofile = flag.String("prof", "", "Output profiling file.")
+	tripleFile    = flag.String("triples", "", "Triple File to load before going to REPL.")
-var queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.")
+	tripleType    = flag.String("format", "cquad", `Triple format to use for loading ("cquad" or "nquad").`)
-var configFile = flag.String("config", "", "Path to an explicit configuration file.")
+	cpuprofile    = flag.String("prof", "", "Output profiling file.")
+	queryLanguage = flag.String("query_lang", "gremlin", "Use this parser as the query language.")
+	configFile    = flag.String("config", "", "Path to an explicit configuration file.")
+)
 
 // Filled in by `go build ldflags="-X main.VERSION `ver`"`.
-var BUILD_DATE string
+var (
-var VERSION string
+	BUILD_DATE string
+	VERSION    string
+)
 
 func Usage() {
 	fmt.Println("Cayley is a graph store and graph query layer.")
@@ -100,40 +116,140 @@ func main() {
 			fmt.Println("Cayley snapshot")
 		}
 		os.Exit(0)
+
 	case "init":
-		err = db.Init(cfg, *tripleFile)
+		err = db.Init(cfg)
+		if err != nil {
+			break
+		}
+		if *tripleFile != "" {
+			ts, err = db.Open(cfg)
+			if err != nil {
+				break
+			}
+			err = load(ts, cfg, *tripleFile, *tripleType)
+			if err != nil {
+				break
+			}
+			ts.Close()
+		}
+
 	case "load":
 		ts, err = db.Open(cfg)
 		if err != nil {
 			break
 		}
-		err = db.Load(ts, cfg, *tripleFile)
+		err = load(ts, cfg, *tripleFile, *tripleType)
 		if err != nil {
 			break
 		}
+
 		ts.Close()
+
 	case "repl":
 		ts, err = db.Open(cfg)
 		if err != nil {
 			break
 		}
-		err = db.Repl(ts, *queryLanguage, cfg)
+		if !graph.IsPersistent(cfg.DatabaseType) {
+			err = load(ts, cfg, "", *tripleType)
 			if err != nil {
 				break
 			}
+		}
+
+		err = db.Repl(ts, *queryLanguage, cfg)
+
 		ts.Close()
+
 	case "http":
 		ts, err = db.Open(cfg)
 		if err != nil {
 			break
 		}
+		if !graph.IsPersistent(cfg.DatabaseType) {
+			err = load(ts, cfg, "", *tripleType)
+			if err != nil {
+				break
+			}
+		}
+
 		http.Serve(ts, cfg)
+
 		ts.Close()
+
 	default:
 		fmt.Println("No command", cmd)
 		flag.Usage()
 	}
 	if err != nil {
-		glog.Fatalln(err)
+		glog.Errorln(err)
+	}
+}
+
+func load(ts graph.TripleStore, cfg *config.Config, path, typ string) error {
+	var r io.Reader
+
+	if path == "" {
+		path = cfg.DatabasePath
+	}
+	u, err := url.Parse(path)
+	if err != nil || u.Scheme == "file" || u.Scheme == "" {
+		// Don't alter relative URL path or non-URL path parameter.
+		if u.Scheme != "" && err == nil {
+			// Recovery heuristic for mistyping "file://path/to/file".
+			path = filepath.Join(u.Host, u.Path)
+		}
+		f, err := os.Open(path)
+		if err != nil {
+			return fmt.Errorf("could not open file %q: %v", path, err)
+		}
+		defer f.Close()
+		r = f
+	} else {
+		res, err := client.Get(path)
+		if err != nil {
+			return fmt.Errorf("could not get resource <%s>: %v", u, err)
+		}
+		defer res.Body.Close()
+		r = res.Body
+	}
+
+	r, err = decompressor(r)
+	if err != nil {
+		return err
+	}
+
+	var dec quad.Unmarshaler
+	switch typ {
+	case "cquad":
+		dec = cquads.NewDecoder(r)
+	case "nquad":
+		dec = nquads.NewDecoder(r)
+	default:
+		return fmt.Errorf("unknown quad format %q", typ)
+	}
+
+	return db.Load(ts, cfg, dec)
+}
+
+const (
+	gzipMagic  = "\x1f\x8b"
+	b2zipMagic = "BZh"
+)
+
+func decompressor(r io.Reader) (io.Reader, error) {
+	br := bufio.NewReader(r)
+	buf, err := br.Peek(3)
+	if err != nil {
+		return nil, err
+	}
+	switch {
+	case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0:
+		return gzip.NewReader(br)
+	case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0:
+		return bzip2.NewReader(br), nil
+	default:
+		return br, nil
 	}
 }

cayley_test.go

@@ -15,6 +15,11 @@
 package main
 
 import (
+	"bytes"
+	"compress/bzip2"
+	"compress/gzip"
+	"io"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -308,6 +313,13 @@ func prepare(t testing.TB) {
 		if err != nil {
 			t.Fatalf("Failed to open %q: %v", cfg.DatabasePath, err)
 		}
+
+		if !graph.IsPersistent(cfg.DatabaseType) {
+			err = load(ts, cfg, "", "cquad")
+			if err != nil {
+				t.Fatalf("Failed to load %q: %v", cfg.DatabasePath, err)
+			}
+		}
 	})
 }
 
@@ -408,3 +420,84 @@ func BenchmarkKeanuOther(b *testing.B) {
 func BenchmarkKeanuBullockOther(b *testing.B) {
 	runBench(8, b)
 }
+
+// reader is a test helper to filter non-io.Reader methods from the contained io.Reader.
+type reader struct {
+	r io.Reader
+}
+
+func (r reader) Read(p []byte) (int, error) {
+	return r.r.Read(p)
+}
+
+var testDecompressor = []struct {
+	message string
+	input   io.Reader
+	expect  []byte
+	err     error
+	readErr error
+}{
+	{
+		message: "text input",
+		input:   strings.NewReader("cayley data\n"),
+		err:     nil,
+		expect:  []byte("cayley data\n"),
+		readErr: nil,
+	},
+	{
+		message: "gzip input",
+		input: bytes.NewReader([]byte{
+			0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad,
+			0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00,
+		}),
+		err:     nil,
+		expect:  []byte("cayley data\n"),
+		readErr: nil,
+	},
+	{
+		message: "bzip2 input",
+		input: bytes.NewReader([]byte{
+			0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00,
+			0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c,
+			0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16,
+			0xa9, 0x7c, 0x78, 0x80,
+		}),
+		err:     nil,
+		expect:  []byte("cayley data\n"),
+		readErr: nil,
+	},
+	{
+		message: "bad gzip input",
+		input:   strings.NewReader("\x1f\x8bcayley data\n"),
+		err:     gzip.ErrHeader,
+		expect:  nil,
+		readErr: nil,
+	},
+	{
+		message: "bad bzip2 input",
+		input:   strings.NewReader("\x42\x5a\x68cayley data\n"),
+		err:     nil,
+		expect:  nil,
+		readErr: bzip2.StructuralError("invalid compression level"),
+	},
+}
+
+func TestDecompressor(t *testing.T) {
+	for _, test := range testDecompressor {
+		r, err := decompressor(test.input)
+		if err != test.err {
+			t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err)
+		}
+		if err != nil {
+			continue
+		}
+		p := make([]byte, len(test.expect)*2)
+		n, err := r.Read(p)
+		if err != test.readErr {
+			t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err)
+		}
+		if bytes.Compare(p[:n], test.expect) != 0 {
+			t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect)
+		}
+	}
+}

db/db.go

@@ -15,47 +15,49 @@
 package db
 
 import (
-	"bytes"
+	"errors"
-	"compress/bzip2"
-	"compress/gzip"
 	"fmt"
 	"io"
-	"os"
 
 	"github.com/barakmich/glog"
+
 	"github.com/google/cayley/config"
 	"github.com/google/cayley/graph"
 	"github.com/google/cayley/quad"
-	"github.com/google/cayley/quad/cquads"
 )
 
-func Load(ts graph.TripleStore, cfg *config.Config, path string) error {
+var ErrNotPersistent = errors.New("database type is not persistent")
-	f, err := os.Open(path)
-	if err != nil {
-		return fmt.Errorf("could not open file %q: %v", path, err)
-	}
-	defer f.Close()
 
-	r, err := decompressor(f)
+func Init(cfg *config.Config) error {
-	if err != nil {
+	if !graph.IsPersistent(cfg.DatabaseType) {
-		glog.Fatalln(err)
+		return fmt.Errorf("ignoring unproductive database initialization request: %v", ErrNotPersistent)
 	}
 
-	dec := cquads.NewDecoder(r)
+	return graph.InitTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions)
+}
 
+func Open(cfg *config.Config) (graph.TripleStore, error) {
+	glog.Infof("Opening database %q at %s", cfg.DatabaseType, cfg.DatabasePath)
+	ts, err := graph.NewTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions)
+	if err != nil {
+		return nil, err
+	}
+
+	return ts, nil
+}
+
+func Load(ts graph.TripleStore, cfg *config.Config, dec quad.Unmarshaler) error {
 	bulker, canBulk := ts.(graph.BulkLoader)
 	if canBulk {
-		err = bulker.BulkLoad(dec)
+		switch err := bulker.BulkLoad(dec); err {
-		if err == nil {
+		case nil:
 			return nil
-		}
+		case graph.ErrCannotBulkLoad:
-		if err == graph.ErrCannotBulkLoad {
+			// Try individual loading.
-			err = nil
+		default:
-		}
-	}
-	if err != nil {
 			return err
 		}
+	}
 
 	block := make([]quad.Quad, 0, cfg.LoadSize)
 	for {
@@ -76,29 +78,3 @@ func Load(ts graph.TripleStore, cfg *config.Config, path string) error {
 
 	return nil
 }
-
-const (
-	gzipMagic  = "\x1f\x8b"
-	b2zipMagic = "BZh"
-)
-
-type readAtReader interface {
-	io.Reader
-	io.ReaderAt
-}
-
-func decompressor(r readAtReader) (io.Reader, error) {
-	var buf [3]byte
-	_, err := r.ReadAt(buf[:], 0)
-	if err != nil {
-		return nil, err
-	}
-	switch {
-	case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0:
-		return gzip.NewReader(r)
-	case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0:
-		return bzip2.NewReader(r), nil
-	default:
-		return r, nil
-	}
-}

db/init.go

@@ -1,39 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package db
-
-import (
-	"github.com/google/cayley/config"
-	"github.com/google/cayley/graph"
-)
-
-func Init(cfg *config.Config, triplePath string) error {
-	err := graph.InitTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions)
-	if err != nil {
-		return err
-	}
-	if triplePath != "" {
-		ts, err := Open(cfg)
-		if err != nil {
-			return err
-		}
-		err = Load(ts, cfg, triplePath)
-		if err != nil {
-			return err
-		}
-		ts.Close()
-	}
-	return err
-}

db/load_test.go

@@ -1,81 +0,0 @@
-package db
-
-import (
-	"bytes"
-	"compress/bzip2"
-	"compress/gzip"
-	"testing"
-)
-
-var testDecompressor = []struct {
-	message string
-	input   []byte
-	expect  []byte
-	err     error
-	readErr error
-}{
-	{
-		message: "text input",
-		input:   []byte("cayley data\n"),
-		err:     nil,
-		expect:  []byte("cayley data\n"),
-		readErr: nil,
-	},
-	{
-		message: "gzip input",
-		input: []byte{
-			0x1f, 0x8b, 0x08, 0x00, 0x5c, 0xbc, 0xcd, 0x53, 0x00, 0x03, 0x4b, 0x4e, 0xac, 0xcc, 0x49, 0xad,
-			0x54, 0x48, 0x49, 0x2c, 0x49, 0xe4, 0x02, 0x00, 0x03, 0xe1, 0xfc, 0xc3, 0x0c, 0x00, 0x00, 0x00,
-		},
-		err:     nil,
-		expect:  []byte("cayley data\n"),
-		readErr: nil,
-	},
-	{
-		message: "bzip2 input",
-		input: []byte{
-			0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59, 0x26, 0x53, 0x59, 0xb5, 0x4b, 0xe3, 0xc4, 0x00, 0x00,
-			0x02, 0xd1, 0x80, 0x00, 0x10, 0x40, 0x00, 0x2e, 0x04, 0x04, 0x20, 0x20, 0x00, 0x31, 0x06, 0x4c,
-			0x41, 0x4c, 0x1e, 0xa7, 0xa9, 0x2a, 0x18, 0x26, 0xb1, 0xc2, 0xee, 0x48, 0xa7, 0x0a, 0x12, 0x16,
-			0xa9, 0x7c, 0x78, 0x80,
-		},
-		err:     nil,
-		expect:  []byte("cayley data\n"),
-		readErr: nil,
-	},
-	{
-		message: "bad gzip input",
-		input:   []byte{0x1f, 0x8b, 'c', 'a', 'y', 'l', 'e', 'y', ' ', 'd', 'a', 't', 'a', '\n'},
-		err:     gzip.ErrHeader,
-		expect:  nil,
-		readErr: nil,
-	},
-	{
-		message: "bad bzip2 input",
-		input:   []byte{0x42, 0x5a, 0x68, 'c', 'a', 'y', 'l', 'e', 'y', ' ', 'd', 'a', 't', 'a', '\n'},
-		err:     nil,
-		expect:  nil,
-		readErr: bzip2.StructuralError("invalid compression level"),
-	},
-}
-
-func TestDecompressor(t *testing.T) {
-	for _, test := range testDecompressor {
-		buf := bytes.NewReader(test.input)
-		r, err := decompressor(buf)
-		if err != test.err {
-			t.Fatalf("Unexpected error for %s, got:%v expect:%v", test.message, err, test.err)
-		}
-		if err != nil {
-			continue
-		}
-		p := make([]byte, len(test.expect)*2)
-		n, err := r.Read(p)
-		if err != test.readErr {
-			t.Fatalf("Unexpected error for reading %s, got:%v expect:%v", test.message, err, test.err)
-		}
-		if bytes.Compare(p[:n], test.expect) != 0 {
-			t.Errorf("Unexpected read result for %s, got:%q expect:%q", test.message, p[:n], test.expect)
-		}
-	}
-}

db/open.go

@@ -1,40 +0,0 @@
-// Copyright 2014 The Cayley Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package db
-
-import (
-	"github.com/barakmich/glog"
-
-	"github.com/google/cayley/config"
-	"github.com/google/cayley/graph"
-)
-
-func Open(cfg *config.Config) (graph.TripleStore, error) {
-	glog.Infof("Opening database %q at %s", cfg.DatabaseType, cfg.DatabasePath)
-	ts, err := graph.NewTripleStore(cfg.DatabaseType, cfg.DatabasePath, cfg.DatabaseOptions)
-	if err != nil {
-		return nil, err
-	}
-
-	// Memstore is not persistent, so it MUST be loaded.
-	if cfg.DatabaseType == "memstore" {
-		err = Load(ts, cfg, cfg.DatabasePath)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return ts, nil
-}

db/repl.go

@@ -15,14 +15,15 @@
 package db
 
 import (
-	"bufio"
-	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"os"
+	"os/signal"
+	"strings"
 	"time"
 
+	"github.com/peterh/liner"
+
 	"github.com/google/cayley/config"
 	"github.com/google/cayley/graph"
 	"github.com/google/cayley/quad/cquads"
@@ -62,6 +63,13 @@ func Run(query string, ses query.Session) {
 	}
 }
 
+const (
+	ps1 = "cayley> "
+	ps2 = "...     "
+
+	history = ".cayley_history"
+)
+
 func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error {
 	var ses query.Session
 	switch queryLanguage {
@@ -74,80 +82,123 @@ func Repl(ts graph.TripleStore, queryLanguage string, cfg *config.Config) error
 	default:
 		ses = gremlin.NewSession(ts, cfg.Timeout, true)
 	}
-	buf := bufio.NewReader(os.Stdin)
+
-	var line []byte
+	term, err := terminal(history)
-	for {
+	if os.IsNotExist(err) {
-		if len(line) == 0 {
+		fmt.Printf("creating new history file: %q\n", history)
-			fmt.Print("cayley> ")
-		} else {
-			fmt.Print("...       ")
 	}
-		l, prefix, err := buf.ReadLine()
+	defer persist(term, history)
-		if err == io.EOF {
+
-			if len(line) != 0 {
+	var (
-				line = line[:0]
+		prompt = ps1
+
+		code string
+	)
+
+	for {
+		if len(code) == 0 {
+			prompt = ps1
 		} else {
+			prompt = ps2
+		}
+		line, err := term.Prompt(prompt)
+		if err != nil {
+			if err == io.EOF {
 				return nil
 			}
+			return err
 		}
-		if err != nil {
+
-			line = line[:0]
+		term.AppendHistory(line)
-		}
+
-		if prefix {
+		line = strings.TrimSpace(line)
-			return errors.New("line too long")
-		}
-		line = append(line, l...)
-		if len(line) == 0 {
-			continue
-		}
-		line = bytes.TrimSpace(line)
 		if len(line) == 0 || line[0] == '#' {
-			line = line[:0]
 			continue
 		}
-		if bytes.HasPrefix(line, []byte(":debug")) {
+
+		if code == "" {
+			switch {
+			case strings.HasPrefix(line, ":debug"):
 				ses.ToggleDebug()
 				fmt.Println("Debug Toggled")
-			line = line[:0]
 				continue
-		}
+
-		if bytes.HasPrefix(line, []byte(":a")) {
+			case strings.HasPrefix(line, ":a"):
-			var tripleStmt = line[3:]
+				triple, err := cquads.Parse(line[3:])
-			triple, err := cquads.Parse(string(tripleStmt))
 				if !triple.IsValid() {
 					if err != nil {
 						fmt.Printf("not a valid triple: %v\n", err)
 					}
-				line = line[:0]
 					continue
 				}
 				ts.AddTriple(triple)
-			line = line[:0]
 				continue
-		}
+
-		if bytes.HasPrefix(line, []byte(":d")) {
+			case strings.HasPrefix(line, ":d"):
-			var tripleStmt = line[3:]
+				triple, err := cquads.Parse(line[3:])
-			triple, err := cquads.Parse(string(tripleStmt))
 				if !triple.IsValid() {
 					if err != nil {
 						fmt.Printf("not a valid triple: %v\n", err)
 					}
-				line = line[:0]
 					continue
 				}
 				ts.RemoveTriple(triple)
-			line = line[:0]
 				continue
 			}
-		result, err := ses.InputParses(string(line))
+		}
+
+		code += line
+
+		result, err := ses.InputParses(code)
 		switch result {
 		case query.Parsed:
-			Run(string(line), ses)
+			Run(code, ses)
-			line = line[:0]
+			code = ""
 		case query.ParseFail:
 			fmt.Println("Error: ", err)
-			line = line[:0]
+			code = ""
 		case query.ParseMore:
 		}
 	}
 }
+
+func terminal(path string) (*liner.State, error) {
+	term := liner.NewLiner()
+
+	go func() {
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt, os.Kill)
+		<-c
+
+		persist(term, history)
+
+		err := term.Close()
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "failed to properly clean up terminal: %v\n", err)
+			os.Exit(1)
+		}
+
+		os.Exit(0)
+	}()
+
+	f, err := os.Open(path)
+	if err != nil {
+		return term, err
+	}
+	defer f.Close()
+	_, err = term.ReadHistory(f)
+	return term, err
+}
+
+func persist(term *liner.State, path string) error {
+	f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
+	if err != nil {
+		return fmt.Errorf("could not open %q to append history: %v", path, err)
+	}
+	defer f.Close()
+	_, err = term.WriteHistory(f)
+	if err != nil {
+		return fmt.Errorf("could not write history to %q: %v", path, err)
+	}
+	return nil
+}

graph/iterator.go

@@ -77,17 +77,21 @@ type Iterator interface {
 	// the iteration interface.
 	//
 	// To get the full results of iteration, do the following:
-	// while (!Next()):
+	//
-	//   emit result
+	//  for graph.Next(it) {
-	//   while (!NextResult()):
+	//  	val := it.Result()
-	//       emit result
+	//  	... do things with val.
+	//  	for it.NextPath() {
+	//  		... find other paths to iterate
+	//  	}
+	//  }
 	//
 	// All of them should set iterator.Last to be the last returned value, to
 	// make results work.
 	//
-	// NextResult() advances iterators that may have more than one valid result,
+	// NextPath() advances iterators that may have more than one valid result,
 	// from the bottom up.
-	NextResult() bool
+	NextPath() bool
 
 	// Contains returns whether the value is within the set held by the iterator.
 	Contains(Value) bool
@@ -135,22 +139,22 @@ type Iterator interface {
 }
 
 type Nexter interface {
-	// Next() advances the iterator and returns the next valid result. Returns
+	// Next advances the iterator to the next value, which will then be available through
-	// (<value>, true) or (nil, false)
+	// the Result method. It returns false if no further advancement is possible.
-	Next() (Value, bool)
+	Next() bool
 
 	Iterator
 }
 
 // Next is a convenience function that conditionally calls the Next method
 // of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next
-// return a nil Value and false.
+// returns false.
-func Next(it Iterator) (Value, bool) {
+func Next(it Iterator) bool {
 	if n, ok := it.(Nexter); ok {
 		return n.Next()
 	}
 	glog.Errorln("Nexting an un-nextable iterator")
-	return nil, false
+	return false
 }
 
 // Height is a convienence function to measure the height of an iterator tree.
@@ -271,7 +275,7 @@ func NextLogIn(it Iterator) {
 	}
 }
 
-func NextLogOut(it Iterator, val Value, ok bool) (Value, bool) {
+func NextLogOut(it Iterator, val Value, ok bool) bool {
 	if glog.V(4) {
 		if ok {
 			glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type().String()), it.UID(), val)
@@ -279,5 +283,5 @@ func NextLogOut(it Iterator, val Value, ok bool) (Value, bool) {
 			glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type().String()), it.UID())
 		}
 	}
-	return val, ok
+	return ok
 }

graph/iterator/all_iterator.go

@@ -87,7 +87,7 @@ func (it *Int64) DebugString(indent int) string {
 
 // Next() on an Int64 all iterator is a simple incrementing counter.
 // Return the next integer, and mark it as the result.
-func (it *Int64) Next() (graph.Value, bool) {
+func (it *Int64) Next() bool {
 	graph.NextLogIn(it)
 	if it.at == -1 {
 		return graph.NextLogOut(it, nil, false)
@@ -110,7 +110,7 @@ func (it *Int64) Result() graph.Value {
 	return it.result
 }
 
-func (it *Int64) NextResult() bool {
+func (it *Int64) NextPath() bool {
 	return false
 }
 

graph/iterator/and_iterator.go

@@ -151,25 +151,20 @@ func (it *And) AddSubIterator(sub graph.Iterator) {
 	it.itCount++
 }
 
-// Returns the Next value from the And iterator. Because the And is the
+// Returns advances the And iterator. Because the And is the intersection of its
-// intersection of its subiterators, it must choose one subiterator to produce a
+// subiterators, it must choose one subiterator to produce a candidate, and check
-// candidate, and check this value against the subiterators. A productive choice
+// this value against the subiterators. A productive choice of primary iterator
-// of primary iterator is therefore very important.
+// is therefore very important.
-func (it *And) Next() (graph.Value, bool) {
+func (it *And) Next() bool {
 	graph.NextLogIn(it)
-	var curr graph.Value
+	for graph.Next(it.primaryIt) {
-	var exists bool
+		curr := it.primaryIt.Result()
-	for {
-		curr, exists = graph.Next(it.primaryIt)
-		if !exists {
-			return graph.NextLogOut(it, nil, false)
-		}
 		if it.subItsContain(curr) {
 			it.result = curr
 			return graph.NextLogOut(it, curr, true)
 		}
 	}
-	panic("unreachable")
+	return graph.NextLogOut(it, nil, false)
 }
 
 func (it *And) Result() graph.Value {
@@ -236,15 +231,15 @@ func (it *And) Size() (int64, bool) {
 	return val, b
 }
 
-// An And has no NextResult of its own -- that is, there are no other values
+// An And has no NextPath of its own -- that is, there are no other values
 // which satisfy our previous result that are not the result itself. Our
 // subiterators might, however, so just pass the call recursively.
-func (it *And) NextResult() bool {
+func (it *And) NextPath() bool {
-	if it.primaryIt.NextResult() {
+	if it.primaryIt.NextPath() {
 		return true
 	}
 	for _, sub := range it.internalIterators {
-		if sub.NextResult() {
+		if sub.NextPath() {
 			return true
 		}
 	}

graph/iterator/and_iterator_test.go

@@ -36,10 +36,10 @@ func TestTag(t *testing.T) {
 		t.Errorf("Cannot get tag back, got %s", out[0])
 	}
 
-	val, ok := and.Next()
+	if !and.Next() {
-	if !ok {
 		t.Errorf("And did not next")
 	}
+	val := and.Result()
 	if val != 234 {
 		t.Errorf("Unexpected value")
 	}
@@ -76,18 +76,15 @@ func TestAndAndFixedIterators(t *testing.T) {
 		t.Error("not accurate")
 	}
 
-	val, ok := and.Next()
+	if !and.Next() || and.Result() != 3 {
-	if val != 3 || ok == false {
 		t.Error("Incorrect first value")
 	}
 
-	val, ok = and.Next()
+	if !and.Next() || and.Result() != 4 {
-	if val != 4 || ok == false {
 		t.Error("Incorrect second value")
 	}
 
-	val, ok = and.Next()
+	if and.Next() {
-	if ok {
 		t.Error("Too many values")
 	}
 
@@ -117,8 +114,7 @@ func TestNonOverlappingFixedIterators(t *testing.T) {
 		t.Error("not accurate")
 	}
 
-	_, ok := and.Next()
+	if and.Next() {
-	if ok {
 		t.Error("Too many values")
 	}
 
@@ -131,18 +127,15 @@ func TestAllIterators(t *testing.T) {
 	and.AddSubIterator(all2)
 	and.AddSubIterator(all1)
 
-	val, ok := and.Next()
+	if !and.Next() || and.Result() != int64(4) {
-	if val.(int64) != 4 || ok == false {
 		t.Error("Incorrect first value")
 	}
 
-	val, ok = and.Next()
+	if !and.Next() || and.Result() != int64(5) {
-	if val.(int64) != 5 || ok == false {
 		t.Error("Incorrect second value")
 	}
 
-	val, ok = and.Next()
+	if and.Next() {
-	if ok {
 		t.Error("Too many values")
 	}
 

graph/iterator/fixed_iterator.go

@@ -135,8 +135,8 @@ func (it *Fixed) Contains(v graph.Value) bool {
 	return graph.ContainsLogOut(it, v, false)
 }
 
-// Return the next stored value from the iterator.
+// Next advances the iterator.
-func (it *Fixed) Next() (graph.Value, bool) {
+func (it *Fixed) Next() bool {
 	graph.NextLogIn(it)
 	if it.lastIndex == len(it.values) {
 		return graph.NextLogOut(it, nil, false)
@@ -156,7 +156,7 @@ func (it *Fixed) Result() graph.Value {
 	return it.result
 }
 
-func (it *Fixed) NextResult() bool {
+func (it *Fixed) NextPath() bool {
 	return false
 }
 

graph/iterator/hasa_iterator.go

@@ -27,9 +27,9 @@ package iterator
 // value to check, it means "Check all predicates that have this value for your
 // direction against the subiterator." This would imply that there's more than
 // one possibility for the same Contains()ed value. While we could return the
-// number of options, it's simpler to return one, and then call NextResult()
+// number of options, it's simpler to return one, and then call NextPath()
 // enough times to enumerate the options. (In fact, one could argue that the
-// raison d'etre for NextResult() is this iterator).
+// raison d'etre for NextPath() is this iterator).
 //
 // Alternatively, can be seen as the dual of the LinksTo iterator.
 
@@ -158,16 +158,13 @@ func (it *HasA) Contains(val graph.Value) bool {
 // result iterator (a triple iterator based on the last checked value) and returns true if
 // another match is made.
 func (it *HasA) NextContains() bool {
-	for {
+	for graph.Next(it.resultIt) {
-		linkVal, ok := graph.Next(it.resultIt)
+		link := it.resultIt.Result()
-		if !ok {
-			break
-		}
 		if glog.V(4) {
-			glog.V(4).Infoln("Quad is", it.ts.Quad(linkVal))
+			glog.V(4).Infoln("Quad is", it.ts.Quad(link))
 		}
-		if it.primaryIt.Contains(linkVal) {
+		if it.primaryIt.Contains(link) {
-			it.result = it.ts.TripleDirection(linkVal, it.dir)
+			it.result = it.ts.TripleDirection(link, it.dir)
 			return true
 		}
 	}
@@ -175,33 +172,33 @@ func (it *HasA) NextContains() bool {
 }
 
 // Get the next result that matches this branch.
-func (it *HasA) NextResult() bool {
+func (it *HasA) NextPath() bool {
-	// Order here is important. If the subiterator has a NextResult, then we
+	// Order here is important. If the subiterator has a NextPath, then we
 	// need do nothing -- there is a next result, and we shouldn't move forward.
 	// However, we then need to get the next result from our last Contains().
 	//
-	// The upshot is, the end of NextResult() bubbles up from the bottom of the
+	// The upshot is, the end of NextPath() bubbles up from the bottom of the
 	// iterator tree up, and we need to respect that.
-	if it.primaryIt.NextResult() {
+	if it.primaryIt.NextPath() {
 		return true
 	}
 	return it.NextContains()
 }
 
-// Get the next result from this iterator. This is simpler than Contains. We have a
+// Next advances the iterator. This is simpler than Contains. We have a
 // subiterator we can get a value from, and we can take that resultant triple,
 // pull our direction out of it, and return that.
-func (it *HasA) Next() (graph.Value, bool) {
+func (it *HasA) Next() bool {
 	graph.NextLogIn(it)
 	if it.resultIt != nil {
 		it.resultIt.Close()
 	}
 	it.resultIt = &Null{}
 
-	tID, ok := graph.Next(it.primaryIt)
+	if !graph.Next(it.primaryIt) {
-	if !ok {
 		return graph.NextLogOut(it, 0, false)
 	}
+	tID := it.primaryIt.Result()
 	name := it.ts.Quad(tID).Get(it.dir)
 	val := it.ts.ValueOf(name)
 	it.result = val

graph/iterator/iterator.go

@@ -79,8 +79,8 @@ func (it *Null) DebugString(indent int) string {
 	return strings.Repeat(" ", indent) + "(null)"
 }
 
-func (it *Null) Next() (graph.Value, bool) {
+func (it *Null) Next() bool {
-	return nil, false
+	return false
 }
 
 func (it *Null) Result() graph.Value {
@@ -95,7 +95,7 @@ func (it *Null) SubIterators() []graph.Iterator {
 	return nil
 }
 
-func (it *Null) NextResult() bool {
+func (it *Null) NextPath() bool {
 	return false
 }
 

graph/iterator/linksto_iterator.go

@@ -153,23 +153,23 @@ func (it *LinksTo) Optimize() (graph.Iterator, bool) {
 }
 
 // Next()ing a LinksTo operates as described above.
-func (it *LinksTo) Next() (graph.Value, bool) {
+func (it *LinksTo) Next() bool {
 	graph.NextLogIn(it)
-	val, ok := graph.Next(it.nextIt)
+	if graph.Next(it.nextIt) {
-	if !ok {
+		it.result = it.nextIt.Result()
+		return graph.NextLogOut(it, it.nextIt, true)
+	}
+
 	// Subiterator is empty, get another one
-		candidate, ok := graph.Next(it.primaryIt)
+	if !graph.Next(it.primaryIt) {
-		if !ok {
 		// We're out of nodes in our subiterator, so we're done as well.
 		return graph.NextLogOut(it, 0, false)
 	}
 	it.nextIt.Close()
-		it.nextIt = it.ts.TripleIterator(it.dir, candidate)
+	it.nextIt = it.ts.TripleIterator(it.dir, it.primaryIt.Result())
+
 	// Recurse -- return the first in the next set.
 	return it.Next()
-	}
-	it.result = val
-	return graph.NextLogOut(it, val, ok)
 }
 
 func (it *LinksTo) Result() graph.Value {
@@ -183,8 +183,8 @@ func (it *LinksTo) Close() {
 }
 
 // We won't ever have a new result, but our subiterators might.
-func (it *LinksTo) NextResult() bool {
+func (it *LinksTo) NextPath() bool {
-	return it.primaryIt.NextResult()
+	return it.primaryIt.NextPath()
 }
 
 // Register the LinksTo.

graph/iterator/linksto_iterator_test.go

@@ -33,10 +33,10 @@ func TestLinksTo(t *testing.T) {
 	}
 	fixed.Add(val)
 	lto := NewLinksTo(ts, fixed, quad.Object)
-	val, ok := lto.Next()
+	if !lto.Next() {
-	if !ok {
 		t.Error("At least one triple matches the fixed object")
 	}
+	val = lto.Result()
 	if val != 2 {
 		t.Errorf("Quad index 2, such as %s, should match %s", ts.Quad(2), ts.Quad(val))
 	}

graph/iterator/materialize_iterator.go

@@ -32,8 +32,12 @@ type result struct {
 	tags map[string]graph.Value
 }
 
-type hasher interface {
+// Keyer provides a method for comparing types that are not otherwise comparable.
-	Hasher() interface{}
+// The Key method must return a dynamic type that is comparable according to the
+// Go language specification. The returned value must be unique for each receiver
+// value.
+type Keyer interface {
+	Key() interface{}
 }
 
 type Materialize struct {
@@ -179,7 +183,7 @@ func (it *Materialize) Stats() graph.IteratorStats {
 	}
 }
 
-func (it *Materialize) Next() (graph.Value, bool) {
+func (it *Materialize) Next() bool {
 	graph.NextLogIn(it)
 	if !it.hasRun {
 		it.materializeSet()
@@ -205,8 +209,8 @@ func (it *Materialize) Contains(v graph.Value) bool {
 		return it.subIt.Contains(v)
 	}
 	key := v
-	if h, ok := v.(hasher); ok {
+	if h, ok := v.(Keyer); ok {
-		key = h.Hasher()
+		key = h.Key()
 	}
 	if i, ok := it.containsMap[key]; ok {
 		it.index = i
@@ -216,12 +220,12 @@ func (it *Materialize) Contains(v graph.Value) bool {
 	return graph.ContainsLogOut(it, v, false)
 }
 
-func (it *Materialize) NextResult() bool {
+func (it *Materialize) NextPath() bool {
 	if !it.hasRun {
 		it.materializeSet()
 	}
 	if it.aborted {
-		return it.subIt.NextResult()
+		return it.subIt.NextPath()
 	}
 
 	it.subindex++
@@ -235,19 +239,16 @@ func (it *Materialize) NextResult() bool {
 
 func (it *Materialize) materializeSet() {
 	i := 0
-	for {
+	for graph.Next(it.subIt) {
-		id, ok := graph.Next(it.subIt)
+		i++
-		if !ok {
-			break
-		}
-		i += 1
 		if i > abortMaterializeAt {
 			it.aborted = true
 			break
 		}
+		id := it.subIt.Result()
 		val := id
-		if h, ok := id.(hasher); ok {
+		if h, ok := id.(Keyer); ok {
-			val = h.Hasher()
+			val = h.Key()
 		}
 		if _, ok := it.containsMap[val]; !ok {
 			it.containsMap[val] = len(it.values)
@@ -257,7 +258,7 @@ func (it *Materialize) materializeSet() {
 		tags := make(map[string]graph.Value)
 		it.subIt.TagResults(tags)
 		it.values[index] = append(it.values[index], result{id: id, tags: tags})
-		for it.subIt.NextResult() == true {
+		for it.subIt.NextPath() {
 			tags := make(map[string]graph.Value)
 			it.subIt.TagResults(tags)
 			it.values[index] = append(it.values[index], result{id: id, tags: tags})

graph/iterator/optional_iterator.go

@@ -51,8 +51,6 @@ func NewOptional(it graph.Iterator) *Optional {
 	}
 }
 
-func (it *Optional) CanNext() bool { return false }
-
 func (it *Optional) UID() uint64 {
 	return it.uid
 }
@@ -88,9 +86,9 @@ func (it *Optional) Result() graph.Value {
 // An optional iterator only has a next result if, (a) last time we checked
 // we had any results whatsoever, and (b) there was another subresult in our
 // optional subbranch.
-func (it *Optional) NextResult() bool {
+func (it *Optional) NextPath() bool {
 	if it.lastCheck {
-		return it.subIt.NextResult()
+		return it.subIt.NextPath()
 	}
 	return false
 }

graph/iterator/or_iterator.go

@@ -141,35 +141,34 @@ func (it *Or) AddSubIterator(sub graph.Iterator) {
 	it.itCount++
 }
 
-// Returns the Next value from the Or graph.iterator. Because the Or is the
+// Next advances the Or graph.iterator. Because the Or is the union of its
-// union of its subiterators, it must produce from all subiterators -- unless
+// subiterators, it must produce from all subiterators -- unless it it
-// it's shortcircuiting, in which case, it's the first one that returns anything.
+// shortcircuiting, in which case, it is the first one that returns anything.
-func (it *Or) Next() (graph.Value, bool) {
+func (it *Or) Next() bool {
 	graph.NextLogIn(it)
-	var curr graph.Value
+	var first bool
-	var exists bool
-	firstTime := false
 	for {
 		if it.currentIterator == -1 {
 			it.currentIterator = 0
-			firstTime = true
+			first = true
 		}
 		curIt := it.internalIterators[it.currentIterator]
-		curr, exists = graph.Next(curIt)
+
-		if !exists {
+		if graph.Next(curIt) {
-			if it.isShortCircuiting && !firstTime {
+			it.result = curIt.Result()
-				return graph.NextLogOut(it, nil, false)
+			return graph.NextLogOut(it, it.result, true)
+		}
+
+		if it.isShortCircuiting && !first {
+			break
 		}
 		it.currentIterator++
 		if it.currentIterator == it.itCount {
+			break
+		}
+	}
+
 	return graph.NextLogOut(it, nil, false)
-			}
-		} else {
-			it.result = curr
-			return graph.NextLogOut(it, curr, true)
-		}
-	}
-	panic("unreachable")
 }
 
 func (it *Or) Result() graph.Value {
@@ -228,13 +227,13 @@ func (it *Or) Size() (int64, bool) {
 	return val, b
 }
 
-// An Or has no NextResult of its own -- that is, there are no other values
+// An Or has no NextPath of its own -- that is, there are no other values
 // which satisfy our previous result that are not the result itself. Our
 // subiterators might, however, so just pass the call recursively. In the case of
 // shortcircuiting, only allow new results from the currently checked graph.iterator
-func (it *Or) NextResult() bool {
+func (it *Or) NextPath() bool {
 	if it.currentIterator != -1 {
-		return it.internalIterators[it.currentIterator].NextResult()
+		return it.internalIterators[it.currentIterator].NextPath()
 	}
 	return false
 }

graph/iterator/or_iterator_test.go

@@ -23,12 +23,8 @@ import (
 
 func iterated(it graph.Iterator) []int {
 	var res []int
-	for {
+	for graph.Next(it) {
-		val, ok := graph.Next(it)
+		res = append(res, it.Result().(int))
-		if !ok {
-			break
-		}
-		res = append(res, val.(int))
 	}
 	return res
 }

graph/iterator/query_shape.go

@@ -129,12 +129,8 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
 		}
 	case graph.Fixed:
 		n.IsFixed = true
-		for {
+		for graph.Next(it) {
-			val, more := graph.Next(it)
+			n.Values = append(n.Values, qs.ts.NameOf(it.Result()))
-			if !more {
-				break
-			}
-			n.Values = append(n.Values, qs.ts.NameOf(val))
 		}
 	case graph.HasA:
 		hasa := it.(*HasA)

graph/iterator/value_comparison_iterator.go

@@ -127,20 +127,15 @@ func (it *Comparison) Clone() graph.Iterator {
 	return out
 }
 
-func (it *Comparison) Next() (graph.Value, bool) {
+func (it *Comparison) Next() bool {
-	var val graph.Value
+	for graph.Next(it.subIt) {
-	var ok bool
+		val := it.subIt.Result()
-	for {
-		val, ok = graph.Next(it.subIt)
-		if !ok {
-			return nil, false
-		}
 		if it.doComparison(val) {
-			break
-		}
-	}
 			it.result = val
-	return val, ok
+			return true
+		}
+	}
+	return false
 }
 
 // DEPRECATED
@@ -152,9 +147,9 @@ func (it *Comparison) Result() graph.Value {
 	return it.result
 }
 
-func (it *Comparison) NextResult() bool {
+func (it *Comparison) NextPath() bool {
 	for {
-		hasNext := it.subIt.NextResult()
+		hasNext := it.subIt.NextPath()
 		if !hasNext {
 			return false
 		}

graph/iterator/value_comparison_iterator_test.go

@@ -69,12 +69,8 @@ func TestValueComparison(t *testing.T) {
 		vc := NewComparison(simpleFixedIterator(), test.operator, test.operand, ts)
 
 		var got []string
-		for {
+		for vc.Next() {
-			val, ok := vc.Next()
+			got = append(got, ts.NameOf(vc.Result()))
-			if !ok {
-				break
-			}
-			got = append(got, ts.NameOf(val))
 		}
 		if !reflect.DeepEqual(got, test.expect) {
 			t.Errorf("Failed to show %s, got:%q expect:%q", test.message, got, test.expect)

graph/leveldb/all_iterator.go

@@ -101,10 +101,10 @@ func (it *AllIterator) Clone() graph.Iterator {
 	return out
 }
 
-func (it *AllIterator) Next() (graph.Value, bool) {
+func (it *AllIterator) Next() bool {
 	if !it.open {
 		it.result = nil
-		return nil, false
+		return false
 	}
 	var out []byte
 	out = make([]byte, len(it.iter.Key()))
@@ -115,10 +115,10 @@ func (it *AllIterator) Next() (graph.Value, bool) {
 	}
 	if !bytes.HasPrefix(out, it.prefix) {
 		it.Close()
-		return nil, false
+		return false
 	}
 	it.result = Token(out)
-	return it.result, true
+	return true
 }
 
 func (it *AllIterator) ResultTree() *graph.ResultTree {
@@ -129,7 +129,7 @@ func (it *AllIterator) Result() graph.Value {
 	return it.result
 }
 
-func (it *AllIterator) NextResult() bool {
+func (it *AllIterator) NextPath() bool {
 	return false
 }
 

graph/leveldb/iterator.go

@@ -125,19 +125,19 @@ func (it *Iterator) isLiveValue(val []byte) bool {
 	return len(entry.History)%2 != 0
 }
 
-func (it *Iterator) Next() (graph.Value, bool) {
+func (it *Iterator) Next() bool {
 	if it.iter == nil {
 		it.result = nil
-		return nil, false
+		return false
 	}
 	if !it.open {
 		it.result = nil
-		return nil, false
+		return false
 	}
 	if !it.iter.Valid() {
 		it.result = nil
 		it.Close()
-		return nil, false
+		return false
 	}
 	if bytes.HasPrefix(it.iter.Key(), it.nextPrefix) {
 		if !it.isLiveValue(it.iter.Value()) {
@@ -150,11 +150,11 @@ func (it *Iterator) Next() (graph.Value, bool) {
 		if !ok {
 			it.Close()
 		}
-		return Token(out), true
+		return true
 	}
 	it.Close()
 	it.result = nil
-	return nil, false
+	return false
 }
 
 func (it *Iterator) ResultTree() *graph.ResultTree {
@@ -165,7 +165,7 @@ func (it *Iterator) Result() graph.Value {
 	return it.result
 }
 
-func (it *Iterator) NextResult() bool {
+func (it *Iterator) NextPath() bool {
 	return false
 }
 

graph/leveldb/leveldb_test.go

@@ -46,12 +46,8 @@ func makeTripleSet() []quad.Quad {
 
 func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []quad.Quad {
 	var res ordered
-	for {
+	for graph.Next(it) {
-		val, ok := graph.Next(it)
+		res = append(res, qs.Quad(it.Result()))
-		if !ok {
-			break
-		}
-		res = append(res, qs.Quad(val))
 	}
 	sort.Sort(res)
 	return res
@@ -86,12 +82,8 @@ func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
 
 func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string {
 	var res []string
-	for {
+	for graph.Next(it) {
-		val, ok := graph.Next(it)
+		res = append(res, qs.NameOf(it.Result()))
-		if !ok {
-			break
-		}
-		res = append(res, qs.NameOf(val))
 	}
 	sort.Strings(res)
 	return res
@@ -271,8 +263,8 @@ func TestIterator(t *testing.T) {
 	it.Reset()
 
 	it = qs.TriplesAllIterator()
-	edge, _ := graph.Next(it)
+	graph.Next(it)
-	triple := qs.Quad(edge)
+	triple := qs.Quad(it.Result())
 	set := makeTripleSet()
 	var ok bool
 	for _, t := range set {

graph/leveldb/triplestore.go

@@ -35,7 +35,7 @@ import (
 )
 
 func init() {
-	graph.RegisterTripleStore("leveldb", newTripleStore, createNewLevelDB)
+	graph.RegisterTripleStore("leveldb", true, newTripleStore, createNewLevelDB)
 }
 
 const (
@@ -45,7 +45,7 @@ const (
 
 type Token []byte
 
-func (t Token) Hasher() interface{} {
+func (t Token) Key() interface{} {
 	return string(t)
 }
 

graph/leveldb/triplestore_iterator_optimize.go

@@ -37,10 +37,10 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
 	if primary.Type() == graph.Fixed {
 		size, _ := primary.Size()
 		if size == 1 {
-			val, ok := graph.Next(primary)
+			if !graph.Next(primary) {
-			if !ok {
+				panic("unexpected size during optimize")
-				panic("Sizes lie")
 			}
+			val := primary.Result()
 			newIt := ts.TripleIterator(it.Direction(), val)
 			nt := newIt.Tagger()
 			nt.CopyFrom(it)

graph/memstore/all_iterator.go

@@ -35,21 +35,19 @@ func NewMemstoreNodesAllIterator(ts *TripleStore) *NodesAllIterator {
 }
 
 // No subiterators.
-func (it *AllIterator) SubIterators() []graph.Iterator {
+func (nit *NodesAllIterator) SubIterators() []graph.Iterator {
 	return nil
 }
 
-func (nit *NodesAllIterator) Next() (graph.Value, bool) {
+func (nit *NodesAllIterator) Next() bool {
-	next, out := nit.Int64.Next()
+	if !nit.Int64.Next() {
-	if !out {
+		return false
-		return next, out
 	}
-	i64 := next.(int64)
+	_, ok := nit.ts.revIdMap[nit.Int64.Result().(int64)]
-	_, ok := nit.ts.revIdMap[i64]
 	if !ok {
 		return nit.Next()
 	}
-	return next, out
+	return true
 }
 
 func NewMemstoreQuadsAllIterator(ts *TripleStore) *QuadsAllIterator {
@@ -59,13 +57,13 @@ func NewMemstoreQuadsAllIterator(ts *TripleStore) *QuadsAllIterator {
 	return &out
 }
 
-func (qit *QuadsAllIterator) Next() (graph.Value, bool) {
+func (qit *QuadsAllIterator) Next() bool {
-	next, out := qit.Int64.Next()
+	out := qit.Int64.Next()
 	if out {
-		i64 := next.(int64)
+		i64 := qit.Int64.Result().(int64)
 		if qit.ts.log[i64].DeletedBy != 0 || qit.ts.log[i64].Action == graph.Delete {
 			return qit.Next()
 		}
 	}
-	return next, out
+	return out
 }

graph/memstore/iterator.go

@@ -99,7 +99,7 @@ func (it *Iterator) checkValid(index int64) bool {
 	return it.ts.log[index].DeletedBy == 0
 }
 
-func (it *Iterator) Next() (graph.Value, bool) {
+func (it *Iterator) Next() bool {
 	graph.NextLogIn(it)
 	if it.tree.Max() == nil || it.iterLast == it.tree.Max().(Int64) {
 		return graph.NextLogOut(it, nil, false)
@@ -120,7 +120,7 @@ func (it *Iterator) Result() graph.Value {
 	return it.result
 }
 
-func (it *Iterator) NextResult() bool {
+func (it *Iterator) NextPath() bool {
 	return false
 }
 

graph/memstore/triplestore.go

@@ -26,7 +26,7 @@ import (
 )
 
 func init() {
-	graph.RegisterTripleStore("memstore", func(string, graph.Options) (graph.TripleStore, error) {
+	graph.RegisterTripleStore("memstore", false, func(string, graph.Options) (graph.TripleStore, error) {
 		return newTripleStore(), nil
 	}, nil)
 }
@@ -145,14 +145,10 @@ func (ts *TripleStore) quadExists(t quad.Quad) (bool, int64) {
 	}
 	it := NewLlrbIterator(smallest_tree, "", ts)
 
-	for {
+	for it.Next() {
-		val, ok := it.Next()
+		val := it.Result()
-		if !ok {
+		if t == ts.log[val.(int64)].Quad {
-			break
+			return true, val.(int64)
-		}
-		ival := val.(int64)
-		if t == ts.log[ival].Quad {
-			return true, ival
 		}
 	}
 	return false, 0

graph/memstore/triplestore_iterator_optimize.go

@@ -37,10 +37,10 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
 	if primary.Type() == graph.Fixed {
 		size, _ := primary.Size()
 		if size == 1 {
-			val, ok := graph.Next(primary)
+			if !graph.Next(primary) {
-			if !ok {
+				panic("unexpected size during optimize")
-				panic("Sizes lie")
 			}
+			val := primary.Result()
 			newIt := ts.TripleIterator(it.Direction(), val)
 			nt := newIt.Tagger()
 			nt.CopyFrom(it)

graph/memstore/triplestore_test.go

@@ -117,10 +117,10 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
 	outerAnd.AddSubIterator(fixed)
 	outerAnd.AddSubIterator(hasa)
 
-	val, ok := outerAnd.Next()
+	if !outerAnd.Next() {
-	if !ok {
 		t.Error("Expected one matching subtree")
 	}
+	val := outerAnd.Result()
 	if ts.NameOf(val) != "C" {
 		t.Errorf("Matching subtree should be %s, got %s", "barak", ts.NameOf(val))
 	}
@@ -131,7 +131,7 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
 	)
 	for {
 		got = append(got, ts.NameOf(all.Result()))
-		if !outerAnd.NextResult() {
+		if !outerAnd.NextPath() {
 			break
 		}
 	}
@@ -141,8 +141,7 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
 		t.Errorf("Unexpected result, got:%q expect:%q", got, expect)
 	}
 
-	val, ok = outerAnd.Next()
+	if outerAnd.Next() {
-	if ok {
 		t.Error("More than one possible top level output?")
 	}
 }
@@ -193,8 +192,7 @@ func TestRemoveTriple(t *testing.T) {
 	hasa := iterator.NewHasA(ts, innerAnd, quad.Object)
 
 	newIt, _ := hasa.Optimize()
-	_, ok := graph.Next(newIt)
+	if graph.Next(newIt) {
-	if ok {
 		t.Error("E should not have any followers.")
 	}
 }

graph/mongo/iterator.go

@@ -138,7 +138,7 @@ func (it *Iterator) Clone() graph.Iterator {
 	return m
 }
 
-func (it *Iterator) Next() (graph.Value, bool) {
+func (it *Iterator) Next() bool {
 	var result struct {
 		Id string "_id"
 		//Sub string "Sub"
@@ -151,10 +151,10 @@ func (it *Iterator) Next() (graph.Value, bool) {
 		if err != nil {
 			glog.Errorln("Error Nexting Iterator: ", err)
 		}
-		return nil, false
+		return false
 	}
 	it.result = result.Id
-	return result.Id, true
+	return true
 }
 
 func (it *Iterator) ResultTree() *graph.ResultTree {
@@ -165,7 +165,7 @@ func (it *Iterator) Result() graph.Value {
 	return it.result
 }
 
-func (it *Iterator) NextResult() bool {
+func (it *Iterator) NextPath() bool {
 	return false
 }
 

graph/mongo/triplestore.go

@@ -30,7 +30,7 @@ import (
 )
 
 func init() {
-	graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
+	graph.RegisterTripleStore("mongo", true, newTripleStore, createNewMongoGraph)
 }
 
 // Guarantee we satisfy graph.Bulkloader.

graph/mongo/triplestore_iterator_optimize.go

@@ -37,10 +37,10 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
 	if primary.Type() == graph.Fixed {
 		size, _ := primary.Size()
 		if size == 1 {
-			val, ok := graph.Next(primary)
+			if !graph.Next(primary) {
-			if !ok {
+				panic("unexpected size during optimize")
-				panic("Sizes lie")
 			}
+			val := primary.Result()
 			newIt := ts.TripleIterator(it.Direction(), val)
 			nt := newIt.Tagger()
 			nt.CopyFrom(it)

graph/result_tree_evaluator.go

@@ -14,7 +14,10 @@
 
 package graph
 
-import "fmt"
+import (
+	"bytes"
+	"fmt"
+)
 
 type ResultTree struct {
 	result   Value
@@ -26,14 +29,13 @@ func NewResultTree(result Value) *ResultTree {
 }
 
 func (t *ResultTree) String() string {
-	base := fmt.Sprintf("(%d", t.result)
+	var buf bytes.Buffer
-	if len(t.subtrees) != 0 {
+	fmt.Fprintf(&buf, "(%d", t.result)
 	for _, sub := range t.subtrees {
-			base += fmt.Sprintf(" %s", sub)
+		fmt.Fprintf(&buf, " %s", sub)
 	}
-	}
+	buf.WriteByte(')')
-	base += ")"
+	return buf.String()
-	return base
 }
 
 func (t *ResultTree) AddSubtree(sub *ResultTree) {
@@ -41,22 +43,15 @@ func (t *ResultTree) AddSubtree(sub *ResultTree) {
 }
 
 func StringResultTreeEvaluator(it Nexter) string {
-	ok := true
+	var buf bytes.Buffer
-	out := ""
+	for it.Next() {
-	for {
+		fmt.Fprintln(&buf, it.ResultTree())
-		_, ok = it.Next()
+		for it.NextPath() {
-		if !ok {
+			buf.WriteByte(' ')
-			break
+			fmt.Fprintln(&buf, it.ResultTree())
-		}
-		out += it.ResultTree().String()
-		out += "\n"
-		for it.NextResult() == true {
-			out += " "
-			out += it.ResultTree().String()
-			out += "\n"
 		}
 	}
-	return out
+	return buf.String()
 }
 
 func PrintResultTreeEvaluator(it Nexter) {

graph/triplestore.go

@@ -37,7 +37,7 @@ import (
 // pointers to structs, or merely triples, or whatever works best for the
 // backing store.
 //
-// These must be comparable, or implement a `Hasher() interface{}` function
+// These must be comparable, or implement a `Key() interface{}` function
 // so that they may be stored in maps.
 type Value interface{}
 
@@ -133,38 +133,45 @@ type BulkLoader interface {
 type NewStoreFunc func(string, Options) (TripleStore, error)
 type InitStoreFunc func(string, Options) error
 
-var storeRegistry = make(map[string]NewStoreFunc)
+type register struct {
-var storeInitRegistry = make(map[string]InitStoreFunc)
+	newFunc      NewStoreFunc
+	initFunc     InitStoreFunc
+	isPersistent bool
+}
 
-func RegisterTripleStore(name string, newFunc NewStoreFunc, initFunc InitStoreFunc) {
+var storeRegistry = make(map[string]register)
+
+func RegisterTripleStore(name string, persists bool, newFunc NewStoreFunc, initFunc InitStoreFunc) {
 	if _, found := storeRegistry[name]; found {
 		panic("already registered TripleStore " + name)
 	}
-	storeRegistry[name] = newFunc
+	storeRegistry[name] = register{
-	if initFunc != nil {
+		newFunc:      newFunc,
-		storeInitRegistry[name] = initFunc
+		initFunc:     initFunc,
+		isPersistent: persists,
 	}
 }
 
 func NewTripleStore(name, dbpath string, opts Options) (TripleStore, error) {
-	newFunc, hasNew := storeRegistry[name]
+	r, registered := storeRegistry[name]
-	if !hasNew {
+	if !registered {
 		return nil, errors.New("triplestore: name '" + name + "' is not registered")
 	}
-	return newFunc(dbpath, opts)
+	return r.newFunc(dbpath, opts)
 }
 
 func InitTripleStore(name, dbpath string, opts Options) error {
-	initFunc, hasInit := storeInitRegistry[name]
+	r, registered := storeRegistry[name]
-	if hasInit {
+	if registered {
-		return initFunc(dbpath, opts)
+		return r.initFunc(dbpath, opts)
-	}
-	if _, isRegistered := storeRegistry[name]; isRegistered {
-		return nil
 	}
 	return errors.New("triplestore: name '" + name + "' is not registered")
 }
 
+func IsPersistent(name string) bool {
+	return storeRegistry[name].isPersistent
+}
+
 func TripleStores() []string {
 	t := make([]string, 0, len(storeRegistry))
 	for n := range storeRegistry {

http/write.go

@@ -26,7 +26,7 @@ import (
 	"github.com/julienschmidt/httprouter"
 
 	"github.com/google/cayley/quad"
-	"github.com/google/cayley/quad/nquads"
+	"github.com/google/cayley/quad/cquads"
 )
 
 func ParseJsonToTripleList(jsonBody []byte) ([]quad.Quad, error) {
@@ -78,7 +78,8 @@ func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params
 		blockSize = int64(api.config.LoadSize)
 	}
 
-	dec := nquads.NewDecoder(formFile)
+	// TODO(kortschak) Make this configurable from the web UI.
+	dec := cquads.NewDecoder(formFile)
 
 	var (
 		n int

query/gremlin/finals.go

@@ -153,8 +153,7 @@ func runIteratorToArray(it graph.Iterator, ses *Session, limit int) []map[string
 			return nil
 		default:
 		}
-		_, ok := graph.Next(it)
+		if !graph.Next(it) {
-		if !ok {
 			break
 		}
 		tags := make(map[string]graph.Value)
@@ -164,7 +163,7 @@ func runIteratorToArray(it graph.Iterator, ses *Session, limit int) []map[string
 		if limit >= 0 && count >= limit {
 			break
 		}
-		for it.NextResult() == true {
+		for it.NextPath() {
 			select {
 			case <-ses.kill:
 				return nil
@@ -193,11 +192,10 @@ func runIteratorToArrayNoTags(it graph.Iterator, ses *Session, limit int) []stri
 			return nil
 		default:
 		}
-		val, ok := graph.Next(it)
+		if !graph.Next(it) {
-		if !ok {
 			break
 		}
-		output = append(output, ses.ts.NameOf(val))
+		output = append(output, ses.ts.NameOf(it.Result()))
 		count++
 		if limit >= 0 && count >= limit {
 			break
@@ -216,8 +214,7 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu
 			return
 		default:
 		}
-		_, ok := graph.Next(it)
+		if !graph.Next(it) {
-		if !ok {
 			break
 		}
 		tags := make(map[string]graph.Value)
@@ -228,7 +225,7 @@ func runIteratorWithCallback(it graph.Iterator, ses *Session, callback otto.Valu
 		if limit >= 0 && count >= limit {
 			break
 		}
-		for it.NextResult() == true {
+		for it.NextPath() {
 			select {
 			case <-ses.kill:
 				return
@@ -260,8 +257,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) {
 			return
 		default:
 		}
-		_, ok := graph.Next(it)
+		if !graph.Next(it) {
-		if !ok {
 			break
 		}
 		tags := make(map[string]graph.Value)
@@ -269,7 +265,7 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) {
 		if !ses.SendResult(&Result{actualResults: &tags}) {
 			break
 		}
-		for it.NextResult() == true {
+		for it.NextPath() {
 			select {
 			case <-ses.kill:
 				return

query/mql/session.go

@@ -88,15 +88,11 @@ func (s *Session) ExecInput(input string, c chan interface{}, limit int) {
 	if glog.V(2) {
 		glog.V(2).Infoln(it.DebugString(0))
 	}
-	for {
+	for graph.Next(it) {
-		_, ok := graph.Next(it)
-		if !ok {
-			break
-		}
 		tags := make(map[string]graph.Value)
 		it.TagResults(tags)
 		c <- tags
-		for it.NextResult() == true {
+		for it.NextPath() == true {
 			tags := make(map[string]graph.Value)
 			it.TagResults(tags)
 			c <- tags

query/sexp/parser_test.go

@@ -67,10 +67,10 @@ func TestMemstoreBackedSexp(t *testing.T) {
 		if it.Type() != test.typ {
 			t.Errorf("Incorrect type for %s, got:%q expect %q", test.message, it.Type(), test.expect)
 		}
-		got, ok := graph.Next(it)
+		if !graph.Next(it) {
-		if !ok {
 			t.Errorf("Failed to %s", test.message)
 		}
+		got := it.Result()
 		if expect := ts.ValueOf(test.expect); got != expect {
 			t.Errorf("Incorrect result for %s, got:%v expect %v", test.message, got, expect)
 		}
@@ -88,10 +88,10 @@ func TestTreeConstraintParse(t *testing.T) {
 	if it.Type() != graph.And {
 		t.Error("Odd iterator tree. Got: %s", it.DebugString(0))
 	}
-	out, ok := graph.Next(it)
+	if !graph.Next(it) {
-	if !ok {
 		t.Error("Got no results")
 	}
+	out := it.Result()
 	if out != ts.ValueOf("i") {
 		t.Errorf("Got %d, expected %d", out, ts.ValueOf("i"))
 	}
@@ -105,8 +105,7 @@ func TestTreeConstraintTagParse(t *testing.T) {
 		"(:like\n" +
 		"($a (:is :good))))"
 	it := BuildIteratorTreeForQuery(ts, query)
-	_, ok := graph.Next(it)
+	if !graph.Next(it) {
-	if !ok {
 		t.Error("Got no results")
 	}
 	tags := make(map[string]graph.Value)
@@ -135,15 +134,14 @@ func TestMultipleConstraintParse(t *testing.T) {
 	if it.Type() != graph.And {
 		t.Error("Odd iterator tree. Got: %s", it.DebugString(0))
 	}
-	out, ok := graph.Next(it)
+	if !graph.Next(it) {
-	if !ok {
 		t.Error("Got no results")
 	}
+	out := it.Result()
 	if out != ts.ValueOf("i") {
 		t.Errorf("Got %d, expected %d", out, ts.ValueOf("i"))
 	}
-	_, ok = graph.Next(it)
+	if graph.Next(it) {
-	if ok {
 		t.Error("Too many results")
 	}
 }

query/sexp/session.go

@@ -77,11 +77,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) {
 		fmt.Println(it.DebugString(0))
 	}
 	nResults := 0
-	for {
+	for graph.Next(it) {
-		_, ok := graph.Next(it)
-		if !ok {
-			break
-		}
 		tags := make(map[string]graph.Value)
 		it.TagResults(tags)
 		out <- &tags
@@ -89,7 +85,7 @@ func (s *Session) ExecInput(input string, out chan interface{}, limit int) {
 		if nResults > limit && limit != -1 {
 			break
 		}
-		for it.NextResult() == true {
+		for it.NextPath() == true {
 			tags := make(map[string]graph.Value)
 			it.TagResults(tags)
 			out <- &tags