Reorganize to go get will work

This makes almost no changes to source, but touches every almost file.

Also fixes error in gremlin test code.

graph/all-iterator.go

@@ -0,0 +1,117 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines one of the base iterators, the All iterator. Which, logically
+// enough, represents all nodes or all links in the graph.
+//
+// This particular file is actually vestigal. It's up to the TripleStore to give
+// us an All iterator that represents all things in the graph. So this is
+// really the All iterator for the MemTripleStore. That said, it *is* one of
+// the base iterators, and it helps just to see it here.
+
+import (
+	"fmt"
+	"strings"
+)
+
+// An All iterator across a range of int64 values, from `max` to `min`.
+type Int64AllIterator struct {
+	BaseIterator
+	max, min int64
+	at       int64
+}
+
+// Creates a new Int64AllIterator with the given range.
+func NewInt64AllIterator(min, max int64) *Int64AllIterator {
+	var all Int64AllIterator
+	BaseIteratorInit(&all.BaseIterator)
+	all.max = max
+	all.min = min
+	all.at = min
+	return &all
+}
+
+// Start back at the beginning
+func (a *Int64AllIterator) Reset() {
+	a.at = a.min
+}
+
+func (a *Int64AllIterator) Close() {
+}
+
+func (a *Int64AllIterator) Clone() Iterator {
+	out := NewInt64AllIterator(a.min, a.max)
+	out.CopyTagsFrom(a)
+	return out
+}
+
+// Prints the All iterator as just an "all".
+func (a *Int64AllIterator) DebugString(indent int) string {
+	return fmt.Sprintf("%s(%s)", strings.Repeat(" ", indent), a.Type())
+}
+
+// Next() on an Int64 all iterator is a simple incrementing counter.
+// Return the next integer, and mark it as the result.
+func (a *Int64AllIterator) Next() (TSVal, bool) {
+	NextLogIn(a)
+	if a.at == -1 {
+		return NextLogOut(a, nil, false)
+	}
+	val := a.at
+	a.at = a.at + 1
+	if a.at > a.max {
+		a.at = -1
+	}
+	a.Last = val
+	return NextLogOut(a, val, true)
+}
+
+// The number of elements in an Int64AllIterator is the size of the range.
+// The size is exact.
+func (a *Int64AllIterator) Size() (int64, bool) {
+	Size := ((a.max - a.min) + 1)
+	return Size, true
+}
+
+// Check() for an Int64AllIterator is merely seeing if the passed value is
+// withing the range, assuming the value is an int64.
+func (a *Int64AllIterator) Check(tsv TSVal) bool {
+	CheckLogIn(a, tsv)
+	v := tsv.(int64)
+	if a.min <= v && v <= a.max {
+		a.Last = v
+		return CheckLogOut(a, v, true)
+	}
+	return CheckLogOut(a, v, false)
+}
+
+// The type of this iterator is an "all". This is important, as it puts it in
+// the class of "all iterators.
+func (a *Int64AllIterator) Type() string { return "all" }
+
+// There's nothing to optimize about this little iterator.
+func (a *Int64AllIterator) Optimize() (Iterator, bool) { return a, false }
+
+// Stats for an Int64AllIterator are simple. Super cheap to do any operation,
+// and as big as the range.
+func (a *Int64AllIterator) GetStats() *IteratorStats {
+	s, _ := a.Size()
+	return &IteratorStats{
+		CheckCost: 1,
+		NextCost:  1,
+		Size:      s,
+	}
+}

graph/and-iterator-optimize.go

@@ -0,0 +1,330 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Perhaps the most tricky file in this entire module. Really a method on the
+// AndIterator, but important enough to deserve its own file.
+//
+// Calling Optimize() on an And iterator, like any iterator, requires that we
+// preserve the underlying meaning. However, the And has many choices, namely,
+// which one of it's subiterators will be the branch that does the Next()ing,
+// and which ordering of the remaining iterators is the most efficient. In
+// short, this is where a lot of the query optimization happens, and there are
+// many wins to be had here, as well as many bad bugs. The worst class of bug
+// changes the meaning of the query. The second worst class makes things really
+// slow.
+//
+// The good news is this: If Optimize() is never called (turned off, perhaps) we can
+// be sure the results are as good as the query language called for.
+//
+// In short, tread lightly.
+
+import (
+	"container/list"
+)
+
+// Optimizes the AndIterator, by picking the most efficient way to Next() and
+// Check() its subiterators. For SQL fans, this is equivalent to JOIN.
+func (and *AndIterator) Optimize() (Iterator, bool) {
+	// First, let's get the list of iterators, in order (first one is Next()ed,
+	// the rest are Check()ed)
+	oldItList := and.GetSubIterators()
+
+	// And call Optimize() on our subtree, replacing each one in the order we
+	// found them. it_list is the newly optimized versions of these, and changed
+	// is another list, of only the ones that have returned replacements and
+	// changed.
+	itList := optimizeSubIterators(oldItList)
+
+	// Close the replaced iterators (they ought to close themselves, but Close()
+	// is idempotent, so this just protects against any machinations).
+	closeIteratorList(oldItList, nil)
+
+	// If we can find only one subiterator which is equivalent to this whole and,
+	// we can replace the And...
+	out := and.optimizeReplacement(itList)
+	if out != nil {
+		// ...Move the tags to the replacement...
+		moveTagsTo(out, and)
+		// ...Close everyone except `out`, our replacement...
+		closeIteratorList(itList, out)
+		// ...And return it.
+		return out, true
+	}
+
+	// And now, without changing any of the iterators, we reorder them. it_list is
+	// now a permutation of itself, but the contents are unchanged.
+	itList = optimizeOrder(itList)
+
+	// Okay! At this point we have an optimized order.
+
+	// The easiest thing to do at this point is merely to create a new And iterator
+	// and replace ourselves with our (reordered, optimized) clone.
+	newAnd := NewAndIterator()
+
+	// Add the subiterators in order.
+	for e := itList.Front(); e != nil; e = e.Next() {
+		newAnd.AddSubIterator(e.Value.(Iterator))
+	}
+
+	// Move the tags hanging on us (like any good replacement).
+	newAnd.CopyTagsFrom(and)
+
+	newAnd.optimizeCheck()
+
+	// And close ourselves but not our subiterators -- some may still be alive in
+	// the new And (they were unchanged upon calling Optimize() on them, at the
+	// start).
+	and.cleanUp()
+	return newAnd, true
+}
+
+// Closes a list of iterators, except the one passed in `except`. Closes all
+// of the iterators in the list if `except` is nil.
+func closeIteratorList(l *list.List, except Iterator) {
+	for e := l.Front(); e != nil; e = e.Next() {
+		it := e.Value.(Iterator)
+		if it != except {
+			e.Value.(Iterator).Close()
+		}
+	}
+}
+
+// Find if there is a single subiterator which is a valid replacement for this
+// AndIterator.
+func (and *AndIterator) optimizeReplacement(itList *list.List) Iterator {
+	// If we were created with no SubIterators, we're as good as Null.
+	if itList.Len() == 0 {
+		return &NullIterator{}
+	}
+	if itList.Len() == 1 {
+		// When there's only one iterator, there's only one choice.
+		return itList.Front().Value.(Iterator)
+	}
+	// If any of our subiterators, post-optimization, are also Null, then
+	// there's no point in continuing the branch, we will have no results
+	// and we are null as well.
+	if hasAnyNullIterators(itList) {
+		return &NullIterator{}
+	}
+
+	// If we have one useful iterator, use that.
+	it := hasOneUsefulIterator(itList)
+	if it != nil {
+		return it
+	}
+	return nil
+}
+
+// optimizeOrder(l) takes a list and returns a list, containing the same contents
+// but with a new ordering, however it wishes.
+func optimizeOrder(l *list.List) *list.List {
+	out := list.New()
+	var bestIt Iterator
+	bestCost := int64(1 << 62)
+	// bad contains iterators that can't be (efficiently) nexted, such as
+	// "optional" or "not". Separate them out and tack them on at the end.
+	bad := list.New()
+
+	// Find the iterator with the projected "best" total cost.
+	// Total cost is defined as The Next()ed iterator's cost to Next() out
+	// all of it's contents, and to Check() each of those against everyone
+	// else.
+	for e := l.Front(); e != nil; e = e.Next() {
+		it := e.Value.(Iterator)
+		if !it.Nextable() {
+			bad.PushBack(it)
+			continue
+		}
+		rootStats := e.Value.(Iterator).GetStats()
+		projectedCost := rootStats.NextCost
+		for f := l.Front(); f != nil; f = f.Next() {
+			if !f.Value.(Iterator).Nextable() {
+				continue
+			}
+			if f == e {
+				continue
+			}
+			stats := f.Value.(Iterator).GetStats()
+			projectedCost += stats.CheckCost
+		}
+		projectedCost = projectedCost * rootStats.Size
+		if projectedCost < bestCost {
+			bestIt = it
+			bestCost = projectedCost
+		}
+	}
+
+	// TODO(barakmich): Optimization of order need not stop here. Picking a smart
+	// Check() order based on probability of getting a false Check() first is
+	// useful (fail faster).
+
+	// Put the best iterator (the one we wish to Next()) at the front...
+	out.PushBack(bestIt)
+	// ...And push everyone else after...
+	for e := l.Front(); e != nil; e = e.Next() {
+		thisIt := e.Value.(Iterator)
+		if !thisIt.Nextable() {
+			continue
+		}
+		if thisIt != bestIt {
+			out.PushBack(thisIt)
+		}
+	}
+	// ...And finally, the difficult children on the end.
+	out.PushBackList(bad)
+	return out
+}
+
+// optimizeCheck(l) creates an alternate check list, containing the same contents
+// but with a new ordering, however it wishes.
+func (and *AndIterator) optimizeCheck() {
+	subIts := and.GetSubIterators()
+	out := list.New()
+
+	// Find the iterator with the lowest Check() cost, push it to the front, repeat.
+	for subIts.Len() != 0 {
+		var best *list.Element
+		bestCost := int64(1 << 62)
+		for e := subIts.Front(); e != nil; e = e.Next() {
+			it := e.Value.(Iterator)
+			rootStats := it.GetStats()
+			projectedCost := rootStats.CheckCost
+			if projectedCost < bestCost {
+				best = e
+				bestCost = projectedCost
+			}
+		}
+		out.PushBack(best.Value)
+		subIts.Remove(best)
+	}
+
+	and.checkList = out
+}
+
+// If we're replacing ourselves by a single iterator, we need to grab the
+// result tags from the iterators that, while still valid and would hold
+// the same values as this and, are not going to stay.
+// getSubTags() returns a map of the tags for all the subiterators.
+func (and *AndIterator) getSubTags() map[string]bool {
+	subs := and.GetSubIterators()
+	tags := make(map[string]bool)
+	for e := subs.Front(); e != nil; e = e.Next() {
+		it := e.Value.(Iterator)
+		for _, tag := range it.Tags() {
+			tags[tag] = true
+		}
+	}
+	for _, tag := range and.Tags() {
+		tags[tag] = true
+	}
+	return tags
+}
+
+// moveTagsTo() gets the tags for all of the And's subiterators and the
+// And itself, and moves them to `out`.
+func moveTagsTo(out Iterator, and *AndIterator) {
+	tagmap := and.getSubTags()
+	for _, tag := range out.Tags() {
+		if tagmap[tag] {
+			delete(tagmap, tag)
+		}
+	}
+	for k, _ := range tagmap {
+		out.AddTag(k)
+	}
+}
+
+// optimizeSubIterators(l) takes a list of iterators and calls Optimize() on all
+// of them. It returns two lists -- the first contains the same list as l, where
+// any replacements are made by Optimize() and the second contains the originals
+// which were replaced.
+func optimizeSubIterators(l *list.List) *list.List {
+	itList := list.New()
+	for e := l.Front(); e != nil; e = e.Next() {
+		it := e.Value.(Iterator)
+		newIt, change := it.Optimize()
+		if change {
+			itList.PushBack(newIt)
+		} else {
+			itList.PushBack(it.Clone())
+		}
+	}
+	return itList
+}
+
+// Check a list of iterators for any Null iterators.
+func hasAnyNullIterators(l *list.List) bool {
+	for e := l.Front(); e != nil; e = e.Next() {
+		it := e.Value.(Iterator)
+		if it.Type() == "null" {
+			return true
+		}
+	}
+	return false
+}
+
+// There are two "not-useful" iterators -- namely "null" which returns
+// nothing, and "all" which returns everything. Particularly, we want
+// to see if we're intersecting with a bunch of "all" iterators, and,
+// if we are, then we have only one useful iterator.
+func hasOneUsefulIterator(l *list.List) Iterator {
+	usefulCount := 0
+	var usefulIt Iterator
+	for e := l.Front(); e != nil; e = e.Next() {
+		it := e.Value.(Iterator)
+		switch it.Type() {
+		case "null", "all":
+			continue
+		case "optional":
+			// Optional is weird -- it's not useful, but we can't optimize
+			// away from it. Therefore, we skip this optimization
+			// if we see one.
+			return nil
+		default:
+			usefulCount++
+			usefulIt = it
+		}
+	}
+
+	if usefulCount == 1 {
+		return usefulIt
+	}
+	return nil
+}
+
+// and.GetStats() lives here in and-iterator-optimize.go because it may
+// in the future return different statistics based on how it is optimized.
+// For now, however, it's pretty static.
+func (and *AndIterator) GetStats() *IteratorStats {
+	primaryStats := and.primaryIt.GetStats()
+	CheckCost := primaryStats.CheckCost
+	NextCost := primaryStats.NextCost
+	Size := primaryStats.Size
+	for _, it := range and.internalIterators {
+		stats := it.GetStats()
+		NextCost += stats.CheckCost
+		CheckCost += stats.CheckCost
+		if Size > stats.Size {
+			Size = stats.Size
+		}
+	}
+	return &IteratorStats{
+		CheckCost: CheckCost,
+		NextCost:  NextCost,
+		Size:      Size,
+	}
+
+}

graph/and-iterator-optimize_test.go

@@ -0,0 +1,111 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Tests relating to methods in and-iterator-optimize. Many are pretty simplistic, but
+// nonetheless cover a lot of basic cases.
+
+import (
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func TestIteratorPromotion(t *testing.T) {
+	all := NewInt64AllIterator(1, 3)
+	fixed := newFixedIterator()
+	fixed.AddValue(3)
+	a := NewAndIterator()
+	a.AddSubIterator(all)
+	a.AddSubIterator(fixed)
+	all.AddTag("a")
+	fixed.AddTag("b")
+	a.AddTag("c")
+	newIt, changed := a.Optimize()
+	if !changed {
+		t.Error("Iterator didn't optimize")
+	}
+	if newIt.Type() != "fixed" {
+		t.Error("Expected fixed iterator")
+	}
+	tagsExpected := []string{"a", "b", "c"}
+	tags := newIt.Tags()
+	sort.Strings(tags)
+	if !reflect.DeepEqual(tags, tagsExpected) {
+		t.Fatal("Tags don't match")
+	}
+}
+
+func TestNullIteratorAnd(t *testing.T) {
+	all := NewInt64AllIterator(1, 3)
+	null := NewNullIterator()
+	a := NewAndIterator()
+	a.AddSubIterator(all)
+	a.AddSubIterator(null)
+	newIt, changed := a.Optimize()
+	if !changed {
+		t.Error("Didn't change")
+	}
+	if newIt.Type() != "null" {
+		t.Error("Expected null iterator, got ", newIt.Type())
+	}
+}
+
+func TestReorderWithTag(t *testing.T) {
+	all := NewInt64AllIterator(100, 300)
+	all.AddTag("good")
+	all2 := NewInt64AllIterator(1, 30000)
+	all2.AddTag("slow")
+	a := NewAndIterator()
+	// Make all2 the default iterator
+	a.AddSubIterator(all2)
+	a.AddSubIterator(all)
+
+	newIt, changed := a.Optimize()
+	if !changed {
+		t.Error("Expected new iterator")
+	}
+	expectedTags := []string{"good", "slow"}
+	tagsOut := make([]string, 0)
+	l := newIt.GetSubIterators()
+	for e := l.Front(); e != nil; e = e.Next() {
+		for _, x := range e.Value.(Iterator).Tags() {
+			tagsOut = append(tagsOut, x)
+		}
+	}
+	if !reflect.DeepEqual(expectedTags, tagsOut) {
+		t.Fatal("Tags don't match")
+	}
+}
+
+func TestAndStatistics(t *testing.T) {
+	all := NewInt64AllIterator(100, 300)
+	all.AddTag("good")
+	all2 := NewInt64AllIterator(1, 30000)
+	all2.AddTag("slow")
+	a := NewAndIterator()
+	// Make all2 the default iterator
+	a.AddSubIterator(all2)
+	a.AddSubIterator(all)
+	stats1 := a.GetStats()
+	newIt, changed := a.Optimize()
+	if !changed {
+		t.Error("Didn't optimize")
+	}
+	stats2 := newIt.GetStats()
+	if stats2.NextCost > stats1.NextCost {
+		t.Error("And didn't optimize. Next cost old ", stats1.NextCost, "and new ", stats2.NextCost)
+	}
+}

graph/and-iterator.go

@@ -0,0 +1,248 @@
+// Defines the And iterator, one of the base iterators. And requires no
+// knowledge of the constituent TripleStore; its sole purpose is to act as an
+// intersection operator across the subiterators it is given. If one iterator
+// contains [1,3,5] and another [2,3,4] -- then And is an iterator that
+// 'contains' [3]
+//
+// It accomplishes this in one of two ways. If it is a Next()ed iterator (that
+// is, it is a top level iterator, or on the "Next() path", then it will Next()
+// it's primary iterator (helpfully, and.primary_it) and Check() the resultant
+// value against it's other iterators. If it matches all of them, then it
+// returns that value. Otherwise, it repeats the process.
+//
+// If it's on a Check() path, it merely Check()s every iterator, and returns the
+// logical AND of each result.
+
+package graph
+
+import (
+	"container/list"
+	"fmt"
+	"strings"
+)
+
+// The And iterator. Consists of a BaseIterator and a number of subiterators, the primary of which will
+// be Next()ed if next is called.
+type AndIterator struct {
+	BaseIterator
+	internalIterators []Iterator
+	itCount           int
+	primaryIt         Iterator
+	checkList         *list.List
+}
+
+// Creates a new And iterator.
+func NewAndIterator() *AndIterator {
+	var and AndIterator
+	BaseIteratorInit(&and.BaseIterator)
+	and.internalIterators = make([]Iterator, 0, 20)
+	and.checkList = nil
+	return &and
+}
+
+// Reset all internal iterators
+func (and *AndIterator) Reset() {
+	and.primaryIt.Reset()
+	for _, it := range and.internalIterators {
+		it.Reset()
+	}
+	and.checkList = nil
+}
+
+func (and *AndIterator) Clone() Iterator {
+	newAnd := NewAndIterator()
+	newAnd.AddSubIterator(and.primaryIt.Clone())
+	newAnd.CopyTagsFrom(and)
+	for _, it := range and.internalIterators {
+		newAnd.AddSubIterator(it.Clone())
+	}
+	if and.checkList != nil {
+		newAnd.optimizeCheck()
+	}
+	return newAnd
+}
+
+// Returns a list.List of the subiterators, in order (primary iterator first).
+func (and *AndIterator) GetSubIterators() *list.List {
+	l := list.New()
+	l.PushBack(and.primaryIt)
+	for _, it := range and.internalIterators {
+		l.PushBack(it)
+	}
+	return l
+}
+
+// Overrides BaseIterator TagResults, as it needs to add it's own results and
+// recurse down it's subiterators.
+func (and *AndIterator) TagResults(out *map[string]TSVal) {
+	and.BaseIterator.TagResults(out)
+	if and.primaryIt != nil {
+		and.primaryIt.TagResults(out)
+	}
+	for _, it := range and.internalIterators {
+		it.TagResults(out)
+	}
+}
+
+// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators.
+func (and *AndIterator) GetResultTree() *ResultTree {
+	tree := NewResultTree(and.LastResult())
+	tree.AddSubtree(and.primaryIt.GetResultTree())
+	for _, it := range and.internalIterators {
+		tree.AddSubtree(it.GetResultTree())
+	}
+	return tree
+}
+
+// Prints information about this iterator.
+func (and *AndIterator) DebugString(indent int) string {
+	var total string
+	for i, it := range and.internalIterators {
+		total += strings.Repeat(" ", indent+2)
+		total += fmt.Sprintf("%d:\n%s\n", i, it.DebugString(indent+4))
+	}
+	var tags string
+	for _, k := range and.Tags() {
+		tags += fmt.Sprintf("%s;", k)
+	}
+	spaces := strings.Repeat(" ", indent+2)
+
+	return fmt.Sprintf("%s(%s %d\n%stags:%s\n%sprimary_it:\n%s\n%sother_its:\n%s)",
+		strings.Repeat(" ", indent),
+		and.Type(),
+		and.GetUid(),
+		spaces,
+		tags,
+		spaces,
+		and.primaryIt.DebugString(indent+4),
+		spaces,
+		total)
+}
+
+// Add a subiterator to this And iterator.
+//
+// The first iterator that is added becomes the primary iterator. This is
+// important. Calling Optimize() is the way to change the order based on
+// subiterator statistics. Without Optimize(), the order added is the order
+// used.
+func (and *AndIterator) AddSubIterator(sub Iterator) {
+	if and.itCount > 0 {
+		and.internalIterators = append(and.internalIterators, sub)
+		and.itCount++
+		return
+	}
+	and.primaryIt = sub
+	and.itCount++
+}
+
+// Returns the Next value from the And iterator. Because the And is the
+// intersection of its subiterators, it must choose one subiterator to produce a
+// candidate, and check this value against the subiterators. A productive choice
+// of primary iterator is therefore very important.
+func (and *AndIterator) Next() (TSVal, bool) {
+	NextLogIn(and)
+	var curr TSVal
+	var exists bool
+	for {
+
+		curr, exists = and.primaryIt.Next()
+		if !exists {
+			return NextLogOut(and, nil, false)
+		}
+		if and.checkSubIts(curr) {
+			and.Last = curr
+			return NextLogOut(and, curr, true)
+		}
+	}
+	panic("Somehow broke out of Next() loop in AndIterator")
+}
+
+// Checks a value against the non-primary iterators, in order.
+func (and *AndIterator) checkSubIts(val TSVal) bool {
+	var subIsGood = true
+	for _, it := range and.internalIterators {
+		subIsGood = it.Check(val)
+		if !subIsGood {
+			break
+		}
+	}
+	return subIsGood
+}
+
+func (and *AndIterator) checkCheckList(val TSVal) bool {
+	var isGood = true
+	for e := and.checkList.Front(); e != nil; e = e.Next() {
+		isGood = e.Value.(Iterator).Check(val)
+		if !isGood {
+			break
+		}
+	}
+	return CheckLogOut(and, val, isGood)
+}
+
+// Check a value against the entire iterator, in order.
+func (and *AndIterator) Check(val TSVal) bool {
+	CheckLogIn(and, val)
+	if and.checkList != nil {
+		return and.checkCheckList(val)
+	}
+	mainGood := and.primaryIt.Check(val)
+	if !mainGood {
+		return CheckLogOut(and, val, false)
+	}
+	othersGood := and.checkSubIts(val)
+	if !othersGood {
+		return CheckLogOut(and, val, false)
+	}
+	and.Last = val
+	return CheckLogOut(and, val, true)
+}
+
+// Returns the approximate size of the And iterator. Because we're dealing
+// with an intersection, we know that the largest we can be is the size of the
+// smallest iterator. This is the heuristic we shall follow. Better heuristics
+// welcome.
+func (and *AndIterator) Size() (int64, bool) {
+	val, b := and.primaryIt.Size()
+	for _, it := range and.internalIterators {
+		newval, newb := it.Size()
+		if val > newval {
+			val = newval
+		}
+		b = newb && b
+	}
+	return val, b
+}
+
+// An And has no NextResult of its own -- that is, there are no other values
+// which satisfy our previous result that are not the result itself. Our
+// subiterators might, however, so just pass the call recursively.
+func (and *AndIterator) NextResult() bool {
+	if and.primaryIt.NextResult() {
+		return true
+	}
+	for _, it := range and.internalIterators {
+		if it.NextResult() {
+			return true
+		}
+	}
+	return false
+}
+
+// Perform and-specific cleanup, of which there currently is none.
+func (and *AndIterator) cleanUp() {
+}
+
+// Close this iterator, and, by extension, close the subiterators.
+// Close should be idempotent, and it follows that if it's subiterators
+// follow this contract, the And follows the contract.
+func (and *AndIterator) Close() {
+	and.cleanUp()
+	and.primaryIt.Close()
+	for _, it := range and.internalIterators {
+		it.Close()
+	}
+}
+
+// Register this as an "and" iterator.
+func (and *AndIterator) Type() string { return "and" }

graph/and-iterator_test.go

@@ -0,0 +1,147 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	"testing"
+)
+
+// Make sure that tags work on the And.
+func TestTag(t *testing.T) {
+	fix1 := newFixedIterator()
+	fix1.AddValue(234)
+	fix1.AddTag("foo")
+	and := NewAndIterator()
+	and.AddSubIterator(fix1)
+	and.AddTag("bar")
+	out := fix1.Tags()
+	if len(out) != 1 {
+		t.Errorf("Expected length 1, got %d", len(out))
+	}
+	if out[0] != "foo" {
+		t.Errorf("Cannot get tag back, got %s", out[0])
+	}
+
+	val, ok := and.Next()
+	if !ok {
+		t.Errorf("And did not next")
+	}
+	if val != 234 {
+		t.Errorf("Unexpected value")
+	}
+	tags := make(map[string]TSVal)
+	and.TagResults(&tags)
+	if tags["bar"] != 234 {
+		t.Errorf("no bar tag")
+	}
+	if tags["foo"] != 234 {
+		t.Errorf("no foo tag")
+	}
+}
+
+// Do a simple itersection of fixed values.
+func TestAndAndFixedIterators(t *testing.T) {
+	fix1 := newFixedIterator()
+	fix1.AddValue(1)
+	fix1.AddValue(2)
+	fix1.AddValue(3)
+	fix1.AddValue(4)
+	fix2 := newFixedIterator()
+	fix2.AddValue(3)
+	fix2.AddValue(4)
+	fix2.AddValue(5)
+	and := NewAndIterator()
+	and.AddSubIterator(fix1)
+	and.AddSubIterator(fix2)
+	// Should be as big as smallest subiterator
+	size, accurate := and.Size()
+	if size != 3 {
+		t.Error("Incorrect size")
+	}
+	if !accurate {
+		t.Error("not accurate")
+	}
+
+	val, ok := and.Next()
+	if val != 3 || ok == false {
+		t.Error("Incorrect first value")
+	}
+
+	val, ok = and.Next()
+	if val != 4 || ok == false {
+		t.Error("Incorrect second value")
+	}
+
+	val, ok = and.Next()
+	if ok {
+		t.Error("Too many values")
+	}
+
+}
+
+// If there's no intersection, the size should still report the same,
+// but there should be nothing to Next()
+func TestNonOverlappingFixedIterators(t *testing.T) {
+	fix1 := newFixedIterator()
+	fix1.AddValue(1)
+	fix1.AddValue(2)
+	fix1.AddValue(3)
+	fix1.AddValue(4)
+	fix2 := newFixedIterator()
+	fix2.AddValue(5)
+	fix2.AddValue(6)
+	fix2.AddValue(7)
+	and := NewAndIterator()
+	and.AddSubIterator(fix1)
+	and.AddSubIterator(fix2)
+	// Should be as big as smallest subiterator
+	size, accurate := and.Size()
+	if size != 3 {
+		t.Error("Incorrect size")
+	}
+	if !accurate {
+		t.Error("not accurate")
+	}
+
+	_, ok := and.Next()
+	if ok {
+		t.Error("Too many values")
+	}
+
+}
+
+func TestAllIterators(t *testing.T) {
+	all1 := NewInt64AllIterator(1, 5)
+	all2 := NewInt64AllIterator(4, 10)
+	and := NewAndIterator()
+	and.AddSubIterator(all2)
+	and.AddSubIterator(all1)
+
+	val, ok := and.Next()
+	if val.(int64) != 4 || ok == false {
+		t.Error("Incorrect first value")
+	}
+
+	val, ok = and.Next()
+	if val.(int64) != 5 || ok == false {
+		t.Error("Incorrect second value")
+	}
+
+	val, ok = and.Next()
+	if ok {
+		t.Error("Too many values")
+	}
+
+}

graph/fixed-iterator.go

@@ -0,0 +1,157 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines one of the base iterators, the Fixed iterator. A fixed iterator is quite simple; it
+// contains an explicit fixed array of values.
+//
+// A fixed iterator requires an Equality function to be passed to it, by reason that TSVal, the
+// opaque Triple store value, may not answer to ==.
+
+import (
+	"fmt"
+	"strings"
+)
+
+// A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and
+// an equality function.
+type FixedIterator struct {
+	BaseIterator
+	values    []TSVal
+	lastIndex int
+	cmp       Equality
+}
+
+// Define the signature of an equality function.
+type Equality func(a, b TSVal) bool
+
+// Define an equality function of purely ==, which works for native types.
+func BasicEquality(a, b TSVal) bool {
+	if a == b {
+		return true
+	}
+	return false
+}
+
+// Creates a new Fixed iterator based around == equality.
+func newFixedIterator() *FixedIterator {
+	return NewFixedIteratorWithCompare(BasicEquality)
+}
+
+// Creates a new Fixed iterator with a custom comparitor.
+func NewFixedIteratorWithCompare(compareFn Equality) *FixedIterator {
+	var it FixedIterator
+	BaseIteratorInit(&it.BaseIterator)
+	it.values = make([]TSVal, 0, 20)
+	it.lastIndex = 0
+	it.cmp = compareFn
+	return &it
+}
+
+func (f *FixedIterator) Reset() {
+	f.lastIndex = 0
+}
+
+func (f *FixedIterator) Close() {
+}
+
+func (f *FixedIterator) Clone() Iterator {
+	out := NewFixedIteratorWithCompare(f.cmp)
+	for _, val := range f.values {
+		out.AddValue(val)
+	}
+	out.CopyTagsFrom(f)
+	return out
+}
+
+// Add a value to the iterator. The array now contains this value.
+// TODO(barakmich): This ought to be a set someday, disallowing repeated values.
+func (f *FixedIterator) AddValue(v TSVal) {
+	f.values = append(f.values, v)
+}
+
+// Print some information about the iterator.
+func (f *FixedIterator) DebugString(indent int) string {
+	value := ""
+	if len(f.values) > 0 {
+		value = fmt.Sprint(f.values[0])
+	}
+	return fmt.Sprintf("%s(%s tags: %s Size: %d id0: %d)",
+		strings.Repeat(" ", indent),
+		f.Type(),
+		f.FixedTags(),
+		len(f.values),
+		value,
+	)
+}
+
+// Register this iterator as a Fixed iterator.
+func (f *FixedIterator) Type() string {
+	return "fixed"
+}
+
+// Check if the passed value is equal to one of the values stored in the iterator.
+func (f *FixedIterator) Check(v TSVal) bool {
+	// Could be optimized by keeping it sorted or using a better datastructure.
+	// However, for fixed iterators, which are by definition kind of tiny, this
+	// isn't a big issue.
+	CheckLogIn(f, v)
+	for _, x := range f.values {
+		if f.cmp(x, v) {
+			f.Last = x
+			return CheckLogOut(f, v, true)
+		}
+	}
+	return CheckLogOut(f, v, false)
+}
+
+// Return the next stored value from the iterator.
+func (f *FixedIterator) Next() (TSVal, bool) {
+	NextLogIn(f)
+	if f.lastIndex == len(f.values) {
+		return NextLogOut(f, nil, false)
+	}
+	out := f.values[f.lastIndex]
+	f.Last = out
+	f.lastIndex++
+	return NextLogOut(f, out, true)
+}
+
+// Optimize() for a Fixed iterator is simple. Returns a Null iterator if it's empty
+// (so that other iterators upstream can treat this as null) or there is no
+// optimization.
+func (f *FixedIterator) Optimize() (Iterator, bool) {
+
+	if len(f.values) == 1 && f.values[0] == nil {
+		return &NullIterator{}, true
+	}
+
+	return f, false
+}
+
+// Size is the number of values stored.
+func (f *FixedIterator) Size() (int64, bool) {
+	return int64(len(f.values)), true
+}
+
+// As we right now have to scan the entire list, Next and Check are linear with the
+// size. However, a better data structure could remove these limits.
+func (a *FixedIterator) GetStats() *IteratorStats {
+	return &IteratorStats{
+		CheckCost: int64(len(a.values)),
+		NextCost:  int64(len(a.values)),
+		Size:      int64(len(a.values)),
+	}
+}

graph/hasa-iterator.go

@@ -0,0 +1,224 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines one of the base iterators, the HasA iterator. The HasA takes a
+// subiterator of links, and acts as an iterator of nodes in the given
+// direction. The name comes from the idea that a "link HasA subject" or a "link
+// HasA predicate".
+//
+// HasA is weird in that it may return the same value twice if on the Next()
+// path. That's okay -- in reality, it can be viewed as returning the value for
+// a new triple, but to make logic much simpler, here we have the HasA.
+//
+// Likewise, it's important to think about Check()ing a HasA. When given a
+// value to check, it means "Check all predicates that have this value for your
+// direction against the subiterator." This would imply that there's more than
+// one possibility for the same Check()ed value. While we could return the
+// number of options, it's simpler to return one, and then call NextResult()
+// enough times to enumerate the options. (In fact, one could argue that the
+// raison d'etre for NextResult() is this iterator).
+//
+// Alternatively, can be seen as the dual of the LinksTo iterator.
+
+import (
+	"container/list"
+	"fmt"
+	"github.com/barakmich/glog"
+	"strings"
+)
+
+// A HasaIterator consists of a reference back to the TripleStore that it references,
+// a primary subiterator, a direction in which the triples for that subiterator point,
+// and a temporary holder for the iterator generated on Check().
+type HasaIterator struct {
+	BaseIterator
+	ts        TripleStore
+	primaryIt Iterator
+	direction string
+	resultIt  Iterator
+}
+
+// Construct a new HasA iterator, given the triple subiterator, and the triple
+// direction for which it stands.
+func NewHasaIterator(ts TripleStore, subIt Iterator, dir string) *HasaIterator {
+	var hasa HasaIterator
+	BaseIteratorInit(&hasa.BaseIterator)
+	hasa.ts = ts
+	hasa.primaryIt = subIt
+	hasa.direction = dir
+	return &hasa
+}
+
+// Return our sole subiterator, in a list.List.
+func (h *HasaIterator) GetSubIterators() *list.List {
+	l := list.New()
+	l.PushBack(h.primaryIt)
+	return l
+}
+
+func (h *HasaIterator) Reset() {
+	h.primaryIt.Reset()
+	if h.resultIt != nil {
+		h.resultIt.Close()
+	}
+}
+
+func (h *HasaIterator) Clone() Iterator {
+	out := NewHasaIterator(h.ts, h.primaryIt.Clone(), h.direction)
+	out.CopyTagsFrom(h)
+	return out
+}
+
+// Direction accessor.
+func (h *HasaIterator) Direction() string { return h.direction }
+
+// Pass the Optimize() call along to the subiterator. If it becomes Null,
+// then the HasA becomes Null (there are no triples that have any directions).
+func (h *HasaIterator) Optimize() (Iterator, bool) {
+
+	newPrimary, changed := h.primaryIt.Optimize()
+	if changed {
+		h.primaryIt = newPrimary
+		if h.primaryIt.Type() == "null" {
+			return h.primaryIt, true
+		}
+	}
+	return h, false
+}
+
+// Pass the TagResults down the chain.
+func (h *HasaIterator) TagResults(out *map[string]TSVal) {
+	h.BaseIterator.TagResults(out)
+	h.primaryIt.TagResults(out)
+}
+
+// DEPRECATED Return results in a ResultTree.
+func (h *HasaIterator) GetResultTree() *ResultTree {
+	tree := NewResultTree(h.LastResult())
+	tree.AddSubtree(h.primaryIt.GetResultTree())
+	return tree
+}
+
+// Print some information about this iterator.
+func (h *HasaIterator) DebugString(indent int) string {
+	var tags string
+	for _, k := range h.Tags() {
+		tags += fmt.Sprintf("%s;", k)
+	}
+	return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), h.Type(), h.GetUid(), tags, h.direction, h.primaryIt.DebugString(indent+4))
+}
+
+// Check a value against our internal iterator. In order to do this, we must first open a new
+// iterator of "triples that have `val` in our direction", given to us by the triple store,
+// and then Next() values out of that iterator and Check() them against our subiterator.
+func (h *HasaIterator) Check(val TSVal) bool {
+	CheckLogIn(h, val)
+	if glog.V(4) {
+		glog.V(4).Infoln("Id is", h.ts.GetNameFor(val))
+	}
+	// TODO(barakmich): Optimize this
+	if h.resultIt != nil {
+		h.resultIt.Close()
+	}
+	h.resultIt = h.ts.GetTripleIterator(h.direction, val)
+	return CheckLogOut(h, val, h.GetCheckResult())
+}
+
+// GetCheckResult() is shared code between Check() and GetNextResult() -- calls next on the
+// result iterator (a triple iterator based on the last checked value) and returns true if
+// another match is made.
+func (h *HasaIterator) GetCheckResult() bool {
+	for {
+		linkVal, ok := h.resultIt.Next()
+		if !ok {
+			break
+		}
+		if glog.V(4) {
+			glog.V(4).Infoln("Triple is", h.ts.GetTriple(linkVal).ToString())
+		}
+		if h.primaryIt.Check(linkVal) {
+			h.Last = h.ts.GetTripleDirection(linkVal, h.direction)
+			return true
+		}
+	}
+	return false
+}
+
+// Get the next result that matches this branch.
+func (h *HasaIterator) NextResult() bool {
+	// Order here is important. If the subiterator has a NextResult, then we
+	// need do nothing -- there is a next result, and we shouldn't move forward.
+	// However, we then need to get the next result from our last Check().
+	//
+	// The upshot is, the end of NextResult() bubbles up from the bottom of the
+	// iterator tree up, and we need to respect that.
+	if h.primaryIt.NextResult() {
+		return true
+	}
+	return h.GetCheckResult()
+}
+
+// Get the next result from this iterator. This is simpler than Check. We have a
+// subiterator we can get a value from, and we can take that resultant triple,
+// pull our direction out of it, and return that.
+func (h *HasaIterator) Next() (TSVal, bool) {
+	NextLogIn(h)
+	if h.resultIt != nil {
+		h.resultIt.Close()
+	}
+	h.resultIt = &NullIterator{}
+
+	tID, ok := h.primaryIt.Next()
+	if !ok {
+		return NextLogOut(h, 0, false)
+	}
+	name := h.ts.GetTriple(tID).Get(h.direction)
+	val := h.ts.GetIdFor(name)
+	h.Last = val
+	return NextLogOut(h, val, true)
+}
+
+// GetStats() returns the statistics on the HasA iterator. This is curious. Next
+// cost is easy, it's an extra call or so on top of the subiterator Next cost.
+// CheckCost involves going to the TripleStore, iterating out values, and hoping
+// one sticks -- potentially expensive, depending on fanout. Size, however, is
+// potentially smaller. we know at worst it's the size of the subiterator, but
+// if there are many repeated values, it could be much smaller in totality.
+func (h *HasaIterator) GetStats() *IteratorStats {
+	subitStats := h.primaryIt.GetStats()
+	// TODO(barakmich): These should really come from the triplestore itself
+	// and be optimized.
+	faninFactor := int64(1)
+	fanoutFactor := int64(30)
+	nextConstant := int64(2)
+	tripleConstant := int64(1)
+	return &IteratorStats{
+		NextCost:  tripleConstant + subitStats.NextCost,
+		CheckCost: (fanoutFactor * nextConstant) * subitStats.CheckCost,
+		Size:      faninFactor * subitStats.Size,
+	}
+}
+
+// Close the subiterator, the result iterator (if any) and the HasA.
+func (h *HasaIterator) Close() {
+	if h.resultIt != nil {
+		h.resultIt.Close()
+	}
+	h.primaryIt.Close()
+}
+
+// Register this iterator as a HasA.
+func (h *HasaIterator) Type() string { return "hasa" }

graph/iterator.go

@@ -0,0 +1,304 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Define the general iterator interface, as well as the BaseIterator which all
+// iterators can "inherit" from to get default iterator functionality.
+
+import (
+	"container/list"
+	"fmt"
+	"github.com/barakmich/glog"
+	"strings"
+)
+
+var iterator_n int = 0
+
+type Iterator interface {
+	// Tags are the way we handle results. By adding a tag to an iterator, we can
+	// "name" it, in a sense, and at each step of iteration, get a named result.
+	// TagResults() is therefore the handy way of walking an iterator tree and
+	// getting the named results.
+	//
+	// Tag Accessors.
+	AddTag(string)
+	Tags() []string
+	AddFixedTag(string, TSVal)
+	FixedTags() map[string]TSVal
+	CopyTagsFrom(Iterator)
+	// Fills a tag-to-result-value map.
+	TagResults(*map[string]TSVal)
+	// Returns the current result.
+	LastResult() TSVal
+	// DEPRECATED -- Fills a ResultTree struct with Result().
+	GetResultTree() *ResultTree
+
+	// These methods are the heart and soul of the iterator, as they constitute
+	// the iteration interface.
+	//
+	// To get the full results of iteraton, do the following:
+	// while (!Next()):
+	//   emit result
+	//   while (!NextResult()):
+	//       emit result
+	//
+	// All of them should set iterator.Last to be the last returned value, to
+	// make results work.
+	//
+	// Next() advances the iterator and returns the next valid result. Returns
+	// (<value>, true) or (nil, false)
+	Next() (TSVal, bool)
+	// NextResult() advances iterators that may have more than one valid result,
+	// from the bottom up.
+	NextResult() bool
+	// Check(), given a value, returns whether or not that value is within the set
+	// held by this iterator.
+	Check(TSVal) bool
+	// Start iteration from the beginning
+	Reset()
+	// Create a new iterator just like this one
+	Clone() Iterator
+	// These methods relate to choosing the right iterator, or optimizing an
+	// iterator tree
+	//
+	// GetStats() returns the relative costs of calling the iteration methods for
+	// this iterator, as well as the size. Roughly, it will take NextCost * Size
+	// "cost units" to get everything out of the iterator. This is a wibbly-wobbly
+	// thing, and not exact, but a useful heuristic.
+	GetStats() *IteratorStats
+	// Helpful accessor for the number of things in the iterator. The first return
+	// value is the size, and the second return value is whether that number is exact,
+	// or a conservative estimate.
+	Size() (int64, bool)
+	// Returns a string relating to what the function of the iterator is. By
+	// knowing the names of the iterators, we can devise optimization strategies.
+	Type() string
+	// Optimizes an iterator. Can replace the iterator, or merely move things
+	// around internally. if it chooses to replace it with a better iterator,
+	// returns (the new iterator, true), if not, it returns (self, false).
+	Optimize() (Iterator, bool)
+	// Return a list of the subiterators for this iterator.
+	GetSubIterators() *list.List
+
+	// Return a string representation of the iterator, indented by the given amount.
+	DebugString(int) string
+	// Return whether this iterator is relaiably nextable. Most iterators are.
+	// However, some iterators, like "not" are, by definition, the whole database
+	// except themselves. Next() on these is unproductive, if impossible.
+	Nextable() bool
+	// Close the iterator and do internal cleanup.
+	Close()
+	GetUid() int
+}
+
+type IteratorStats struct {
+	CheckCost int64
+	NextCost  int64
+	Size      int64
+}
+
+// The Base iterator is the iterator other iterators inherit from to get some
+// default functionality.
+type BaseIterator struct {
+	Last      TSVal
+	tags      []string
+	fixedTags map[string]TSVal
+	nextable  bool
+	uid       int
+}
+
+// Called by subclases.
+func BaseIteratorInit(b *BaseIterator) {
+	// Your basic iterator is nextable
+	b.nextable = true
+	b.uid = iterator_n
+	if glog.V(2) {
+		iterator_n++
+	}
+}
+
+func (b *BaseIterator) GetUid() int {
+	return b.uid
+}
+
+// Adds a tag to the iterator. Most iterators don't need to override.
+func (b *BaseIterator) AddTag(tag string) {
+	if b.tags == nil {
+		b.tags = make([]string, 0)
+	}
+	b.tags = append(b.tags, tag)
+}
+
+func (b *BaseIterator) AddFixedTag(tag string, value TSVal) {
+	if b.fixedTags == nil {
+		b.fixedTags = make(map[string]TSVal)
+	}
+	b.fixedTags[tag] = value
+}
+
+// Returns the tags.
+func (b *BaseIterator) Tags() []string {
+	return b.tags
+}
+
+func (b *BaseIterator) FixedTags() map[string]TSVal {
+	return b.fixedTags
+}
+
+func (b *BaseIterator) CopyTagsFrom(other_it Iterator) {
+	for _, tag := range other_it.Tags() {
+		b.AddTag(tag)
+	}
+
+	for k, v := range other_it.FixedTags() {
+		b.AddFixedTag(k, v)
+	}
+
+}
+
+// Prints a silly debug string. Most classes override.
+func (n *BaseIterator) DebugString(indent int) string {
+	return fmt.Sprintf("%s(base)", strings.Repeat(" ", indent))
+}
+
+// Nothing in a base iterator.
+func (n *BaseIterator) Check(v TSVal) bool {
+	return false
+}
+
+// Base iterators should never appear in a tree if they are, select against
+// them.
+func (n *BaseIterator) GetStats() *IteratorStats {
+	return &IteratorStats{100000, 100000, 100000}
+}
+
+// DEPRECATED
+func (b *BaseIterator) GetResultTree() *ResultTree {
+	tree := NewResultTree(b.LastResult())
+	return tree
+}
+
+// Nothing in a base iterator.
+func (n *BaseIterator) Next() (TSVal, bool) {
+	return nil, false
+}
+
+func (n *BaseIterator) NextResult() bool {
+	return false
+}
+
+// Returns the last result of an iterator.
+func (n *BaseIterator) LastResult() TSVal {
+	return n.Last
+}
+
+// If you're empty and you know it, clap your hands.
+func (n *BaseIterator) Size() (int64, bool) {
+	return 0, true
+}
+
+// No subiterators. Only those with subiterators need to do anything here.
+func (n *BaseIterator) GetSubIterators() *list.List {
+	return nil
+}
+
+// Accessor
+func (b *BaseIterator) Nextable() bool { return b.nextable }
+
+// Fill the map based on the tags assigned to this iterator. Default
+// functionality works well for most iterators.
+func (a *BaseIterator) TagResults(out_map *map[string]TSVal) {
+	for _, tag := range a.Tags() {
+		(*out_map)[tag] = a.LastResult()
+	}
+
+	for tag, value := range a.FixedTags() {
+		(*out_map)[tag] = value
+	}
+}
+
+// Nothing to clean up.
+//func (a *BaseIterator) Close() {}
+func (a *NullIterator) Close() {}
+
+func (a *BaseIterator) Reset() {}
+
+// Here we define the simplest base iterator -- the Null iterator. It contains nothing.
+// It is the empty set. Often times, queries that contain one of these match nothing,
+// so it's important to give it a special iterator.
+type NullIterator struct {
+	BaseIterator
+}
+
+// Fairly useless New function.
+func NewNullIterator() *NullIterator {
+	var n NullIterator
+	return &n
+}
+
+func (n *NullIterator) Clone() Iterator { return NewNullIterator() }
+
+// Name the null iterator.
+func (n *NullIterator) Type() string { return "null" }
+
+// A good iterator will close itself when it returns true.
+// Null has nothing it needs to do.
+func (n *NullIterator) Optimize() (Iterator, bool) { return n, false }
+
+// Print the null iterator.
+func (n *NullIterator) DebugString(indent int) string {
+	return strings.Repeat(" ", indent) + "(null)"
+}
+
+// A null iterator costs nothing. Use it!
+func (n *NullIterator) GetStats() *IteratorStats {
+	return &IteratorStats{0, 0, 0}
+}
+
+// Utility logging functions for when an iterator gets called Next upon, or Check upon, as
+// well as what they return. Highly useful for tracing the execution path of a query.
+func CheckLogIn(it Iterator, val TSVal) {
+	if glog.V(4) {
+		glog.V(4).Infof("%s %d CHECK %d", strings.ToUpper(it.Type()), it.GetUid(), val)
+	}
+}
+
+func CheckLogOut(it Iterator, val TSVal, good bool) bool {
+	if glog.V(4) {
+		if good {
+			glog.V(4).Infof("%s %d CHECK %d GOOD", strings.ToUpper(it.Type()), it.GetUid(), val)
+		} else {
+			glog.V(4).Infof("%s %d CHECK %d BAD", strings.ToUpper(it.Type()), it.GetUid(), val)
+		}
+	}
+	return good
+}
+
+func NextLogIn(it Iterator) {
+	if glog.V(4) {
+		glog.V(4).Infof("%s %d NEXT", strings.ToUpper(it.Type()), it.GetUid())
+	}
+}
+
+func NextLogOut(it Iterator, val TSVal, ok bool) (TSVal, bool) {
+	if glog.V(4) {
+		if ok {
+			glog.V(4).Infof("%s %d NEXT IS %d", strings.ToUpper(it.Type()), it.GetUid(), val)
+		} else {
+			glog.V(4).Infof("%s %d NEXT DONE", strings.ToUpper(it.Type()), it.GetUid())
+		}
+	}
+	return val, ok
+}

graph/linksto-iterator.go

@@ -0,0 +1,184 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines one of the base iterators, the LinksTo iterator. A LinksTo takes a
+// subiterator of nodes, and contains an iteration of links which "link to"
+// those nodes in a given direction.
+//
+// Next()ing a LinksTo is straightforward -- iterate through all links to //
+// things in the subiterator, and then advance the subiterator, and do it again.
+// LinksTo is therefore sensitive to growing with a fanout. (A small-sized
+// subiterator could cause LinksTo to be large).
+//
+// Check()ing a LinksTo means, given a link, take the direction we care about
+// and check if it's in our subiterator. Checking is therefore fairly cheap, and
+// similar to checking the subiterator alone.
+//
+// Can be seen as the dual of the HasA iterator.
+
+import (
+	"container/list"
+	"fmt"
+	"strings"
+)
+
+// A LinksTo has a reference back to the TripleStore (to create the iterators
+// for each node) the subiterator, and the direction the iterator comes from.
+// `next_it` is the tempoarary iterator held per result in `primary_it`.
+type LinksToIterator struct {
+	BaseIterator
+	ts        TripleStore
+	primaryIt Iterator
+	direction string
+	nextIt    Iterator
+}
+
+// Construct a new LinksTo iterator around a direction and a subiterator of
+// nodes.
+func NewLinksToIterator(ts TripleStore, it Iterator, dir string) *LinksToIterator {
+	var lto LinksToIterator
+	BaseIteratorInit(&lto.BaseIterator)
+	lto.ts = ts
+	lto.primaryIt = it
+	lto.direction = dir
+	lto.nextIt = &NullIterator{}
+	return &lto
+}
+
+func (l *LinksToIterator) Reset() {
+	l.primaryIt.Reset()
+	if l.nextIt != nil {
+		l.nextIt.Close()
+	}
+	l.nextIt = &NullIterator{}
+}
+
+func (l *LinksToIterator) Clone() Iterator {
+	out := NewLinksToIterator(l.ts, l.primaryIt.Clone(), l.direction)
+	out.CopyTagsFrom(l)
+	return out
+}
+
+// Return the direction under consideration.
+func (l *LinksToIterator) Direction() string { return l.direction }
+
+// Tag these results, and our subiterator's results.
+func (l *LinksToIterator) TagResults(out *map[string]TSVal) {
+	l.BaseIterator.TagResults(out)
+	l.primaryIt.TagResults(out)
+}
+
+// DEPRECATED
+func (l *LinksToIterator) GetResultTree() *ResultTree {
+	tree := NewResultTree(l.LastResult())
+	tree.AddSubtree(l.primaryIt.GetResultTree())
+	return tree
+}
+
+// Print the iterator.
+func (l *LinksToIterator) DebugString(indent int) string {
+	return fmt.Sprintf("%s(%s %d direction:%s\n%s)",
+		strings.Repeat(" ", indent),
+		l.Type(), l.GetUid(), l.direction, l.primaryIt.DebugString(indent+4))
+}
+
+// If it checks in the right direction for the subiterator, it is a valid link
+// for the LinksTo.
+func (l *LinksToIterator) Check(val TSVal) bool {
+	CheckLogIn(l, val)
+	node := l.ts.GetTripleDirection(val, l.direction)
+	if l.primaryIt.Check(node) {
+		l.Last = val
+		return CheckLogOut(l, val, true)
+	}
+	return CheckLogOut(l, val, false)
+}
+
+// Return a list containing only our subiterator.
+func (lto *LinksToIterator) GetSubIterators() *list.List {
+	l := list.New()
+	l.PushBack(lto.primaryIt)
+	return l
+}
+
+// Optimize the LinksTo, by replacing it if it can be.
+func (lto *LinksToIterator) Optimize() (Iterator, bool) {
+	newPrimary, changed := lto.primaryIt.Optimize()
+	if changed {
+		lto.primaryIt = newPrimary
+		if lto.primaryIt.Type() == "null" {
+			lto.nextIt.Close()
+			return lto.primaryIt, true
+		}
+	}
+	// Ask the TripleStore if we can be replaced. Often times, this is a great
+	// optimization opportunity (there's a fixed iterator underneath us, for
+	// example).
+	newReplacement, hasOne := lto.ts.OptimizeIterator(lto)
+	if hasOne {
+		lto.Close()
+		return newReplacement, true
+	}
+	return lto, false
+}
+
+// Next()ing a LinksTo operates as described above.
+func (l *LinksToIterator) Next() (TSVal, bool) {
+	NextLogIn(l)
+	val, ok := l.nextIt.Next()
+	if !ok {
+		// Subiterator is empty, get another one
+		candidate, ok := l.primaryIt.Next()
+		if !ok {
+			// We're out of nodes in our subiterator, so we're done as well.
+			return NextLogOut(l, 0, false)
+		}
+		l.nextIt.Close()
+		l.nextIt = l.ts.GetTripleIterator(l.direction, candidate)
+		// Recurse -- return the first in the next set.
+		return l.Next()
+	}
+	l.Last = val
+	return NextLogOut(l, val, ok)
+}
+
+// Close our subiterators.
+func (l *LinksToIterator) Close() {
+	l.nextIt.Close()
+	l.primaryIt.Close()
+}
+
+// We won't ever have a new result, but our subiterators might.
+func (l *LinksToIterator) NextResult() bool {
+	return l.primaryIt.NextResult()
+}
+
+// Register the LinksTo.
+func (l *LinksToIterator) Type() string { return "linksto" }
+
+// Return a guess as to how big or costly it is to next the iterator.
+func (l *LinksToIterator) GetStats() *IteratorStats {
+	subitStats := l.primaryIt.GetStats()
+	// TODO(barakmich): These should really come from the triplestore itself
+	fanoutFactor := int64(20)
+	checkConstant := int64(1)
+	nextConstant := int64(2)
+	return &IteratorStats{
+		NextCost:  nextConstant + subitStats.NextCost,
+		CheckCost: checkConstant + subitStats.CheckCost,
+		Size:      fanoutFactor * subitStats.Size,
+	}
+}

graph/linksto-iterator_test.go

@@ -0,0 +1,37 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	"testing"
+)
+
+func TestLinksTo(t *testing.T) {
+	ts := new(TestTripleStore)
+	tsFixed := newFixedIterator()
+	tsFixed.AddValue(2)
+	ts.On("GetIdFor", "cool").Return(1)
+	ts.On("GetTripleIterator", "o", 1).Return(tsFixed)
+	fixed := newFixedIterator()
+	fixed.AddValue(ts.GetIdFor("cool"))
+	lto := NewLinksToIterator(ts, fixed, "o")
+	val, ok := lto.Next()
+	if !ok {
+		t.Error("At least one triple matches the fixed object")
+	}
+	if val != 2 {
+		t.Errorf("Triple index 2, such as %s, should match %s", ts.GetTriple(2), ts.GetTriple(val))
+	}
+}

graph/memstore/llrb-iterator.go

@@ -0,0 +1,119 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memstore
+
+import (
+	"fmt"
+	"math"
+	"strings"
+
+	"github.com/petar/GoLLRB/llrb"
+
+	"github.com/google/cayley/graph"
+)
+
+type LlrbIterator struct {
+	graph.BaseIterator
+	tree      *llrb.LLRB
+	data      string
+	isRunning bool
+	iterLast  Int64
+}
+
+type Int64 int64
+
+func (i Int64) Less(than llrb.Item) bool {
+	return i < than.(Int64)
+}
+
+func IterateOne(tree *llrb.LLRB, last Int64) Int64 {
+	var next Int64
+	tree.AscendGreaterOrEqual(last, func(i llrb.Item) bool {
+		if i.(Int64) == last {
+			return true
+		} else {
+			next = i.(Int64)
+			return false
+		}
+	})
+	return next
+}
+
+func NewLlrbIterator(tree *llrb.LLRB, data string) *LlrbIterator {
+	var it LlrbIterator
+	graph.BaseIteratorInit(&it.BaseIterator)
+	it.tree = tree
+	it.iterLast = Int64(-1)
+	it.data = data
+	return &it
+}
+
+func (it *LlrbIterator) Reset() {
+	it.iterLast = Int64(-1)
+}
+
+func (it *LlrbIterator) Clone() graph.Iterator {
+	var new_it = NewLlrbIterator(it.tree, it.data)
+	new_it.CopyTagsFrom(it)
+	return new_it
+}
+
+func (it *LlrbIterator) Close() {}
+
+func (it *LlrbIterator) Next() (graph.TSVal, bool) {
+	graph.NextLogIn(it)
+	if it.tree.Max() == nil || it.Last == int64(it.tree.Max().(Int64)) {
+		return graph.NextLogOut(it, nil, false)
+	}
+	it.iterLast = IterateOne(it.tree, it.iterLast)
+	it.Last = int64(it.iterLast)
+	return graph.NextLogOut(it, it.Last, true)
+}
+
+func (it *LlrbIterator) Size() (int64, bool) {
+	return int64(it.tree.Len()), true
+}
+
+func (it *LlrbIterator) Check(v graph.TSVal) bool {
+	graph.CheckLogIn(it, v)
+	if it.tree.Has(Int64(v.(int64))) {
+		it.Last = v
+		return graph.CheckLogOut(it, v, true)
+	}
+	return graph.CheckLogOut(it, v, false)
+}
+
+func (it *LlrbIterator) DebugString(indent int) string {
+	size, _ := it.Size()
+	return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data)
+}
+
+func (it *LlrbIterator) Type() string {
+	return "llrb"
+}
+func (it *LlrbIterator) Sorted() bool {
+	return true
+}
+func (it *LlrbIterator) Optimize() (graph.Iterator, bool) {
+	return it, false
+}
+
+func (it *LlrbIterator) GetStats() *graph.IteratorStats {
+	return &graph.IteratorStats{
+		CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1,
+		NextCost:  1,
+		Size:      int64(it.tree.Len()),
+	}
+}

graph/memstore/memstore-all-iterator.go

@@ -0,0 +1,45 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memstore
+
+import (
+	"github.com/google/cayley/graph"
+)
+
+type MemstoreAllIterator struct {
+	graph.Int64AllIterator
+	ts *MemTripleStore
+}
+
+func NewMemstoreAllIterator(ts *MemTripleStore) *MemstoreAllIterator {
+	var out MemstoreAllIterator
+	out.Int64AllIterator = *graph.NewInt64AllIterator(1, ts.idCounter-1)
+	out.ts = ts
+	return &out
+}
+
+func (memall *MemstoreAllIterator) Next() (graph.TSVal, bool) {
+	next, out := memall.Int64AllIterator.Next()
+	if !out {
+		return next, out
+	}
+	i64 := next.(int64)
+	_, ok := memall.ts.revIdMap[i64]
+	if !ok {
+		return memall.Next()
+	}
+	memall.Last = next
+	return next, out
+}

graph/memstore/memtriplestore-iterator-optimize.go

@@ -0,0 +1,53 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memstore
+
+import (
+	"github.com/google/cayley/graph"
+)
+
+func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
+	switch it.Type() {
+	case "linksto":
+		return ts.optimizeLinksTo(it.(*graph.LinksToIterator))
+
+	}
+	return it, false
+}
+
+func (ts *MemTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) {
+	l := it.GetSubIterators()
+	if l.Len() != 1 {
+		return it, false
+	}
+	primaryIt := l.Front().Value.(graph.Iterator)
+	if primaryIt.Type() == "fixed" {
+		size, _ := primaryIt.Size()
+		if size == 1 {
+			val, ok := primaryIt.Next()
+			if !ok {
+				panic("Sizes lie")
+			}
+			newIt := ts.GetTripleIterator(it.Direction(), val)
+			newIt.CopyTagsFrom(it)
+			for _, tag := range primaryIt.Tags() {
+				newIt.AddFixedTag(tag, val)
+			}
+			return newIt, true
+		}
+	}
+	it.Close()
+	return it, false
+}

graph/memstore/memtriplestore.go

@@ -0,0 +1,268 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memstore
+
+import (
+	"fmt"
+
+	"github.com/barakmich/glog"
+	"github.com/google/cayley/graph"
+
+	"github.com/petar/GoLLRB/llrb"
+)
+
+type TripleDirectionIndex struct {
+	subject    map[int64]*llrb.LLRB
+	predicate  map[int64]*llrb.LLRB
+	object     map[int64]*llrb.LLRB
+	provenance map[int64]*llrb.LLRB
+}
+
+func NewTripleDirectionIndex() *TripleDirectionIndex {
+	var tdi TripleDirectionIndex
+	tdi.subject = make(map[int64]*llrb.LLRB)
+	tdi.predicate = make(map[int64]*llrb.LLRB)
+	tdi.object = make(map[int64]*llrb.LLRB)
+	tdi.provenance = make(map[int64]*llrb.LLRB)
+	return &tdi
+}
+
+func (tdi *TripleDirectionIndex) GetForDir(s string) map[int64]*llrb.LLRB {
+	if s == "s" {
+		return tdi.subject
+	} else if s == "o" {
+		return tdi.object
+	} else if s == "p" {
+		return tdi.predicate
+	} else if s == "c" {
+		return tdi.provenance
+	}
+	panic("Bad direction")
+}
+
+func (tdi *TripleDirectionIndex) GetOrCreate(dir string, id int64) *llrb.LLRB {
+	directionIndex := tdi.GetForDir(dir)
+	if _, ok := directionIndex[id]; !ok {
+		directionIndex[id] = llrb.New()
+	}
+	return directionIndex[id]
+}
+
+func (tdi *TripleDirectionIndex) Get(dir string, id int64) (*llrb.LLRB, bool) {
+	directionIndex := tdi.GetForDir(dir)
+	tree, exists := directionIndex[id]
+	return tree, exists
+}
+
+type MemTripleStore struct {
+	idCounter       int64
+	tripleIdCounter int64
+	idMap           map[string]int64
+	revIdMap        map[int64]string
+	triples         []graph.Triple
+	size            int64
+	index           TripleDirectionIndex
+	// vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree
+}
+
+func NewMemTripleStore() *MemTripleStore {
+	var ts MemTripleStore
+	ts.idMap = make(map[string]int64)
+	ts.revIdMap = make(map[int64]string)
+	ts.triples = make([]graph.Triple, 1, 200)
+
+	// Sentinel null triple so triple indices start at 1
+	ts.triples[0] = graph.Triple{}
+	ts.size = 1
+	ts.index = *NewTripleDirectionIndex()
+	ts.idCounter = 1
+	ts.tripleIdCounter = 1
+	return &ts
+}
+
+func (ts *MemTripleStore) AddTripleSet(triples []*graph.Triple) {
+	for _, t := range triples {
+		ts.AddTriple(t)
+	}
+}
+
+func (ts *MemTripleStore) tripleExists(t *graph.Triple) (bool, int64) {
+	smallest := -1
+	var smallest_tree *llrb.LLRB
+	for _, dir := range graph.TripleDirections {
+		sid := t.Get(dir)
+		if dir == "c" && sid == "" {
+			continue
+		}
+		id, ok := ts.idMap[sid]
+		// If we've never heard about a node, it most not exist
+		if !ok {
+			return false, 0
+		}
+		index, exists := ts.index.Get(dir, id)
+		if !exists {
+			// If it's never been indexed in this direction, it can't exist.
+			return false, 0
+		}
+		if smallest == -1 || index.Len() < smallest {
+			smallest = index.Len()
+			smallest_tree = index
+		}
+	}
+	it := NewLlrbIterator(smallest_tree, "")
+
+	for {
+		val, ok := it.Next()
+		if !ok {
+			break
+		}
+		if t.Equals(&ts.triples[val.(int64)]) {
+			return true, val.(int64)
+		}
+	}
+	return false, 0
+}
+
+func (ts *MemTripleStore) AddTriple(t *graph.Triple) {
+	if exists, _ := ts.tripleExists(t); exists {
+		return
+	}
+	var tripleID int64
+	ts.triples = append(ts.triples, *t)
+	tripleID = ts.tripleIdCounter
+	ts.size++
+	ts.tripleIdCounter++
+
+	for _, dir := range graph.TripleDirections {
+		sid := t.Get(dir)
+		if dir == "c" && sid == "" {
+			continue
+		}
+		if _, ok := ts.idMap[sid]; !ok {
+			ts.idMap[sid] = ts.idCounter
+			ts.revIdMap[ts.idCounter] = sid
+			ts.idCounter++
+		}
+	}
+
+	for _, dir := range graph.TripleDirections {
+		if dir == "c" && t.Get(dir) == "" {
+			continue
+		}
+		id := ts.idMap[t.Get(dir)]
+		tree := ts.index.GetOrCreate(dir, id)
+		tree.ReplaceOrInsert(Int64(tripleID))
+	}
+
+	// TODO(barakmich): Add VIP indexing
+}
+
+func (ts *MemTripleStore) RemoveTriple(t *graph.Triple) {
+	var tripleID int64
+	var exists bool
+	tripleID = 0
+	if exists, tripleID = ts.tripleExists(t); !exists {
+		return
+	}
+
+	ts.triples[tripleID] = graph.Triple{}
+	ts.size--
+
+	for _, dir := range graph.TripleDirections {
+		if dir == "c" && t.Get(dir) == "" {
+			continue
+		}
+		id := ts.idMap[t.Get(dir)]
+		tree := ts.index.GetOrCreate(dir, id)
+		tree.Delete(Int64(tripleID))
+	}
+
+	for _, dir := range graph.TripleDirections {
+		if dir == "c" && t.Get(dir) == "" {
+			continue
+		}
+		id, ok := ts.idMap[t.Get(dir)]
+		if !ok {
+			continue
+		}
+		stillExists := false
+		for _, dir := range graph.TripleDirections {
+			if dir == "c" && t.Get(dir) == "" {
+				continue
+			}
+			nodeTree := ts.index.GetOrCreate(dir, id)
+			if nodeTree.Len() != 0 {
+				stillExists = true
+				break
+			}
+		}
+		if !stillExists {
+			delete(ts.idMap, t.Get(dir))
+			delete(ts.revIdMap, id)
+		}
+	}
+}
+
+func (ts *MemTripleStore) GetTriple(index graph.TSVal) *graph.Triple {
+	return &ts.triples[index.(int64)]
+}
+
+func (ts *MemTripleStore) GetTripleIterator(direction string, value graph.TSVal) graph.Iterator {
+	index, ok := ts.index.Get(direction, value.(int64))
+	data := fmt.Sprintf("dir:%s val:%d", direction, value.(int64))
+	if ok {
+		return NewLlrbIterator(index, data)
+	}
+	return &graph.NullIterator{}
+}
+
+func (ts *MemTripleStore) Size() int64 {
+	return ts.size - 1 // Don't count the sentinel
+}
+
+func (ts *MemTripleStore) DebugPrint() {
+	for i, t := range ts.triples {
+		if i == 0 {
+			continue
+		}
+		glog.V(2).Infoln("%d: %s", i, t.ToString())
+	}
+}
+
+func (ts *MemTripleStore) GetIdFor(name string) graph.TSVal {
+	return ts.idMap[name]
+}
+
+func (ts *MemTripleStore) GetNameFor(id graph.TSVal) string {
+	return ts.revIdMap[id.(int64)]
+}
+
+func (ts *MemTripleStore) GetTriplesAllIterator() graph.Iterator {
+	return graph.NewInt64AllIterator(0, ts.Size())
+}
+
+func (ts *MemTripleStore) MakeFixed() *graph.FixedIterator {
+	return graph.NewFixedIteratorWithCompare(graph.BasicEquality)
+}
+
+func (ts *MemTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal {
+	name := ts.GetTriple(val).Get(direction)
+	return ts.GetIdFor(name)
+}
+
+func (ts *MemTripleStore) GetNodesAllIterator() graph.Iterator {
+	return NewMemstoreAllIterator(ts)
+}
+func (ts *MemTripleStore) Close() {}

graph/memstore/memtriplestore_test.go

@@ -0,0 +1,138 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memstore
+
+import (
+	"sort"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph"
+)
+
+func TestMemstore(t *testing.T) {
+	Convey("With a simple memstore", t, func() {
+		ts := MakeTestingMemstore()
+		Convey("It should have a reasonable size", func() {
+			So(ts.Size(), ShouldEqual, 11)
+		})
+		Convey("It should have an Id Space that makes sense", func() {
+			v := ts.GetIdFor("C")
+			So(v.(int64), ShouldEqual, 4)
+		})
+	})
+}
+
+func TestIteratorsAndNextResultOrderA(t *testing.T) {
+	ts := MakeTestingMemstore()
+	fixed := ts.MakeFixed()
+	fixed.AddValue(ts.GetIdFor("C"))
+	all := ts.GetNodesAllIterator()
+	lto := graph.NewLinksToIterator(ts, all, "o")
+	innerAnd := graph.NewAndIterator()
+
+	fixed2 := ts.MakeFixed()
+	fixed2.AddValue(ts.GetIdFor("follows"))
+	lto2 := graph.NewLinksToIterator(ts, fixed2, "p")
+	innerAnd.AddSubIterator(lto2)
+	innerAnd.AddSubIterator(lto)
+	hasa := graph.NewHasaIterator(ts, innerAnd, "s")
+	outerAnd := graph.NewAndIterator()
+	outerAnd.AddSubIterator(fixed)
+	outerAnd.AddSubIterator(hasa)
+	val, ok := outerAnd.Next()
+	if !ok {
+		t.Error("Expected one matching subtree")
+	}
+	if ts.GetNameFor(val) != "C" {
+		t.Errorf("Matching subtree should be %s, got %s", "barak", ts.GetNameFor(val))
+	}
+	expected := make([]string, 2)
+	expected[0] = "B"
+	expected[1] = "D"
+	actualOut := make([]string, 2)
+	actualOut[0] = ts.GetNameFor(all.LastResult())
+	nresultOk := outerAnd.NextResult()
+	if !nresultOk {
+		t.Error("Expected two results got one")
+	}
+	actualOut[1] = ts.GetNameFor(all.LastResult())
+	nresultOk = outerAnd.NextResult()
+	if nresultOk {
+		t.Error("Expected two results got three")
+	}
+	CompareStringSlices(t, expected, actualOut)
+	val, ok = outerAnd.Next()
+	if ok {
+		t.Error("More than one possible top level output?")
+	}
+}
+
+func CompareStringSlices(t *testing.T, expected []string, actual []string) {
+	if len(expected) != len(actual) {
+		t.Error("String slices are not the same length")
+	}
+	sort.Strings(expected)
+	sort.Strings(actual)
+	for i := 0; i < len(expected); i++ {
+		if expected[i] != actual[i] {
+			t.Errorf("At index %d, expected \"%s\" and got \"%s\"", i, expected[i], actual[i])
+		}
+	}
+}
+
+func TestLinksToOptimization(t *testing.T) {
+	ts := MakeTestingMemstore()
+	fixed := ts.MakeFixed()
+	fixed.AddValue(ts.GetIdFor("cool"))
+	lto := graph.NewLinksToIterator(ts, fixed, "o")
+	lto.AddTag("foo")
+	newIt, changed := lto.Optimize()
+	if !changed {
+		t.Error("Iterator didn't change")
+	}
+	if newIt.Type() != "llrb" {
+		t.Fatal("Didn't swap out to LLRB")
+	}
+	v := newIt.(*LlrbIterator)
+	v_clone := v.Clone()
+	if v_clone.DebugString(0) != v.DebugString(0) {
+		t.Fatal("Wrong iterator. Got ", v_clone.DebugString(0))
+	}
+	if len(v_clone.Tags()) < 1 || v_clone.Tags()[0] != "foo" {
+		t.Fatal("Tag on LinksTo did not persist")
+	}
+}
+
+func TestRemoveTriple(t *testing.T) {
+	ts := MakeTestingMemstore()
+	ts.RemoveTriple(graph.MakeTriple("E", "follows", "F", ""))
+	fixed := ts.MakeFixed()
+	fixed.AddValue(ts.GetIdFor("E"))
+	lto := graph.NewLinksToIterator(ts, fixed, "s")
+	fixed2 := ts.MakeFixed()
+	fixed2.AddValue(ts.GetIdFor("follows"))
+	lto2 := graph.NewLinksToIterator(ts, fixed2, "p")
+	innerAnd := graph.NewAndIterator()
+	innerAnd.AddSubIterator(lto2)
+	innerAnd.AddSubIterator(lto)
+	hasa := graph.NewHasaIterator(ts, innerAnd, "o")
+	newIt, _ := hasa.Optimize()
+	_, ok := newIt.Next()
+	if ok {
+		t.Error("E should not have any followers.")
+	}
+}

graph/memstore/testing_memstore.go

@@ -0,0 +1,45 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memstore
+
+import "github.com/google/cayley/graph"
+
+//    +---+                        +---+
+//    | A |-------               ->| F |<--
+//    +---+       \------>+---+-/  +---+   \--+---+
+//                 ------>|#B#|      |        | E |
+//    +---+-------/      >+---+      |        +---+
+//    | C |             /            v
+//    +---+           -/           +---+
+//      ----    +---+/             |#G#|
+//          \-->|#D#|------------->+---+
+//              +---+
+//
+
+func MakeTestingMemstore() *MemTripleStore {
+	ts := NewMemTripleStore()
+	ts.AddTriple(graph.MakeTriple("A", "follows", "B", ""))
+	ts.AddTriple(graph.MakeTriple("C", "follows", "B", ""))
+	ts.AddTriple(graph.MakeTriple("C", "follows", "D", ""))
+	ts.AddTriple(graph.MakeTriple("D", "follows", "B", ""))
+	ts.AddTriple(graph.MakeTriple("B", "follows", "F", ""))
+	ts.AddTriple(graph.MakeTriple("F", "follows", "G", ""))
+	ts.AddTriple(graph.MakeTriple("D", "follows", "G", ""))
+	ts.AddTriple(graph.MakeTriple("E", "follows", "F", ""))
+	ts.AddTriple(graph.MakeTriple("B", "status", "cool", "status_graph"))
+	ts.AddTriple(graph.MakeTriple("D", "status", "cool", "status_graph"))
+	ts.AddTriple(graph.MakeTriple("G", "status", "cool", "status_graph"))
+	return ts
+}

graph/mock_ts.go

@@ -0,0 +1,58 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// A quickly mocked version of the TripleStore interface, for use in tests.
+// Can better used Mock.Called but will fill in as needed.
+
+import (
+	"github.com/stretchrcom/testify/mock"
+)
+
+type TestTripleStore struct {
+	mock.Mock
+}
+
+func (ts *TestTripleStore) GetIdFor(s string) TSVal {
+	args := ts.Mock.Called(s)
+	return args.Get(0)
+}
+func (ts *TestTripleStore) AddTriple(*Triple)       {}
+func (ts *TestTripleStore) AddTripleSet([]*Triple)  {}
+func (ts *TestTripleStore) GetTriple(TSVal) *Triple { return &Triple{} }
+func (ts *TestTripleStore) GetTripleIterator(s string, i TSVal) Iterator {
+	args := ts.Mock.Called(s, i)
+	return args.Get(0).(Iterator)
+}
+func (ts *TestTripleStore) GetNodesAllIterator() Iterator   { return &NullIterator{} }
+func (ts *TestTripleStore) GetTriplesAllIterator() Iterator { return &NullIterator{} }
+func (ts *TestTripleStore) GetIteratorByString(string, string, string) Iterator {
+	return &NullIterator{}
+}
+func (ts *TestTripleStore) GetNameFor(v TSVal) string {
+	args := ts.Mock.Called(v)
+	return args.Get(0).(string)
+}
+func (ts *TestTripleStore) Size() int64 { return 0 }
+func (ts *TestTripleStore) DebugPrint() {}
+func (ts *TestTripleStore) OptimizeIterator(it Iterator) (Iterator, bool) {
+	return &NullIterator{}, false
+}
+func (ts *TestTripleStore) MakeFixed() *FixedIterator {
+	return NewFixedIteratorWithCompare(BasicEquality)
+}
+func (ts *TestTripleStore) Close()                                 {}
+func (ts *TestTripleStore) GetTripleDirection(TSVal, string) TSVal { return 0 }
+func (ts *TestTripleStore) RemoveTriple(t *Triple)                 {}

graph/mongo/lru.go

@@ -0,0 +1,62 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+	"container/list"
+)
+
+type IDLru struct {
+	cache    map[string]*list.Element
+	priority *list.List
+	maxSize  int
+}
+
+type KV struct {
+	key   string
+	value string
+}
+
+func NewIDLru(size int) *IDLru {
+	var lru IDLru
+	lru.maxSize = size
+	lru.priority = list.New()
+	lru.cache = make(map[string]*list.Element)
+	return &lru
+}
+
+func (lru *IDLru) Put(key string, value string) {
+	if _, ok := lru.Get(key); ok {
+		return
+	}
+	if len(lru.cache) == lru.maxSize {
+		lru.removeOldest()
+	}
+	lru.priority.PushFront(KV{key: key, value: value})
+	lru.cache[key] = lru.priority.Front()
+}
+
+func (lru *IDLru) Get(key string) (string, bool) {
+	if element, ok := lru.cache[key]; ok {
+		lru.priority.MoveToFront(element)
+		return element.Value.(KV).value, true
+	}
+	return "", false
+}
+
+func (lru *IDLru) removeOldest() {
+	last := lru.priority.Remove(lru.priority.Back())
+	delete(lru.cache, last.(KV).key)
+}

graph/mongo/mongo-iterator.go

@@ -0,0 +1,181 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/barakmich/glog"
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+
+	"github.com/google/cayley/graph"
+)
+
+type MongoIterator struct {
+	graph.BaseIterator
+	ts         *MongoTripleStore
+	dir        string
+	iter       *mgo.Iter
+	hash       string
+	name       string
+	size       int64
+	isAll      bool
+	constraint bson.M
+	collection string
+}
+
+func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator {
+	var m MongoIterator
+	graph.BaseIteratorInit(&m.BaseIterator)
+
+	m.name = ts.GetNameFor(val)
+	m.collection = collection
+	switch dir {
+
+	case "s":
+		m.constraint = bson.M{"Sub": m.name}
+	case "p":
+		m.constraint = bson.M{"Pred": m.name}
+	case "o":
+		m.constraint = bson.M{"Obj": m.name}
+	case "c":
+		m.constraint = bson.M{"Provenance": m.name}
+	}
+
+	m.ts = ts
+	m.dir = dir
+	m.iter = ts.db.C(collection).Find(m.constraint).Iter()
+	size, err := ts.db.C(collection).Find(m.constraint).Count()
+	if err != nil {
+		glog.Errorln("Trouble getting size for iterator! ", err)
+		return nil
+	}
+	m.size = int64(size)
+	m.hash = val.(string)
+	m.isAll = false
+	return &m
+}
+
+func NewMongoAllIterator(ts *MongoTripleStore, collection string) *MongoIterator {
+	var m MongoIterator
+	m.ts = ts
+	m.dir = "all"
+	m.constraint = nil
+	m.collection = collection
+	m.iter = ts.db.C(collection).Find(nil).Iter()
+	size, err := ts.db.C(collection).Count()
+	if err != nil {
+		glog.Errorln("Trouble getting size for iterator! ", err)
+		return nil
+	}
+	m.size = int64(size)
+	m.hash = ""
+	m.isAll = true
+	return &m
+}
+
+func (m *MongoIterator) Reset() {
+	m.iter.Close()
+	m.iter = m.ts.db.C(m.collection).Find(m.constraint).Iter()
+
+}
+
+func (m *MongoIterator) Close() {
+	m.iter.Close()
+}
+
+func (m *MongoIterator) Clone() graph.Iterator {
+	var newM graph.Iterator
+	if m.isAll {
+		newM = NewMongoAllIterator(m.ts, m.collection)
+	} else {
+		newM = NewMongoIterator(m.ts, m.collection, m.dir, m.hash)
+	}
+	newM.CopyTagsFrom(m)
+	return newM
+}
+
+func (m *MongoIterator) Next() (graph.TSVal, bool) {
+	var result struct {
+		Id string "_id"
+		//Sub string "Sub"
+		//Pred string "Pred"
+		//Obj string "Obj"
+	}
+	found := m.iter.Next(&result)
+	if !found {
+		err := m.iter.Err()
+		if err != nil {
+			glog.Errorln("Error Nexting MongoIterator: ", err)
+		}
+		return nil, false
+	}
+	m.Last = result.Id
+	return result.Id, true
+}
+
+func (m *MongoIterator) Check(v graph.TSVal) bool {
+	graph.CheckLogIn(m, v)
+	if m.isAll {
+		m.Last = v
+		return graph.CheckLogOut(m, v, true)
+	}
+	var offset int
+	switch m.dir {
+	case "s":
+		offset = 0
+	case "p":
+		offset = (m.ts.hasher.Size() * 2)
+	case "o":
+		offset = (m.ts.hasher.Size() * 2) * 2
+	case "c":
+		offset = (m.ts.hasher.Size() * 2) * 3
+	}
+	val := v.(string)[offset : m.ts.hasher.Size()*2+offset]
+	if val == m.hash {
+		m.Last = v
+		return graph.CheckLogOut(m, v, true)
+	}
+	return graph.CheckLogOut(m, v, false)
+}
+
+func (m *MongoIterator) Size() (int64, bool) {
+	return m.size, true
+}
+
+func (m *MongoIterator) Type() string {
+	if m.isAll {
+		return "all"
+	}
+	return "mongo"
+}
+func (m *MongoIterator) Sorted() bool                     { return true }
+func (m *MongoIterator) Optimize() (graph.Iterator, bool) { return m, false }
+
+func (m *MongoIterator) DebugString(indent int) string {
+	size, _ := m.Size()
+	return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), m.Type(), size, m.hash, m.name)
+}
+
+func (m *MongoIterator) GetStats() *graph.IteratorStats {
+	size, _ := m.Size()
+	return &graph.IteratorStats{
+		CheckCost: 1,
+		NextCost:  5,
+		Size:      size,
+	}
+}

graph/mongo/mongo-triplestore-iterator-optimize.go

@@ -0,0 +1,53 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+	"github.com/google/cayley/graph"
+)
+
+func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
+	switch it.Type() {
+	case "linksto":
+		return ts.optimizeLinksTo(it.(*graph.LinksToIterator))
+
+	}
+	return it, false
+}
+
+func (ts *MongoTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) {
+	l := it.GetSubIterators()
+	if l.Len() != 1 {
+		return it, false
+	}
+	primaryIt := l.Front().Value.(graph.Iterator)
+	if primaryIt.Type() == "fixed" {
+		size, _ := primaryIt.Size()
+		if size == 1 {
+			val, ok := primaryIt.Next()
+			if !ok {
+				panic("Sizes lie")
+			}
+			newIt := ts.GetTripleIterator(it.Direction(), val)
+			newIt.CopyTagsFrom(it)
+			for _, tag := range primaryIt.Tags() {
+				newIt.AddFixedTag(tag, val)
+			}
+			it.Close()
+			return newIt, true
+		}
+	}
+	return it, false
+}

graph/mongo/mongo-triplestore.go

@@ -0,0 +1,329 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mongo
+
+import (
+	"crypto/sha1"
+	"encoding/hex"
+	"hash"
+	"log"
+
+	"labix.org/v2/mgo"
+	"labix.org/v2/mgo/bson"
+
+	"github.com/barakmich/glog"
+	"github.com/google/cayley/graph"
+)
+
+const DefaultDBName = "cayley"
+
+type MongoTripleStore struct {
+	session *mgo.Session
+	db      *mgo.Database
+	hasher  hash.Hash
+	idCache *IDLru
+}
+
+func CreateNewMongoGraph(addr string, options graph.OptionsDict) bool {
+	conn, err := mgo.Dial(addr)
+	if err != nil {
+		glog.Fatal("Error connecting: ", err)
+		return false
+	}
+	conn.SetSafe(&mgo.Safe{})
+	dbName := DefaultDBName
+	if val, ok := options.GetStringKey("database_name"); ok {
+		dbName = val
+	}
+	db := conn.DB(dbName)
+	indexOpts := mgo.Index{
+		Key:        []string{"Sub"},
+		Unique:     false,
+		DropDups:   false,
+		Background: true,
+		Sparse:     true,
+	}
+	db.C("triples").EnsureIndex(indexOpts)
+	indexOpts.Key = []string{"Pred"}
+	db.C("triples").EnsureIndex(indexOpts)
+	indexOpts.Key = []string{"Obj"}
+	db.C("triples").EnsureIndex(indexOpts)
+	indexOpts.Key = []string{"Provenance"}
+	db.C("triples").EnsureIndex(indexOpts)
+	return true
+}
+
+func NewMongoTripleStore(addr string, options graph.OptionsDict) *MongoTripleStore {
+	var ts MongoTripleStore
+	conn, err := mgo.Dial(addr)
+	if err != nil {
+		glog.Fatal("Error connecting: ", err)
+	}
+	conn.SetSafe(&mgo.Safe{})
+	dbName := DefaultDBName
+	if val, ok := options.GetStringKey("database_name"); ok {
+		dbName = val
+	}
+	ts.db = conn.DB(dbName)
+	ts.session = conn
+	ts.hasher = sha1.New()
+	ts.idCache = NewIDLru(1 << 16)
+	return &ts
+}
+
+func (ts *MongoTripleStore) getIdForTriple(t *graph.Triple) string {
+	id := ts.ConvertStringToByteHash(t.Sub)
+	id += ts.ConvertStringToByteHash(t.Pred)
+	id += ts.ConvertStringToByteHash(t.Obj)
+	id += ts.ConvertStringToByteHash(t.Provenance)
+	return id
+}
+
+func (ts *MongoTripleStore) ConvertStringToByteHash(s string) string {
+	ts.hasher.Reset()
+	key := make([]byte, 0, ts.hasher.Size())
+	ts.hasher.Write([]byte(s))
+	key = ts.hasher.Sum(key)
+	return hex.EncodeToString(key)
+}
+
+type MongoNode struct {
+	Id   string "_id"
+	Name string "Name"
+	Size int    "Size"
+}
+
+func (ts *MongoTripleStore) updateNodeBy(node_name string, inc int) {
+	var size MongoNode
+	node := ts.GetIdFor(node_name)
+	err := ts.db.C("nodes").FindId(node).One(&size)
+	if err != nil {
+		if err.Error() == "not found" {
+			// Not found. Okay.
+			size.Id = node.(string)
+			size.Name = node_name
+			size.Size = inc
+		} else {
+			glog.Error("Error:", err)
+			return
+		}
+	} else {
+		size.Id = node.(string)
+		size.Name = node_name
+		size.Size += inc
+	}
+
+	// Removing something...
+	if inc < 0 {
+		if size.Size <= 0 {
+			err := ts.db.C("nodes").RemoveId(node)
+			if err != nil {
+				glog.Error("Error: ", err, " while removing node ", node_name)
+				return
+			}
+		}
+	}
+
+	_, err2 := ts.db.C("nodes").UpsertId(node, size)
+	if err2 != nil {
+		glog.Error("Error: ", err)
+	}
+}
+
+func (ts *MongoTripleStore) writeTriple(t *graph.Triple) bool {
+	tripledoc := bson.M{"_id": ts.getIdForTriple(t), "Sub": t.Sub, "Pred": t.Pred, "Obj": t.Obj, "Provenance": t.Provenance}
+	err := ts.db.C("triples").Insert(tripledoc)
+	if err != nil {
+		// Among the reasons I hate MongoDB. "Errors don't happen! Right guys?"
+		if err.(*mgo.LastError).Code == 11000 {
+			return false
+		}
+		glog.Error("Error: ", err)
+		return false
+	}
+	return true
+}
+
+func (ts *MongoTripleStore) AddTriple(t *graph.Triple) {
+	_ = ts.writeTriple(t)
+	ts.updateNodeBy(t.Sub, 1)
+	ts.updateNodeBy(t.Pred, 1)
+	ts.updateNodeBy(t.Obj, 1)
+	if t.Provenance != "" {
+		ts.updateNodeBy(t.Provenance, 1)
+	}
+}
+
+func (ts *MongoTripleStore) AddTripleSet(in []*graph.Triple) {
+	ts.session.SetSafe(nil)
+	idMap := make(map[string]int)
+	for _, t := range in {
+		wrote := ts.writeTriple(t)
+		if wrote {
+			idMap[t.Sub]++
+			idMap[t.Obj]++
+			idMap[t.Pred]++
+			if t.Provenance != "" {
+				idMap[t.Provenance]++
+			}
+		}
+	}
+	for k, v := range idMap {
+		ts.updateNodeBy(k, v)
+	}
+	ts.session.SetSafe(&mgo.Safe{})
+}
+
+func (ts *MongoTripleStore) RemoveTriple(t *graph.Triple) {
+	err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t))
+	if err == mgo.ErrNotFound {
+		return
+	} else if err != nil {
+		log.Println("Error: ", err, " while removing triple ", t)
+		return
+	}
+	ts.updateNodeBy(t.Sub, -1)
+	ts.updateNodeBy(t.Pred, -1)
+	ts.updateNodeBy(t.Obj, -1)
+	if t.Provenance != "" {
+		ts.updateNodeBy(t.Provenance, -1)
+	}
+}
+
+func (ts *MongoTripleStore) GetTriple(val graph.TSVal) *graph.Triple {
+	var bsonDoc bson.M
+	err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc)
+	if err != nil {
+		log.Println("Error: Couldn't retrieve triple", val.(string), err)
+	}
+	return graph.MakeTriple(
+		bsonDoc["Sub"].(string),
+		bsonDoc["Pred"].(string),
+		bsonDoc["Obj"].(string),
+		bsonDoc["Provenance"].(string))
+}
+
+func (ts *MongoTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator {
+	return NewMongoIterator(ts, "triples", dir, val)
+}
+
+func (ts *MongoTripleStore) GetNodesAllIterator() graph.Iterator {
+	return NewMongoAllIterator(ts, "nodes")
+}
+
+func (ts *MongoTripleStore) GetTriplesAllIterator() graph.Iterator {
+	return NewMongoAllIterator(ts, "triples")
+}
+
+func (ts *MongoTripleStore) GetIdFor(s string) graph.TSVal {
+	return ts.ConvertStringToByteHash(s)
+}
+
+func (ts *MongoTripleStore) GetNameFor(v graph.TSVal) string {
+	val, ok := ts.idCache.Get(v.(string))
+	if ok {
+		return val
+	}
+	var node MongoNode
+	err := ts.db.C("nodes").FindId(v.(string)).One(&node)
+	if err != nil {
+		log.Println("Error: Couldn't retrieve node", v.(string), err)
+	}
+	ts.idCache.Put(v.(string), node.Name)
+	return node.Name
+}
+
+func (ts *MongoTripleStore) Size() int64 {
+	count, err := ts.db.C("triples").Count()
+	if err != nil {
+		glog.Error("Error: ", err)
+		return 0
+	}
+	return int64(count)
+}
+
+func compareStrings(a, b graph.TSVal) bool {
+	return a.(string) == b.(string)
+}
+
+func (ts *MongoTripleStore) MakeFixed() *graph.FixedIterator {
+	return graph.NewFixedIteratorWithCompare(compareStrings)
+}
+
+func (ts *MongoTripleStore) Close() {
+	ts.db.Session.Close()
+}
+
+func (ts *MongoTripleStore) GetTripleDirection(in graph.TSVal, dir string) graph.TSVal {
+	// Maybe do the trick here
+	var offset int
+	switch dir {
+	case "s":
+		offset = 0
+	case "p":
+		offset = (ts.hasher.Size() * 2)
+	case "o":
+		offset = (ts.hasher.Size() * 2) * 2
+	case "c":
+		offset = (ts.hasher.Size() * 2) * 3
+	}
+	val := in.(string)[offset : ts.hasher.Size()*2+offset]
+	return val
+}
+
+func (ts *MongoTripleStore) BulkLoad(t_chan chan *graph.Triple) {
+	ts.session.SetSafe(nil)
+	for triple := range t_chan {
+		ts.writeTriple(triple)
+	}
+	outputTo := bson.M{"replace": "nodes", "sharded": true}
+	glog.Infoln("Mapreducing")
+	job := mgo.MapReduce{
+		Map: `function() {
+      var len = this["_id"].length
+      var s_key = this["_id"].slice(0, len / 4)
+      var p_key = this["_id"].slice(len / 4, 2 * len / 4)
+      var o_key = this["_id"].slice(2 * len / 4, 3 * len / 4)
+      var c_key = this["_id"].slice(3 * len / 4)
+      emit(s_key, {"_id": s_key, "Name" : this.Sub, "Size" : 1})
+      emit(p_key, {"_id": p_key, "Name" : this.Pred, "Size" : 1})
+      emit(o_key, {"_id": o_key, "Name" : this.Obj, "Size" : 1})
+			if (this.Provenance != "") {
+				emit(c_key, {"_id": c_key, "Name" : this.Provenance, "Size" : 1})
+			}
+    }
+    `,
+		Reduce: `
+      function(key, value_list) {
+        out = {"_id": key, "Name": value_list[0].Name}
+        count = 0
+        for (var i = 0; i < value_list.length; i++) {
+          count = count + value_list[i].Size
+
+        }
+        out["Size"] = count
+        return out
+      }
+    `,
+		Out: outputTo,
+	}
+	ts.db.C("triples").Find(nil).MapReduce(&job, nil)
+	glog.Infoln("Fixing")
+	ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
+    db.nodes.update({"_id": result._id}, result.value)
+  }) }`}, {"args", bson.D{}}}, nil)
+
+	ts.session.SetSafe(&mgo.Safe{})
+}

graph/optional-iterator.go

@@ -0,0 +1,134 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// "Optional" is kind of odd. It's not an iterator in the strictest sense, but
+// it's easier to implement as an iterator.
+//
+// Consider what it means. It means that we have a subconstraint which we do
+// not want to constrain the query -- we just want it to return the matching
+// subgraph if one matches at all. By analogy to regular expressions, it is the
+// '?' operator.
+//
+// If it were a proper iterator of its own (and indeed, a reasonable refactor
+// of this iterator would be to make it such) it would contain an all iterator
+// -- all things in the graph. It matches everything (as does the regex "(a)?")
+
+import (
+	"fmt"
+	"github.com/barakmich/glog"
+	"strings"
+)
+
+// An optional iterator has the subconstraint iterator we wish to be optional
+// and whether the last check we received was true or false.
+type OptionalIterator struct {
+	BaseIterator
+	subIt     Iterator
+	lastCheck bool
+}
+
+// Creates a new optional iterator.
+func NewOptionalIterator(it Iterator) *OptionalIterator {
+	var o OptionalIterator
+	BaseIteratorInit(&o.BaseIterator)
+	o.nextable = false
+	o.subIt = it
+	return &o
+}
+
+func (o *OptionalIterator) Reset() {
+	o.subIt.Reset()
+	o.lastCheck = false
+}
+
+func (o *OptionalIterator) Close() {
+	o.subIt.Close()
+}
+
+func (o *OptionalIterator) Clone() Iterator {
+	out := NewOptionalIterator(o.subIt.Clone())
+	out.CopyTagsFrom(o)
+	return out
+}
+
+// Nexting the iterator is unsupported -- error and return an empty set.
+// (As above, a reasonable alternative would be to Next() an all iterator)
+func (o *OptionalIterator) Next() (TSVal, bool) {
+	glog.Errorln("Nexting an un-nextable iterator")
+	return nil, false
+}
+
+// An optional iterator only has a next result if, (a) last time we checked
+// we had any results whatsoever, and (b) there was another subresult in our
+// optional subbranch.
+func (o *OptionalIterator) NextResult() bool {
+	if o.lastCheck {
+		return o.subIt.NextResult()
+	}
+	return false
+}
+
+// Check() is the real hack of this iterator. It always returns true, regardless
+// of whether the subiterator matched. But we keep track of whether the subiterator
+// matched for results purposes.
+func (o *OptionalIterator) Check(val TSVal) bool {
+	checked := o.subIt.Check(val)
+	o.lastCheck = checked
+	o.Last = val
+	return true
+}
+
+// If we failed the check, then the subiterator should not contribute to the result
+// set. Otherwise, go ahead and tag it.
+func (o *OptionalIterator) TagResults(out *map[string]TSVal) {
+	if o.lastCheck == false {
+		return
+	}
+	o.subIt.TagResults(out)
+}
+
+// Registers the optional iterator.
+func (o *OptionalIterator) Type() string { return "optional" }
+
+// Prints the optional and it's subiterator.
+func (o *OptionalIterator) DebugString(indent int) string {
+	return fmt.Sprintf("%s(%s tags:%s\n%s)",
+		strings.Repeat(" ", indent),
+		o.Type(),
+		o.Tags(),
+		o.subIt.DebugString(indent+4))
+}
+
+// There's nothing to optimize for an optional. Optimize the subiterator and
+// potentially replace it.
+func (o *OptionalIterator) Optimize() (Iterator, bool) {
+	newSub, changed := o.subIt.Optimize()
+	if changed {
+		o.subIt.Close()
+		o.subIt = newSub
+	}
+	return o, false
+}
+
+// We're only as expensive as our subiterator. Except, we can't be nexted.
+func (o *OptionalIterator) GetStats() *IteratorStats {
+	subStats := o.subIt.GetStats()
+	return &IteratorStats{
+		CheckCost: subStats.CheckCost,
+		NextCost:  int64(1 << 62),
+		Size:      subStats.Size,
+	}
+}

graph/or-iterator.go

@@ -0,0 +1,287 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines the or and short-circuiting or iterator. Or is the union operator for it's subiterators.
+// Short-circuiting-or is a little different. It will return values from the first iterator that returns
+// values at all, and then stops.
+//
+// Never reorders the iterators from the order they arrive. It is either the union or the first one.
+// May return the same value twice -- once for each branch.
+
+import (
+	"container/list"
+	"fmt"
+	"strings"
+)
+
+type OrIterator struct {
+	BaseIterator
+	isShortCircuiting bool
+	internalIterators []Iterator
+	itCount           int
+	currentIterator   int
+}
+
+func NewOrIterator() *OrIterator {
+	var or OrIterator
+	BaseIteratorInit(&or.BaseIterator)
+	or.internalIterators = make([]Iterator, 0, 20)
+	or.isShortCircuiting = false
+	or.currentIterator = -1
+	return &or
+}
+
+func NewShortCircuitOrIterator() *OrIterator {
+	var or OrIterator
+	BaseIteratorInit(&or.BaseIterator)
+	or.internalIterators = make([]Iterator, 0, 20)
+	or.isShortCircuiting = true
+	or.currentIterator = -1
+	return &or
+}
+
+// Reset all internal iterators
+func (or *OrIterator) Reset() {
+	for _, it := range or.internalIterators {
+		it.Reset()
+	}
+	or.currentIterator = -1
+}
+
+func (or *OrIterator) Clone() Iterator {
+	var newOr *OrIterator
+	if or.isShortCircuiting {
+		newOr = NewShortCircuitOrIterator()
+	} else {
+		newOr = NewOrIterator()
+	}
+	for _, it := range or.internalIterators {
+		newOr.AddSubIterator(it.Clone())
+	}
+	or.CopyTagsFrom(or)
+	return newOr
+}
+
+// Returns a list.List of the subiterators, in order.
+func (or *OrIterator) GetSubIterators() *list.List {
+	l := list.New()
+	for _, it := range or.internalIterators {
+		l.PushBack(it)
+	}
+	return l
+}
+
+// Overrides BaseIterator TagResults, as it needs to add it's own results and
+// recurse down it's subiterators.
+func (or *OrIterator) TagResults(out *map[string]TSVal) {
+	or.BaseIterator.TagResults(out)
+	or.internalIterators[or.currentIterator].TagResults(out)
+}
+
+// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators.
+func (or *OrIterator) GetResultTree() *ResultTree {
+	tree := NewResultTree(or.LastResult())
+	for _, it := range or.internalIterators {
+		tree.AddSubtree(it.GetResultTree())
+	}
+	return tree
+}
+
+// Prints information about this iterator.
+func (or *OrIterator) DebugString(indent int) string {
+	var total string
+	for i, it := range or.internalIterators {
+		total += strings.Repeat(" ", indent+2)
+		total += fmt.Sprintf("%d:\n%s\n", i, it.DebugString(indent+4))
+	}
+	var tags string
+	for _, k := range or.Tags() {
+		tags += fmt.Sprintf("%s;", k)
+	}
+	spaces := strings.Repeat(" ", indent+2)
+
+	return fmt.Sprintf("%s(%s\n%stags:%s\n%sits:\n%s)",
+		strings.Repeat(" ", indent),
+		or.Type(),
+		spaces,
+		tags,
+		spaces,
+		total)
+}
+
+// Add a subiterator to this Or iterator. Order matters.
+func (or *OrIterator) AddSubIterator(sub Iterator) {
+	or.internalIterators = append(or.internalIterators, sub)
+	or.itCount++
+}
+
+// Returns the Next value from the Or iterator. Because the Or is the
+// union of its subiterators, it must produce from all subiterators -- unless
+// it's shortcircuiting, in which case, it's the first one that returns anything.
+func (or *OrIterator) Next() (TSVal, bool) {
+	NextLogIn(or)
+	var curr TSVal
+	var exists bool
+	firstTime := false
+	for {
+		if or.currentIterator == -1 {
+			or.currentIterator = 0
+			firstTime = true
+		}
+		curIt := or.internalIterators[or.currentIterator]
+		curr, exists = curIt.Next()
+		if !exists {
+			if or.isShortCircuiting && !firstTime {
+				return NextLogOut(or, nil, false)
+			}
+			or.currentIterator++
+			if or.currentIterator == or.itCount {
+				return NextLogOut(or, nil, false)
+			}
+		} else {
+			or.Last = curr
+			return NextLogOut(or, curr, true)
+		}
+	}
+	panic("Somehow broke out of Next() loop in OrIterator")
+}
+
+// Checks a value against the iterators, in order.
+func (or *OrIterator) checkSubIts(val TSVal) bool {
+	var subIsGood = false
+	for i, it := range or.internalIterators {
+		subIsGood = it.Check(val)
+		if subIsGood {
+			or.currentIterator = i
+			break
+		}
+	}
+	return subIsGood
+}
+
+// Check a value against the entire iterator, in order.
+func (or *OrIterator) Check(val TSVal) bool {
+	CheckLogIn(or, val)
+	anyGood := or.checkSubIts(val)
+	if !anyGood {
+		return CheckLogOut(or, val, false)
+	}
+	or.Last = val
+	return CheckLogOut(or, val, true)
+}
+
+// Returns the approximate size of the Or iterator. Because we're dealing
+// with a union, we know that the largest we can be is the sum of all the iterators,
+// or in the case of short-circuiting, the longest.
+func (or *OrIterator) Size() (int64, bool) {
+	var val int64
+	var b bool
+	if or.isShortCircuiting {
+		val = 0
+		b = true
+		for _, it := range or.internalIterators {
+			newval, newb := it.Size()
+			if val < newval {
+				val = newval
+			}
+			b = newb && b
+		}
+	} else {
+		val = 0
+		b = true
+		for _, it := range or.internalIterators {
+			newval, newb := it.Size()
+			val += newval
+			b = newb && b
+		}
+	}
+	return val, b
+}
+
+// An Or has no NextResult of its own -- that is, there are no other values
+// which satisfy our previous result that are not the result itself. Our
+// subiterators might, however, so just pass the call recursively. In the case of
+// shortcircuiting, only allow new results from the currently checked iterator
+func (or *OrIterator) NextResult() bool {
+	if or.currentIterator != -1 {
+		return or.internalIterators[or.currentIterator].NextResult()
+	}
+	return false
+}
+
+// Perform or-specific cleanup, of which there currently is none.
+func (or *OrIterator) cleanUp() {}
+
+// Close this iterator, and, by extension, close the subiterators.
+// Close should be idempotent, and it follows that if it's subiterators
+// follow this contract, the And follows the contract.
+func (or *OrIterator) Close() {
+	or.cleanUp()
+	for _, it := range or.internalIterators {
+		it.Close()
+	}
+}
+
+func (or *OrIterator) Optimize() (Iterator, bool) {
+	oldItList := or.GetSubIterators()
+	itList := optimizeSubIterators(oldItList)
+	// Close the replaced iterators (they ought to close themselves, but Close()
+	// is idempotent, so this just protects against any machinations).
+	closeIteratorList(oldItList, nil)
+	newOr := NewOrIterator()
+	newOr.isShortCircuiting = or.isShortCircuiting
+
+	// Add the subiterators in order.
+	for e := itList.Front(); e != nil; e = e.Next() {
+		newOr.AddSubIterator(e.Value.(Iterator))
+	}
+
+	// Move the tags hanging on us (like any good replacement).
+	newOr.CopyTagsFrom(or)
+
+	// And close ourselves but not our subiterators -- some may still be alive in
+	// the new And (they were unchanged upon calling Optimize() on them, at the
+	// start).
+	or.cleanUp()
+	return newOr, true
+}
+
+func (or *OrIterator) GetStats() *IteratorStats {
+	CheckCost := int64(0)
+	NextCost := int64(0)
+	Size := int64(0)
+	for _, it := range or.internalIterators {
+		stats := it.GetStats()
+		NextCost += stats.NextCost
+		CheckCost += stats.CheckCost
+		if or.isShortCircuiting {
+			if Size < stats.Size {
+				Size = stats.Size
+			}
+		} else {
+			Size += stats.Size
+		}
+	}
+	return &IteratorStats{
+		CheckCost: CheckCost,
+		NextCost:  NextCost,
+		Size:      Size,
+	}
+
+}
+
+// Register this as an "or" iterator.
+func (or *OrIterator) Type() string { return "or" }

graph/or-iterator_test.go

@@ -0,0 +1,142 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func extractNumbersFromIterator(it Iterator) []int {
+	var outputNumbers []int
+	for {
+		val, ok := it.Next()
+		if !ok {
+			break
+		}
+		outputNumbers = append(outputNumbers, val.(int))
+	}
+	return outputNumbers
+}
+
+func TestOrIteratorBasics(t *testing.T) {
+	var orIt *OrIterator
+
+	Convey("Given an Or Iterator of two fixed iterators", t, func() {
+		orIt = NewOrIterator()
+		fixed1 := newFixedIterator()
+		fixed1.AddValue(1)
+		fixed1.AddValue(2)
+		fixed1.AddValue(3)
+		fixed2 := newFixedIterator()
+		fixed2.AddValue(3)
+		fixed2.AddValue(9)
+		fixed2.AddValue(20)
+		fixed2.AddValue(21)
+		orIt.AddSubIterator(fixed1)
+		orIt.AddSubIterator(fixed2)
+
+		Convey("It should guess its size.", func() {
+			v, _ := orIt.Size()
+			So(v, ShouldEqual, 7)
+		})
+
+		Convey("It should extract all the numbers, potentially twice.", func() {
+			allNumbers := []int{1, 2, 3, 3, 9, 20, 21}
+			So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers)
+			orIt.Reset()
+			So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers)
+			// Optimization works
+			newOr, _ := orIt.Optimize()
+			So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers)
+		})
+
+		Convey("It should check that numbers in either iterator exist.", func() {
+			So(orIt.Check(2), ShouldEqual, true)
+			So(orIt.Check(3), ShouldEqual, true)
+			So(orIt.Check(21), ShouldEqual, true)
+		})
+
+		Convey("It should check that numbers not in either iterator are false.", func() {
+			So(orIt.Check(22), ShouldEqual, false)
+			So(orIt.Check(5), ShouldEqual, false)
+			So(orIt.Check(0), ShouldEqual, false)
+		})
+
+	})
+
+}
+
+func TestShortCircuitingOrBasics(t *testing.T) {
+	var orIt *OrIterator
+
+	Convey("Given a short-circuiting Or of two fixed iterators", t, func() {
+		orIt = NewShortCircuitOrIterator()
+		fixed1 := newFixedIterator()
+		fixed1.AddValue(1)
+		fixed1.AddValue(2)
+		fixed1.AddValue(3)
+		fixed2 := newFixedIterator()
+		fixed2.AddValue(3)
+		fixed2.AddValue(9)
+		fixed2.AddValue(20)
+		fixed2.AddValue(21)
+
+		Convey("It should guess its size.", func() {
+			orIt.AddSubIterator(fixed1)
+			orIt.AddSubIterator(fixed2)
+			v, _ := orIt.Size()
+			So(v, ShouldEqual, 4)
+		})
+
+		Convey("It should extract the first iterators' numbers.", func() {
+			orIt.AddSubIterator(fixed1)
+			orIt.AddSubIterator(fixed2)
+			allNumbers := []int{1, 2, 3}
+			So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers)
+			orIt.Reset()
+			So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers)
+			// Optimization works
+			newOr, _ := orIt.Optimize()
+			So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers)
+		})
+
+		Convey("It should check that numbers in either iterator exist.", func() {
+			orIt.AddSubIterator(fixed1)
+			orIt.AddSubIterator(fixed2)
+			So(orIt.Check(2), ShouldEqual, true)
+			So(orIt.Check(3), ShouldEqual, true)
+			So(orIt.Check(21), ShouldEqual, true)
+			So(orIt.Check(22), ShouldEqual, false)
+			So(orIt.Check(5), ShouldEqual, false)
+			So(orIt.Check(0), ShouldEqual, false)
+
+		})
+
+		Convey("It should check that it pulls the second iterator's numbers if the first is empty.", func() {
+			orIt.AddSubIterator(newFixedIterator())
+			orIt.AddSubIterator(fixed2)
+			allNumbers := []int{3, 9, 20, 21}
+			So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers)
+			orIt.Reset()
+			So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers)
+			// Optimization works
+			newOr, _ := orIt.Optimize()
+			So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers)
+		})
+
+	})
+
+}

graph/query-shape.go

@@ -0,0 +1,189 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+type Node struct {
+	Id         int      `json:"id"`
+	Tags       []string `json:"tags,omitempty"`
+	Values     []string `json:"values,omitempty"`
+	IsLinkNode bool     `json:"is_link_node"`
+	IsFixed    bool     `json:"is_fixed"`
+}
+
+type Link struct {
+	Source   int `json:"source"`
+	Target   int `json:"target"`
+	Pred     int `json:"type"`
+	LinkNode int `json:"link_node"`
+}
+
+type queryShape struct {
+	nodes    []Node
+	links    []Link
+	ts       TripleStore
+	nodeId   int
+	hasaIds  []int
+	hasaDirs []string
+}
+
+func OutputQueryShapeForIterator(it Iterator, ts TripleStore, outputMap *map[string]interface{}) {
+	qs := &queryShape{}
+	qs.nodes = make([]Node, 0)
+	qs.links = make([]Link, 0)
+	qs.hasaIds = make([]int, 0)
+	qs.hasaDirs = make([]string, 0)
+	qs.ts = ts
+	qs.nodeId = 1
+
+	node := qs.MakeNode(it.Clone())
+	qs.AddNode(node)
+	(*outputMap)["nodes"] = qs.nodes
+	(*outputMap)["links"] = qs.links
+}
+
+func (qs *queryShape) AddNode(n *Node) {
+	qs.nodes = append(qs.nodes, *n)
+}
+
+func (qs *queryShape) AddLink(l *Link) {
+	qs.links = append(qs.links, *l)
+}
+
+func (qs *queryShape) LastHasa() (int, string) {
+	return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1]
+}
+
+func (qs *queryShape) PushHasa(i int, s string) {
+	qs.hasaIds = append(qs.hasaIds, i)
+	qs.hasaDirs = append(qs.hasaDirs, s)
+}
+
+func (qs *queryShape) RemoveHasa() {
+	qs.hasaIds = qs.hasaIds[:len(qs.hasaIds)-1]
+	qs.hasaDirs = qs.hasaDirs[:len(qs.hasaDirs)-1]
+}
+
+func (qs *queryShape) StealNode(left *Node, right *Node) {
+	for _, v := range right.Values {
+		left.Values = append(left.Values, v)
+	}
+	for _, v := range right.Tags {
+		left.Tags = append(left.Tags, v)
+	}
+	left.IsLinkNode = left.IsLinkNode || right.IsLinkNode
+	left.IsFixed = left.IsFixed || right.IsFixed
+	for i, link := range qs.links {
+		rewrite := false
+		if link.LinkNode == right.Id {
+			link.LinkNode = left.Id
+			rewrite = true
+		}
+		if link.Source == right.Id {
+			link.Source = left.Id
+			rewrite = true
+		}
+		if link.Target == right.Id {
+			link.Target = left.Id
+			rewrite = true
+		}
+		if rewrite {
+			qs.links = append(append(qs.links[:i], qs.links[i+1:]...), link)
+		}
+	}
+}
+
+func (qs *queryShape) MakeNode(it Iterator) *Node {
+	var n Node
+	n.IsLinkNode = false
+	n.IsFixed = false
+	n.Id = qs.nodeId
+	n.Tags = make([]string, 0)
+	n.Values = make([]string, 0)
+	for _, tag := range it.Tags() {
+		n.Tags = append(n.Tags, tag)
+	}
+	for k, _ := range it.FixedTags() {
+		n.Tags = append(n.Tags, k)
+	}
+
+	switch it.Type() {
+	case "and":
+		list := it.GetSubIterators()
+		for e := list.Front(); e != nil; e = e.Next() {
+			subit := e.Value.(Iterator)
+			qs.nodeId++
+			newNode := qs.MakeNode(subit)
+			if subit.Type() != "or" {
+				qs.StealNode(&n, newNode)
+			} else {
+				qs.AddNode(newNode)
+				qs.AddLink(&Link{n.Id, newNode.Id, 0, 0})
+			}
+		}
+	case "fixed":
+		n.IsFixed = true
+		for {
+			val, more := it.Next()
+			if !more {
+				break
+			}
+			n.Values = append(n.Values, qs.ts.GetNameFor(val))
+		}
+	case "hasa":
+		hasa := it.(*HasaIterator)
+		qs.PushHasa(n.Id, hasa.direction)
+		qs.nodeId++
+		newNode := qs.MakeNode(hasa.primaryIt)
+		qs.AddNode(newNode)
+		qs.RemoveHasa()
+	case "or":
+		list := it.GetSubIterators()
+		for e := list.Front(); e != nil; e = e.Next() {
+			subit := e.Value.(Iterator)
+			qs.nodeId++
+			newNode := qs.MakeNode(subit)
+			if subit.Type() == "or" {
+				qs.StealNode(&n, newNode)
+			} else {
+				qs.AddNode(newNode)
+				qs.AddLink(&Link{n.Id, newNode.Id, 0, 0})
+			}
+		}
+	case "linksto":
+		n.IsLinkNode = true
+		lto := it.(*LinksToIterator)
+		qs.nodeId++
+		newNode := qs.MakeNode(lto.primaryIt)
+		hasaID, hasaDir := qs.LastHasa()
+		if (hasaDir == "s" && lto.direction == "o") ||
+			(hasaDir == "o" && lto.direction == "s") {
+			qs.AddNode(newNode)
+			if hasaDir == "s" {
+				qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id})
+			} else {
+				qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id})
+			}
+		} else if lto.primaryIt.Type() == "fixed" {
+			qs.StealNode(&n, newNode)
+		} else {
+			qs.AddNode(newNode)
+		}
+	case "optional":
+		// Unsupported, for the moment
+		fallthrough
+	case "all":
+	}
+	return &n
+}

graph/query-shape_test.go

@@ -0,0 +1,124 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	. "github.com/smartystreets/goconvey/convey"
+	"testing"
+)
+
+func buildHasaWithTag(ts TripleStore, tag string, target string) *HasaIterator {
+	fixed_obj := ts.MakeFixed()
+	fixed_pred := ts.MakeFixed()
+	fixed_obj.AddValue(ts.GetIdFor(target))
+	fixed_pred.AddValue(ts.GetIdFor("status"))
+	fixed_obj.AddTag(tag)
+	lto1 := NewLinksToIterator(ts, fixed_obj, "o")
+	lto2 := NewLinksToIterator(ts, fixed_pred, "p")
+	and := NewAndIterator()
+	and.AddSubIterator(lto1)
+	and.AddSubIterator(lto2)
+	hasa := NewHasaIterator(ts, and, "s")
+	return hasa
+}
+
+func TestQueryShape(t *testing.T) {
+	var queryShape map[string]interface{}
+	var ts *TestTripleStore
+	ts = new(TestTripleStore)
+	ts.On("GetIdFor", "cool").Return(1)
+	ts.On("GetNameFor", 1).Return("cool")
+	ts.On("GetIdFor", "status").Return(2)
+	ts.On("GetNameFor", 2).Return("status")
+	ts.On("GetIdFor", "fun").Return(3)
+	ts.On("GetNameFor", 3).Return("fun")
+	ts.On("GetIdFor", "name").Return(4)
+	ts.On("GetNameFor", 4).Return("name")
+
+	Convey("Given a single linkage iterator's shape", t, func() {
+		queryShape = make(map[string]interface{})
+		hasa := buildHasaWithTag(ts, "tag", "cool")
+		hasa.AddTag("top")
+		OutputQueryShapeForIterator(hasa, ts, &queryShape)
+
+		Convey("It should have three nodes and one link", func() {
+			nodes := queryShape["nodes"].([]Node)
+			links := queryShape["links"].([]Link)
+			So(len(nodes), ShouldEqual, 3)
+			So(len(links), ShouldEqual, 1)
+		})
+
+		Convey("These nodes should be correctly tagged", func() {
+			nodes := queryShape["nodes"].([]Node)
+			So(nodes[0].Tags, ShouldResemble, []string{"tag"})
+			So(nodes[1].IsLinkNode, ShouldEqual, true)
+			So(nodes[2].Tags, ShouldResemble, []string{"top"})
+
+		})
+
+		Convey("The link should be correctly typed", func() {
+			nodes := queryShape["nodes"].([]Node)
+			links := queryShape["links"].([]Link)
+			So(links[0].Source, ShouldEqual, nodes[2].Id)
+			So(links[0].Target, ShouldEqual, nodes[0].Id)
+			So(links[0].LinkNode, ShouldEqual, nodes[1].Id)
+			So(links[0].Pred, ShouldEqual, 0)
+
+		})
+
+	})
+
+	Convey("Given a name-of-an-and-iterator's shape", t, func() {
+		queryShape = make(map[string]interface{})
+		hasa1 := buildHasaWithTag(ts, "tag1", "cool")
+		hasa1.AddTag("hasa1")
+		hasa2 := buildHasaWithTag(ts, "tag2", "fun")
+		hasa1.AddTag("hasa2")
+		andInternal := NewAndIterator()
+		andInternal.AddSubIterator(hasa1)
+		andInternal.AddSubIterator(hasa2)
+		fixed_pred := ts.MakeFixed()
+		fixed_pred.AddValue(ts.GetIdFor("name"))
+		lto1 := NewLinksToIterator(ts, andInternal, "s")
+		lto2 := NewLinksToIterator(ts, fixed_pred, "p")
+		and := NewAndIterator()
+		and.AddSubIterator(lto1)
+		and.AddSubIterator(lto2)
+		hasa := NewHasaIterator(ts, and, "o")
+		OutputQueryShapeForIterator(hasa, ts, &queryShape)
+
+		Convey("It should have seven nodes and three links", func() {
+			nodes := queryShape["nodes"].([]Node)
+			links := queryShape["links"].([]Link)
+			So(len(nodes), ShouldEqual, 7)
+			So(len(links), ShouldEqual, 3)
+		})
+
+		Convey("Three of the nodes are link nodes, four aren't", func() {
+			nodes := queryShape["nodes"].([]Node)
+			count := 0
+			for _, node := range nodes {
+				if node.IsLinkNode {
+					count++
+				}
+			}
+			So(count, ShouldEqual, 3)
+		})
+
+		Convey("These nodes should be correctly tagged", nil)
+
+	})
+
+}

graph/result-tree-evaluator.go

@@ -0,0 +1,70 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	"container/list"
+	"fmt"
+)
+
+type ResultTree struct {
+	result   TSVal
+	subtrees *list.List
+}
+
+func NewResultTree(result TSVal) *ResultTree {
+	var tree ResultTree
+	tree.subtrees = list.New()
+	tree.result = result
+	return &tree
+}
+
+func (tree *ResultTree) ToString() string {
+	base := fmt.Sprintf("(%d", tree.result)
+	if tree.subtrees.Len() != 0 {
+		for e := tree.subtrees.Front(); e != nil; e = e.Next() {
+			base += fmt.Sprintf(" %s", (e.Value.(*ResultTree)).ToString())
+		}
+	}
+	base += ")"
+	return base
+}
+
+func (tree *ResultTree) AddSubtree(sub *ResultTree) {
+	tree.subtrees.PushBack(sub)
+}
+
+func StringResultTreeEvaluator(it Iterator) string {
+	ok := true
+	out := ""
+	for {
+		_, ok = it.Next()
+		if !ok {
+			break
+		}
+		out += it.GetResultTree().ToString()
+		out += "\n"
+		for it.NextResult() == true {
+			out += " "
+			out += it.GetResultTree().ToString()
+			out += "\n"
+		}
+	}
+	return out
+}
+
+func PrintResultTreeEvaluator(it Iterator) {
+	fmt.Print(StringResultTreeEvaluator(it))
+}

graph/result-tree-evaluator_test.go

@@ -0,0 +1,42 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	"testing"
+)
+
+func TestSingleIterator(t *testing.T) {
+	all := NewInt64AllIterator(1, 3)
+	result := StringResultTreeEvaluator(all)
+	expected := "(1)\n(2)\n(3)\n"
+	if expected != result {
+		t.Errorf("Expected \"%s\" got \"%s\"", expected, result)
+	}
+}
+
+func TestAndIterator(t *testing.T) {
+	all1 := NewInt64AllIterator(1, 3)
+	all2 := NewInt64AllIterator(3, 5)
+	and := NewAndIterator()
+	and.AddSubIterator(all1)
+	and.AddSubIterator(all2)
+
+	result := StringResultTreeEvaluator(and)
+	expected := "(3 (3) (3))\n"
+	if expected != result {
+		t.Errorf("Expected \"%s\" got \"%s\"", expected, result)
+	}
+}

graph/session.go

@@ -0,0 +1,45 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines the graph session interface general to all query languages.
+
+type ParseResult int
+
+const (
+	Parsed ParseResult = iota
+	ParseMore
+	ParseFail
+)
+
+type Session interface {
+	// Return whether the string is a valid expression.
+	InputParses(string) (ParseResult, error)
+	ExecInput(string, chan interface{}, int)
+	ToText(interface{}) string
+	ToggleDebug()
+}
+
+type HttpSession interface {
+	// Return whether the string is a valid expression.
+	InputParses(string) (ParseResult, error)
+	// Runs the query and returns individual results on the channel.
+	ExecInput(string, chan interface{}, int)
+	GetQuery(string, chan map[string]interface{})
+	BuildJson(interface{})
+	GetJson() (interface{}, error)
+	ClearJson()
+	ToggleDebug()
+}

graph/sexp/parser.go

@@ -0,0 +1,271 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sexp
+
+import (
+	"github.com/badgerodon/peg"
+
+	"github.com/google/cayley/graph"
+)
+
+func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator {
+	tree := parseQuery(query)
+	return buildIteratorTree(tree, ts)
+}
+
+func ParseString(input string) string {
+	return parseQuery(input).String()
+}
+
+func parseQuery(input string) *peg.ExpressionTree {
+	parser := peg.NewParser()
+
+	start := parser.NonTerminal("Start")
+	whitespace := parser.NonTerminal("Whitespace")
+	quotedString := parser.NonTerminal("QuotedString")
+	rootConstraint := parser.NonTerminal("RootConstraint")
+
+	constraint := parser.NonTerminal("Constraint")
+	colonIdentifier := parser.NonTerminal("ColonIdentifier")
+	variable := parser.NonTerminal("Variable")
+	identifier := parser.NonTerminal("Identifier")
+	fixedNode := parser.NonTerminal("FixedNode")
+	nodeIdent := parser.NonTerminal("NodeIdentifier")
+	predIdent := parser.NonTerminal("PredIdentifier")
+	reverse := parser.NonTerminal("Reverse")
+	predKeyword := parser.NonTerminal("PredicateKeyword")
+	optional := parser.NonTerminal("OptionalKeyword")
+
+	start.Expression = rootConstraint
+
+	whitespace.Expression = parser.OneOrMore(
+		parser.OrderedChoice(
+			parser.Terminal(' '),
+			parser.Terminal('\t'),
+			parser.Terminal('\n'),
+			parser.Terminal('\r'),
+		),
+	)
+
+	quotedString.Expression = parser.Sequence(
+		parser.Terminal('"'),
+		parser.OneOrMore(
+			parser.OrderedChoice(
+				parser.Range('0', '9'),
+				parser.Range('a', 'z'),
+				parser.Range('A', 'Z'),
+				parser.Terminal('_'),
+				parser.Terminal('/'),
+				parser.Terminal(':'),
+				parser.Terminal(' '),
+				parser.Terminal('\''),
+			),
+		),
+		parser.Terminal('"'),
+	)
+
+	predKeyword.Expression = parser.OrderedChoice(
+		optional,
+	)
+
+	optional.Expression = parser.Sequence(
+		parser.Terminal('o'),
+		parser.Terminal('p'),
+		parser.Terminal('t'),
+		parser.Terminal('i'),
+		parser.Terminal('o'),
+		parser.Terminal('n'),
+		parser.Terminal('a'),
+		parser.Terminal('l'),
+	)
+
+	identifier.Expression = parser.OneOrMore(
+		parser.OrderedChoice(
+			parser.Range('0', '9'),
+			parser.Range('a', 'z'),
+			parser.Range('A', 'Z'),
+			parser.Terminal('_'),
+			parser.Terminal('.'),
+			parser.Terminal('/'),
+			parser.Terminal(':'),
+			parser.Terminal('#'),
+		),
+	)
+
+	reverse.Expression = parser.Terminal('!')
+
+	variable.Expression = parser.Sequence(
+		parser.Terminal('$'),
+		identifier,
+	)
+
+	colonIdentifier.Expression = parser.Sequence(
+		parser.Terminal(':'),
+		identifier,
+	)
+
+	fixedNode.Expression = parser.OrderedChoice(
+		colonIdentifier,
+		quotedString,
+	)
+
+	nodeIdent.Expression = parser.OrderedChoice(
+		variable,
+		fixedNode,
+	)
+
+	predIdent.Expression = parser.Sequence(
+		parser.Optional(reverse),
+		parser.OrderedChoice(
+			nodeIdent,
+			constraint,
+		),
+	)
+
+	constraint.Expression = parser.Sequence(
+		parser.Terminal('('),
+		parser.Optional(whitespace),
+		predIdent,
+		parser.Optional(whitespace),
+		parser.Optional(predKeyword),
+		parser.Optional(whitespace),
+		parser.OrderedChoice(
+			nodeIdent,
+			rootConstraint,
+		),
+		parser.Optional(whitespace),
+		parser.Terminal(')'),
+	)
+
+	rootConstraint.Expression = parser.Sequence(
+		parser.Terminal('('),
+		parser.Optional(whitespace),
+		nodeIdent,
+		parser.Optional(whitespace),
+		parser.ZeroOrMore(parser.Sequence(
+			constraint,
+			parser.Optional(whitespace),
+		)),
+		parser.Terminal(')'),
+	)
+
+	tree := parser.Parse(input)
+	return tree
+}
+
+func getIdentString(tree *peg.ExpressionTree) string {
+	out := ""
+	if len(tree.Children) > 0 {
+		for _, child := range tree.Children {
+			out += getIdentString(child)
+		}
+	} else {
+		if tree.Value != '"' {
+			out += string(tree.Value)
+		}
+	}
+	return out
+}
+
+func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Iterator {
+	switch tree.Name {
+	case "Start":
+		return buildIteratorTree(tree.Children[0], ts)
+	case "NodeIdentifier":
+		var out graph.Iterator
+		nodeID := getIdentString(tree)
+		if tree.Children[0].Name == "Variable" {
+			allIt := ts.GetNodesAllIterator()
+			allIt.AddTag(nodeID)
+			out = allIt
+		} else {
+			n := nodeID
+			if tree.Children[0].Children[0].Name == "ColonIdentifier" {
+				n = nodeID[1:]
+			}
+			fixed := ts.MakeFixed()
+			fixed.AddValue(ts.GetIdFor(n))
+			out = fixed
+		}
+		return out
+	case "PredIdentifier":
+		i := 0
+		if tree.Children[0].Name == "Reverse" {
+			//Taken care of below
+			i++
+		}
+		it := buildIteratorTree(tree.Children[i], ts)
+		lto := graph.NewLinksToIterator(ts, it, "p")
+		return lto
+	case "RootConstraint":
+		constraintCount := 0
+		and := graph.NewAndIterator()
+		for _, c := range tree.Children {
+			switch c.Name {
+			case "NodeIdentifier":
+				fallthrough
+			case "Constraint":
+				it := buildIteratorTree(c, ts)
+				and.AddSubIterator(it)
+				constraintCount++
+				continue
+			default:
+				continue
+			}
+		}
+		return and
+	case "Constraint":
+		var hasa *graph.HasaIterator
+		topLevelDir := "s"
+		subItDir := "o"
+		subAnd := graph.NewAndIterator()
+		isOptional := false
+		for _, c := range tree.Children {
+			switch c.Name {
+			case "PredIdentifier":
+				if c.Children[0].Name == "Reverse" {
+					topLevelDir = "o"
+					subItDir = "s"
+				}
+				it := buildIteratorTree(c, ts)
+				subAnd.AddSubIterator(it)
+				continue
+			case "PredicateKeyword":
+				switch c.Children[0].Name {
+				case "OptionalKeyword":
+					isOptional = true
+				}
+			case "NodeIdentifier":
+				fallthrough
+			case "RootConstraint":
+				it := buildIteratorTree(c, ts)
+				l := graph.NewLinksToIterator(ts, it, subItDir)
+				subAnd.AddSubIterator(l)
+				continue
+			default:
+				continue
+			}
+		}
+		hasa = graph.NewHasaIterator(ts, subAnd, topLevelDir)
+		if isOptional {
+			optional := graph.NewOptionalIterator(hasa)
+			return optional
+		}
+		return hasa
+	default:
+		return &graph.NullIterator{}
+	}
+	panic("Not reached")
+}

graph/sexp/parser_test.go

@@ -0,0 +1,129 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sexp
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/graph/memstore"
+)
+
+func TestBadParse(t *testing.T) {
+	str := ParseString("()")
+	if str != "" {
+		t.Errorf("It parsed! Got \"%s\"", str)
+	}
+}
+
+func TestParseSexpWithMemstore(t *testing.T) {
+	Convey("With a Memstore", t, func() {
+		ts := memstore.NewMemTripleStore()
+
+		Convey("It should parse an empty query", func() {
+			it := BuildIteratorTreeForQuery(ts, "()")
+			So(it.Type(), ShouldEqual, "null")
+		})
+
+		Convey("It should get a single triple linkage", func() {
+			ts.AddTriple(graph.MakeTriple("i", "can", "win", ""))
+			query := "($a (:can \"win\"))"
+			So(len(query), ShouldEqual, 17)
+			it := BuildIteratorTreeForQuery(ts, query)
+			So(it.Type(), ShouldEqual, "and")
+			out, ok := it.Next()
+			So(ok, ShouldBeTrue)
+			So(out, ShouldEqual, ts.GetIdFor("i"))
+		})
+
+		Convey("It can get an internal linkage", func() {
+			ts.AddTriple(graph.MakeTriple("i", "can", "win", ""))
+			query := "(\"i\" (:can $a))"
+			it := BuildIteratorTreeForQuery(ts, query)
+			So(it.Type(), ShouldEqual, "and")
+			out, ok := it.Next()
+			So(ok, ShouldBeTrue)
+			So(out, ShouldEqual, ts.GetIdFor("i"))
+		})
+
+	})
+}
+
+func TestTreeConstraintParse(t *testing.T) {
+	ts := memstore.NewMemTripleStore()
+	ts.AddTriple(graph.MakeTriple("i", "like", "food", ""))
+	ts.AddTriple(graph.MakeTriple("food", "is", "good", ""))
+	query := "(\"i\"\n" +
+		"(:like\n" +
+		"($a (:is :good))))"
+	it := BuildIteratorTreeForQuery(ts, query)
+	if it.Type() != "and" {
+		t.Error("Odd iterator tree. Got: %s", it.DebugString(0))
+	}
+	out, ok := it.Next()
+	if !ok {
+		t.Error("Got no results")
+	}
+	if out != ts.GetIdFor("i") {
+		t.Errorf("Got %d, expected %d", out, ts.GetIdFor("i"))
+	}
+}
+
+func TestTreeConstraintTagParse(t *testing.T) {
+	ts := memstore.NewMemTripleStore()
+	ts.AddTriple(graph.MakeTriple("i", "like", "food", ""))
+	ts.AddTriple(graph.MakeTriple("food", "is", "good", ""))
+	query := "(\"i\"\n" +
+		"(:like\n" +
+		"($a (:is :good))))"
+	it := BuildIteratorTreeForQuery(ts, query)
+	_, ok := it.Next()
+	if !ok {
+		t.Error("Got no results")
+	}
+	tags := make(map[string]graph.TSVal)
+	it.TagResults(&tags)
+	if ts.GetNameFor(tags["$a"]) != "food" {
+		t.Errorf("Got %s, expected food", ts.GetNameFor(tags["$a"]))
+	}
+
+}
+
+func TestMultipleConstraintParse(t *testing.T) {
+	ts := memstore.NewMemTripleStore()
+	ts.AddTriple(graph.MakeTriple("i", "like", "food", ""))
+	ts.AddTriple(graph.MakeTriple("i", "like", "beer", ""))
+	ts.AddTriple(graph.MakeTriple("you", "like", "beer", ""))
+	query := "($a \n" +
+		"(:like :beer)\n" +
+		"(:like \"food\"))"
+	it := BuildIteratorTreeForQuery(ts, query)
+	if it.Type() != "and" {
+		t.Error("Odd iterator tree. Got: %s", it.DebugString(0))
+	}
+	out, ok := it.Next()
+	if !ok {
+		t.Error("Got no results")
+	}
+	if out != ts.GetIdFor("i") {
+		t.Errorf("Got %d, expected %d", out, ts.GetIdFor("i"))
+	}
+	_, ok = it.Next()
+	if ok {
+		t.Error("Too many results")
+	}
+}

graph/sexp/sexp-session.go

@@ -0,0 +1,121 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sexp
+
+// Defines a running session of the sexp query language.
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	"github.com/google/cayley/graph"
+)
+
+type SexpSession struct {
+	ts    graph.TripleStore
+	debug bool
+}
+
+func NewSexpSession(inputTripleStore graph.TripleStore) *SexpSession {
+	var s SexpSession
+	s.ts = inputTripleStore
+	return &s
+}
+
+func (s *SexpSession) ToggleDebug() {
+	s.debug = !s.debug
+}
+
+func (s *SexpSession) InputParses(input string) (graph.ParseResult, error) {
+	var parenDepth int
+	for i, x := range input {
+		if x == '(' {
+			parenDepth++
+		}
+		if x == ')' {
+			parenDepth--
+			if parenDepth < 0 {
+				min := 0
+				if (i - 10) > min {
+					min = i - 10
+				}
+				return graph.ParseFail, errors.New(fmt.Sprintf("Too many close parens at char %d: %s", i, input[min:i]))
+			}
+		}
+	}
+	if parenDepth > 0 {
+		return graph.ParseMore, nil
+	}
+	if len(ParseString(input)) > 0 {
+		return graph.Parsed, nil
+	}
+	return graph.ParseFail, errors.New("Invalid Syntax")
+}
+
+func (s *SexpSession) ExecInput(input string, out chan interface{}, limit int) {
+	it := BuildIteratorTreeForQuery(s.ts, input)
+	newIt, changed := it.Optimize()
+	if changed {
+		it = newIt
+	}
+
+	if s.debug {
+		fmt.Println(it.DebugString(0))
+	}
+	nResults := 0
+	for {
+		_, ok := it.Next()
+		if !ok {
+			break
+		}
+		tags := make(map[string]graph.TSVal)
+		it.TagResults(&tags)
+		out <- &tags
+		nResults++
+		if nResults > limit && limit != -1 {
+			break
+		}
+		for it.NextResult() == true {
+			tags := make(map[string]graph.TSVal)
+			it.TagResults(&tags)
+			out <- &tags
+			nResults++
+			if nResults > limit && limit != -1 {
+				break
+			}
+		}
+	}
+	close(out)
+}
+
+func (s *SexpSession) ToText(result interface{}) string {
+	out := fmt.Sprintln("****")
+	tags := result.(*map[string]graph.TSVal)
+	tagKeys := make([]string, len(*tags))
+	i := 0
+	for k, _ := range *tags {
+		tagKeys[i] = k
+		i++
+	}
+	sort.Strings(tagKeys)
+	for _, k := range tagKeys {
+		if k == "$_" {
+			continue
+		}
+		out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k]))
+	}
+	return out
+}

graph/triple.go

@@ -0,0 +1,109 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines the struct which makes the TripleStore possible -- the triple.
+//
+// At its heart, it consists of three fields -- Subject, Predicate, and Object.
+// Three IDs that relate to each other. That's all there is to it. The triples
+// are the links in the graph, and the existence of node IDs is defined by the
+// fact that some triple in the graph mentions them.
+//
+// This means that a complete representation of the graph is equivalent to a
+// list of triples. The rest is just indexing for speed.
+//
+// Adding fields to the triple is not to be taken lightly. You'll see I mention
+// provenance, but don't as yet use it in any backing store. In general, there
+// can be features that can be turned on or off for any store, but I haven't
+// decided how to allow/disallow them yet. Another such example would be to add
+// a forward and reverse index field -- forward being "order the list of
+// objects pointed at by this subject with this predicate" such as first and
+// second children, top billing, what have you.
+//
+// There will never be that much in this file except for the definition, but
+// the consequences are not to be taken lightly. But do suggest cool features!
+
+import (
+	"fmt"
+	"reflect"
+)
+
+// Our triple struct, used throughout.
+type Triple struct {
+	Sub        string `json:"subject"`
+	Pred       string `json:"predicate"`
+	Obj        string `json:"object"`
+	Provenance string `json:"provenance,omitempty"`
+}
+
+func NewTriple() *Triple {
+	return &Triple{}
+}
+
+func MakeTriple(sub string, pred string, obj string, provenance string) *Triple {
+	return &Triple{sub, pred, obj, provenance}
+}
+
+// List of the valid directions of a triple.
+// TODO(barakmich): Replace all instances of "dir string" in the codebase
+// with an enum of valid directions, to make this less stringly typed.
+var TripleDirections = [4]string{"s", "p", "o", "c"}
+
+// Per-field accessor for triples
+func (t *Triple) Get(dir string) string {
+	if dir == "s" {
+		return t.Sub
+	} else if dir == "p" {
+		return t.Pred
+	} else if dir == "prov" || dir == "c" {
+		return t.Provenance
+	} else if dir == "o" {
+		return t.Obj
+	} else {
+		panic(fmt.Sprintf("No Such Triple Direction, %s", dir))
+	}
+}
+
+func (t *Triple) Equals(other *Triple) bool {
+	return reflect.DeepEqual(t, other)
+}
+
+// Pretty-prints a triple.
+func (t *Triple) ToString() string {
+	return fmt.Sprintf("%s -- %s -> %s\n", t.Sub, t.Pred, t.Obj)
+}
+
+func (t *Triple) IsValid() bool {
+	if t.Sub == "" {
+		return false
+	}
+	if t.Pred == "" {
+		return false
+	}
+	if t.Obj == "" {
+		return false
+	}
+	return true
+}
+
+// Prints a triple in N-Triple format.
+func (t *Triple) ToNTriple() string {
+	if t.Provenance == "" {
+		//TODO(barakmich): Proper escaping.
+		return fmt.Sprintf("%s %s %s .", t.Sub, t.Pred, t.Obj)
+	} else {
+		return fmt.Sprintf("%s %s %s %s .", t.Sub, t.Pred, t.Obj, t.Provenance)
+	}
+}

graph/triplestore.go

@@ -0,0 +1,119 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// Defines the TripleStore interface. Every backing store must implement at
+// least this interface.
+//
+// Most of these are pretty straightforward. As long as we can surface this
+// interface, the rest of the stack will "just work" and we can connect to any
+// triple backing store we prefer.
+
+import (
+	"github.com/barakmich/glog"
+)
+
+// Defines an opaque "triple store value" type. However the backend wishes to
+// implement it, a TSVal is merely a token to a triple or a node that the backing
+// store itself understands, and the base iterators pass around.
+//
+// For example, in a very traditional, graphd-style graph, these are int64s
+// (guids of the primitives). In a very direct sort of graph, these could be
+// pointers to structs, or merely triples, or whatever works best for the
+// backing store.
+type TSVal interface{}
+
+type TripleStore interface {
+	// Add a triple to the store.
+	AddTriple(*Triple)
+
+	// Add a set of triples to the store, atomically if possible.
+	AddTripleSet([]*Triple)
+
+	// Removes a triple matching the given one  from the database,
+	// if it exists. Does nothing otherwise.
+	RemoveTriple(*Triple)
+
+	// Given an opaque token, returns the triple for that token from the store.
+	GetTriple(TSVal) *Triple
+
+	// Given a direction and a token, creates an iterator of links which have
+	// that node token in that directional field.
+	GetTripleIterator(string, TSVal) Iterator
+
+	// Returns an iterator enumerating all nodes in the graph.
+	GetNodesAllIterator() Iterator
+
+	// Returns an iterator enumerating all links in the graph.
+	GetTriplesAllIterator() Iterator
+
+	// Given a node ID, return the opaque token used by the TripleStore
+	// to represent that id.
+	GetIdFor(string) TSVal
+
+	// Given an opaque token, return the node that it represents.
+	GetNameFor(TSVal) string
+
+	// Returns the number of triples currently stored.
+	Size() int64
+
+	// Creates a Fixed iterator which can compare TSVals
+	MakeFixed() *FixedIterator
+
+	// Optimize an iterator in the context of the triple store.
+	// Suppose we have a better index for the passed tree; this
+	// gives the TripleStore the oppotunity to replace it
+	// with a more efficient iterator.
+	OptimizeIterator(it Iterator) (Iterator, bool)
+
+	// Close the triple store and clean up. (Flush to disk, cleanly
+	// sever connections, etc)
+	Close()
+
+	// Convienence function for speed. Given a triple token and a direction
+	// return the node token for that direction. Sometimes, a TripleStore
+	// can do this without going all the way to the backing store, and
+	// gives the TripleStore the opportunity to make this optimization.
+	//
+	// Iterators will call this. At worst, a valid implementation is
+	// self.GetIdFor(self.GetTriple(triple_id).Get(dir))
+	GetTripleDirection(triple_id TSVal, dir string) TSVal
+}
+
+type OptionsDict map[string]interface{}
+
+func (d OptionsDict) GetIntKey(key string) (int, bool) {
+	if val, ok := d[key]; ok {
+		switch vv := val.(type) {
+		case float64:
+			return int(vv), true
+		default:
+			glog.Fatalln("Invalid", key, "parameter type from config.")
+		}
+	}
+	return 0, false
+}
+
+func (d OptionsDict) GetStringKey(key string) (string, bool) {
+	if val, ok := d[key]; ok {
+		switch vv := val.(type) {
+		case string:
+			return vv, true
+		default:
+			glog.Fatalln("Invalid", key, "parameter type from config.")
+		}
+	}
+	return "", false
+}

graph/value-comparison-iterator.go

@@ -0,0 +1,193 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+// "Value Comparison" is a unary operator -- a filter across the values in the
+// relevant subiterator.
+//
+// This is hugely useful for things like provenance, but value ranges in general
+// come up from time to time. At *worst* we're as big as our underlying iterator.
+// At best, we're the null iterator.
+//
+// This is ripe for backend-side optimization. If you can run a value iterator,
+// from a sorted set -- some sort of value index, then go for it.
+//
+// In MQL terms, this is the [{"age>=": 21}] concept.
+
+import (
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+)
+
+type ComparisonOperator int
+
+const (
+	kCompareLT ComparisonOperator = iota
+	kCompareLTE
+	kCompareGT
+	kCompareGTE
+	// Why no Equals? Because that's usually an AndIterator.
+)
+
+type ValueComparisonIterator struct {
+	BaseIterator
+	subIt           Iterator
+	op              ComparisonOperator
+	comparisonValue interface{}
+	ts              TripleStore
+}
+
+func NewValueComparisonIterator(
+	subIt Iterator,
+	operator ComparisonOperator,
+	value interface{},
+	ts TripleStore) *ValueComparisonIterator {
+
+	var vc ValueComparisonIterator
+	BaseIteratorInit(&vc.BaseIterator)
+	vc.subIt = subIt
+	vc.op = operator
+	vc.comparisonValue = value
+	vc.ts = ts
+	return &vc
+}
+
+// Here's the non-boilerplate part of the ValueComparison iterator. Given a value
+// and our operator, determine whether or not we meet the requirement.
+func (vc *ValueComparisonIterator) doComparison(val TSVal) bool {
+	//TODO(barakmich): Implement string comparison.
+	nodeStr := vc.ts.GetNameFor(val)
+	switch cVal := vc.comparisonValue.(type) {
+	case int:
+		cInt := int64(cVal)
+		intVal, err := strconv.ParseInt(nodeStr, 10, 64)
+		if err != nil {
+			return false
+		}
+		return RunIntOp(intVal, vc.op, cInt)
+	case int64:
+		intVal, err := strconv.ParseInt(nodeStr, 10, 64)
+		if err != nil {
+			return false
+		}
+		return RunIntOp(intVal, vc.op, cVal)
+	default:
+		return true
+	}
+}
+
+func (vc *ValueComparisonIterator) Close() {
+	vc.subIt.Close()
+}
+
+func RunIntOp(a int64, op ComparisonOperator, b int64) bool {
+	switch op {
+	case kCompareLT:
+		return a < b
+	case kCompareLTE:
+		return a <= b
+	case kCompareGT:
+		return a > b
+	case kCompareGTE:
+		return a >= b
+	default:
+		log.Fatal("Unknown operator type")
+		return false
+	}
+}
+
+func (vc *ValueComparisonIterator) Reset() {
+	vc.subIt.Reset()
+}
+
+func (vc *ValueComparisonIterator) Clone() Iterator {
+	out := NewValueComparisonIterator(vc.subIt.Clone(), vc.op, vc.comparisonValue, vc.ts)
+	out.CopyTagsFrom(vc)
+	return out
+}
+
+func (vc *ValueComparisonIterator) Next() (TSVal, bool) {
+	var val TSVal
+	var ok bool
+	for {
+		val, ok = vc.subIt.Next()
+		if !ok {
+			return nil, false
+		}
+		if vc.doComparison(val) {
+			break
+		}
+	}
+	vc.Last = val
+	return val, ok
+}
+
+func (vc *ValueComparisonIterator) NextResult() bool {
+	for {
+		hasNext := vc.subIt.NextResult()
+		if !hasNext {
+			return false
+		}
+		if vc.doComparison(vc.subIt.LastResult()) {
+			return true
+		}
+	}
+	vc.Last = vc.subIt.LastResult()
+	return true
+}
+
+func (vc *ValueComparisonIterator) Check(val TSVal) bool {
+	if !vc.doComparison(val) {
+		return false
+	}
+	return vc.subIt.Check(val)
+}
+
+// If we failed the check, then the subiterator should not contribute to the result
+// set. Otherwise, go ahead and tag it.
+func (vc *ValueComparisonIterator) TagResults(out *map[string]TSVal) {
+	vc.BaseIterator.TagResults(out)
+	vc.subIt.TagResults(out)
+}
+
+// Registers the value-comparison iterator.
+func (vc *ValueComparisonIterator) Type() string { return "value-comparison" }
+
+// Prints the value-comparison and its subiterator.
+func (vc *ValueComparisonIterator) DebugString(indent int) string {
+	return fmt.Sprintf("%s(%s\n%s)",
+		strings.Repeat(" ", indent),
+		vc.Type(), vc.subIt.DebugString(indent+4))
+}
+
+// There's nothing to optimize, locally, for a value-comparison iterator.
+// Replace the underlying iterator if need be.
+// potentially replace it.
+func (vc *ValueComparisonIterator) Optimize() (Iterator, bool) {
+	newSub, changed := vc.subIt.Optimize()
+	if changed {
+		vc.subIt.Close()
+		vc.subIt = newSub
+	}
+	return vc, false
+}
+
+// We're only as expensive as our subiterator.
+// Again, optimized value comparison iterators should do better.
+func (vc *ValueComparisonIterator) GetStats() *IteratorStats {
+	return vc.subIt.GetStats()
+}

graph/value-comparison-iterator_test.go

@@ -0,0 +1,126 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph
+
+import (
+	"testing"
+)
+
+func SetupMockTripleStore(nameMap map[string]int) *TestTripleStore {
+	ts := new(TestTripleStore)
+	for k, v := range nameMap {
+		ts.On("GetIdFor", k).Return(v)
+		ts.On("GetNameFor", v).Return(k)
+	}
+	return ts
+}
+
+func SimpleValueTripleStore() *TestTripleStore {
+	ts := SetupMockTripleStore(map[string]int{
+		"0": 0,
+		"1": 1,
+		"2": 2,
+		"3": 3,
+		"4": 4,
+		"5": 5,
+	})
+	return ts
+}
+
+func BuildFixedIterator() *FixedIterator {
+	fixed := newFixedIterator()
+	fixed.AddValue(0)
+	fixed.AddValue(1)
+	fixed.AddValue(2)
+	fixed.AddValue(3)
+	fixed.AddValue(4)
+	return fixed
+}
+
+func checkIteratorContains(ts TripleStore, it Iterator, expected []string, t *testing.T) {
+	var actual []string
+	actual = nil
+	for {
+		val, ok := it.Next()
+		if !ok {
+			break
+		}
+		actual = append(actual, ts.GetNameFor(val))
+	}
+	actualSet := actual[:]
+	for _, a := range expected {
+		found := false
+		for j, b := range actualSet {
+			if a == b {
+				actualSet = append(actualSet[:j], actualSet[j+1:]...)
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Error("Couldn't find", a, "in actual output.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet)
+			return
+		}
+	}
+	if len(actualSet) != 0 {
+		t.Error("Actual output has more than expected.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet)
+	}
+}
+
+func TestWorkingIntValueComparison(t *testing.T) {
+	ts := SimpleValueTripleStore()
+	fixed := BuildFixedIterator()
+	vc := NewValueComparisonIterator(fixed, kCompareLT, int64(3), ts)
+	checkIteratorContains(ts, vc, []string{"0", "1", "2"}, t)
+}
+
+func TestFailingIntValueComparison(t *testing.T) {
+	ts := SimpleValueTripleStore()
+	fixed := BuildFixedIterator()
+	vc := NewValueComparisonIterator(fixed, kCompareLT, int64(0), ts)
+	checkIteratorContains(ts, vc, []string{}, t)
+}
+
+func TestWorkingGT(t *testing.T) {
+	ts := SimpleValueTripleStore()
+	fixed := BuildFixedIterator()
+	vc := NewValueComparisonIterator(fixed, kCompareGT, int64(2), ts)
+	checkIteratorContains(ts, vc, []string{"3", "4"}, t)
+}
+
+func TestWorkingGTE(t *testing.T) {
+	ts := SimpleValueTripleStore()
+	fixed := BuildFixedIterator()
+	vc := NewValueComparisonIterator(fixed, kCompareGTE, int64(2), ts)
+	checkIteratorContains(ts, vc, []string{"2", "3", "4"}, t)
+}
+
+func TestVCICheck(t *testing.T) {
+	ts := SimpleValueTripleStore()
+	fixed := BuildFixedIterator()
+	vc := NewValueComparisonIterator(fixed, kCompareGTE, int64(2), ts)
+	if vc.Check(1) {
+		t.Error("1 is less than 2, should be GTE")
+	}
+	if !vc.Check(2) {
+		t.Error("2 is GTE 2")
+	}
+	if !vc.Check(3) {
+		t.Error("3 is GTE 2")
+	}
+	if vc.Check(5) {
+		t.Error("5 is not in the underlying iterator")
+	}
+}