barak/cayley
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

update to master

Browse source
This commit is contained in:
Barak Michener 2014-08-04 00:39:19 -04:00
commit cedaac35d0
93 changed files with 15683 additions and 2361 deletions
Download patch file Download diff file Expand all files Collapse all files

106
graph/iterator.go
View file

@ -14,8 +14,7 @@
package graph
// Define the general iterator interface, as well as the Base iterator which all
// iterators can "inherit" from to get default iterator functionality.
// Define the general iterator interface.
import (
"strings"
@ -24,18 +23,46 @@ import (
"github.com/barakmich/glog"
)
type Tagger struct {
tags []string
fixedTags map[string]Value
}
// Adds a tag to the iterator.
func (t *Tagger) Add(tag string) {
t.tags = append(t.tags, tag)
}
func (t *Tagger) AddFixed(tag string, value Value) {
if t.fixedTags == nil {
t.fixedTags = make(map[string]Value)
}
t.fixedTags[tag] = value
}
// Returns the tags. The returned value must not be mutated.
func (t *Tagger) Tags() []string {
return t.tags
}
// Returns the fixed tags. The returned value must not be mutated.
func (t *Tagger) Fixed() map[string]Value {
return t.fixedTags
}
func (t *Tagger) CopyFrom(src Iterator) {
for _, tag := range src.Tagger().Tags() {
t.Add(tag)
}
for k, v := range src.Tagger().Fixed() {
t.AddFixed(k, v)
}
}
type Iterator interface {
// Tags are the way we handle results. By adding a tag to an iterator, we can
// "name" it, in a sense, and at each step of iteration, get a named result.
// TagResults() is therefore the handy way of walking an iterator tree and
// getting the named results.
//
// Tag Accessors.
AddTag(string)
Tags() []string
AddFixedTag(string, Value)
FixedTags() map[string]Value
CopyTagsFrom(Iterator)
Tagger() *Tagger
// Fills a tag-to-result-value map.
TagResults(map[string]Value)
@ -58,22 +85,12 @@ type Iterator interface {
// All of them should set iterator.Last to be the last returned value, to
// make results work.
//
// Next() advances the iterator and returns the next valid result. Returns
// (<value>, true) or (nil, false)
Next() (Value, bool)
// NextResult() advances iterators that may have more than one valid result,
// from the bottom up.
NextResult() bool
// Return whether this iterator is reliably nextable. Most iterators are.
// However, some iterators, like "not" are, by definition, the whole database
// except themselves. Next() on these is unproductive, if impossible.
CanNext() bool
// Check(), given a value, returns whether or not that value is within the set
// held by this iterator.
Check(Value) bool
// Contains returns whether the value is within the set held by the iterator.
Contains(Value) bool
// Start iteration from the beginning
Reset()
@ -114,7 +131,26 @@ type Iterator interface {
Close()
// UID returns the unique identifier of the iterator.
UID() uintptr
UID() uint64
}
type Nexter interface {
// Next() advances the iterator and returns the next valid result. Returns
// (<value>, true) or (nil, false)
Next() (Value, bool)
Iterator
}
// Next is a convenience function that conditionally calls the Next method
// of an Iterator if it is a Nexter. If the Iterator is not a Nexter, Next
// return a nil Value and false.
func Next(it Iterator) (Value, bool) {
if n, ok := it.(Nexter); ok {
return n.Next()
}
glog.Errorln("Nexting an un-nextable iterator")
return nil, false
}
// FixedIterator wraps iterators that are modifiable by addition of fixed value sets.
@ -124,9 +160,9 @@ type FixedIterator interface {
}
type IteratorStats struct {
CheckCost int64
NextCost int64
Size int64
ContainsCost int64
NextCost int64
Size int64
}
// Type enumerates the set of Iterator types.
@ -192,20 +228,20 @@ func (t Type) String() string {
return types[t]
}
// Utility logging functions for when an iterator gets called Next upon, or Check upon, as
// Utility logging functions for when an iterator gets called Next upon, or Contains upon, as
// well as what they return. Highly useful for tracing the execution path of a query.
func CheckLogIn(it Iterator, val Value) {
func ContainsLogIn(it Iterator, val Value) {
if glog.V(4) {
glog.V(4).Infof("%s %d CHECK %d", strings.ToUpper(it.Type().String()), it.UID(), val)
glog.V(4).Infof("%s %d CHECK CONTAINS %d", strings.ToUpper(it.Type().String()), it.UID(), val)
}
}
func CheckLogOut(it Iterator, val Value, good bool) bool {
func ContainsLogOut(it Iterator, val Value, good bool) bool {
if glog.V(4) {
if good {
glog.V(4).Infof("%s %d CHECK %d GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
glog.V(4).Infof("%s %d CHECK CONTAINS %d GOOD", strings.ToUpper(it.Type().String()), it.UID(), val)
} else {
glog.V(4).Infof("%s %d CHECK %d BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
glog.V(4).Infof("%s %d CHECK CONTAINS %d BAD", strings.ToUpper(it.Type().String()), it.UID(), val)
}
}
return good

77
graph/iterator/all_iterator.go
View file

@ -31,19 +31,25 @@ import (
// An All iterator across a range of int64 values, from `max` to `min`.
type Int64 struct {
Base
uid uint64
tags graph.Tagger
max, min int64
at int64
result graph.Value
}
// Creates a new Int64 with the given range.
func NewInt64(min, max int64) *Int64 {
var all Int64
BaseInit(&all.Base)
all.max = max
all.min = min
all.at = min
return &all
return &Int64{
uid: NextUID(),
min: min,
max: max,
at: min,
}
}
func (it *Int64) UID() uint64 {
return it.uid
}
// Start back at the beginning
@ -55,13 +61,28 @@ func (it *Int64) Close() {}
func (it *Int64) Clone() graph.Iterator {
out := NewInt64(it.min, it.max)
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
func (it *Int64) Tagger() *graph.Tagger {
return &it.tags
}
// Fill the map based on the tags assigned to this iterator.
func (it *Int64) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
// Prints the All iterator as just an "all".
func (it *Int64) DebugString(indent int) string {
return fmt.Sprintf("%s(%s tags: %v)", strings.Repeat(" ", indent), it.Type(), it.Tags())
return fmt.Sprintf("%s(%s tags: %v)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags())
}
// Next() on an Int64 all iterator is a simple incrementing counter.
@ -76,10 +97,28 @@ func (it *Int64) Next() (graph.Value, bool) {
if it.at > it.max {
it.at = -1
}
it.Last = val
it.result = val
return graph.NextLogOut(it, val, true)
}
// DEPRECATED
func (it *Int64) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Int64) Result() graph.Value {
return it.result
}
func (it *Int64) NextResult() bool {
return false
}
// No sub-iterators.
func (it *Int64) SubIterators() []graph.Iterator {
return nil
}
// The number of elements in an Int64 is the size of the range.
// The size is exact.
func (it *Int64) Size() (int64, bool) {
@ -87,16 +126,16 @@ func (it *Int64) Size() (int64, bool) {
return Size, true
}
// Check() for an Int64 is merely seeing if the passed value is
// Contains() for an Int64 is merely seeing if the passed value is
// withing the range, assuming the value is an int64.
func (it *Int64) Check(tsv graph.Value) bool {
graph.CheckLogIn(it, tsv)
func (it *Int64) Contains(tsv graph.Value) bool {
graph.ContainsLogIn(it, tsv)
v := tsv.(int64)
if it.min <= v && v <= it.max {
it.Last = v
return graph.CheckLogOut(it, v, true)
it.result = v
return graph.ContainsLogOut(it, v, true)
}
return graph.CheckLogOut(it, v, false)
return graph.ContainsLogOut(it, v, false)
}
// The type of this iterator is an "all". This is important, as it puts it in
@ -111,8 +150,8 @@ func (it *Int64) Optimize() (graph.Iterator, bool) { return it, false }
func (it *Int64) Stats() graph.IteratorStats {
s, _ := it.Size()
return graph.IteratorStats{
CheckCost: 1,
NextCost: 1,
Size: s,
ContainsCost: 1,
NextCost: 1,
Size: s,
}
}

106
graph/iterator/and_iterator.go
View file

@ -6,11 +6,11 @@
//
// It accomplishes this in one of two ways. If it is a Next()ed iterator (that
// is, it is a top level iterator, or on the "Next() path", then it will Next()
// it's primary iterator (helpfully, and.primary_it) and Check() the resultant
// it's primary iterator (helpfully, and.primary_it) and Contains() the resultant
// value against it's other iterators. If it matches all of them, then it
// returns that value. Otherwise, it repeats the process.
//
// If it's on a Check() path, it merely Check()s every iterator, and returns the
// If it's on a Contains() path, it merely Contains()s every iterator, and returns the
// logical AND of each result.
package iterator
@ -22,23 +22,28 @@ import (
"github.com/google/cayley/graph"
)
// The And iterator. Consists of a Base and a number of subiterators, the primary of which will
// The And iterator. Consists of a number of subiterators, the primary of which will
// be Next()ed if next is called.
type And struct {
Base
uid uint64
tags graph.Tagger
internalIterators []graph.Iterator
itCount int
primaryIt graph.Iterator
checkList []graph.Iterator
result graph.Value
}
// Creates a new And iterator.
func NewAnd() *And {
var and And
BaseInit(&and.Base)
and.internalIterators = make([]graph.Iterator, 0, 20)
and.checkList = nil
return &and
return &And{
uid: NextUID(),
internalIterators: make([]graph.Iterator, 0, 20),
}
}
func (it *And) UID() uint64 {
return it.uid
}
// Reset all internal iterators
@ -50,15 +55,38 @@ func (it *And) Reset() {
it.checkList = nil
}
func (it *And) Tagger() *graph.Tagger {
return &it.tags
}
// An extended TagResults, as it needs to add it's own results and
// recurse down it's subiterators.
func (it *And) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
if it.primaryIt != nil {
it.primaryIt.TagResults(dst)
}
for _, sub := range it.internalIterators {
sub.TagResults(dst)
}
}
func (it *And) Clone() graph.Iterator {
and := NewAnd()
and.AddSubIterator(it.primaryIt.Clone())
and.CopyTagsFrom(it)
and.tags.CopyFrom(it)
for _, sub := range it.internalIterators {
and.AddSubIterator(sub.Clone())
}
if it.checkList != nil {
and.optimizeCheck()
and.optimizeContains()
}
return and
}
@ -71,18 +99,6 @@ func (it *And) SubIterators() []graph.Iterator {
return iters
}
// Overrides Base TagResults, as it needs to add it's own results and
// recurse down it's subiterators.
func (it *And) TagResults(dst map[string]graph.Value) {
it.Base.TagResults(dst)
if it.primaryIt != nil {
it.primaryIt.TagResults(dst)
}
for _, sub := range it.internalIterators {
sub.TagResults(dst)
}
}
// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators.
func (it *And) ResultTree() *graph.ResultTree {
tree := graph.NewResultTree(it.Result())
@ -101,7 +117,7 @@ func (it *And) DebugString(indent int) string {
total += fmt.Sprintf("%d:\n%s\n", i, sub.DebugString(indent+4))
}
var tags string
for _, k := range it.Tags() {
for _, k := range it.tags.Tags() {
tags += fmt.Sprintf("%s;", k)
}
spaces := strings.Repeat(" ", indent+2)
@ -144,23 +160,27 @@ func (it *And) Next() (graph.Value, bool) {
var curr graph.Value
var exists bool
for {
curr, exists = it.primaryIt.Next()
curr, exists = graph.Next(it.primaryIt)
if !exists {
return graph.NextLogOut(it, nil, false)
}
if it.checkSubIts(curr) {
it.Last = curr
if it.subItsContain(curr) {
it.result = curr
return graph.NextLogOut(it, curr, true)
}
}
panic("Somehow broke out of Next() loop in And")
panic("unreachable")
}
func (it *And) Result() graph.Value {
return it.result
}
// Checks a value against the non-primary iterators, in order.
func (it *And) checkSubIts(val graph.Value) bool {
func (it *And) subItsContain(val graph.Value) bool {
var subIsGood = true
for _, sub := range it.internalIterators {
subIsGood = sub.Check(val)
subIsGood = sub.Contains(val)
if !subIsGood {
break
}
@ -168,36 +188,36 @@ func (it *And) checkSubIts(val graph.Value) bool {
return subIsGood
}
func (it *And) checkCheckList(val graph.Value) bool {
func (it *And) checkContainsList(val graph.Value) bool {
ok := true
for _, c := range it.checkList {
ok = c.Check(val)
ok = c.Contains(val)
if !ok {
break
}
}
if ok {
it.Last = val
it.result = val
}
return graph.CheckLogOut(it, val, ok)
return graph.ContainsLogOut(it, val, ok)
}
// Check a value against the entire iterator, in order.
func (it *And) Check(val graph.Value) bool {
graph.CheckLogIn(it, val)
func (it *And) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
if it.checkList != nil {
return it.checkCheckList(val)
return it.checkContainsList(val)
}
mainGood := it.primaryIt.Check(val)
mainGood := it.primaryIt.Contains(val)
if !mainGood {
return graph.CheckLogOut(it, val, false)
return graph.ContainsLogOut(it, val, false)
}
othersGood := it.checkSubIts(val)
othersGood := it.subItsContain(val)
if !othersGood {
return graph.CheckLogOut(it, val, false)
return graph.ContainsLogOut(it, val, false)
}
it.Last = val
return graph.CheckLogOut(it, val, true)
it.result = val
return graph.ContainsLogOut(it, val, true)
}
// Returns the approximate size of the And iterator. Because we're dealing

47
graph/iterator/and_iterator_optimize.go
View file

@ -38,10 +38,10 @@ import (
// In short, tread lightly.
// Optimizes the And, by picking the most efficient way to Next() and
// Check() its subiterators. For SQL fans, this is equivalent to JOIN.
// Contains() its subiterators. For SQL fans, this is equivalent to JOIN.
func (it *And) Optimize() (graph.Iterator, bool) {
// First, let's get the slice of iterators, in order (first one is Next()ed,
// the rest are Check()ed)
// the rest are Contains()ed)
old := it.SubIterators()
// And call Optimize() on our subtree, replacing each one in the order we
@ -82,9 +82,9 @@ func (it *And) Optimize() (graph.Iterator, bool) {
}
// Move the tags hanging on us (like any good replacement).
newAnd.CopyTagsFrom(it)
newAnd.tags.CopyFrom(it)
newAnd.optimizeCheck()
newAnd.optimizeContains()
// And close ourselves but not our subiterators -- some may still be alive in
// the new And (they were unchanged upon calling Optimize() on them, at the
@ -142,24 +142,24 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
// Find the iterator with the projected "best" total cost.
// Total cost is defined as The Next()ed iterator's cost to Next() out
// all of it's contents, and to Check() each of those against everyone
// all of it's contents, and to Contains() each of those against everyone
// else.
for _, it := range its {
if !it.CanNext() {
if _, canNext := it.(graph.Nexter); !canNext {
bad = append(bad, it)
continue
}
rootStats := it.Stats()
cost := rootStats.NextCost
for _, f := range its {
if !f.CanNext() {
if _, canNext := it.(graph.Nexter); !canNext {
continue
}
if f == it {
continue
}
stats := f.Stats()
cost += stats.CheckCost
cost += stats.ContainsCost
}
cost *= rootStats.Size
if cost < bestCost {
@ -169,7 +169,7 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
}
// TODO(barakmich): Optimization of order need not stop here. Picking a smart
// Check() order based on probability of getting a false Check() first is
// Contains() order based on probability of getting a false Contains() first is
// useful (fail faster).
// Put the best iterator (the one we wish to Next()) at the front...
@ -177,7 +177,7 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
// ... push everyone else after...
for _, it := range its {
if !it.CanNext() {
if _, canNext := it.(graph.Nexter); !canNext {
continue
}
if it != best {
@ -192,12 +192,12 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator {
type byCost []graph.Iterator
func (c byCost) Len() int { return len(c) }
func (c byCost) Less(i, j int) bool { return c[i].Stats().CheckCost < c[j].Stats().CheckCost }
func (c byCost) Less(i, j int) bool { return c[i].Stats().ContainsCost < c[j].Stats().ContainsCost }
func (c byCost) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
// optimizeCheck(l) creates an alternate check list, containing the same contents
// optimizeContains() creates an alternate check list, containing the same contents
// but with a new ordering, however it wishes.
func (it *And) optimizeCheck() {
func (it *And) optimizeContains() {
// GetSubIterators allocates, so this is currently safe.
// TODO(kortschak) Reuse it.checkList if possible.
// This involves providing GetSubIterators with a slice to fill.
@ -213,11 +213,11 @@ func (it *And) optimizeCheck() {
func (it *And) getSubTags() map[string]struct{} {
tags := make(map[string]struct{})
for _, sub := range it.SubIterators() {
for _, tag := range sub.Tags() {
for _, tag := range sub.Tagger().Tags() {
tags[tag] = struct{}{}
}
}
for _, tag := range it.Tags() {
for _, tag := range it.tags.Tags() {
tags[tag] = struct{}{}
}
return tags
@ -227,13 +227,14 @@ func (it *And) getSubTags() map[string]struct{} {
// src itself, and moves them to dst.
func moveTagsTo(dst graph.Iterator, src *And) {
tags := src.getSubTags()
for _, tag := range dst.Tags() {
for _, tag := range dst.Tagger().Tags() {
if _, ok := tags[tag]; ok {
delete(tags, tag)
}
}
dt := dst.Tagger()
for k := range tags {
dst.AddTag(k)
dt.Add(k)
}
}
@ -297,21 +298,21 @@ func hasOneUsefulIterator(its []graph.Iterator) graph.Iterator {
// For now, however, it's pretty static.
func (it *And) Stats() graph.IteratorStats {
primaryStats := it.primaryIt.Stats()
CheckCost := primaryStats.CheckCost
ContainsCost := primaryStats.ContainsCost
NextCost := primaryStats.NextCost
Size := primaryStats.Size
for _, sub := range it.internalIterators {
stats := sub.Stats()
NextCost += stats.CheckCost
CheckCost += stats.CheckCost
NextCost += stats.ContainsCost
ContainsCost += stats.ContainsCost
if Size > stats.Size {
Size = stats.Size
}
}
return graph.IteratorStats{
CheckCost: CheckCost,
NextCost: NextCost,
Size: Size,
ContainsCost: ContainsCost,
NextCost: NextCost,
Size: Size,
}
}

18
graph/iterator/and_iterator_optimize_test.go
View file

@ -32,9 +32,9 @@ func TestIteratorPromotion(t *testing.T) {
a := NewAnd()
a.AddSubIterator(all)
a.AddSubIterator(fixed)
all.AddTag("a")
fixed.AddTag("b")
a.AddTag("c")
all.Tagger().Add("a")
fixed.Tagger().Add("b")
a.Tagger().Add("c")
newIt, changed := a.Optimize()
if !changed {
t.Error("Iterator didn't optimize")
@ -43,7 +43,7 @@ func TestIteratorPromotion(t *testing.T) {
t.Error("Expected fixed iterator")
}
tagsExpected := []string{"a", "b", "c"}
tags := newIt.Tags()
tags := newIt.Tagger().Tags()
sort.Strings(tags)
if !reflect.DeepEqual(tags, tagsExpected) {
t.Fatal("Tags don't match")
@ -67,9 +67,9 @@ func TestNullIteratorAnd(t *testing.T) {
func TestReorderWithTag(t *testing.T) {
all := NewInt64(100, 300)
all.AddTag("good")
all.Tagger().Add("good")
all2 := NewInt64(1, 30000)
all2.AddTag("slow")
all2.Tagger().Add("slow")
a := NewAnd()
// Make all2 the default iterator
a.AddSubIterator(all2)
@ -82,7 +82,7 @@ func TestReorderWithTag(t *testing.T) {
expectedTags := []string{"good", "slow"}
tagsOut := make([]string, 0)
for _, sub := range newIt.SubIterators() {
for _, x := range sub.Tags() {
for _, x := range sub.Tagger().Tags() {
tagsOut = append(tagsOut, x)
}
}
@ -93,9 +93,9 @@ func TestReorderWithTag(t *testing.T) {
func TestAndStatistics(t *testing.T) {
all := NewInt64(100, 300)
all.AddTag("good")
all.Tagger().Add("good")
all2 := NewInt64(1, 30000)
all2.AddTag("slow")
all2.Tagger().Add("slow")
a := NewAnd()
// Make all2 the default iterator
a.AddSubIterator(all2)

6
graph/iterator/and_iterator_test.go
View file

@ -24,11 +24,11 @@ import (
func TestTag(t *testing.T) {
fix1 := newFixed()
fix1.Add(234)
fix1.AddTag("foo")
fix1.Tagger().Add("foo")
and := NewAnd()
and.AddSubIterator(fix1)
and.AddTag("bar")
out := fix1.Tags()
and.Tagger().Add("bar")
out := fix1.Tagger().Tags()
if len(out) != 1 {
t.Errorf("Expected length 1, got %d", len(out))
}

75
graph/iterator/fixed_iterator.go
View file

@ -30,10 +30,12 @@ import (
// A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and
// an equality function.
type Fixed struct {
Base
uid uint64
tags graph.Tagger
values []graph.Value
lastIndex int
cmp Equality
result graph.Value
}
// Define the signature of an equality function.
@ -54,12 +56,15 @@ func newFixed() *Fixed {
// Creates a new Fixed iterator with a custom comparitor.
func NewFixedIteratorWithCompare(compareFn Equality) *Fixed {
var it Fixed
BaseInit(&it.Base)
it.values = make([]graph.Value, 0, 20)
it.lastIndex = 0
it.cmp = compareFn
return &it
return &Fixed{
uid: NextUID(),
values: make([]graph.Value, 0, 20),
cmp: compareFn,
}
}
func (it *Fixed) UID() uint64 {
return it.uid
}
func (it *Fixed) Reset() {
@ -68,12 +73,26 @@ func (it *Fixed) Reset() {
func (it *Fixed) Close() {}
func (it *Fixed) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Fixed) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
func (it *Fixed) Clone() graph.Iterator {
out := NewFixedIteratorWithCompare(it.cmp)
for _, val := range it.values {
out.Add(val)
}
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
@ -92,7 +111,7 @@ func (it *Fixed) DebugString(indent int) string {
return fmt.Sprintf("%s(%s tags: %s Size: %d id0: %d)",
strings.Repeat(" ", indent),
it.Type(),
it.FixedTags(),
it.tags.Fixed(),
len(it.values),
value,
)
@ -102,18 +121,18 @@ func (it *Fixed) DebugString(indent int) string {
func (it *Fixed) Type() graph.Type { return graph.Fixed }
// Check if the passed value is equal to one of the values stored in the iterator.
func (it *Fixed) Check(v graph.Value) bool {
func (it *Fixed) Contains(v graph.Value) bool {
// Could be optimized by keeping it sorted or using a better datastructure.
// However, for fixed iterators, which are by definition kind of tiny, this
// isn't a big issue.
graph.CheckLogIn(it, v)
graph.ContainsLogIn(it, v)
for _, x := range it.values {
if it.cmp(x, v) {
it.Last = x
return graph.CheckLogOut(it, v, true)
it.result = x
return graph.ContainsLogOut(it, v, true)
}
}
return graph.CheckLogOut(it, v, false)
return graph.ContainsLogOut(it, v, false)
}
// Return the next stored value from the iterator.
@ -123,11 +142,29 @@ func (it *Fixed) Next() (graph.Value, bool) {
return graph.NextLogOut(it, nil, false)
}
out := it.values[it.lastIndex]
it.Last = out
it.result = out
it.lastIndex++
return graph.NextLogOut(it, out, true)
}
// DEPRECATED
func (it *Fixed) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Fixed) Result() graph.Value {
return it.result
}
func (it *Fixed) NextResult() bool {
return false
}
// No sub-iterators.
func (it *Fixed) SubIterators() []graph.Iterator {
return nil
}
// Optimize() for a Fixed iterator is simple. Returns a Null iterator if it's empty
// (so that other iterators upstream can treat this as null) or there is no
// optimization.
@ -144,12 +181,12 @@ func (it *Fixed) Size() (int64, bool) {
return int64(len(it.values)), true
}
// As we right now have to scan the entire list, Next and Check are linear with the
// As we right now have to scan the entire list, Next and Contains are linear with the
// size. However, a better data structure could remove these limits.
func (it *Fixed) Stats() graph.IteratorStats {
return graph.IteratorStats{
CheckCost: int64(len(it.values)),
NextCost: int64(len(it.values)),
Size: int64(len(it.values)),
ContainsCost: int64(len(it.values)),
NextCost: int64(len(it.values)),
Size: int64(len(it.values)),
}
}

98
graph/iterator/hasa_iterator.go
View file

@ -23,10 +23,10 @@ package iterator
// path. That's okay -- in reality, it can be viewed as returning the value for
// a new triple, but to make logic much simpler, here we have the HasA.
//
// Likewise, it's important to think about Check()ing a HasA. When given a
// Likewise, it's important to think about Contains()ing a HasA. When given a
// value to check, it means "Check all predicates that have this value for your
// direction against the subiterator." This would imply that there's more than
// one possibility for the same Check()ed value. While we could return the
// one possibility for the same Contains()ed value. While we could return the
// number of options, it's simpler to return one, and then call NextResult()
// enough times to enumerate the options. (In fact, one could argue that the
// raison d'etre for NextResult() is this iterator).
@ -40,28 +40,35 @@ import (
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
// A HasA consists of a reference back to the graph.TripleStore that it references,
// a primary subiterator, a direction in which the triples for that subiterator point,
// and a temporary holder for the iterator generated on Check().
// and a temporary holder for the iterator generated on Contains().
type HasA struct {
Base
uid uint64
tags graph.Tagger
ts graph.TripleStore
primaryIt graph.Iterator
dir graph.Direction
dir quad.Direction
resultIt graph.Iterator
result graph.Value
}
// Construct a new HasA iterator, given the triple subiterator, and the triple
// direction for which it stands.
func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d graph.Direction) *HasA {
var hasa HasA
BaseInit(&hasa.Base)
hasa.ts = ts
hasa.primaryIt = subIt
hasa.dir = d
return &hasa
func NewHasA(ts graph.TripleStore, subIt graph.Iterator, d quad.Direction) *HasA {
return &HasA{
uid: NextUID(),
ts: ts,
primaryIt: subIt,
dir: d,
}
}
func (it *HasA) UID() uint64 {
return it.uid
}
// Return our sole subiterator.
@ -76,14 +83,18 @@ func (it *HasA) Reset() {
}
}
func (it *HasA) Tagger() *graph.Tagger {
return &it.tags
}
func (it *HasA) Clone() graph.Iterator {
out := NewHasA(it.ts, it.primaryIt.Clone(), it.dir)
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
// Direction accessor.
func (it *HasA) Direction() graph.Direction { return it.dir }
func (it *HasA) Direction() quad.Direction { return it.dir }
// Pass the Optimize() call along to the subiterator. If it becomes Null,
// then the HasA becomes Null (there are no triples that have any directions).
@ -100,7 +111,14 @@ func (it *HasA) Optimize() (graph.Iterator, bool) {
// Pass the TagResults down the chain.
func (it *HasA) TagResults(dst map[string]graph.Value) {
it.Base.TagResults(dst)
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
it.primaryIt.TagResults(dst)
}
@ -114,7 +132,7 @@ func (it *HasA) ResultTree() *graph.ResultTree {
// Print some information about this iterator.
func (it *HasA) DebugString(indent int) string {
var tags string
for _, k := range it.Tags() {
for _, k := range it.tags.Tags() {
tags += fmt.Sprintf("%s;", k)
}
return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), it.Type(), it.UID(), tags, it.dir, it.primaryIt.DebugString(indent+4))
@ -122,9 +140,9 @@ func (it *HasA) DebugString(indent int) string {
// Check a value against our internal iterator. In order to do this, we must first open a new
// iterator of "triples that have `val` in our direction", given to us by the triple store,
// and then Next() values out of that iterator and Check() them against our subiterator.
func (it *HasA) Check(val graph.Value) bool {
graph.CheckLogIn(it, val)
// and then Next() values out of that iterator and Contains() them against our subiterator.
func (it *HasA) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
if glog.V(4) {
glog.V(4).Infoln("Id is", it.ts.NameOf(val))
}
@ -133,23 +151,23 @@ func (it *HasA) Check(val graph.Value) bool {
it.resultIt.Close()
}
it.resultIt = it.ts.TripleIterator(it.dir, val)
return graph.CheckLogOut(it, val, it.GetCheckResult())
return graph.ContainsLogOut(it, val, it.NextContains())
}
// GetCheckResult() is shared code between Check() and GetNextResult() -- calls next on the
// NextContains() is shared code between Contains() and GetNextResult() -- calls next on the
// result iterator (a triple iterator based on the last checked value) and returns true if
// another match is made.
func (it *HasA) GetCheckResult() bool {
func (it *HasA) NextContains() bool {
for {
linkVal, ok := it.resultIt.Next()
linkVal, ok := graph.Next(it.resultIt)
if !ok {
break
}
if glog.V(4) {
glog.V(4).Infoln("Triple is", it.ts.Triple(linkVal))
glog.V(4).Infoln("Quad is", it.ts.Quad(linkVal))
}
if it.primaryIt.Check(linkVal) {
it.Last = it.ts.TripleDirection(linkVal, it.dir)
if it.primaryIt.Contains(linkVal) {
it.result = it.ts.TripleDirection(linkVal, it.dir)
return true
}
}
@ -160,17 +178,17 @@ func (it *HasA) GetCheckResult() bool {
func (it *HasA) NextResult() bool {
// Order here is important. If the subiterator has a NextResult, then we
// need do nothing -- there is a next result, and we shouldn't move forward.
// However, we then need to get the next result from our last Check().
// However, we then need to get the next result from our last Contains().
//
// The upshot is, the end of NextResult() bubbles up from the bottom of the
// iterator tree up, and we need to respect that.
if it.primaryIt.NextResult() {
return true
}
return it.GetCheckResult()
return it.NextContains()
}
// Get the next result from this iterator. This is simpler than Check. We have a
// Get the next result from this iterator. This is simpler than Contains. We have a
// subiterator we can get a value from, and we can take that resultant triple,
// pull our direction out of it, and return that.
func (it *HasA) Next() (graph.Value, bool) {
@ -180,19 +198,23 @@ func (it *HasA) Next() (graph.Value, bool) {
}
it.resultIt = &Null{}
tID, ok := it.primaryIt.Next()
tID, ok := graph.Next(it.primaryIt)
if !ok {
return graph.NextLogOut(it, 0, false)
}
name := it.ts.Triple(tID).Get(it.dir)
name := it.ts.Quad(tID).Get(it.dir)
val := it.ts.ValueOf(name)
it.Last = val
it.result = val
return graph.NextLogOut(it, val, true)
}
func (it *HasA) Result() graph.Value {
return it.result
}
// GetStats() returns the statistics on the HasA iterator. This is curious. Next
// cost is easy, it's an extra call or so on top of the subiterator Next cost.
// CheckCost involves going to the graph.TripleStore, iterating out values, and hoping
// ContainsCost involves going to the graph.TripleStore, iterating out values, and hoping
// one sticks -- potentially expensive, depending on fanout. Size, however, is
// potentially smaller. we know at worst it's the size of the subiterator, but
// if there are many repeated values, it could be much smaller in totality.
@ -205,9 +227,9 @@ func (it *HasA) Stats() graph.IteratorStats {
nextConstant := int64(2)
tripleConstant := int64(1)
return graph.IteratorStats{
NextCost: tripleConstant + subitStats.NextCost,
CheckCost: (fanoutFactor * nextConstant) * subitStats.CheckCost,
Size: faninFactor * subitStats.Size,
NextCost: tripleConstant + subitStats.NextCost,
ContainsCost: (fanoutFactor * nextConstant) * subitStats.ContainsCost,
Size: faninFactor * subitStats.Size,
}
}
@ -221,3 +243,7 @@ func (it *HasA) Close() {
// Register this iterator as a HasA.
func (it *HasA) Type() graph.Type { return graph.HasA }
func (it *HasA) Size() (int64, bool) {
return 0, true
}

196
graph/iterator/iterator.go
View file

@ -14,161 +14,55 @@
package iterator
// Define the general iterator interface, as well as the Base which all
// iterators can "inherit" from to get default iterator functionality.
// Define the general iterator interface.
import (
"fmt"
"strings"
"sync/atomic"
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
)
var nextIteratorID uintptr
var nextIteratorID uint64
func nextID() uintptr {
return atomic.AddUintptr(&nextIteratorID, 1) - 1
func NextUID() uint64 {
return atomic.AddUint64(&nextIteratorID, 1) - 1
}
// The Base iterator is the iterator other iterators inherit from to get some
// default functionality.
type Base struct {
Last graph.Value
tags []string
fixedTags map[string]graph.Value
canNext bool
uid uintptr
}
// Called by subclases.
func BaseInit(it *Base) {
// Your basic iterator is nextable
it.canNext = true
if glog.V(2) {
it.uid = nextID()
}
}
func (it *Base) UID() uintptr {
return it.uid
}
// Adds a tag to the iterator. Most iterators don't need to override.
func (it *Base) AddTag(tag string) {
if it.tags == nil {
it.tags = make([]string, 0)
}
it.tags = append(it.tags, tag)
}
func (it *Base) AddFixedTag(tag string, value graph.Value) {
if it.fixedTags == nil {
it.fixedTags = make(map[string]graph.Value)
}
it.fixedTags[tag] = value
}
// Returns the tags.
func (it *Base) Tags() []string {
return it.tags
}
func (it *Base) FixedTags() map[string]graph.Value {
return it.fixedTags
}
func (it *Base) CopyTagsFrom(other_it graph.Iterator) {
for _, tag := range other_it.Tags() {
it.AddTag(tag)
}
for k, v := range other_it.FixedTags() {
it.AddFixedTag(k, v)
}
}
// Prints a silly debug string. Most classes override.
func (it *Base) DebugString(indent int) string {
return fmt.Sprintf("%s(base)", strings.Repeat(" ", indent))
}
// Nothing in a base iterator.
func (it *Base) Check(v graph.Value) bool {
return false
}
// Base iterators should never appear in a tree if they are, select against
// them.
func (it *Base) Stats() graph.IteratorStats {
return graph.IteratorStats{100000, 100000, 100000}
}
// DEPRECATED
func (it *Base) ResultTree() *graph.ResultTree {
tree := graph.NewResultTree(it.Result())
return tree
}
// Nothing in a base iterator.
func (it *Base) Next() (graph.Value, bool) {
return nil, false
}
func (it *Base) NextResult() bool {
return false
}
// Returns the last result of an iterator.
func (it *Base) Result() graph.Value {
return it.Last
}
// If you're empty and you know it, clap your hands.
func (it *Base) Size() (int64, bool) {
return 0, true
}
// No subiterators. Only those with subiterators need to do anything here.
func (it *Base) SubIterators() []graph.Iterator {
return nil
}
// Accessor
func (it *Base) CanNext() bool { return it.canNext }
// Fill the map based on the tags assigned to this iterator. Default
// functionality works well for most iterators.
func (it *Base) TagResults(dst map[string]graph.Value) {
for _, tag := range it.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.FixedTags() {
dst[tag] = value
}
}
// Nothing to clean up.
// func (it *Base) Close() {}
func (it *Null) Close() {}
func (it *Base) Reset() {}
// Here we define the simplest base iterator -- the Null iterator. It contains nothing.
// Here we define the simplest iterator -- the Null iterator. It contains nothing.
// It is the empty set. Often times, queries that contain one of these match nothing,
// so it's important to give it a special iterator.
type Null struct {
Base
uid uint64
tags graph.Tagger
}
// Fairly useless New function.
func NewNull() *Null {
return &Null{}
return &Null{uid: NextUID()}
}
func (it *Null) UID() uint64 {
return it.uid
}
func (it *Null) Tagger() *graph.Tagger {
return &it.tags
}
// Fill the map based on the tags assigned to this iterator.
func (it *Null) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
func (it *Null) Contains(graph.Value) bool {
return false
}
func (it *Null) Clone() graph.Iterator { return NewNull() }
@ -185,6 +79,34 @@ func (it *Null) DebugString(indent int) string {
return strings.Repeat(" ", indent) + "(null)"
}
func (it *Null) Next() (graph.Value, bool) {
return nil, false
}
func (it *Null) Result() graph.Value {
return nil
}
func (it *Null) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Null) SubIterators() []graph.Iterator {
return nil
}
func (it *Null) NextResult() bool {
return false
}
func (it *Null) Size() (int64, bool) {
return 0, true
}
func (it *Null) Reset() {}
func (it *Null) Close() {}
// A null iterator costs nothing. Use it!
func (it *Null) Stats() graph.IteratorStats {
return graph.IteratorStats{}

78
graph/iterator/linksto_iterator.go
View file

@ -23,7 +23,7 @@ package iterator
// LinksTo is therefore sensitive to growing with a fanout. (A small-sized
// subiterator could cause LinksTo to be large).
//
// Check()ing a LinksTo means, given a link, take the direction we care about
// Contains()ing a LinksTo means, given a link, take the direction we care about
// and check if it's in our subiterator. Checking is therefore fairly cheap, and
// similar to checking the subiterator alone.
//
@ -34,29 +34,36 @@ import (
"strings"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
// A LinksTo has a reference back to the graph.TripleStore (to create the iterators
// for each node) the subiterator, and the direction the iterator comes from.
// `next_it` is the tempoarary iterator held per result in `primary_it`.
type LinksTo struct {
Base
uid uint64
tags graph.Tagger
ts graph.TripleStore
primaryIt graph.Iterator
dir graph.Direction
dir quad.Direction
nextIt graph.Iterator
result graph.Value
}
// Construct a new LinksTo iterator around a direction and a subiterator of
// nodes.
func NewLinksTo(ts graph.TripleStore, it graph.Iterator, d graph.Direction) *LinksTo {
var lto LinksTo
BaseInit(&lto.Base)
lto.ts = ts
lto.primaryIt = it
lto.dir = d
lto.nextIt = &Null{}
return &lto
func NewLinksTo(ts graph.TripleStore, it graph.Iterator, d quad.Direction) *LinksTo {
return &LinksTo{
uid: NextUID(),
ts: ts,
primaryIt: it,
dir: d,
nextIt: &Null{},
}
}
func (it *LinksTo) UID() uint64 {
return it.uid
}
func (it *LinksTo) Reset() {
@ -67,18 +74,29 @@ func (it *LinksTo) Reset() {
it.nextIt = &Null{}
}
func (it *LinksTo) Tagger() *graph.Tagger {
return &it.tags
}
func (it *LinksTo) Clone() graph.Iterator {
out := NewLinksTo(it.ts, it.primaryIt.Clone(), it.dir)
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
// Return the direction under consideration.
func (it *LinksTo) Direction() graph.Direction { return it.dir }
func (it *LinksTo) Direction() quad.Direction { return it.dir }
// Tag these results, and our subiterator's results.
func (it *LinksTo) TagResults(dst map[string]graph.Value) {
it.Base.TagResults(dst)
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
it.primaryIt.TagResults(dst)
}
@ -98,14 +116,14 @@ func (it *LinksTo) DebugString(indent int) string {
// If it checks in the right direction for the subiterator, it is a valid link
// for the LinksTo.
func (it *LinksTo) Check(val graph.Value) bool {
graph.CheckLogIn(it, val)
func (it *LinksTo) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
node := it.ts.TripleDirection(val, it.dir)
if it.primaryIt.Check(node) {
it.Last = val
return graph.CheckLogOut(it, val, true)
if it.primaryIt.Contains(node) {
it.result = val
return graph.ContainsLogOut(it, val, true)
}
return graph.CheckLogOut(it, val, false)
return graph.ContainsLogOut(it, val, false)
}
// Return a list containing only our subiterator.
@ -137,10 +155,10 @@ func (it *LinksTo) Optimize() (graph.Iterator, bool) {
// Next()ing a LinksTo operates as described above.
func (it *LinksTo) Next() (graph.Value, bool) {
graph.NextLogIn(it)
val, ok := it.nextIt.Next()
val, ok := graph.Next(it.nextIt)
if !ok {
// Subiterator is empty, get another one
candidate, ok := it.primaryIt.Next()
candidate, ok := graph.Next(it.primaryIt)
if !ok {
// We're out of nodes in our subiterator, so we're done as well.
return graph.NextLogOut(it, 0, false)
@ -150,10 +168,14 @@ func (it *LinksTo) Next() (graph.Value, bool) {
// Recurse -- return the first in the next set.
return it.Next()
}
it.Last = val
it.result = val
return graph.NextLogOut(it, val, ok)
}
func (it *LinksTo) Result() graph.Value {
return it.result
}
// Close our subiterators.
func (it *LinksTo) Close() {
it.nextIt.Close()
@ -176,8 +198,12 @@ func (it *LinksTo) Stats() graph.IteratorStats {
checkConstant := int64(1)
nextConstant := int64(2)
return graph.IteratorStats{
NextCost: nextConstant + subitStats.NextCost,
CheckCost: checkConstant + subitStats.CheckCost,
Size: fanoutFactor * subitStats.Size,
NextCost: nextConstant + subitStats.NextCost,
ContainsCost: checkConstant + subitStats.ContainsCost,
Size: fanoutFactor * subitStats.Size,
}
}
func (it *LinksTo) Size() (int64, bool) {
return 0, true
}

6
graph/iterator/linksto_iterator_test.go
View file

@ -17,7 +17,7 @@ package iterator
import (
"testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
func TestLinksTo(t *testing.T) {
@ -32,12 +32,12 @@ func TestLinksTo(t *testing.T) {
t.Fatalf("Failed to return correct value, got:%v expect:1", val)
}
fixed.Add(val)
lto := NewLinksTo(ts, fixed, graph.Object)
lto := NewLinksTo(ts, fixed, quad.Object)
val, ok := lto.Next()
if !ok {
t.Error("At least one triple matches the fixed object")
}
if val != 2 {
t.Errorf("Triple index 2, such as %s, should match %s", ts.Triple(2), ts.Triple(val))
t.Errorf("Quad index 2, such as %s, should match %s", ts.Quad(2), ts.Quad(val))
}
}

43
graph/iterator/mock_ts_test.go
View file

@ -17,15 +17,18 @@ package iterator
// A quickly mocked version of the TripleStore interface, for use in tests.
// Can better used Mock.Called but will fill in as needed.
import "github.com/google/cayley/graph"
import (
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
type store struct {
data []string
iter graph.Iterator
}
func (ts *store) ValueOf(s string) graph.Value {
for i, v := range ts.data {
func (qs *store) ValueOf(s string) graph.Value {
for i, v := range qs.data {
if s == v {
return i
}
@ -33,42 +36,42 @@ func (ts *store) ValueOf(s string) graph.Value {
return nil
}
func (ts *store) AddTriple(*graph.Triple) {}
func (qs *store) AddTriple(*quad.Quad) {}
func (ts *store) AddTripleSet([]*graph.Triple) {}
func (qs *store) AddTripleSet([]*quad.Quad) {}
func (ts *store) Triple(graph.Value) *graph.Triple { return &graph.Triple{} }
func (qs *store) Quad(graph.Value) *quad.Quad { return &quad.Quad{} }
func (ts *store) TripleIterator(d graph.Direction, i graph.Value) graph.Iterator {
return ts.iter
func (qs *store) TripleIterator(d quad.Direction, i graph.Value) graph.Iterator {
return qs.iter
}
func (ts *store) NodesAllIterator() graph.Iterator { return &Null{} }
func (qs *store) NodesAllIterator() graph.Iterator { return &Null{} }
func (ts *store) TriplesAllIterator() graph.Iterator { return &Null{} }
func (qs *store) TriplesAllIterator() graph.Iterator { return &Null{} }
func (ts *store) NameOf(v graph.Value) string {
func (qs *store) NameOf(v graph.Value) string {
i := v.(int)
if i < 0 || i >= len(ts.data) {
if i < 0 || i >= len(qs.data) {
return ""
}
return ts.data[i]
return qs.data[i]
}
func (ts *store) Size() int64 { return 0 }
func (qs *store) Size() int64 { return 0 }
func (ts *store) DebugPrint() {}
func (qs *store) DebugPrint() {}
func (ts *store) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
func (qs *store) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) {
return &Null{}, false
}
func (ts *store) FixedIterator() graph.FixedIterator {
func (qs *store) FixedIterator() graph.FixedIterator {
return NewFixedIteratorWithCompare(BasicEquality)
}
func (ts *store) Close() {}
func (qs *store) Close() {}
func (ts *store) TripleDirection(graph.Value, graph.Direction) graph.Value { return 0 }
func (qs *store) TripleDirection(graph.Value, quad.Direction) graph.Value { return 0 }
func (ts *store) RemoveTriple(t *graph.Triple) {}
func (qs *store) RemoveTriple(t *quad.Quad) {}

67
graph/iterator/optional_iterator.go
View file

@ -30,26 +30,31 @@ import (
"fmt"
"strings"
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
)
// An optional iterator has the subconstraint iterator we wish to be optional
// An optional iterator has the sub-constraint iterator we wish to be optional
// and whether the last check we received was true or false.
type Optional struct {
Base
uid uint64
tags graph.Tagger
subIt graph.Iterator
lastCheck bool
result graph.Value
}
// Creates a new optional iterator.
func NewOptional(it graph.Iterator) *Optional {
var o Optional
BaseInit(&o.Base)
o.canNext = false
o.subIt = it
return &o
return &Optional{
uid: NextUID(),
subIt: it,
}
}
func (it *Optional) CanNext() bool { return false }
func (it *Optional) UID() uint64 {
return it.uid
}
func (it *Optional) Reset() {
@ -61,17 +66,23 @@ func (it *Optional) Close() {
it.subIt.Close()
}
func (it *Optional) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Optional) Clone() graph.Iterator {
out := NewOptional(it.subIt.Clone())
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
// Nexting the iterator is unsupported -- error and return an empty set.
// (As above, a reasonable alternative would be to Next() an all iterator)
func (it *Optional) Next() (graph.Value, bool) {
glog.Errorln("Nexting an un-nextable iterator")
return nil, false
// DEPRECATED
func (it *Optional) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Optional) Result() graph.Value {
return it.result
}
// An optional iterator only has a next result if, (a) last time we checked
@ -84,13 +95,18 @@ func (it *Optional) NextResult() bool {
return false
}
// Check() is the real hack of this iterator. It always returns true, regardless
// No subiterators.
func (it *Optional) SubIterators() []graph.Iterator {
return nil
}
// Contains() is the real hack of this iterator. It always returns true, regardless
// of whether the subiterator matched. But we keep track of whether the subiterator
// matched for results purposes.
func (it *Optional) Check(val graph.Value) bool {
checked := it.subIt.Check(val)
func (it *Optional) Contains(val graph.Value) bool {
checked := it.subIt.Contains(val)
it.lastCheck = checked
it.Last = val
it.result = val
return true
}
@ -111,7 +127,7 @@ func (it *Optional) DebugString(indent int) string {
return fmt.Sprintf("%s(%s tags:%s\n%s)",
strings.Repeat(" ", indent),
it.Type(),
it.Tags(),
it.tags.Tags(),
it.subIt.DebugString(indent+4))
}
@ -130,8 +146,13 @@ func (it *Optional) Optimize() (graph.Iterator, bool) {
func (it *Optional) Stats() graph.IteratorStats {
subStats := it.subIt.Stats()
return graph.IteratorStats{
CheckCost: subStats.CheckCost,
NextCost: int64(1 << 62),
Size: subStats.Size,
ContainsCost: subStats.ContainsCost,
NextCost: int64(1 << 62),
Size: subStats.Size,
}
}
// If you're empty and you know it, clap your hands.
func (it *Optional) Size() (int64, bool) {
return 0, true
}

86
graph/iterator/or_iterator.go
View file

@ -29,29 +29,34 @@ import (
)
type Or struct {
Base
uid uint64
tags graph.Tagger
isShortCircuiting bool
internalIterators []graph.Iterator
itCount int
currentIterator int
result graph.Value
}
func NewOr() *Or {
var or Or
BaseInit(&or.Base)
or.internalIterators = make([]graph.Iterator, 0, 20)
or.isShortCircuiting = false
or.currentIterator = -1
return &or
return &Or{
uid: NextUID(),
internalIterators: make([]graph.Iterator, 0, 20),
currentIterator: -1,
}
}
func NewShortCircuitOr() *Or {
var or Or
BaseInit(&or.Base)
or.internalIterators = make([]graph.Iterator, 0, 20)
or.isShortCircuiting = true
or.currentIterator = -1
return &or
return &Or{
uid: NextUID(),
internalIterators: make([]graph.Iterator, 0, 20),
isShortCircuiting: true,
currentIterator: -1,
}
}
func (it *Or) UID() uint64 {
return it.uid
}
// Reset all internal iterators
@ -62,6 +67,10 @@ func (it *Or) Reset() {
it.currentIterator = -1
}
func (it *Or) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Or) Clone() graph.Iterator {
var or *Or
if it.isShortCircuiting {
@ -72,7 +81,7 @@ func (it *Or) Clone() graph.Iterator {
for _, sub := range it.internalIterators {
or.AddSubIterator(sub.Clone())
}
or.CopyTagsFrom(it)
or.tags.CopyFrom(it)
return or
}
@ -84,7 +93,14 @@ func (it *Or) SubIterators() []graph.Iterator {
// Overrides BaseIterator TagResults, as it needs to add it's own results and
// recurse down it's subiterators.
func (it *Or) TagResults(dst map[string]graph.Value) {
it.Base.TagResults(dst)
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
it.internalIterators[it.currentIterator].TagResults(dst)
}
@ -105,7 +121,7 @@ func (it *Or) DebugString(indent int) string {
total += fmt.Sprintf("%d:\n%s\n", i, sub.DebugString(indent+4))
}
var tags string
for _, k := range it.Tags() {
for _, k := range it.tags.Tags() {
tags += fmt.Sprintf("%s;", k)
}
spaces := strings.Repeat(" ", indent+2)
@ -139,7 +155,7 @@ func (it *Or) Next() (graph.Value, bool) {
firstTime = true
}
curIt := it.internalIterators[it.currentIterator]
curr, exists = curIt.Next()
curr, exists = graph.Next(curIt)
if !exists {
if it.isShortCircuiting && !firstTime {
return graph.NextLogOut(it, nil, false)
@ -149,18 +165,22 @@ func (it *Or) Next() (graph.Value, bool) {
return graph.NextLogOut(it, nil, false)
}
} else {
it.Last = curr
it.result = curr
return graph.NextLogOut(it, curr, true)
}
}
panic("Somehow broke out of Next() loop in Or")
panic("unreachable")
}
func (it *Or) Result() graph.Value {
return it.result
}
// Checks a value against the iterators, in order.
func (it *Or) checkSubIts(val graph.Value) bool {
func (it *Or) subItsContain(val graph.Value) bool {
var subIsGood = false
for i, sub := range it.internalIterators {
subIsGood = sub.Check(val)
subIsGood = sub.Contains(val)
if subIsGood {
it.currentIterator = i
break
@ -170,14 +190,14 @@ func (it *Or) checkSubIts(val graph.Value) bool {
}
// Check a value against the entire graph.iterator, in order.
func (it *Or) Check(val graph.Value) bool {
graph.CheckLogIn(it, val)
anyGood := it.checkSubIts(val)
func (it *Or) Contains(val graph.Value) bool {
graph.ContainsLogIn(it, val)
anyGood := it.subItsContain(val)
if !anyGood {
return graph.CheckLogOut(it, val, false)
return graph.ContainsLogOut(it, val, false)
}
it.Last = val
return graph.CheckLogOut(it, val, true)
it.result = val
return graph.ContainsLogOut(it, val, true)
}
// Returns the approximate size of the Or graph.iterator. Because we're dealing
@ -247,7 +267,7 @@ func (it *Or) Optimize() (graph.Iterator, bool) {
}
// Move the tags hanging on us (like any good replacement).
newOr.CopyTagsFrom(it)
newOr.tags.CopyFrom(it)
// And close ourselves but not our subiterators -- some may still be alive in
// the new And (they were unchanged upon calling Optimize() on them, at the
@ -257,13 +277,13 @@ func (it *Or) Optimize() (graph.Iterator, bool) {
}
func (it *Or) Stats() graph.IteratorStats {
CheckCost := int64(0)
ContainsCost := int64(0)
NextCost := int64(0)
Size := int64(0)
for _, sub := range it.internalIterators {
stats := sub.Stats()
NextCost += stats.NextCost
CheckCost += stats.CheckCost
ContainsCost += stats.ContainsCost
if it.isShortCircuiting {
if Size < stats.Size {
Size = stats.Size
@ -273,9 +293,9 @@ func (it *Or) Stats() graph.IteratorStats {
}
}
return graph.IteratorStats{
CheckCost: CheckCost,
NextCost: NextCost,
Size: Size,
ContainsCost: ContainsCost,
NextCost: NextCost,
Size: Size,
}
}

10
graph/iterator/or_iterator_test.go
View file

@ -24,7 +24,7 @@ import (
func iterated(it graph.Iterator) []int {
var res []int
for {
val, ok := it.Next()
val, ok := graph.Next(it)
if !ok {
break
}
@ -66,13 +66,13 @@ func TestOrIteratorBasics(t *testing.T) {
}
for _, v := range []int{2, 3, 21} {
if !or.Check(v) {
if !or.Contains(v) {
t.Errorf("Failed to correctly check %d as true", v)
}
}
for _, v := range []int{22, 5, 0} {
if or.Check(v) {
if or.Contains(v) {
t.Errorf("Failed to correctly check %d as false", v)
}
}
@ -125,12 +125,12 @@ func TestShortCircuitingOrBasics(t *testing.T) {
or.AddSubIterator(f1)
or.AddSubIterator(f2)
for _, v := range []int{2, 3, 21} {
if !or.Check(v) {
if !or.Contains(v) {
t.Errorf("Failed to correctly check %d as true", v)
}
}
for _, v := range []int{22, 5, 0} {
if or.Check(v) {
if or.Contains(v) {
t.Errorf("Failed to correctly check %d as false", v)
}
}

19
graph/iterator/query_shape.go
View file

@ -16,6 +16,7 @@ package iterator
import (
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
type Node struct {
@ -39,7 +40,7 @@ type queryShape struct {
ts graph.TripleStore
nodeId int
hasaIds []int
hasaDirs []graph.Direction
hasaDirs []quad.Direction
}
func OutputQueryShapeForIterator(it graph.Iterator, ts graph.TripleStore, outputMap map[string]interface{}) {
@ -62,11 +63,11 @@ func (qs *queryShape) AddLink(l *Link) {
qs.links = append(qs.links, *l)
}
func (qs *queryShape) LastHasa() (int, graph.Direction) {
func (qs *queryShape) LastHasa() (int, quad.Direction) {
return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1]
}
func (qs *queryShape) PushHasa(i int, d graph.Direction) {
func (qs *queryShape) PushHasa(i int, d quad.Direction) {
qs.hasaIds = append(qs.hasaIds, i)
qs.hasaDirs = append(qs.hasaDirs, d)
}
@ -107,10 +108,10 @@ func (qs *queryShape) StealNode(left *Node, right *Node) {
func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
n := Node{Id: qs.nodeId}
for _, tag := range it.Tags() {
for _, tag := range it.Tagger().Tags() {
n.Tags = append(n.Tags, tag)
}
for k, _ := range it.FixedTags() {
for k, _ := range it.Tagger().Fixed() {
n.Tags = append(n.Tags, k)
}
@ -129,7 +130,7 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
case graph.Fixed:
n.IsFixed = true
for {
val, more := it.Next()
val, more := graph.Next(it)
if !more {
break
}
@ -159,10 +160,10 @@ func (qs *queryShape) MakeNode(it graph.Iterator) *Node {
qs.nodeId++
newNode := qs.MakeNode(lto.primaryIt)
hasaID, hasaDir := qs.LastHasa()
if (hasaDir == graph.Subject && lto.dir == graph.Object) ||
(hasaDir == graph.Object && lto.dir == graph.Subject) {
if (hasaDir == quad.Subject && lto.dir == quad.Object) ||
(hasaDir == quad.Object && lto.dir == quad.Subject) {
qs.AddNode(newNode)
if hasaDir == graph.Subject {
if hasaDir == quad.Subject {
qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id})
} else {
qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id})

21
graph/iterator/query_shape_test.go
View file

@ -19,6 +19,7 @@ import (
"testing"
"github.com/google/cayley/graph"
"github.com/google/cayley/quad"
)
func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA {
@ -26,14 +27,14 @@ func hasaWithTag(ts graph.TripleStore, tag string, target string) *HasA {
obj := ts.FixedIterator()
obj.Add(ts.ValueOf(target))
obj.AddTag(tag)
and.AddSubIterator(NewLinksTo(ts, obj, graph.Object))
obj.Tagger().Add(tag)
and.AddSubIterator(NewLinksTo(ts, obj, quad.Object))
pred := ts.FixedIterator()
pred.Add(ts.ValueOf("status"))
and.AddSubIterator(NewLinksTo(ts, pred, graph.Predicate))
and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate))
return NewHasA(ts, and, graph.Subject)
return NewHasA(ts, and, quad.Subject)
}
func TestQueryShape(t *testing.T) {
@ -48,7 +49,7 @@ func TestQueryShape(t *testing.T) {
// Given a single linkage iterator's shape.
hasa := hasaWithTag(ts, "tag", "cool")
hasa.AddTag("top")
hasa.Tagger().Add("top")
shape := make(map[string]interface{})
OutputQueryShapeForIterator(hasa, ts, shape)
@ -93,22 +94,22 @@ func TestQueryShape(t *testing.T) {
andInternal := NewAnd()
hasa1 := hasaWithTag(ts, "tag1", "cool")
hasa1.AddTag("hasa1")
hasa1.Tagger().Add("hasa1")
andInternal.AddSubIterator(hasa1)
hasa2 := hasaWithTag(ts, "tag2", "fun")
hasa2.AddTag("hasa2")
hasa2.Tagger().Add("hasa2")
andInternal.AddSubIterator(hasa2)
pred := ts.FixedIterator()
pred.Add(ts.ValueOf("name"))
and := NewAnd()
and.AddSubIterator(NewLinksTo(ts, andInternal, graph.Subject))
and.AddSubIterator(NewLinksTo(ts, pred, graph.Predicate))
and.AddSubIterator(NewLinksTo(ts, andInternal, quad.Subject))
and.AddSubIterator(NewLinksTo(ts, pred, quad.Predicate))
shape = make(map[string]interface{})
OutputQueryShapeForIterator(NewHasA(ts, and, graph.Object), ts, shape)
OutputQueryShapeForIterator(NewHasA(ts, and, quad.Object), ts, shape)
links = shape["links"].([]Link)
if len(links) != 3 {

75
graph/iterator/value_comparison_iterator.go
View file

@ -17,7 +17,7 @@ package iterator
// "Value Comparison" is a unary operator -- a filter across the values in the
// relevant subiterator.
//
// This is hugely useful for things like provenance, but value ranges in general
// This is hugely useful for things like label, but value ranges in general
// come up from time to time. At *worst* we're as big as our underlying iterator.
// At best, we're the null iterator.
//
@ -46,21 +46,27 @@ const (
)
type Comparison struct {
Base
subIt graph.Iterator
op Operator
val interface{}
ts graph.TripleStore
uid uint64
tags graph.Tagger
subIt graph.Iterator
op Operator
val interface{}
ts graph.TripleStore
result graph.Value
}
func NewComparison(sub graph.Iterator, op Operator, val interface{}, ts graph.TripleStore) *Comparison {
var vc Comparison
BaseInit(&vc.Base)
vc.subIt = sub
vc.op = op
vc.val = val
vc.ts = ts
return &vc
return &Comparison{
uid: NextUID(),
subIt: sub,
op: op,
val: val,
ts: ts,
}
}
func (it *Comparison) UID() uint64 {
return it.uid
}
// Here's the non-boilerplate part of the ValueComparison iterator. Given a value
@ -111,9 +117,13 @@ func (it *Comparison) Reset() {
it.subIt.Reset()
}
func (it *Comparison) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Comparison) Clone() graph.Iterator {
out := NewComparison(it.subIt.Clone(), it.op, it.val, it.ts)
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
@ -121,7 +131,7 @@ func (it *Comparison) Next() (graph.Value, bool) {
var val graph.Value
var ok bool
for {
val, ok = it.subIt.Next()
val, ok = graph.Next(it.subIt)
if !ok {
return nil, false
}
@ -129,10 +139,19 @@ func (it *Comparison) Next() (graph.Value, bool) {
break
}
}
it.Last = val
it.result = val
return val, ok
}
// DEPRECATED
func (it *Comparison) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Comparison) Result() graph.Value {
return it.result
}
func (it *Comparison) NextResult() bool {
for {
hasNext := it.subIt.NextResult()
@ -143,21 +162,33 @@ func (it *Comparison) NextResult() bool {
return true
}
}
it.Last = it.subIt.Result()
it.result = it.subIt.Result()
return true
}
func (it *Comparison) Check(val graph.Value) bool {
// No subiterators.
func (it *Comparison) SubIterators() []graph.Iterator {
return nil
}
func (it *Comparison) Contains(val graph.Value) bool {
if !it.doComparison(val) {
return false
}
return it.subIt.Check(val)
return it.subIt.Contains(val)
}
// If we failed the check, then the subiterator should not contribute to the result
// set. Otherwise, go ahead and tag it.
func (it *Comparison) TagResults(dst map[string]graph.Value) {
it.Base.TagResults(dst)
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
it.subIt.TagResults(dst)
}
@ -188,3 +219,7 @@ func (it *Comparison) Optimize() (graph.Iterator, bool) {
func (it *Comparison) Stats() graph.IteratorStats {
return it.subIt.Stats()
}
func (it *Comparison) Size() (int64, bool) {
return 0, true
}

8
graph/iterator/value_comparison_iterator_test.go
View file

@ -82,7 +82,7 @@ func TestValueComparison(t *testing.T) {
}
}
var vciCheckTests = []struct {
var vciContainsTests = []struct {
message string
operator Operator
check graph.Value
@ -114,10 +114,10 @@ var vciCheckTests = []struct {
},
}
func TestVCICheck(t *testing.T) {
for _, test := range vciCheckTests {
func TestVCIContains(t *testing.T) {
for _, test := range vciContainsTests {
vc := NewComparison(simpleFixedIterator(), test.operator, int64(2), simpleStore)
if vc.Check(test.check) != test.expect {
if vc.Contains(test.check) != test.expect {
t.Errorf("Failed to show %s", test.message)
}
}

88
graph/leveldb/all_iterator.go
View file

@ -24,36 +24,51 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
type AllIterator struct {
iterator.Base
uid uint64
tags graph.Tagger
prefix []byte
dir graph.Direction
dir quad.Direction
open bool
iter ldbit.Iterator
ts *TripleStore
ro *opt.ReadOptions
result graph.Value
}
func NewAllIterator(prefix string, d graph.Direction, ts *TripleStore) *AllIterator {
var it AllIterator
iterator.BaseInit(&it.Base)
it.ro = &opt.ReadOptions{}
it.ro.DontFillCache = true
it.iter = ts.db.NewIterator(nil, it.ro)
it.prefix = []byte(prefix)
it.dir = d
it.open = true
it.ts = ts
func NewAllIterator(prefix string, d quad.Direction, ts *TripleStore) *AllIterator {
opts := &opt.ReadOptions{
DontFillCache: true,
}
it := AllIterator{
uid: iterator.NextUID(),
ro: opts,
iter: ts.db.NewIterator(nil, opts),
prefix: []byte(prefix),
dir: d,
open: true,
ts: ts,
}
it.iter.Seek(it.prefix)
if !it.iter.Valid() {
// FIXME(kortschak) What are the semantics here? Is this iterator usable?
// If not, we should return nil *Iterator and an error.
it.open = false
it.iter.Release()
}
return &it
}
func (it *AllIterator) UID() uint64 {
return it.uid
}
func (it *AllIterator) Reset() {
if !it.open {
it.iter = it.ts.db.NewIterator(nil, it.ro)
@ -66,15 +81,29 @@ func (it *AllIterator) Reset() {
}
}
func (it *AllIterator) Tagger() *graph.Tagger {
return &it.tags
}
func (it *AllIterator) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
func (it *AllIterator) Clone() graph.Iterator {
out := NewAllIterator(string(it.prefix), it.dir, it.ts)
out.CopyTagsFrom(it)
out.tags.CopyFrom(it)
return out
}
func (it *AllIterator) Next() (graph.Value, bool) {
if !it.open {
it.Last = nil
it.result = nil
return nil, false
}
var out []byte
@ -88,12 +117,29 @@ func (it *AllIterator) Next() (graph.Value, bool) {
it.Close()
return nil, false
}
it.Last = out
it.result = out
return out, true
}
func (it *AllIterator) Check(v graph.Value) bool {
it.Last = v
func (it *AllIterator) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *AllIterator) Result() graph.Value {
return it.result
}
func (it *AllIterator) NextResult() bool {
return false
}
// No subiterators.
func (it *AllIterator) SubIterators() []graph.Iterator {
return nil
}
func (it *AllIterator) Contains(v graph.Value) bool {
it.result = v
return true
}
@ -115,7 +161,7 @@ func (it *AllIterator) Size() (int64, bool) {
func (it *AllIterator) DebugString(indent int) string {
size, _ := it.Size()
return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.dir, it)
return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags(), size, it.dir, it)
}
func (it *AllIterator) Type() graph.Type { return graph.All }
@ -128,8 +174,8 @@ func (it *AllIterator) Optimize() (graph.Iterator, bool) {
func (it *AllIterator) Stats() graph.IteratorStats {
s, _ := it.Size()
return graph.IteratorStats{
CheckCost: 1,
NextCost: 2,
Size: s,
ContainsCost: 1,
NextCost: 2,
Size: s,
}
}

169
graph/leveldb/iterator.go
View file

@ -24,45 +24,63 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
type Iterator struct {
iterator.Base
uid uint64
tags graph.Tagger
nextPrefix []byte
checkId []byte
dir graph.Direction
dir quad.Direction
open bool
iter ldbit.Iterator
ts *TripleStore
qs *TripleStore
ro *opt.ReadOptions
originalPrefix string
result graph.Value
}
func NewIterator(prefix string, d graph.Direction, value graph.Value, ts *TripleStore) *Iterator {
var it Iterator
iterator.BaseInit(&it.Base)
it.checkId = value.([]byte)
it.dir = d
it.originalPrefix = prefix
it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size())
it.nextPrefix = append(it.nextPrefix, []byte(prefix)...)
it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...)
it.ro = &opt.ReadOptions{}
it.ro.DontFillCache = true
it.iter = ts.db.NewIterator(nil, it.ro)
it.open = true
it.ts = ts
func NewIterator(prefix string, d quad.Direction, value graph.Value, qs *TripleStore) *Iterator {
vb := value.([]byte)
p := make([]byte, 0, 2+qs.hasher.Size())
p = append(p, []byte(prefix)...)
p = append(p, []byte(vb[1:])...)
opts := &opt.ReadOptions{
DontFillCache: true,
}
it := Iterator{
uid: iterator.NextUID(),
nextPrefix: p,
checkId: vb,
dir: d,
originalPrefix: prefix,
ro: opts,
iter: qs.db.NewIterator(nil, opts),
open: true,
qs: qs,
}
ok := it.iter.Seek(it.nextPrefix)
if !ok {
// FIXME(kortschak) What are the semantics here? Is this iterator usable?
// If not, we should return nil *Iterator and an error.
it.open = false
it.iter.Release()
}
return &it
}
func (it *Iterator) UID() uint64 {
return it.uid
}
func (it *Iterator) Reset() {
if !it.open {
it.iter = it.ts.db.NewIterator(nil, it.ro)
it.iter = it.qs.db.NewIterator(nil, it.ro)
it.open = true
}
ok := it.iter.Seek(it.nextPrefix)
@ -72,9 +90,23 @@ func (it *Iterator) Reset() {
}
}
func (it *Iterator) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Iterator) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
func (it *Iterator) Clone() graph.Iterator {
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.ts)
out.CopyTagsFrom(it)
out := NewIterator(it.originalPrefix, it.dir, it.checkId, it.qs)
out.tags.CopyFrom(it)
return out
}
@ -87,22 +119,22 @@ func (it *Iterator) Close() {
func (it *Iterator) Next() (graph.Value, bool) {
if it.iter == nil {
it.Last = nil
it.result = nil
return nil, false
}
if !it.open {
it.Last = nil
it.result = nil
return nil, false
}
if !it.iter.Valid() {
it.Last = nil
it.result = nil
it.Close()
return nil, false
}
if bytes.HasPrefix(it.iter.Key(), it.nextPrefix) {
out := make([]byte, len(it.iter.Key()))
copy(out, it.iter.Key())
it.Last = out
it.result = out
ok := it.iter.Next()
if !ok {
it.Close()
@ -110,75 +142,92 @@ func (it *Iterator) Next() (graph.Value, bool) {
return out, true
}
it.Close()
it.Last = nil
it.result = nil
return nil, false
}
func PositionOf(prefix []byte, d graph.Direction, ts *TripleStore) int {
func (it *Iterator) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Iterator) Result() graph.Value {
return it.result
}
func (it *Iterator) NextResult() bool {
return false
}
// No subiterators.
func (it *Iterator) SubIterators() []graph.Iterator {
return nil
}
func PositionOf(prefix []byte, d quad.Direction, qs *TripleStore) int {
if bytes.Equal(prefix, []byte("sp")) {
switch d {
case graph.Subject:
case quad.Subject:
return 2
case graph.Predicate:
return ts.hasher.Size() + 2
case graph.Object:
return 2*ts.hasher.Size() + 2
case graph.Provenance:
case quad.Predicate:
return qs.hasher.Size() + 2
case quad.Object:
return 2*qs.hasher.Size() + 2
case quad.Label:
return -1
}
}
if bytes.Equal(prefix, []byte("po")) {
switch d {
case graph.Subject:
return 2*ts.hasher.Size() + 2
case graph.Predicate:
case quad.Subject:
return 2*qs.hasher.Size() + 2
case quad.Predicate:
return 2
case graph.Object:
return ts.hasher.Size() + 2
case graph.Provenance:
case quad.Object:
return qs.hasher.Size() + 2
case quad.Label:
return -1
}
}
if bytes.Equal(prefix, []byte("os")) {
switch d {
case graph.Subject:
return ts.hasher.Size() + 2
case graph.Predicate:
return 2*ts.hasher.Size() + 2
case graph.Object:
case quad.Subject:
return qs.hasher.Size() + 2
case quad.Predicate:
return 2*qs.hasher.Size() + 2
case quad.Object:
return 2
case graph.Provenance:
case quad.Label:
return -1
}
}
if bytes.Equal(prefix, []byte("cp")) {
switch d {
case graph.Subject:
return 2*ts.hasher.Size() + 2
case graph.Predicate:
return ts.hasher.Size() + 2
case graph.Object:
return 3*ts.hasher.Size() + 2
case graph.Provenance:
case quad.Subject:
return 2*qs.hasher.Size() + 2
case quad.Predicate:
return qs.hasher.Size() + 2
case quad.Object:
return 3*qs.hasher.Size() + 2
case quad.Label:
return 2
}
}
panic("unreachable")
}
func (it *Iterator) Check(v graph.Value) bool {
func (it *Iterator) Contains(v graph.Value) bool {
val := v.([]byte)
if val[0] == 'z' {
return false
}
offset := PositionOf(val[0:2], it.dir, it.ts)
offset := PositionOf(val[0:2], it.dir, it.qs)
if offset != -1 {
if bytes.HasPrefix(val[offset:], it.checkId[1:]) {
return true
}
} else {
nameForDir := it.ts.Triple(v).Get(it.dir)
hashForDir := it.ts.ValueOf(nameForDir).([]byte)
nameForDir := it.qs.Quad(v).Get(it.dir)
hashForDir := it.qs.ValueOf(nameForDir).([]byte)
if bytes.Equal(hashForDir, it.checkId) {
return true
}
@ -187,12 +236,12 @@ func (it *Iterator) Check(v graph.Value) bool {
}
func (it *Iterator) Size() (int64, bool) {
return it.ts.SizeOf(it.checkId), true
return it.qs.SizeOf(it.checkId), true
}
func (it *Iterator) DebugString(indent int) string {
size, _ := it.Size()
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.Tags(), it.dir, size, it.ts.NameOf(it.checkId))
return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.UID(), it.tags.Tags(), it.dir, size, it.qs.NameOf(it.checkId))
}
var levelDBType graph.Type
@ -213,8 +262,8 @@ func (it *Iterator) Optimize() (graph.Iterator, bool) {
func (it *Iterator) Stats() graph.IteratorStats {
s, _ := it.Size()
return graph.IteratorStats{
CheckCost: 1,
NextCost: 2,
Size: s,
ContainsCost: 1,
NextCost: 2,
Size: s,
}
}

161
graph/leveldb/leveldb_test.go
View file

@ -23,10 +23,11 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
func makeTripleSet() []*graph.Triple {
tripleSet := []*graph.Triple{
func makeTripleSet() []*quad.Quad {
tripleSet := []*quad.Quad{
{"A", "follows", "B", ""},
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
@ -42,20 +43,20 @@ func makeTripleSet() []*graph.Triple {
return tripleSet
}
func iteratedTriples(ts graph.TripleStore, it graph.Iterator) []*graph.Triple {
func iteratedTriples(qs graph.TripleStore, it graph.Iterator) []*quad.Quad {
var res ordered
for {
val, ok := it.Next()
val, ok := graph.Next(it)
if !ok {
break
}
res = append(res, ts.Triple(val))
res = append(res, qs.Quad(val))
}
sort.Sort(res)
return res
}
type ordered []*graph.Triple
type ordered []*quad.Quad
func (o ordered) Len() int { return len(o) }
func (o ordered) Less(i, j int) bool {
@ -72,7 +73,7 @@ func (o ordered) Less(i, j int) bool {
o[i].Subject == o[j].Subject &&
o[i].Predicate == o[j].Predicate &&
o[i].Object == o[j].Object &&
o[i].Provenance < o[j].Provenance:
o[i].Label < o[j].Label:
return true
@ -82,14 +83,14 @@ func (o ordered) Less(i, j int) bool {
}
func (o ordered) Swap(i, j int) { o[i], o[j] = o[j], o[i] }
func iteratedNames(ts graph.TripleStore, it graph.Iterator) []string {
func iteratedNames(qs graph.TripleStore, it graph.Iterator) []string {
var res []string
for {
val, ok := it.Next()
val, ok := graph.Next(it)
if !ok {
break
}
res = append(res, ts.NameOf(val))
res = append(res, qs.NameOf(val))
}
sort.Strings(res)
return res
@ -107,14 +108,14 @@ func TestCreateDatabase(t *testing.T) {
t.Fatal("Failed to create LevelDB database.")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
if s := ts.Size(); s != 0 {
if s := qs.Size(); s != 0 {
t.Errorf("Unexpected size, got:%d expected:0", s)
}
ts.Close()
qs.Close()
err = createNewLevelDB("/dev/null/some terrible path", nil)
if err == nil {
@ -137,53 +138,53 @@ func TestLoadDatabase(t *testing.T) {
t.Fatal("Failed to create LevelDB database.")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts.AddTriple(&graph.Triple{"Something", "points_to", "Something Else", "context"})
qs.AddTriple(&quad.Quad{"Something", "points_to", "Something Else", "context"})
for _, pq := range []string{"Something", "points_to", "Something Else", "context"} {
if got := ts.NameOf(ts.ValueOf(pq)); got != pq {
if got := qs.NameOf(qs.ValueOf(pq)); got != pq {
t.Errorf("Failed to roundtrip %q, got:%q expect:%q", pq, got, pq)
}
}
if s := ts.Size(); s != 1 {
if s := qs.Size(); s != 1 {
t.Errorf("Unexpected triplestore size, got:%d expect:1", s)
}
ts.Close()
qs.Close()
err = createNewLevelDB(tmpDir, nil)
if err != nil {
t.Fatal("Failed to create LevelDB database.")
}
ts, err = newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err = newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts2, didConvert := ts.(*TripleStore)
ts2, didConvert := qs.(*TripleStore)
if !didConvert {
t.Errorf("Could not convert from generic to LevelDB TripleStore")
}
ts.AddTripleSet(makeTripleSet())
if s := ts.Size(); s != 11 {
qs.AddTripleSet(makeTripleSet())
if s := qs.Size(); s != 11 {
t.Errorf("Unexpected triplestore size, got:%d expect:11", s)
}
if s := ts2.SizeOf(ts.ValueOf("B")); s != 5 {
if s := ts2.SizeOf(qs.ValueOf("B")); s != 5 {
t.Errorf("Unexpected triplestore size, got:%d expect:5", s)
}
ts.RemoveTriple(&graph.Triple{"A", "follows", "B", ""})
if s := ts.Size(); s != 10 {
qs.RemoveTriple(&quad.Quad{"A", "follows", "B", ""})
if s := qs.Size(); s != 10 {
t.Errorf("Unexpected triplestore size after RemoveTriple, got:%d expect:10", s)
}
if s := ts2.SizeOf(ts.ValueOf("B")); s != 4 {
if s := ts2.SizeOf(qs.ValueOf("B")); s != 4 {
t.Errorf("Unexpected triplestore size, got:%d expect:4", s)
}
ts.Close()
qs.Close()
}
func TestIterator(t *testing.T) {
@ -199,14 +200,14 @@ func TestIterator(t *testing.T) {
t.Fatal("Failed to create LevelDB database.")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts.AddTripleSet(makeTripleSet())
qs.AddTripleSet(makeTripleSet())
var it graph.Iterator
it = ts.NodesAllIterator()
it = qs.NodesAllIterator()
if it == nil {
t.Fatal("Got nil iterator.")
}
@ -241,7 +242,7 @@ func TestIterator(t *testing.T) {
}
sort.Strings(expect)
for i := 0; i < 2; i++ {
got := iteratedNames(ts, it)
got := iteratedNames(qs, it)
sort.Strings(got)
if !reflect.DeepEqual(got, expect) {
t.Errorf("Unexpected iterated result on repeat %d, got:%v expect:%v", i, got, expect)
@ -250,23 +251,23 @@ func TestIterator(t *testing.T) {
}
for _, pq := range expect {
if !it.Check(ts.ValueOf(pq)) {
if !it.Contains(qs.ValueOf(pq)) {
t.Errorf("Failed to find and check %q correctly", pq)
}
}
// FIXME(kortschak) Why does this fail?
/*
for _, pq := range []string{"baller"} {
if it.Check(ts.ValueOf(pq)) {
if it.Contains(qs.ValueOf(pq)) {
t.Errorf("Failed to check %q correctly", pq)
}
}
*/
it.Reset()
it = ts.TriplesAllIterator()
edge, _ := it.Next()
triple := ts.Triple(edge)
it = qs.TriplesAllIterator()
edge, _ := graph.Next(it)
triple := qs.Quad(edge)
set := makeTripleSet()
var ok bool
for _, t := range set {
@ -279,7 +280,7 @@ func TestIterator(t *testing.T) {
t.Errorf("Failed to find %q during iteration, got:%q", triple, set)
}
ts.Close()
qs.Close()
}
func TestSetIterator(t *testing.T) {
@ -292,95 +293,95 @@ func TestSetIterator(t *testing.T) {
t.Fatalf("Failed to create working directory")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
defer ts.Close()
defer qs.Close()
ts.AddTripleSet(makeTripleSet())
qs.AddTripleSet(makeTripleSet())
expect := []*graph.Triple{
expect := []*quad.Quad{
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
}
sort.Sort(ordered(expect))
// Subject iterator.
it := ts.TripleIterator(graph.Subject, ts.ValueOf("C"))
it := qs.TripleIterator(quad.Subject, qs.ValueOf("C"))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
}
it.Reset()
and := iterator.NewAnd()
and.AddSubIterator(ts.TriplesAllIterator())
and.AddSubIterator(qs.TriplesAllIterator())
and.AddSubIterator(it)
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
// Object iterator.
it = ts.TripleIterator(graph.Object, ts.ValueOf("F"))
it = qs.TripleIterator(quad.Object, qs.ValueOf("F"))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "follows", "F", ""},
{"E", "follows", "F", ""},
}
sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results, got:%v expect:%v", got, expect)
}
and = iterator.NewAnd()
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B")))
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it)
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "follows", "F", ""},
}
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
// Predicate iterator.
it = ts.TripleIterator(graph.Predicate, ts.ValueOf("status"))
it = qs.TripleIterator(quad.Predicate, qs.ValueOf("status"))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
}
sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect)
}
// Provenance iterator.
it = ts.TripleIterator(graph.Provenance, ts.ValueOf("status_graph"))
// Label iterator.
it = qs.TripleIterator(quad.Label, qs.ValueOf("status_graph"))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
{"D", "status", "cool", "status_graph"},
{"G", "status", "cool", "status_graph"},
}
sort.Sort(ordered(expect))
if got := iteratedTriples(ts, it); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, it); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get expected results from predicate iterator, got:%v expect:%v", got, expect)
}
it.Reset()
// Order is important
and = iterator.NewAnd()
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B")))
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
and.AddSubIterator(it)
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
}
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
it.Reset()
@ -388,12 +389,12 @@ func TestSetIterator(t *testing.T) {
// Order is important
and = iterator.NewAnd()
and.AddSubIterator(it)
and.AddSubIterator(ts.TripleIterator(graph.Subject, ts.ValueOf("B")))
and.AddSubIterator(qs.TripleIterator(quad.Subject, qs.ValueOf("B")))
expect = []*graph.Triple{
expect = []*quad.Quad{
{"B", "status", "cool", "status_graph"},
}
if got := iteratedTriples(ts, and); !reflect.DeepEqual(got, expect) {
if got := iteratedTriples(qs, and); !reflect.DeepEqual(got, expect) {
t.Errorf("Failed to get confirm expected results, got:%v expect:%v", got, expect)
}
}
@ -406,17 +407,17 @@ func TestOptimize(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create working directory")
}
ts, err := newTripleStore(tmpDir, nil)
if ts == nil || err != nil {
qs, err := newTripleStore(tmpDir, nil)
if qs == nil || err != nil {
t.Error("Failed to create leveldb TripleStore.")
}
ts.AddTripleSet(makeTripleSet())
qs.AddTripleSet(makeTripleSet())
// With an linksto-fixed pair
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("F"))
fixed.AddTag("internal")
lto := iterator.NewLinksTo(ts, fixed, graph.Object)
fixed := qs.FixedIterator()
fixed.Add(qs.ValueOf("F"))
fixed.Tagger().Add("internal")
lto := iterator.NewLinksTo(qs, fixed, quad.Object)
oldIt := lto.Clone()
newIt, ok := lto.Optimize()
@ -427,16 +428,16 @@ func TestOptimize(t *testing.T) {
t.Errorf("Optimized iterator type does not match original, got:%v expect:%v", newIt.Type(), Type())
}
newTriples := iteratedTriples(ts, newIt)
oldTriples := iteratedTriples(ts, oldIt)
newTriples := iteratedTriples(qs, newIt)
oldTriples := iteratedTriples(qs, oldIt)
if !reflect.DeepEqual(newTriples, oldTriples) {
t.Errorf("Optimized iteration does not match original")
}
oldIt.Next()
graph.Next(oldIt)
oldResults := make(map[string]graph.Value)
oldIt.TagResults(oldResults)
newIt.Next()
graph.Next(newIt)
newResults := make(map[string]graph.Value)
newIt.TagResults(newResults)
if !reflect.DeepEqual(newResults, oldResults) {

293
graph/leveldb/triplestore.go
View file

File diff suppressed because it is too large Load diff

9
graph/leveldb/triplestore_iterator_optimize.go
View file

@ -37,14 +37,15 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
if primary.Type() == graph.Fixed {
size, _ := primary.Size()
if size == 1 {
val, ok := primary.Next()
val, ok := graph.Next(primary)
if !ok {
panic("Sizes lie")
}
newIt := ts.TripleIterator(it.Direction(), val)
newIt.CopyTagsFrom(it)
for _, tag := range primary.Tags() {
newIt.AddFixedTag(tag, val)
nt := newIt.Tagger()
nt.CopyFrom(it)
for _, tag := range primary.Tagger().Tags() {
nt.AddFixed(tag, val)
}
it.Close()
return newIt, true

6
graph/memstore/all_iterator.go
View file

@ -31,6 +31,11 @@ func NewMemstoreAllIterator(ts *TripleStore) *AllIterator {
return &out
}
// No subiterators.
func (it *AllIterator) SubIterators() []graph.Iterator {
return nil
}
func (it *AllIterator) Next() (graph.Value, bool) {
next, out := it.Int64.Next()
if !out {
@ -41,6 +46,5 @@ func (it *AllIterator) Next() (graph.Value, bool) {
if !ok {
return it.Next()
}
it.Last = next
return next, out
}

81
graph/memstore/iterator.go
View file

@ -26,11 +26,13 @@ import (
)
type Iterator struct {
iterator.Base
uid uint64
tags graph.Tagger
tree *llrb.LLRB
data string
isRunning bool
iterLast Int64
result graph.Value
}
type Int64 int64
@ -53,52 +55,87 @@ func IterateOne(tree *llrb.LLRB, last Int64) Int64 {
}
func NewLlrbIterator(tree *llrb.LLRB, data string) *Iterator {
var it Iterator
iterator.BaseInit(&it.Base)
it.tree = tree
it.iterLast = Int64(-1)
it.data = data
return &it
return &Iterator{
uid: iterator.NextUID(),
tree: tree,
iterLast: Int64(-1),
data: data,
}
}
func (it *Iterator) UID() uint64 {
return it.uid
}
func (it *Iterator) Reset() {
it.iterLast = Int64(-1)
}
func (it *Iterator) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Iterator) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
func (it *Iterator) Clone() graph.Iterator {
var new_it = NewLlrbIterator(it.tree, it.data)
new_it.CopyTagsFrom(it)
return new_it
m := NewLlrbIterator(it.tree, it.data)
m.tags.CopyFrom(it)
return m
}
func (it *Iterator) Close() {}
func (it *Iterator) Next() (graph.Value, bool) {
graph.NextLogIn(it)
if it.tree.Max() == nil || it.Last == int64(it.tree.Max().(Int64)) {
if it.tree.Max() == nil || it.result == int64(it.tree.Max().(Int64)) {
return graph.NextLogOut(it, nil, false)
}
it.iterLast = IterateOne(it.tree, it.iterLast)
it.Last = int64(it.iterLast)
return graph.NextLogOut(it, it.Last, true)
it.result = int64(it.iterLast)
return graph.NextLogOut(it, it.result, true)
}
func (it *Iterator) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Iterator) Result() graph.Value {
return it.result
}
func (it *Iterator) NextResult() bool {
return false
}
// No subiterators.
func (it *Iterator) SubIterators() []graph.Iterator {
return nil
}
func (it *Iterator) Size() (int64, bool) {
return int64(it.tree.Len()), true
}
func (it *Iterator) Check(v graph.Value) bool {
graph.CheckLogIn(it, v)
func (it *Iterator) Contains(v graph.Value) bool {
graph.ContainsLogIn(it, v)
if it.tree.Has(Int64(v.(int64))) {
it.Last = v
return graph.CheckLogOut(it, v, true)
it.result = v
return graph.ContainsLogOut(it, v, true)
}
return graph.CheckLogOut(it, v, false)
return graph.ContainsLogOut(it, v, false)
}
func (it *Iterator) DebugString(indent int) string {
size, _ := it.Size()
return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data)
return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.tags.Tags(), size, it.data)
}
var memType graph.Type
@ -119,8 +156,8 @@ func (it *Iterator) Optimize() (graph.Iterator, bool) {
func (it *Iterator) Stats() graph.IteratorStats {
return graph.IteratorStats{
CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1,
NextCost: 1,
Size: int64(it.tree.Len()),
ContainsCost: int64(math.Log(float64(it.tree.Len()))) + 1,
NextCost: 1,
Size: int64(it.tree.Len()),
}
}

89
graph/memstore/triplestore.go
View file

@ -20,15 +20,22 @@ import (
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
"github.com/petar/GoLLRB/llrb"
)
func init() {
graph.RegisterTripleStore("memstore", func(string, graph.Options) (graph.TripleStore, error) {
return newTripleStore(), nil
}, nil)
}
type TripleDirectionIndex struct {
subject map[int64]*llrb.LLRB
predicate map[int64]*llrb.LLRB
object map[int64]*llrb.LLRB
provenance map[int64]*llrb.LLRB
subject map[int64]*llrb.LLRB
predicate map[int64]*llrb.LLRB
object map[int64]*llrb.LLRB
label map[int64]*llrb.LLRB
}
func NewTripleDirectionIndex() *TripleDirectionIndex {
@ -36,25 +43,25 @@ func NewTripleDirectionIndex() *TripleDirectionIndex {
tdi.subject = make(map[int64]*llrb.LLRB)
tdi.predicate = make(map[int64]*llrb.LLRB)
tdi.object = make(map[int64]*llrb.LLRB)
tdi.provenance = make(map[int64]*llrb.LLRB)
tdi.label = make(map[int64]*llrb.LLRB)
return &tdi
}
func (tdi *TripleDirectionIndex) GetForDir(d graph.Direction) map[int64]*llrb.LLRB {
func (tdi *TripleDirectionIndex) GetForDir(d quad.Direction) map[int64]*llrb.LLRB {
switch d {
case graph.Subject:
case quad.Subject:
return tdi.subject
case graph.Object:
case quad.Object:
return tdi.object
case graph.Predicate:
case quad.Predicate:
return tdi.predicate
case graph.Provenance:
return tdi.provenance
case quad.Label:
return tdi.label
}
panic("illegal direction")
}
func (tdi *TripleDirectionIndex) GetOrCreate(d graph.Direction, id int64) *llrb.LLRB {
func (tdi *TripleDirectionIndex) GetOrCreate(d quad.Direction, id int64) *llrb.LLRB {
directionIndex := tdi.GetForDir(d)
if _, ok := directionIndex[id]; !ok {
directionIndex[id] = llrb.New()
@ -62,7 +69,7 @@ func (tdi *TripleDirectionIndex) GetOrCreate(d graph.Direction, id int64) *llrb.
return directionIndex[id]
}
func (tdi *TripleDirectionIndex) Get(d graph.Direction, id int64) (*llrb.LLRB, bool) {
func (tdi *TripleDirectionIndex) Get(d quad.Direction, id int64) (*llrb.LLRB, bool) {
directionIndex := tdi.GetForDir(d)
tree, exists := directionIndex[id]
return tree, exists
@ -73,7 +80,7 @@ type TripleStore struct {
tripleIdCounter int64
idMap map[string]int64
revIdMap map[int64]string
triples []graph.Triple
triples []quad.Quad
size int64
index TripleDirectionIndex
// vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree
@ -83,10 +90,10 @@ func newTripleStore() *TripleStore {
var ts TripleStore
ts.idMap = make(map[string]int64)
ts.revIdMap = make(map[int64]string)
ts.triples = make([]graph.Triple, 1, 200)
ts.triples = make([]quad.Quad, 1, 200)
// Sentinel null triple so triple indices start at 1
ts.triples[0] = graph.Triple{}
ts.triples[0] = quad.Quad{}
ts.size = 1
ts.index = *NewTripleDirectionIndex()
ts.idCounter = 1
@ -94,18 +101,18 @@ func newTripleStore() *TripleStore {
return &ts
}
func (ts *TripleStore) AddTripleSet(triples []*graph.Triple) {
func (ts *TripleStore) AddTripleSet(triples []*quad.Quad) {
for _, t := range triples {
ts.AddTriple(t)
}
}
func (ts *TripleStore) tripleExists(t *graph.Triple) (bool, int64) {
func (ts *TripleStore) tripleExists(t *quad.Quad) (bool, int64) {
smallest := -1
var smallest_tree *llrb.LLRB
for d := graph.Subject; d <= graph.Provenance; d++ {
for d := quad.Subject; d <= quad.Label; d++ {
sid := t.Get(d)
if d == graph.Provenance && sid == "" {
if d == quad.Label && sid == "" {
continue
}
id, ok := ts.idMap[sid]
@ -137,7 +144,7 @@ func (ts *TripleStore) tripleExists(t *graph.Triple) (bool, int64) {
return false, 0
}
func (ts *TripleStore) AddTriple(t *graph.Triple) {
func (ts *TripleStore) AddTriple(t *quad.Quad) {
if exists, _ := ts.tripleExists(t); exists {
return
}
@ -147,9 +154,9 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
ts.size++
ts.tripleIdCounter++
for d := graph.Subject; d <= graph.Provenance; d++ {
for d := quad.Subject; d <= quad.Label; d++ {
sid := t.Get(d)
if d == graph.Provenance && sid == "" {
if d == quad.Label && sid == "" {
continue
}
if _, ok := ts.idMap[sid]; !ok {
@ -159,8 +166,8 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
}
}
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Label; d++ {
if d == quad.Label && t.Get(d) == "" {
continue
}
id := ts.idMap[t.Get(d)]
@ -171,7 +178,7 @@ func (ts *TripleStore) AddTriple(t *graph.Triple) {
// TODO(barakmich): Add VIP indexing
}
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
func (ts *TripleStore) RemoveTriple(t *quad.Quad) {
var tripleID int64
var exists bool
tripleID = 0
@ -179,11 +186,11 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
return
}
ts.triples[tripleID] = graph.Triple{}
ts.triples[tripleID] = quad.Quad{}
ts.size--
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Label; d++ {
if d == quad.Label && t.Get(d) == "" {
continue
}
id := ts.idMap[t.Get(d)]
@ -191,8 +198,8 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
tree.Delete(Int64(tripleID))
}
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Label; d++ {
if d == quad.Label && t.Get(d) == "" {
continue
}
id, ok := ts.idMap[t.Get(d)]
@ -200,8 +207,8 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
continue
}
stillExists := false
for d := graph.Subject; d <= graph.Provenance; d++ {
if d == graph.Provenance && t.Get(d) == "" {
for d := quad.Subject; d <= quad.Label; d++ {
if d == quad.Label && t.Get(d) == "" {
continue
}
nodeTree := ts.index.GetOrCreate(d, id)
@ -217,11 +224,11 @@ func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
}
}
func (ts *TripleStore) Triple(index graph.Value) *graph.Triple {
func (ts *TripleStore) Quad(index graph.Value) *quad.Quad {
return &ts.triples[index.(int64)]
}
func (ts *TripleStore) TripleIterator(d graph.Direction, value graph.Value) graph.Iterator {
func (ts *TripleStore) TripleIterator(d quad.Direction, value graph.Value) graph.Iterator {
index, ok := ts.index.Get(d, value.(int64))
data := fmt.Sprintf("dir:%s val:%d", d, value.(int64))
if ok {
@ -239,7 +246,7 @@ func (ts *TripleStore) DebugPrint() {
if i == 0 {
continue
}
glog.V(2).Infoln("%d: %s", i, t)
glog.V(2).Infof("%d: %s", i, t)
}
}
@ -259,8 +266,8 @@ func (ts *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(iterator.BasicEquality)
}
func (ts *TripleStore) TripleDirection(val graph.Value, d graph.Direction) graph.Value {
name := ts.Triple(val).Get(d)
func (ts *TripleStore) TripleDirection(val graph.Value, d quad.Direction) graph.Value {
name := ts.Quad(val).Get(d)
return ts.ValueOf(name)
}
@ -268,9 +275,3 @@ func (ts *TripleStore) NodesAllIterator() graph.Iterator {
return NewMemstoreAllIterator(ts)
}
func (ts *TripleStore) Close() {}
func init() {
graph.RegisterTripleStore("memstore", func(string, graph.Options) (graph.TripleStore, error) {
return newTripleStore(), nil
}, nil)
}

9
graph/memstore/triplestore_iterator_optimize.go
View file

@ -37,14 +37,15 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
if primary.Type() == graph.Fixed {
size, _ := primary.Size()
if size == 1 {
val, ok := primary.Next()
val, ok := graph.Next(primary)
if !ok {
panic("Sizes lie")
}
newIt := ts.TripleIterator(it.Direction(), val)
newIt.CopyTagsFrom(it)
for _, tag := range primary.Tags() {
newIt.AddFixedTag(tag, val)
nt := newIt.Tagger()
nt.CopyFrom(it)
for _, tag := range primary.Tagger().Tags() {
nt.AddFixed(tag, val)
}
return newIt, true
}

30
graph/memstore/triplestore_test.go
View file

@ -21,6 +21,7 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
// This is a simple test graph.
@ -36,7 +37,7 @@ import (
// \-->|#D#|------------->+---+
// +---+
//
var simpleGraph = []*graph.Triple{
var simpleGraph = []*quad.Quad{
{"A", "follows", "B", ""},
{"C", "follows", "B", ""},
{"C", "follows", "D", ""},
@ -50,7 +51,7 @@ var simpleGraph = []*graph.Triple{
{"G", "status", "cool", "status_graph"},
}
func makeTestStore(data []*graph.Triple) (*TripleStore, []pair) {
func makeTestStore(data []*quad.Quad) (*TripleStore, []pair) {
seen := make(map[string]struct{})
ts := newTripleStore()
var (
@ -58,7 +59,7 @@ func makeTestStore(data []*graph.Triple) (*TripleStore, []pair) {
ind []pair
)
for _, t := range data {
for _, qp := range []string{t.Subject, t.Predicate, t.Object, t.Provenance} {
for _, qp := range []string{t.Subject, t.Predicate, t.Object, t.Label} {
if _, ok := seen[qp]; !ok && qp != "" {
val++
ind = append(ind, pair{qp, val})
@ -105,10 +106,10 @@ func TestIteratorsAndNextResultOrderA(t *testing.T) {
all := ts.NodesAllIterator()
innerAnd := iterator.NewAnd()
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, graph.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, all, graph.Object))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, quad.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, all, quad.Object))
hasa := iterator.NewHasA(ts, innerAnd, graph.Subject)
hasa := iterator.NewHasA(ts, innerAnd, quad.Subject)
outerAnd := iterator.NewAnd()
outerAnd.AddSubIterator(fixed)
outerAnd.AddSubIterator(hasa)
@ -149,8 +150,8 @@ func TestLinksToOptimization(t *testing.T) {
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("cool"))
lto := iterator.NewLinksTo(ts, fixed, graph.Object)
lto.AddTag("foo")
lto := iterator.NewLinksTo(ts, fixed, quad.Object)
lto.Tagger().Add("foo")
newIt, changed := lto.Optimize()
if !changed {
@ -165,7 +166,8 @@ func TestLinksToOptimization(t *testing.T) {
if v_clone.DebugString(0) != v.DebugString(0) {
t.Fatal("Wrong iterator. Got ", v_clone.DebugString(0))
}
if len(v_clone.Tags()) < 1 || v_clone.Tags()[0] != "foo" {
vt := v_clone.Tagger()
if len(vt.Tags()) < 1 || vt.Tags()[0] != "foo" {
t.Fatal("Tag on LinksTo did not persist")
}
}
@ -173,7 +175,7 @@ func TestLinksToOptimization(t *testing.T) {
func TestRemoveTriple(t *testing.T) {
ts, _ := makeTestStore(simpleGraph)
ts.RemoveTriple(&graph.Triple{"E", "follows", "F", ""})
ts.RemoveTriple(&quad.Quad{"E", "follows", "F", ""})
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf("E"))
@ -182,13 +184,13 @@ func TestRemoveTriple(t *testing.T) {
fixed2.Add(ts.ValueOf("follows"))
innerAnd := iterator.NewAnd()
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, graph.Subject))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, graph.Predicate))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed, quad.Subject))
innerAnd.AddSubIterator(iterator.NewLinksTo(ts, fixed2, quad.Predicate))
hasa := iterator.NewHasA(ts, innerAnd, graph.Object)
hasa := iterator.NewHasA(ts, innerAnd, quad.Object)
newIt, _ := hasa.Optimize()
_, ok := newIt.Next()
_, ok := graph.Next(newIt)
if ok {
t.Error("E should not have any followers.")
}

174
graph/mongo/iterator.go
View file

@ -24,12 +24,14 @@ import (
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
type Iterator struct {
iterator.Base
ts *TripleStore
dir graph.Direction
uid uint64
tags graph.Tagger
qs *TripleStore
dir quad.Direction
iter *mgo.Iter
hash string
name string
@ -37,60 +39,73 @@ type Iterator struct {
isAll bool
constraint bson.M
collection string
result graph.Value
}
func NewIterator(ts *TripleStore, collection string, d graph.Direction, val graph.Value) *Iterator {
var m Iterator
iterator.BaseInit(&m.Base)
func NewIterator(qs *TripleStore, collection string, d quad.Direction, val graph.Value) *Iterator {
name := qs.NameOf(val)
m.name = ts.NameOf(val)
m.collection = collection
var constraint bson.M
switch d {
case graph.Subject:
m.constraint = bson.M{"Subject": m.name}
case graph.Predicate:
m.constraint = bson.M{"Predicate": m.name}
case graph.Object:
m.constraint = bson.M{"Object": m.name}
case graph.Provenance:
m.constraint = bson.M{"Provenance": m.name}
case quad.Subject:
constraint = bson.M{"Subject": name}
case quad.Predicate:
constraint = bson.M{"Predicate": name}
case quad.Object:
constraint = bson.M{"Object": name}
case quad.Label:
constraint = bson.M{"Label": name}
}
m.ts = ts
m.dir = d
m.iter = ts.db.C(collection).Find(m.constraint).Iter()
size, err := ts.db.C(collection).Find(m.constraint).Count()
size, err := qs.db.C(collection).Find(constraint).Count()
if err != nil {
// FIXME(kortschak) This should be passed back rather than just logging.
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
}
m.size = int64(size)
m.hash = val.(string)
m.isAll = false
return &m
return &Iterator{
uid: iterator.NextUID(),
name: name,
constraint: constraint,
collection: collection,
qs: qs,
dir: d,
iter: qs.db.C(collection).Find(constraint).Iter(),
size: int64(size),
hash: val.(string),
isAll: false,
}
}
func NewAllIterator(ts *TripleStore, collection string) *Iterator {
var m Iterator
m.ts = ts
m.dir = graph.Any
m.constraint = nil
m.collection = collection
m.iter = ts.db.C(collection).Find(nil).Iter()
size, err := ts.db.C(collection).Count()
func NewAllIterator(qs *TripleStore, collection string) *Iterator {
size, err := qs.db.C(collection).Count()
if err != nil {
// FIXME(kortschak) This should be passed back rather than just logging.
glog.Errorln("Trouble getting size for iterator! ", err)
return nil
}
m.size = int64(size)
m.hash = ""
m.isAll = true
return &m
return &Iterator{
uid: iterator.NextUID(),
qs: qs,
dir: quad.Any,
constraint: nil,
collection: collection,
iter: qs.db.C(collection).Find(nil).Iter(),
size: int64(size),
hash: "",
isAll: true,
}
}
func (it *Iterator) UID() uint64 {
return it.uid
}
func (it *Iterator) Reset() {
it.iter.Close()
it.iter = it.ts.db.C(it.collection).Find(it.constraint).Iter()
it.iter = it.qs.db.C(it.collection).Find(it.constraint).Iter()
}
@ -98,15 +113,29 @@ func (it *Iterator) Close() {
it.iter.Close()
}
func (it *Iterator) Clone() graph.Iterator {
var newM graph.Iterator
if it.isAll {
newM = NewAllIterator(it.ts, it.collection)
} else {
newM = NewIterator(it.ts, it.collection, it.dir, it.hash)
func (it *Iterator) Tagger() *graph.Tagger {
return &it.tags
}
func (it *Iterator) TagResults(dst map[string]graph.Value) {
for _, tag := range it.tags.Tags() {
dst[tag] = it.Result()
}
newM.CopyTagsFrom(it)
return newM
for tag, value := range it.tags.Fixed() {
dst[tag] = value
}
}
func (it *Iterator) Clone() graph.Iterator {
var m *Iterator
if it.isAll {
m = NewAllIterator(it.qs, it.collection)
} else {
m = NewIterator(it.qs, it.collection, it.dir, it.hash)
}
m.tags.CopyFrom(it)
return m
}
func (it *Iterator) Next() (graph.Value, bool) {
@ -124,33 +153,50 @@ func (it *Iterator) Next() (graph.Value, bool) {
}
return nil, false
}
it.Last = result.Id
it.result = result.Id
return result.Id, true
}
func (it *Iterator) Check(v graph.Value) bool {
graph.CheckLogIn(it, v)
func (it *Iterator) ResultTree() *graph.ResultTree {
return graph.NewResultTree(it.Result())
}
func (it *Iterator) Result() graph.Value {
return it.result
}
func (it *Iterator) NextResult() bool {
return false
}
// No subiterators.
func (it *Iterator) SubIterators() []graph.Iterator {
return nil
}
func (it *Iterator) Contains(v graph.Value) bool {
graph.ContainsLogIn(it, v)
if it.isAll {
it.Last = v
return graph.CheckLogOut(it, v, true)
it.result = v
return graph.ContainsLogOut(it, v, true)
}
var offset int
switch it.dir {
case graph.Subject:
case quad.Subject:
offset = 0
case graph.Predicate:
offset = (it.ts.hasher.Size() * 2)
case graph.Object:
offset = (it.ts.hasher.Size() * 2) * 2
case graph.Provenance:
offset = (it.ts.hasher.Size() * 2) * 3
case quad.Predicate:
offset = (it.qs.hasher.Size() * 2)
case quad.Object:
offset = (it.qs.hasher.Size() * 2) * 2
case quad.Label:
offset = (it.qs.hasher.Size() * 2) * 3
}
val := v.(string)[offset : it.ts.hasher.Size()*2+offset]
val := v.(string)[offset : it.qs.hasher.Size()*2+offset]
if val == it.hash {
it.Last = v
return graph.CheckLogOut(it, v, true)
it.result = v
return graph.ContainsLogOut(it, v, true)
}
return graph.CheckLogOut(it, v, false)
return graph.ContainsLogOut(it, v, false)
}
func (it *Iterator) Size() (int64, bool) {
@ -183,8 +229,8 @@ func (it *Iterator) DebugString(indent int) string {
func (it *Iterator) Stats() graph.IteratorStats {
size, _ := it.Size()
return graph.IteratorStats{
CheckCost: 1,
NextCost: 5,
Size: size,
ContainsCost: 1,
NextCost: 5,
Size: size,
}
}

221
graph/mongo/triplestore.go
View file

@ -18,7 +18,7 @@ import (
"crypto/sha1"
"encoding/hex"
"hash"
"log"
"io"
"gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
@ -26,8 +26,16 @@ import (
"github.com/barakmich/glog"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
"github.com/google/cayley/quad"
)
func init() {
graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
}
// Guarantee we satisfy graph.Bulkloader.
var _ graph.BulkLoader = (*TripleStore)(nil)
const DefaultDBName = "cayley"
type TripleStore struct {
@ -60,13 +68,13 @@ func createNewMongoGraph(addr string, options graph.Options) error {
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Obj"}
db.C("triples").EnsureIndex(indexOpts)
indexOpts.Key = []string{"Provenance"}
indexOpts.Key = []string{"Label"}
db.C("triples").EnsureIndex(indexOpts)
return nil
}
func newTripleStore(addr string, options graph.Options) (graph.TripleStore, error) {
var ts TripleStore
var qs TripleStore
conn, err := mgo.Dial(addr)
if err != nil {
return nil, err
@ -76,26 +84,26 @@ func newTripleStore(addr string, options graph.Options) (graph.TripleStore, erro
if val, ok := options.StringKey("database_name"); ok {
dbName = val
}
ts.db = conn.DB(dbName)
ts.session = conn
ts.hasher = sha1.New()
ts.idCache = NewIDLru(1 << 16)
return &ts, nil
qs.db = conn.DB(dbName)
qs.session = conn
qs.hasher = sha1.New()
qs.idCache = NewIDLru(1 << 16)
return &qs, nil
}
func (ts *TripleStore) getIdForTriple(t *graph.Triple) string {
id := ts.ConvertStringToByteHash(t.Subject)
id += ts.ConvertStringToByteHash(t.Predicate)
id += ts.ConvertStringToByteHash(t.Object)
id += ts.ConvertStringToByteHash(t.Provenance)
func (qs *TripleStore) getIdForTriple(t *quad.Quad) string {
id := qs.ConvertStringToByteHash(t.Subject)
id += qs.ConvertStringToByteHash(t.Predicate)
id += qs.ConvertStringToByteHash(t.Object)
id += qs.ConvertStringToByteHash(t.Label)
return id
}
func (ts *TripleStore) ConvertStringToByteHash(s string) string {
ts.hasher.Reset()
key := make([]byte, 0, ts.hasher.Size())
ts.hasher.Write([]byte(s))
key = ts.hasher.Sum(key)
func (qs *TripleStore) ConvertStringToByteHash(s string) string {
qs.hasher.Reset()
key := make([]byte, 0, qs.hasher.Size())
qs.hasher.Write([]byte(s))
key = qs.hasher.Sum(key)
return hex.EncodeToString(key)
}
@ -105,10 +113,10 @@ type MongoNode struct {
Size int "Size"
}
func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
func (qs *TripleStore) updateNodeBy(node_name string, inc int) {
var size MongoNode
node := ts.ValueOf(node_name)
err := ts.db.C("nodes").FindId(node).One(&size)
node := qs.ValueOf(node_name)
err := qs.db.C("nodes").FindId(node).One(&size)
if err != nil {
if err.Error() == "not found" {
// Not found. Okay.
@ -116,7 +124,7 @@ func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
size.Name = node_name
size.Size = inc
} else {
glog.Error("Error:", err)
glog.Errorf("Error: %v", err)
return
}
} else {
@ -128,134 +136,134 @@ func (ts *TripleStore) updateNodeBy(node_name string, inc int) {
// Removing something...
if inc < 0 {
if size.Size <= 0 {
err := ts.db.C("nodes").RemoveId(node)
err := qs.db.C("nodes").RemoveId(node)
if err != nil {
glog.Error("Error: ", err, " while removing node ", node_name)
glog.Errorf("Error: %v while removing node %s", err, node_name)
return
}
}
}
_, err2 := ts.db.C("nodes").UpsertId(node, size)
_, err2 := qs.db.C("nodes").UpsertId(node, size)
if err2 != nil {
glog.Error("Error: ", err)
glog.Errorf("Error: %v", err)
}
}
func (ts *TripleStore) writeTriple(t *graph.Triple) bool {
func (qs *TripleStore) writeTriple(t *quad.Quad) bool {
tripledoc := bson.M{
"_id": ts.getIdForTriple(t),
"Subject": t.Subject,
"Predicate": t.Predicate,
"Object": t.Object,
"Provenance": t.Provenance,
"_id": qs.getIdForTriple(t),
"Subject": t.Subject,
"Predicate": t.Predicate,
"Object": t.Object,
"Label": t.Label,
}
err := ts.db.C("triples").Insert(tripledoc)
err := qs.db.C("triples").Insert(tripledoc)
if err != nil {
// Among the reasons I hate MongoDB. "Errors don't happen! Right guys?"
if err.(*mgo.LastError).Code == 11000 {
return false
}
glog.Error("Error: ", err)
glog.Errorf("Error: %v", err)
return false
}
return true
}
func (ts *TripleStore) AddTriple(t *graph.Triple) {
_ = ts.writeTriple(t)
ts.updateNodeBy(t.Subject, 1)
ts.updateNodeBy(t.Predicate, 1)
ts.updateNodeBy(t.Object, 1)
if t.Provenance != "" {
ts.updateNodeBy(t.Provenance, 1)
func (qs *TripleStore) AddTriple(t *quad.Quad) {
_ = qs.writeTriple(t)
qs.updateNodeBy(t.Subject, 1)
qs.updateNodeBy(t.Predicate, 1)
qs.updateNodeBy(t.Object, 1)
if t.Label != "" {
qs.updateNodeBy(t.Label, 1)
}
}
func (ts *TripleStore) AddTripleSet(in []*graph.Triple) {
ts.session.SetSafe(nil)
func (qs *TripleStore) AddTripleSet(in []*quad.Quad) {
qs.session.SetSafe(nil)
ids := make(map[string]int)
for _, t := range in {
wrote := ts.writeTriple(t)
wrote := qs.writeTriple(t)
if wrote {
ids[t.Subject]++
ids[t.Object]++
ids[t.Predicate]++
if t.Provenance != "" {
ids[t.Provenance]++
if t.Label != "" {
ids[t.Label]++
}
}
}
for k, v := range ids {
ts.updateNodeBy(k, v)
qs.updateNodeBy(k, v)
}
ts.session.SetSafe(&mgo.Safe{})
qs.session.SetSafe(&mgo.Safe{})
}
func (ts *TripleStore) RemoveTriple(t *graph.Triple) {
err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t))
func (qs *TripleStore) RemoveTriple(t *quad.Quad) {
err := qs.db.C("triples").RemoveId(qs.getIdForTriple(t))
if err == mgo.ErrNotFound {
return
} else if err != nil {
log.Println("Error: ", err, " while removing triple ", t)
glog.Errorf("Error: %v while removing triple %v", err, t)
return
}
ts.updateNodeBy(t.Subject, -1)
ts.updateNodeBy(t.Predicate, -1)
ts.updateNodeBy(t.Object, -1)
if t.Provenance != "" {
ts.updateNodeBy(t.Provenance, -1)
qs.updateNodeBy(t.Subject, -1)
qs.updateNodeBy(t.Predicate, -1)
qs.updateNodeBy(t.Object, -1)
if t.Label != "" {
qs.updateNodeBy(t.Label, -1)
}
}
func (ts *TripleStore) Triple(val graph.Value) *graph.Triple {
func (qs *TripleStore) Quad(val graph.Value) *quad.Quad {
var bsonDoc bson.M
err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc)
err := qs.db.C("triples").FindId(val.(string)).One(&bsonDoc)
if err != nil {
log.Println("Error: Couldn't retrieve triple", val.(string), err)
glog.Errorf("Error: Couldn't retrieve triple %s %v", val, err)
}
return &graph.Triple{
return &quad.Quad{
bsonDoc["Subject"].(string),
bsonDoc["Predicate"].(string),
bsonDoc["Object"].(string),
bsonDoc["Provenance"].(string),
bsonDoc["Label"].(string),
}
}
func (ts *TripleStore) TripleIterator(d graph.Direction, val graph.Value) graph.Iterator {
return NewIterator(ts, "triples", d, val)
func (qs *TripleStore) TripleIterator(d quad.Direction, val graph.Value) graph.Iterator {
return NewIterator(qs, "triples", d, val)
}
func (ts *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator(ts, "nodes")
func (qs *TripleStore) NodesAllIterator() graph.Iterator {
return NewAllIterator(qs, "nodes")
}
func (ts *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator(ts, "triples")
func (qs *TripleStore) TriplesAllIterator() graph.Iterator {
return NewAllIterator(qs, "triples")
}
func (ts *TripleStore) ValueOf(s string) graph.Value {
return ts.ConvertStringToByteHash(s)
func (qs *TripleStore) ValueOf(s string) graph.Value {
return qs.ConvertStringToByteHash(s)
}
func (ts *TripleStore) NameOf(v graph.Value) string {
val, ok := ts.idCache.Get(v.(string))
func (qs *TripleStore) NameOf(v graph.Value) string {
val, ok := qs.idCache.Get(v.(string))
if ok {
return val
}
var node MongoNode
err := ts.db.C("nodes").FindId(v.(string)).One(&node)
err := qs.db.C("nodes").FindId(v.(string)).One(&node)
if err != nil {
log.Println("Error: Couldn't retrieve node", v.(string), err)
glog.Errorf("Error: Couldn't retrieve node %s %v", v, err)
}
ts.idCache.Put(v.(string), node.Name)
qs.idCache.Put(v.(string), node.Name)
return node.Name
}
func (ts *TripleStore) Size() int64 {
count, err := ts.db.C("triples").Count()
func (qs *TripleStore) Size() int64 {
count, err := qs.db.C("triples").Count()
if err != nil {
glog.Error("Error: ", err)
glog.Errorf("Error: %v", err)
return 0
}
return int64(count)
@ -265,40 +273,48 @@ func compareStrings(a, b graph.Value) bool {
return a.(string) == b.(string)
}
func (ts *TripleStore) FixedIterator() graph.FixedIterator {
func (qs *TripleStore) FixedIterator() graph.FixedIterator {
return iterator.NewFixedIteratorWithCompare(compareStrings)
}
func (ts *TripleStore) Close() {
ts.db.Session.Close()
func (qs *TripleStore) Close() {
qs.db.Session.Close()
}
func (ts *TripleStore) TripleDirection(in graph.Value, d graph.Direction) graph.Value {
func (qs *TripleStore) TripleDirection(in graph.Value, d quad.Direction) graph.Value {
// Maybe do the trick here
var offset int
switch d {
case graph.Subject:
case quad.Subject:
offset = 0
case graph.Predicate:
offset = (ts.hasher.Size() * 2)
case graph.Object:
offset = (ts.hasher.Size() * 2) * 2
case graph.Provenance:
offset = (ts.hasher.Size() * 2) * 3
case quad.Predicate:
offset = (qs.hasher.Size() * 2)
case quad.Object:
offset = (qs.hasher.Size() * 2) * 2
case quad.Label:
offset = (qs.hasher.Size() * 2) * 3
}
val := in.(string)[offset : ts.hasher.Size()*2+offset]
val := in.(string)[offset : qs.hasher.Size()*2+offset]
return val
}
func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool {
if ts.Size() != 0 {
return false
func (qs *TripleStore) BulkLoad(dec quad.Unmarshaler) error {
if qs.Size() != 0 {
return graph.ErrCannotBulkLoad
}
ts.session.SetSafe(nil)
for triple := range t_chan {
ts.writeTriple(triple)
qs.session.SetSafe(nil)
for {
q, err := dec.Unmarshal()
if err != nil {
if err != io.EOF {
return err
}
break
}
qs.writeTriple(q)
}
outputTo := bson.M{"replace": "nodes", "sharded": true}
glog.Infoln("Mapreducing")
job := mgo.MapReduce{
@ -311,8 +327,8 @@ func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool {
emit(s_key, {"_id": s_key, "Name" : this.Subject, "Size" : 1})
emit(p_key, {"_id": p_key, "Name" : this.Predicate, "Size" : 1})
emit(o_key, {"_id": o_key, "Name" : this.Object, "Size" : 1})
if (this.Provenance != "") {
emit(c_key, {"_id": c_key, "Name" : this.Provenance, "Size" : 1})
if (this.Label != "") {
emit(c_key, {"_id": c_key, "Name" : this.Label, "Size" : 1})
}
}
`,
@ -330,16 +346,13 @@ func (ts *TripleStore) BulkLoad(t_chan chan *graph.Triple) bool {
`,
Out: outputTo,
}
ts.db.C("triples").Find(nil).MapReduce(&job, nil)
qs.db.C("triples").Find(nil).MapReduce(&job, nil)
glog.Infoln("Fixing")
ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
qs.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) {
db.nodes.update({"_id": result._id}, result.value)
}) }`}, {"args", bson.D{}}}, nil)
ts.session.SetSafe(&mgo.Safe{})
return true
}
qs.session.SetSafe(&mgo.Safe{})
func init() {
graph.RegisterTripleStore("mongo", newTripleStore, createNewMongoGraph)
return nil
}

9
graph/mongo/triplestore_iterator_optimize.go
View file

@ -37,14 +37,15 @@ func (ts *TripleStore) optimizeLinksTo(it *iterator.LinksTo) (graph.Iterator, bo
if primary.Type() == graph.Fixed {
size, _ := primary.Size()
if size == 1 {
val, ok := primary.Next()
val, ok := graph.Next(primary)
if !ok {
panic("Sizes lie")
}
newIt := ts.TripleIterator(it.Direction(), val)
newIt.CopyTagsFrom(it)
for _, tag := range primary.Tags() {
newIt.AddFixedTag(tag, val)
nt := newIt.Tagger()
nt.CopyFrom(it)
for _, tag := range primary.Tagger().Tags() {
nt.AddFixed(tag, val)
}
it.Close()
return newIt, true

4
graph/result_tree_evaluator.go
View file

@ -40,7 +40,7 @@ func (t *ResultTree) AddSubtree(sub *ResultTree) {
t.subtrees = append(t.subtrees, sub)
}
func StringResultTreeEvaluator(it Iterator) string {
func StringResultTreeEvaluator(it Nexter) string {
ok := true
out := ""
for {
@ -59,6 +59,6 @@ func StringResultTreeEvaluator(it Iterator) string {
return out
}
func PrintResultTreeEvaluator(it Iterator) {
func PrintResultTreeEvaluator(it Nexter) {
fmt.Print(StringResultTreeEvaluator(it))
}

4
graph/result_tree_evaluator_test.go
View file

@ -26,7 +26,7 @@ func TestSingleIterator(t *testing.T) {
result := StringResultTreeEvaluator(all)
expected := "(1)\n(2)\n(3)\n"
if expected != result {
t.Errorf("Expected \"%s\" got \"%s\"", expected, result)
t.Errorf("Expected %q got %q", expected, result)
}
}
@ -40,6 +40,6 @@ func TestAndIterator(t *testing.T) {
result := StringResultTreeEvaluator(and)
expected := "(3 (3) (3))\n"
if expected != result {
t.Errorf("Expected \"%s\" got \"%s\"", expected, result)
t.Errorf("Expected %q got %q", expected, result)
}
}

45
graph/session.go
View file

@ -1,45 +0,0 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package graph
// Defines the graph session interface general to all query languages.
type ParseResult int
const (
Parsed ParseResult = iota
ParseMore
ParseFail
)
type Session interface {
// Return whether the string is a valid expression.
InputParses(string) (ParseResult, error)
ExecInput(string, chan interface{}, int)
ToText(interface{}) string
ToggleDebug()
}
type HttpSession interface {
// Return whether the string is a valid expression.
InputParses(string) (ParseResult, error)
// Runs the query and returns individual results on the channel.
ExecInput(string, chan interface{}, int)
GetQuery(string, chan map[string]interface{})
BuildJson(interface{})
GetJson() (interface{}, error)
ClearJson()
ToggleDebug()
}

272
graph/sexp/parser.go
View file

@ -1,272 +0,0 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sexp
import (
"github.com/badgerodon/peg"
"github.com/google/cayley/graph"
"github.com/google/cayley/graph/iterator"
)
func BuildIteratorTreeForQuery(ts graph.TripleStore, query string) graph.Iterator {
tree := parseQuery(query)
return buildIteratorTree(tree, ts)
}
func ParseString(input string) string {
return parseQuery(input).String()
}
func parseQuery(input string) *peg.ExpressionTree {
parser := peg.NewParser()
start := parser.NonTerminal("Start")
whitespace := parser.NonTerminal("Whitespace")
quotedString := parser.NonTerminal("QuotedString")
rootConstraint := parser.NonTerminal("RootConstraint")
constraint := parser.NonTerminal("Constraint")
colonIdentifier := parser.NonTerminal("ColonIdentifier")
variable := parser.NonTerminal("Variable")
identifier := parser.NonTerminal("Identifier")
fixedNode := parser.NonTerminal("FixedNode")
nodeIdent := parser.NonTerminal("NodeIdentifier")
predIdent := parser.NonTerminal("PredIdentifier")
reverse := parser.NonTerminal("Reverse")
predKeyword := parser.NonTerminal("PredicateKeyword")
optional := parser.NonTerminal("OptionalKeyword")
start.Expression = rootConstraint
whitespace.Expression = parser.OneOrMore(
parser.OrderedChoice(
parser.Terminal(' '),
parser.Terminal('\t'),
parser.Terminal('\n'),
parser.Terminal('\r'),
),
)
quotedString.Expression = parser.Sequence(
parser.Terminal('"'),
parser.OneOrMore(
parser.OrderedChoice(
parser.Range('0', '9'),
parser.Range('a', 'z'),
parser.Range('A', 'Z'),
parser.Terminal('_'),
parser.Terminal('/'),
parser.Terminal(':'),
parser.Terminal(' '),
parser.Terminal('\''),
),
),
parser.Terminal('"'),
)
predKeyword.Expression = parser.OrderedChoice(
optional,
)
optional.Expression = parser.Sequence(
parser.Terminal('o'),
parser.Terminal('p'),
parser.Terminal('t'),
parser.Terminal('i'),
parser.Terminal('o'),
parser.Terminal('n'),
parser.Terminal('a'),
parser.Terminal('l'),
)
identifier.Expression = parser.OneOrMore(
parser.OrderedChoice(
parser.Range('0', '9'),
parser.Range('a', 'z'),
parser.Range('A', 'Z'),
parser.Terminal('_'),
parser.Terminal('.'),
parser.Terminal('/'),
parser.Terminal(':'),
parser.Terminal('#'),
),
)
reverse.Expression = parser.Terminal('!')
variable.Expression = parser.Sequence(
parser.Terminal('$'),
identifier,
)
colonIdentifier.Expression = parser.Sequence(
parser.Terminal(':'),
identifier,
)
fixedNode.Expression = parser.OrderedChoice(
colonIdentifier,
quotedString,
)
nodeIdent.Expression = parser.OrderedChoice(
variable,
fixedNode,
)
predIdent.Expression = parser.Sequence(
parser.Optional(reverse),
parser.OrderedChoice(
nodeIdent,
constraint,
),
)
constraint.Expression = parser.Sequence(
parser.Terminal('('),
parser.Optional(whitespace),
predIdent,
parser.Optional(whitespace),
parser.Optional(predKeyword),
parser.Optional(whitespace),
parser.OrderedChoice(
nodeIdent,
rootConstraint,
),
parser.Optional(whitespace),
parser.Terminal(')'),
)
rootConstraint.Expression = parser.Sequence(
parser.Terminal('('),
parser.Optional(whitespace),
nodeIdent,
parser.Optional(whitespace),
parser.ZeroOrMore(parser.Sequence(
constraint,
parser.Optional(whitespace),
)),
parser.Terminal(')'),
)
tree := parser.Parse(input)
return tree
}
func getIdentString(tree *peg.ExpressionTree) string {
out := ""
if len(tree.Children) > 0 {
for _, child := range tree.Children {
out += getIdentString(child)
}
} else {
if tree.Value != '"' {
out += string(tree.Value)
}
}
return out
}
func buildIteratorTree(tree *peg.ExpressionTree, ts graph.TripleStore) graph.Iterator {
switch tree.Name {
case "Start":
return buildIteratorTree(tree.Children[0], ts)
case "NodeIdentifier":
var out graph.Iterator
nodeID := getIdentString(tree)
if tree.Children[0].Name == "Variable" {
allIt := ts.NodesAllIterator()
allIt.AddTag(nodeID)
out = allIt
} else {
n := nodeID
if tree.Children[0].Children[0].Name == "ColonIdentifier" {
n = nodeID[1:]
}
fixed := ts.FixedIterator()
fixed.Add(ts.ValueOf(n))
out = fixed
}
return out
case "PredIdentifier":
i := 0
if tree.Children[0].Name == "Reverse" {
//Taken care of below
i++
}
it := buildIteratorTree(tree.Children[i], ts)
lto := iterator.NewLinksTo(ts, it, graph.Predicate)
return lto
case "RootConstraint":
constraintCount := 0
and := iterator.NewAnd()
for _, c := range tree.Children {
switch c.Name {
case "NodeIdentifier":
fallthrough
case "Constraint":
it := buildIteratorTree(c, ts)
and.AddSubIterator(it)
constraintCount++
continue
default:
continue
}
}
return and
case "Constraint":
var hasa *iterator.HasA
topLevelDir := graph.Subject
subItDir := graph.Object
subAnd := iterator.NewAnd()
isOptional := false
for _, c := range tree.Children {
switch c.Name {
case "PredIdentifier":
if c.Children[0].Name == "Reverse" {
topLevelDir = graph.Object
subItDir = graph.Subject
}
it := buildIteratorTree(c, ts)
subAnd.AddSubIterator(it)
continue
case "PredicateKeyword":
switch c.Children[0].Name {
case "OptionalKeyword":
isOptional = true
}
case "NodeIdentifier":
fallthrough
case "RootConstraint":
it := buildIteratorTree(c, ts)
l := iterator.NewLinksTo(ts, it, subItDir)
subAnd.AddSubIterator(l)
continue
default:
continue
}
}
hasa = iterator.NewHasA(ts, subAnd, topLevelDir)
if isOptional {
optional := iterator.NewOptional(hasa)
return optional
}
return hasa
default:
return &iterator.Null{}
}
panic("Not reached")
}

147
graph/sexp/parser_test.go
View file

@ -1,147 +0,0 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sexp
import (
"testing"
"github.com/google/cayley/graph"
_ "github.com/google/cayley/graph/memstore"
)
func TestBadParse(t *testing.T) {
str := ParseString("()")
if str != "" {
t.Errorf("Unexpected parse result, got:%q", str)
}
}
var testQueries = []struct {
message string
add *graph.Triple
query string
typ graph.Type
expect string
}{
{
message: "get a single triple linkage",
add: &graph.Triple{"i", "can", "win", ""},
query: "($a (:can \"win\"))",
typ: graph.And,
expect: "i",
},
{
message: "get a single triple linkage",
add: &graph.Triple{"i", "can", "win", ""},
query: "(\"i\" (:can $a))",
typ: graph.And,
expect: "i",
},
}
func TestMemstoreBackedSexp(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
it := BuildIteratorTreeForQuery(ts, "()")
if it.Type() != graph.Null {
t.Errorf(`Incorrect type for empty query, got:%q expect: "null"`, it.Type())
}
for _, test := range testQueries {
if test.add != nil {
ts.AddTriple(test.add)
}
it := BuildIteratorTreeForQuery(ts, test.query)
if it.Type() != test.typ {
t.Errorf("Incorrect type for %s, got:%q expect %q", test.message, it.Type(), test.expect)
}
got, ok := it.Next()
if !ok {
t.Errorf("Failed to %s", test.message)
}
if expect := ts.ValueOf(test.expect); got != expect {
t.Errorf("Incorrect result for %s, got:%v expect %v", test.message, got, expect)
}
}
}
func TestTreeConstraintParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
ts.AddTriple(&graph.Triple{"i", "like", "food", ""})
ts.AddTriple(&graph.Triple{"food", "is", "good", ""})
query := "(\"i\"\n" +
"(:like\n" +
"($a (:is :good))))"
it := BuildIteratorTreeForQuery(ts, query)
if it.Type() != graph.And {
t.Error("Odd iterator tree. Got: %s", it.DebugString(0))
}
out, ok := it.Next()
if !ok {
t.Error("Got no results")
}
if out != ts.ValueOf("i") {
t.Errorf("Got %d, expected %d", out, ts.ValueOf("i"))
}
}
func TestTreeConstraintTagParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
ts.AddTriple(&graph.Triple{"i", "like", "food", ""})
ts.AddTriple(&graph.Triple{"food", "is", "good", ""})
query := "(\"i\"\n" +
"(:like\n" +
"($a (:is :good))))"
it := BuildIteratorTreeForQuery(ts, query)
_, ok := it.Next()
if !ok {
t.Error("Got no results")
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
if ts.NameOf(tags["$a"]) != "food" {
t.Errorf("Got %s, expected food", ts.NameOf(tags["$a"]))
}
}
func TestMultipleConstraintParse(t *testing.T) {
ts, _ := graph.NewTripleStore("memstore", "", nil)
for _, tv := range []*graph.Triple{
{"i", "like", "food", ""},
{"i", "like", "beer", ""},
{"you", "like", "beer", ""},
} {
ts.AddTriple(tv)
}
query := `(
$a
(:like :beer)
(:like "food")
)`
it := BuildIteratorTreeForQuery(ts, query)
if it.Type() != graph.And {
t.Error("Odd iterator tree. Got: %s", it.DebugString(0))
}
out, ok := it.Next()
if !ok {
t.Error("Got no results")
}
if out != ts.ValueOf("i") {
t.Errorf("Got %d, expected %d", out, ts.ValueOf("i"))
}
_, ok = it.Next()
if ok {
t.Error("Too many results")
}
}

121
graph/sexp/session.go
View file

@ -1,121 +0,0 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sexp
// Defines a running session of the sexp query language.
import (
"errors"
"fmt"
"sort"
"github.com/google/cayley/graph"
)
type Session struct {
ts graph.TripleStore
debug bool
}
func NewSession(inputTripleStore graph.TripleStore) *Session {
var s Session
s.ts = inputTripleStore
return &s
}
func (s *Session) ToggleDebug() {
s.debug = !s.debug
}
func (s *Session) InputParses(input string) (graph.ParseResult, error) {
var parenDepth int
for i, x := range input {
if x == '(' {
parenDepth++
}
if x == ')' {
parenDepth--
if parenDepth < 0 {
min := 0
if (i - 10) > min {
min = i - 10
}
return graph.ParseFail, errors.New(fmt.Sprintf("Too many close parens at char %d: %s", i, input[min:i]))
}
}
}
if parenDepth > 0 {
return graph.ParseMore, nil
}
if len(ParseString(input)) > 0 {
return graph.Parsed, nil
}
return graph.ParseFail, errors.New("Invalid Syntax")
}
func (s *Session) ExecInput(input string, out chan interface{}, limit int) {
it := BuildIteratorTreeForQuery(s.ts, input)
newIt, changed := it.Optimize()
if changed {
it = newIt
}
if s.debug {
fmt.Println(it.DebugString(0))
}
nResults := 0
for {
_, ok := it.Next()
if !ok {
break
}
tags := make(map[string]graph.Value)
it.TagResults(tags)
out <- &tags
nResults++
if nResults > limit && limit != -1 {
break
}
for it.NextResult() == true {
tags := make(map[string]graph.Value)
it.TagResults(tags)
out <- &tags
nResults++
if nResults > limit && limit != -1 {
break
}
}
}
close(out)
}
func (s *Session) ToText(result interface{}) string {
out := fmt.Sprintln("****")
tags := result.(map[string]graph.Value)
tagKeys := make([]string, len(tags))
i := 0
for k := range tags {
tagKeys[i] = k
i++
}
sort.Strings(tagKeys)
for _, k := range tagKeys {
if k == "$_" {
continue
}
out += fmt.Sprintf("%s : %s\n", k, s.ts.NameOf(tags[k]))
}
return out
}

142
graph/triple.go
View file

@ -1,142 +0,0 @@
// Copyright 2014 The Cayley Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package graph
// Defines the struct which makes the TripleStore possible -- the triple.
//
// At its heart, it consists of three fields -- Subject, Predicate, and Object.
// Three IDs that relate to each other. That's all there is to it. The triples
// are the links in the graph, and the existence of node IDs is defined by the
// fact that some triple in the graph mentions them.
//
// This means that a complete representation of the graph is equivalent to a
// list of triples. The rest is just indexing for speed.
//
// Adding fields to the triple is not to be taken lightly. You'll see I mention
// provenance, but don't as yet use it in any backing store. In general, there
// can be features that can be turned on or off for any store, but I haven't
// decided how to allow/disallow them yet. Another such example would be to add
// a forward and reverse index field -- forward being "order the list of
// objects pointed at by this subject with this predicate" such as first and
// second children, top billing, what have you.
//
// There will never be that much in this file except for the definition, but
// the consequences are not to be taken lightly. But do suggest cool features!
import "fmt"
// TODO(kortschak) Consider providing MashalJSON and UnmarshalJSON
// instead of using struct tags.
// Our triple struct, used throughout.
type Triple struct {
Subject string `json:"subject"`
Predicate string `json:"predicate"`
Object string `json:"object"`
Provenance string `json:"provenance,omitempty"`
}
// Direction specifies an edge's type.
type Direction byte
// List of the valid directions of a triple.
const (
Any Direction = iota
Subject
Predicate
Object
Provenance
)
func (d Direction) Prefix() byte {
switch d {
case Any:
return 'a'
case Subject:
return 's'
case Predicate:
return 'p'
case Provenance:
return 'c'
case Object:
return 'o'
default:
return '\x00'
}
}
func (d Direction) String() string {
switch d {
case Any:
return "any"
case Subject:
return "subject"
case Predicate:
return "predicate"
case Provenance:
return "provenance"
case Object:
return "object"
default:
return fmt.Sprint("illegal direction:", byte(d))
}
}
// TODO(kortschak) Consider writing methods onto the concrete type
// instead of the pointer. This needs benchmarking to make the decision.
// Per-field accessor for triples
func (t *Triple) Get(d Direction) string {
switch d {
case Subject:
return t.Subject
case Predicate:
return t.Predicate
case Provenance:
return t.Provenance
case Object:
return t.Object
default:
panic(d.String())
}
}
func (t *Triple) Equals(o *Triple) bool {
return *t == *o
}
// Pretty-prints a triple.
func (t *Triple) String() string {
// TODO(kortschak) String methods should generally not terminate in '\n'.
return fmt.Sprintf("%s -- %s -> %s\n", t.Subject, t.Predicate, t.Object)
}
func (t *Triple) IsValid() bool {
return t.Subject != "" && t.Predicate != "" && t.Object != ""
}
// TODO(kortschak) NTriple looks like a good candidate for conversion
// to MarshalText() (text []byte, err error) and then move parsing code
// from nquads to here to provide UnmarshalText(text []byte) error.
// Prints a triple in N-Triple format.
func (t *Triple) NTriple() string {
if t.Provenance == "" {
//TODO(barakmich): Proper escaping.
return fmt.Sprintf("%s %s %s .", t.Subject, t.Predicate, t.Object)
} else {
return fmt.Sprintf("%s %s %s %s .", t.Subject, t.Predicate, t.Object, t.Provenance)
}
}

18
graph/triplestore.go
View file

@ -23,7 +23,9 @@ package graph
import (
"errors"
"github.com/barakmich/glog"
"github.com/google/cayley/quad"
)
// Defines an opaque "triple store value" type. However the backend wishes to
@ -42,11 +44,11 @@ type TripleStore interface {
ApplyTransactions([]*Transaction) error
// Given an opaque token, returns the triple for that token from the store.
Triple(Value) *Triple
Quad(Value) *quad.Quad
// Given a direction and a token, creates an iterator of links which have
// that node token in that directional field.
TripleIterator(Direction, Value) Iterator
TripleIterator(quad.Direction, Value) Iterator
// Returns an iterator enumerating all nodes in the graph.
NodesAllIterator() Iterator
@ -86,8 +88,8 @@ type TripleStore interface {
// gives the TripleStore the opportunity to make this optimization.
//
// Iterators will call this. At worst, a valid implementation is
// ts.IdFor(ts.Triple(triple_id).Get(dir))
TripleDirection(triple_id Value, d Direction) Value
// ts.IdFor(ts.quad.Quad(id).Get(dir))
TripleDirection(id Value, d quad.Direction) Value
}
type Options map[string]interface{}
@ -119,14 +121,10 @@ func (d Options) StringKey(key string) (string, bool) {
var ErrCannotBulkLoad = errors.New("triplestore: cannot bulk load")
type BulkLoader interface {
// BulkLoad loads Triples from a TripleUnmarshaler in bulk to the TripleStore.
// BulkLoad loads Quads from a quad.Unmarshaler in bulk to the TripleStore.
// It returns ErrCannotBulkLoad if bulk loading is not possible. For example if
// you cannot load in bulk to a non-empty database, and the db is non-empty.
BulkLoad(TripleUnmarshaler) error
}
type TripleUnmarshaler interface {
Unmarshal() (*Triple, error)
BulkLoad(quad.Unmarshaler) error
}
type NewStoreFunc func(string, Options) (TripleStore, error)