From 0c9de6141339ff014cec4972defec1a9bd7cd7f6 Mon Sep 17 00:00:00 2001 From: kortschak Date: Thu, 26 Jun 2014 09:10:57 +0930 Subject: [PATCH] Use Go-conventional filenaming --- appengine/appengine.go | 67 +++++ appengine/cayley-appengine.go | 67 ----- graph/all-iterator.go | 117 -------- graph/all_iterator.go | 117 ++++++++ graph/and-iterator-optimize.go | 330 --------------------- graph/and-iterator-optimize_test.go | 111 ------- graph/and-iterator.go | 248 ---------------- graph/and-iterator_test.go | 147 --------- graph/and_iterator.go | 248 ++++++++++++++++ graph/and_iterator_optimize.go | 330 +++++++++++++++++++++ graph/and_iterator_optimize_test.go | 111 +++++++ graph/and_iterator_test.go | 147 +++++++++ graph/fixed-iterator.go | 157 ---------- graph/fixed_iterator.go | 157 ++++++++++ graph/hasa-iterator.go | 224 -------------- graph/hasa_iterator.go | 224 ++++++++++++++ graph/linksto-iterator.go | 184 ------------ graph/linksto-iterator_test.go | 37 --- graph/linksto_iterator.go | 184 ++++++++++++ graph/linksto_iterator_test.go | 37 +++ graph/memstore/llrb-iterator.go | 119 -------- graph/memstore/llrb_iterator.go | 119 ++++++++ graph/memstore/memstore-all-iterator.go | 45 --- graph/memstore/memstore_all_iterator.go | 45 +++ graph/memstore/memtriplestore-iterator-optimize.go | 53 ---- graph/memstore/memtriplestore_iterator_optimize.go | 53 ++++ graph/mongo/mongo-iterator.go | 181 ----------- graph/mongo/mongo-triplestore-iterator-optimize.go | 53 ---- graph/mongo/mongo-triplestore.go | 329 -------------------- graph/mongo/mongo_iterator.go | 181 +++++++++++ graph/mongo/mongo_triplestore.go | 329 ++++++++++++++++++++ graph/mongo/mongo_triplestore_iterator_optimize.go | 53 ++++ graph/optional-iterator.go | 134 --------- graph/optional_iterator.go | 134 +++++++++ graph/or-iterator.go | 287 ------------------ graph/or-iterator_test.go | 142 --------- graph/or_iterator.go | 287 ++++++++++++++++++ graph/or_iterator_test.go | 142 +++++++++ graph/query-shape.go | 189 ------------ graph/query-shape_test.go | 124 -------- graph/query_shape.go | 189 ++++++++++++ graph/query_shape_test.go | 124 ++++++++ graph/result-tree-evaluator.go | 70 ----- graph/result-tree-evaluator_test.go | 42 --- graph/result_tree_evaluator.go | 70 +++++ graph/result_tree_evaluator_test.go | 42 +++ graph/sexp/sexp-session.go | 121 -------- graph/sexp/sexp_session.go | 121 ++++++++ graph/value-comparison-iterator.go | 193 ------------ graph/value-comparison-iterator_test.go | 126 -------- graph/value_comparison_iterator.go | 193 ++++++++++++ graph/value_comparison_iterator_test.go | 126 ++++++++ gremlin/gremlin-build-iterator.go | 315 -------------------- gremlin/gremlin-env.go | 95 ------ gremlin/gremlin-finals.go | 274 ----------------- gremlin/gremlin-functional_test.go | 230 -------------- gremlin/gremlin-session.go | 266 ----------------- gremlin/gremlin-traversals.go | 184 ------------ gremlin/gremlin_build_iterator.go | 315 ++++++++++++++++++++ gremlin/gremlin_env.go | 95 ++++++ gremlin/gremlin_finals.go | 274 +++++++++++++++++ gremlin/gremlin_functional_test.go | 230 ++++++++++++++ gremlin/gremlin_session.go | 266 +++++++++++++++++ gremlin/gremlin_traversals.go | 184 ++++++++++++ http/cayley-http-docs.go | 73 ----- http/cayley-http-query.go | 153 ---------- http/cayley-http-write.go | 119 -------- http/cayley-http.go | 113 ------- http/cayley-http_test.go | 53 ---- http/cayley_http.go | 113 +++++++ http/cayley_http_docs.go | 73 +++++ http/cayley_http_query.go | 153 ++++++++++ http/cayley_http_test.go | 53 ++++ http/cayley_http_write.go | 119 ++++++++ mql/mql-build-iterator.go | 181 ----------- mql/mql-fill.go | 114 ------- mql/mql-functional_test.go | 264 ----------------- mql/mql-query.go | 111 ------- mql/mql-session.go | 144 --------- mql/mql_build_iterator.go | 181 +++++++++++ mql/mql_fill.go | 114 +++++++ mql/mql_functional_test.go | 264 +++++++++++++++++ mql/mql_query.go | 111 +++++++ mql/mql_session.go | 144 +++++++++ 84 files changed, 6519 insertions(+), 6519 deletions(-) create mode 100644 appengine/appengine.go delete mode 100644 appengine/cayley-appengine.go delete mode 100644 graph/all-iterator.go create mode 100644 graph/all_iterator.go delete mode 100644 graph/and-iterator-optimize.go delete mode 100644 graph/and-iterator-optimize_test.go delete mode 100644 graph/and-iterator.go delete mode 100644 graph/and-iterator_test.go create mode 100644 graph/and_iterator.go create mode 100644 graph/and_iterator_optimize.go create mode 100644 graph/and_iterator_optimize_test.go create mode 100644 graph/and_iterator_test.go delete mode 100644 graph/fixed-iterator.go create mode 100644 graph/fixed_iterator.go delete mode 100644 graph/hasa-iterator.go create mode 100644 graph/hasa_iterator.go delete mode 100644 graph/linksto-iterator.go delete mode 100644 graph/linksto-iterator_test.go create mode 100644 graph/linksto_iterator.go create mode 100644 graph/linksto_iterator_test.go delete mode 100644 graph/memstore/llrb-iterator.go create mode 100644 graph/memstore/llrb_iterator.go delete mode 100644 graph/memstore/memstore-all-iterator.go create mode 100644 graph/memstore/memstore_all_iterator.go delete mode 100644 graph/memstore/memtriplestore-iterator-optimize.go create mode 100644 graph/memstore/memtriplestore_iterator_optimize.go delete mode 100644 graph/mongo/mongo-iterator.go delete mode 100644 graph/mongo/mongo-triplestore-iterator-optimize.go delete mode 100644 graph/mongo/mongo-triplestore.go create mode 100644 graph/mongo/mongo_iterator.go create mode 100644 graph/mongo/mongo_triplestore.go create mode 100644 graph/mongo/mongo_triplestore_iterator_optimize.go delete mode 100644 graph/optional-iterator.go create mode 100644 graph/optional_iterator.go delete mode 100644 graph/or-iterator.go delete mode 100644 graph/or-iterator_test.go create mode 100644 graph/or_iterator.go create mode 100644 graph/or_iterator_test.go delete mode 100644 graph/query-shape.go delete mode 100644 graph/query-shape_test.go create mode 100644 graph/query_shape.go create mode 100644 graph/query_shape_test.go delete mode 100644 graph/result-tree-evaluator.go delete mode 100644 graph/result-tree-evaluator_test.go create mode 100644 graph/result_tree_evaluator.go create mode 100644 graph/result_tree_evaluator_test.go delete mode 100644 graph/sexp/sexp-session.go create mode 100644 graph/sexp/sexp_session.go delete mode 100644 graph/value-comparison-iterator.go delete mode 100644 graph/value-comparison-iterator_test.go create mode 100644 graph/value_comparison_iterator.go create mode 100644 graph/value_comparison_iterator_test.go delete mode 100644 gremlin/gremlin-build-iterator.go delete mode 100644 gremlin/gremlin-env.go delete mode 100644 gremlin/gremlin-finals.go delete mode 100644 gremlin/gremlin-functional_test.go delete mode 100644 gremlin/gremlin-session.go delete mode 100644 gremlin/gremlin-traversals.go create mode 100644 gremlin/gremlin_build_iterator.go create mode 100644 gremlin/gremlin_env.go create mode 100644 gremlin/gremlin_finals.go create mode 100644 gremlin/gremlin_functional_test.go create mode 100644 gremlin/gremlin_session.go create mode 100644 gremlin/gremlin_traversals.go delete mode 100644 http/cayley-http-docs.go delete mode 100644 http/cayley-http-query.go delete mode 100644 http/cayley-http-write.go delete mode 100644 http/cayley-http.go delete mode 100644 http/cayley-http_test.go create mode 100644 http/cayley_http.go create mode 100644 http/cayley_http_docs.go create mode 100644 http/cayley_http_query.go create mode 100644 http/cayley_http_test.go create mode 100644 http/cayley_http_write.go delete mode 100644 mql/mql-build-iterator.go delete mode 100644 mql/mql-fill.go delete mode 100644 mql/mql-functional_test.go delete mode 100644 mql/mql-query.go delete mode 100644 mql/mql-session.go create mode 100644 mql/mql_build_iterator.go create mode 100644 mql/mql_fill.go create mode 100644 mql/mql_functional_test.go create mode 100644 mql/mql_query.go create mode 100644 mql/mql_session.go diff --git a/appengine/appengine.go b/appengine/appengine.go new file mode 100644 index 0000000..c915932 --- /dev/null +++ b/appengine/appengine.go @@ -0,0 +1,67 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cayleyappengine + +import ( + "os" + + "github.com/barakmich/glog" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/memstore" + "github.com/google/cayley/http" + "github.com/google/cayley/nquads" +) + +func init() { + glog.SetToStderr(true) + config := cfg.ParseConfigFromFile("cayley_appengine.cfg") + ts := memstore.NewMemTripleStore() + glog.Errorln(config) + LoadTriplesFromFileInto(ts, config.DatabasePath, config.LoadSize) + http.SetupRoutes(ts, config) +} + +func ReadTriplesFromFile(c chan *graph.Triple, tripleFile string) { + f, err := os.Open(tripleFile) + if err != nil { + glog.Fatalln("Couldn't open file", tripleFile) + } + + defer func() { + if err := f.Close(); err != nil { + glog.Fatalln(err) + } + }() + + nquads.ReadNQuadsFromReader(c, f) +} + +func LoadTriplesFromFileInto(ts graph.TripleStore, filename string, loadSize int) { + tChan := make(chan *graph.Triple) + go ReadTriplesFromFile(tChan, filename) + tripleblock := make([]*graph.Triple, loadSize) + i := 0 + for t := range tChan { + tripleblock[i] = t + i++ + if i == loadSize { + ts.AddTripleSet(tripleblock) + i = 0 + } + } + ts.AddTripleSet(tripleblock[0:i]) +} diff --git a/appengine/cayley-appengine.go b/appengine/cayley-appengine.go deleted file mode 100644 index c915932..0000000 --- a/appengine/cayley-appengine.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cayleyappengine - -import ( - "os" - - "github.com/barakmich/glog" - - cfg "github.com/google/cayley/config" - "github.com/google/cayley/graph" - "github.com/google/cayley/graph/memstore" - "github.com/google/cayley/http" - "github.com/google/cayley/nquads" -) - -func init() { - glog.SetToStderr(true) - config := cfg.ParseConfigFromFile("cayley_appengine.cfg") - ts := memstore.NewMemTripleStore() - glog.Errorln(config) - LoadTriplesFromFileInto(ts, config.DatabasePath, config.LoadSize) - http.SetupRoutes(ts, config) -} - -func ReadTriplesFromFile(c chan *graph.Triple, tripleFile string) { - f, err := os.Open(tripleFile) - if err != nil { - glog.Fatalln("Couldn't open file", tripleFile) - } - - defer func() { - if err := f.Close(); err != nil { - glog.Fatalln(err) - } - }() - - nquads.ReadNQuadsFromReader(c, f) -} - -func LoadTriplesFromFileInto(ts graph.TripleStore, filename string, loadSize int) { - tChan := make(chan *graph.Triple) - go ReadTriplesFromFile(tChan, filename) - tripleblock := make([]*graph.Triple, loadSize) - i := 0 - for t := range tChan { - tripleblock[i] = t - i++ - if i == loadSize { - ts.AddTripleSet(tripleblock) - i = 0 - } - } - ts.AddTripleSet(tripleblock[0:i]) -} diff --git a/graph/all-iterator.go b/graph/all-iterator.go deleted file mode 100644 index 6068746..0000000 --- a/graph/all-iterator.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Defines one of the base iterators, the All iterator. Which, logically -// enough, represents all nodes or all links in the graph. -// -// This particular file is actually vestigal. It's up to the TripleStore to give -// us an All iterator that represents all things in the graph. So this is -// really the All iterator for the MemTripleStore. That said, it *is* one of -// the base iterators, and it helps just to see it here. - -import ( - "fmt" - "strings" -) - -// An All iterator across a range of int64 values, from `max` to `min`. -type Int64AllIterator struct { - BaseIterator - max, min int64 - at int64 -} - -// Creates a new Int64AllIterator with the given range. -func NewInt64AllIterator(min, max int64) *Int64AllIterator { - var all Int64AllIterator - BaseIteratorInit(&all.BaseIterator) - all.max = max - all.min = min - all.at = min - return &all -} - -// Start back at the beginning -func (a *Int64AllIterator) Reset() { - a.at = a.min -} - -func (a *Int64AllIterator) Close() { -} - -func (a *Int64AllIterator) Clone() Iterator { - out := NewInt64AllIterator(a.min, a.max) - out.CopyTagsFrom(a) - return out -} - -// Prints the All iterator as just an "all". -func (a *Int64AllIterator) DebugString(indent int) string { - return fmt.Sprintf("%s(%s)", strings.Repeat(" ", indent), a.Type()) -} - -// Next() on an Int64 all iterator is a simple incrementing counter. -// Return the next integer, and mark it as the result. -func (a *Int64AllIterator) Next() (TSVal, bool) { - NextLogIn(a) - if a.at == -1 { - return NextLogOut(a, nil, false) - } - val := a.at - a.at = a.at + 1 - if a.at > a.max { - a.at = -1 - } - a.Last = val - return NextLogOut(a, val, true) -} - -// The number of elements in an Int64AllIterator is the size of the range. -// The size is exact. -func (a *Int64AllIterator) Size() (int64, bool) { - Size := ((a.max - a.min) + 1) - return Size, true -} - -// Check() for an Int64AllIterator is merely seeing if the passed value is -// withing the range, assuming the value is an int64. -func (a *Int64AllIterator) Check(tsv TSVal) bool { - CheckLogIn(a, tsv) - v := tsv.(int64) - if a.min <= v && v <= a.max { - a.Last = v - return CheckLogOut(a, v, true) - } - return CheckLogOut(a, v, false) -} - -// The type of this iterator is an "all". This is important, as it puts it in -// the class of "all iterators. -func (a *Int64AllIterator) Type() string { return "all" } - -// There's nothing to optimize about this little iterator. -func (a *Int64AllIterator) Optimize() (Iterator, bool) { return a, false } - -// Stats for an Int64AllIterator are simple. Super cheap to do any operation, -// and as big as the range. -func (a *Int64AllIterator) GetStats() *IteratorStats { - s, _ := a.Size() - return &IteratorStats{ - CheckCost: 1, - NextCost: 1, - Size: s, - } -} diff --git a/graph/all_iterator.go b/graph/all_iterator.go new file mode 100644 index 0000000..6068746 --- /dev/null +++ b/graph/all_iterator.go @@ -0,0 +1,117 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Defines one of the base iterators, the All iterator. Which, logically +// enough, represents all nodes or all links in the graph. +// +// This particular file is actually vestigal. It's up to the TripleStore to give +// us an All iterator that represents all things in the graph. So this is +// really the All iterator for the MemTripleStore. That said, it *is* one of +// the base iterators, and it helps just to see it here. + +import ( + "fmt" + "strings" +) + +// An All iterator across a range of int64 values, from `max` to `min`. +type Int64AllIterator struct { + BaseIterator + max, min int64 + at int64 +} + +// Creates a new Int64AllIterator with the given range. +func NewInt64AllIterator(min, max int64) *Int64AllIterator { + var all Int64AllIterator + BaseIteratorInit(&all.BaseIterator) + all.max = max + all.min = min + all.at = min + return &all +} + +// Start back at the beginning +func (a *Int64AllIterator) Reset() { + a.at = a.min +} + +func (a *Int64AllIterator) Close() { +} + +func (a *Int64AllIterator) Clone() Iterator { + out := NewInt64AllIterator(a.min, a.max) + out.CopyTagsFrom(a) + return out +} + +// Prints the All iterator as just an "all". +func (a *Int64AllIterator) DebugString(indent int) string { + return fmt.Sprintf("%s(%s)", strings.Repeat(" ", indent), a.Type()) +} + +// Next() on an Int64 all iterator is a simple incrementing counter. +// Return the next integer, and mark it as the result. +func (a *Int64AllIterator) Next() (TSVal, bool) { + NextLogIn(a) + if a.at == -1 { + return NextLogOut(a, nil, false) + } + val := a.at + a.at = a.at + 1 + if a.at > a.max { + a.at = -1 + } + a.Last = val + return NextLogOut(a, val, true) +} + +// The number of elements in an Int64AllIterator is the size of the range. +// The size is exact. +func (a *Int64AllIterator) Size() (int64, bool) { + Size := ((a.max - a.min) + 1) + return Size, true +} + +// Check() for an Int64AllIterator is merely seeing if the passed value is +// withing the range, assuming the value is an int64. +func (a *Int64AllIterator) Check(tsv TSVal) bool { + CheckLogIn(a, tsv) + v := tsv.(int64) + if a.min <= v && v <= a.max { + a.Last = v + return CheckLogOut(a, v, true) + } + return CheckLogOut(a, v, false) +} + +// The type of this iterator is an "all". This is important, as it puts it in +// the class of "all iterators. +func (a *Int64AllIterator) Type() string { return "all" } + +// There's nothing to optimize about this little iterator. +func (a *Int64AllIterator) Optimize() (Iterator, bool) { return a, false } + +// Stats for an Int64AllIterator are simple. Super cheap to do any operation, +// and as big as the range. +func (a *Int64AllIterator) GetStats() *IteratorStats { + s, _ := a.Size() + return &IteratorStats{ + CheckCost: 1, + NextCost: 1, + Size: s, + } +} diff --git a/graph/and-iterator-optimize.go b/graph/and-iterator-optimize.go deleted file mode 100644 index 950d681..0000000 --- a/graph/and-iterator-optimize.go +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Perhaps the most tricky file in this entire module. Really a method on the -// AndIterator, but important enough to deserve its own file. -// -// Calling Optimize() on an And iterator, like any iterator, requires that we -// preserve the underlying meaning. However, the And has many choices, namely, -// which one of it's subiterators will be the branch that does the Next()ing, -// and which ordering of the remaining iterators is the most efficient. In -// short, this is where a lot of the query optimization happens, and there are -// many wins to be had here, as well as many bad bugs. The worst class of bug -// changes the meaning of the query. The second worst class makes things really -// slow. -// -// The good news is this: If Optimize() is never called (turned off, perhaps) we can -// be sure the results are as good as the query language called for. -// -// In short, tread lightly. - -import ( - "container/list" -) - -// Optimizes the AndIterator, by picking the most efficient way to Next() and -// Check() its subiterators. For SQL fans, this is equivalent to JOIN. -func (and *AndIterator) Optimize() (Iterator, bool) { - // First, let's get the list of iterators, in order (first one is Next()ed, - // the rest are Check()ed) - oldItList := and.GetSubIterators() - - // And call Optimize() on our subtree, replacing each one in the order we - // found them. it_list is the newly optimized versions of these, and changed - // is another list, of only the ones that have returned replacements and - // changed. - itList := optimizeSubIterators(oldItList) - - // Close the replaced iterators (they ought to close themselves, but Close() - // is idempotent, so this just protects against any machinations). - closeIteratorList(oldItList, nil) - - // If we can find only one subiterator which is equivalent to this whole and, - // we can replace the And... - out := and.optimizeReplacement(itList) - if out != nil { - // ...Move the tags to the replacement... - moveTagsTo(out, and) - // ...Close everyone except `out`, our replacement... - closeIteratorList(itList, out) - // ...And return it. - return out, true - } - - // And now, without changing any of the iterators, we reorder them. it_list is - // now a permutation of itself, but the contents are unchanged. - itList = optimizeOrder(itList) - - // Okay! At this point we have an optimized order. - - // The easiest thing to do at this point is merely to create a new And iterator - // and replace ourselves with our (reordered, optimized) clone. - newAnd := NewAndIterator() - - // Add the subiterators in order. - for e := itList.Front(); e != nil; e = e.Next() { - newAnd.AddSubIterator(e.Value.(Iterator)) - } - - // Move the tags hanging on us (like any good replacement). - newAnd.CopyTagsFrom(and) - - newAnd.optimizeCheck() - - // And close ourselves but not our subiterators -- some may still be alive in - // the new And (they were unchanged upon calling Optimize() on them, at the - // start). - and.cleanUp() - return newAnd, true -} - -// Closes a list of iterators, except the one passed in `except`. Closes all -// of the iterators in the list if `except` is nil. -func closeIteratorList(l *list.List, except Iterator) { - for e := l.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - if it != except { - e.Value.(Iterator).Close() - } - } -} - -// Find if there is a single subiterator which is a valid replacement for this -// AndIterator. -func (and *AndIterator) optimizeReplacement(itList *list.List) Iterator { - // If we were created with no SubIterators, we're as good as Null. - if itList.Len() == 0 { - return &NullIterator{} - } - if itList.Len() == 1 { - // When there's only one iterator, there's only one choice. - return itList.Front().Value.(Iterator) - } - // If any of our subiterators, post-optimization, are also Null, then - // there's no point in continuing the branch, we will have no results - // and we are null as well. - if hasAnyNullIterators(itList) { - return &NullIterator{} - } - - // If we have one useful iterator, use that. - it := hasOneUsefulIterator(itList) - if it != nil { - return it - } - return nil -} - -// optimizeOrder(l) takes a list and returns a list, containing the same contents -// but with a new ordering, however it wishes. -func optimizeOrder(l *list.List) *list.List { - out := list.New() - var bestIt Iterator - bestCost := int64(1 << 62) - // bad contains iterators that can't be (efficiently) nexted, such as - // "optional" or "not". Separate them out and tack them on at the end. - bad := list.New() - - // Find the iterator with the projected "best" total cost. - // Total cost is defined as The Next()ed iterator's cost to Next() out - // all of it's contents, and to Check() each of those against everyone - // else. - for e := l.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - if !it.Nextable() { - bad.PushBack(it) - continue - } - rootStats := e.Value.(Iterator).GetStats() - projectedCost := rootStats.NextCost - for f := l.Front(); f != nil; f = f.Next() { - if !f.Value.(Iterator).Nextable() { - continue - } - if f == e { - continue - } - stats := f.Value.(Iterator).GetStats() - projectedCost += stats.CheckCost - } - projectedCost = projectedCost * rootStats.Size - if projectedCost < bestCost { - bestIt = it - bestCost = projectedCost - } - } - - // TODO(barakmich): Optimization of order need not stop here. Picking a smart - // Check() order based on probability of getting a false Check() first is - // useful (fail faster). - - // Put the best iterator (the one we wish to Next()) at the front... - out.PushBack(bestIt) - // ...And push everyone else after... - for e := l.Front(); e != nil; e = e.Next() { - thisIt := e.Value.(Iterator) - if !thisIt.Nextable() { - continue - } - if thisIt != bestIt { - out.PushBack(thisIt) - } - } - // ...And finally, the difficult children on the end. - out.PushBackList(bad) - return out -} - -// optimizeCheck(l) creates an alternate check list, containing the same contents -// but with a new ordering, however it wishes. -func (and *AndIterator) optimizeCheck() { - subIts := and.GetSubIterators() - out := list.New() - - // Find the iterator with the lowest Check() cost, push it to the front, repeat. - for subIts.Len() != 0 { - var best *list.Element - bestCost := int64(1 << 62) - for e := subIts.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - rootStats := it.GetStats() - projectedCost := rootStats.CheckCost - if projectedCost < bestCost { - best = e - bestCost = projectedCost - } - } - out.PushBack(best.Value) - subIts.Remove(best) - } - - and.checkList = out -} - -// If we're replacing ourselves by a single iterator, we need to grab the -// result tags from the iterators that, while still valid and would hold -// the same values as this and, are not going to stay. -// getSubTags() returns a map of the tags for all the subiterators. -func (and *AndIterator) getSubTags() map[string]bool { - subs := and.GetSubIterators() - tags := make(map[string]bool) - for e := subs.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - for _, tag := range it.Tags() { - tags[tag] = true - } - } - for _, tag := range and.Tags() { - tags[tag] = true - } - return tags -} - -// moveTagsTo() gets the tags for all of the And's subiterators and the -// And itself, and moves them to `out`. -func moveTagsTo(out Iterator, and *AndIterator) { - tagmap := and.getSubTags() - for _, tag := range out.Tags() { - if tagmap[tag] { - delete(tagmap, tag) - } - } - for k, _ := range tagmap { - out.AddTag(k) - } -} - -// optimizeSubIterators(l) takes a list of iterators and calls Optimize() on all -// of them. It returns two lists -- the first contains the same list as l, where -// any replacements are made by Optimize() and the second contains the originals -// which were replaced. -func optimizeSubIterators(l *list.List) *list.List { - itList := list.New() - for e := l.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - newIt, change := it.Optimize() - if change { - itList.PushBack(newIt) - } else { - itList.PushBack(it.Clone()) - } - } - return itList -} - -// Check a list of iterators for any Null iterators. -func hasAnyNullIterators(l *list.List) bool { - for e := l.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - if it.Type() == "null" { - return true - } - } - return false -} - -// There are two "not-useful" iterators -- namely "null" which returns -// nothing, and "all" which returns everything. Particularly, we want -// to see if we're intersecting with a bunch of "all" iterators, and, -// if we are, then we have only one useful iterator. -func hasOneUsefulIterator(l *list.List) Iterator { - usefulCount := 0 - var usefulIt Iterator - for e := l.Front(); e != nil; e = e.Next() { - it := e.Value.(Iterator) - switch it.Type() { - case "null", "all": - continue - case "optional": - // Optional is weird -- it's not useful, but we can't optimize - // away from it. Therefore, we skip this optimization - // if we see one. - return nil - default: - usefulCount++ - usefulIt = it - } - } - - if usefulCount == 1 { - return usefulIt - } - return nil -} - -// and.GetStats() lives here in and-iterator-optimize.go because it may -// in the future return different statistics based on how it is optimized. -// For now, however, it's pretty static. -func (and *AndIterator) GetStats() *IteratorStats { - primaryStats := and.primaryIt.GetStats() - CheckCost := primaryStats.CheckCost - NextCost := primaryStats.NextCost - Size := primaryStats.Size - for _, it := range and.internalIterators { - stats := it.GetStats() - NextCost += stats.CheckCost - CheckCost += stats.CheckCost - if Size > stats.Size { - Size = stats.Size - } - } - return &IteratorStats{ - CheckCost: CheckCost, - NextCost: NextCost, - Size: Size, - } - -} diff --git a/graph/and-iterator-optimize_test.go b/graph/and-iterator-optimize_test.go deleted file mode 100644 index 4ecee81..0000000 --- a/graph/and-iterator-optimize_test.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Tests relating to methods in and-iterator-optimize. Many are pretty simplistic, but -// nonetheless cover a lot of basic cases. - -import ( - "reflect" - "sort" - "testing" -) - -func TestIteratorPromotion(t *testing.T) { - all := NewInt64AllIterator(1, 3) - fixed := newFixedIterator() - fixed.AddValue(3) - a := NewAndIterator() - a.AddSubIterator(all) - a.AddSubIterator(fixed) - all.AddTag("a") - fixed.AddTag("b") - a.AddTag("c") - newIt, changed := a.Optimize() - if !changed { - t.Error("Iterator didn't optimize") - } - if newIt.Type() != "fixed" { - t.Error("Expected fixed iterator") - } - tagsExpected := []string{"a", "b", "c"} - tags := newIt.Tags() - sort.Strings(tags) - if !reflect.DeepEqual(tags, tagsExpected) { - t.Fatal("Tags don't match") - } -} - -func TestNullIteratorAnd(t *testing.T) { - all := NewInt64AllIterator(1, 3) - null := NewNullIterator() - a := NewAndIterator() - a.AddSubIterator(all) - a.AddSubIterator(null) - newIt, changed := a.Optimize() - if !changed { - t.Error("Didn't change") - } - if newIt.Type() != "null" { - t.Error("Expected null iterator, got ", newIt.Type()) - } -} - -func TestReorderWithTag(t *testing.T) { - all := NewInt64AllIterator(100, 300) - all.AddTag("good") - all2 := NewInt64AllIterator(1, 30000) - all2.AddTag("slow") - a := NewAndIterator() - // Make all2 the default iterator - a.AddSubIterator(all2) - a.AddSubIterator(all) - - newIt, changed := a.Optimize() - if !changed { - t.Error("Expected new iterator") - } - expectedTags := []string{"good", "slow"} - tagsOut := make([]string, 0) - l := newIt.GetSubIterators() - for e := l.Front(); e != nil; e = e.Next() { - for _, x := range e.Value.(Iterator).Tags() { - tagsOut = append(tagsOut, x) - } - } - if !reflect.DeepEqual(expectedTags, tagsOut) { - t.Fatal("Tags don't match") - } -} - -func TestAndStatistics(t *testing.T) { - all := NewInt64AllIterator(100, 300) - all.AddTag("good") - all2 := NewInt64AllIterator(1, 30000) - all2.AddTag("slow") - a := NewAndIterator() - // Make all2 the default iterator - a.AddSubIterator(all2) - a.AddSubIterator(all) - stats1 := a.GetStats() - newIt, changed := a.Optimize() - if !changed { - t.Error("Didn't optimize") - } - stats2 := newIt.GetStats() - if stats2.NextCost > stats1.NextCost { - t.Error("And didn't optimize. Next cost old ", stats1.NextCost, "and new ", stats2.NextCost) - } -} diff --git a/graph/and-iterator.go b/graph/and-iterator.go deleted file mode 100644 index a3458aa..0000000 --- a/graph/and-iterator.go +++ /dev/null @@ -1,248 +0,0 @@ -// Defines the And iterator, one of the base iterators. And requires no -// knowledge of the constituent TripleStore; its sole purpose is to act as an -// intersection operator across the subiterators it is given. If one iterator -// contains [1,3,5] and another [2,3,4] -- then And is an iterator that -// 'contains' [3] -// -// It accomplishes this in one of two ways. If it is a Next()ed iterator (that -// is, it is a top level iterator, or on the "Next() path", then it will Next() -// it's primary iterator (helpfully, and.primary_it) and Check() the resultant -// value against it's other iterators. If it matches all of them, then it -// returns that value. Otherwise, it repeats the process. -// -// If it's on a Check() path, it merely Check()s every iterator, and returns the -// logical AND of each result. - -package graph - -import ( - "container/list" - "fmt" - "strings" -) - -// The And iterator. Consists of a BaseIterator and a number of subiterators, the primary of which will -// be Next()ed if next is called. -type AndIterator struct { - BaseIterator - internalIterators []Iterator - itCount int - primaryIt Iterator - checkList *list.List -} - -// Creates a new And iterator. -func NewAndIterator() *AndIterator { - var and AndIterator - BaseIteratorInit(&and.BaseIterator) - and.internalIterators = make([]Iterator, 0, 20) - and.checkList = nil - return &and -} - -// Reset all internal iterators -func (and *AndIterator) Reset() { - and.primaryIt.Reset() - for _, it := range and.internalIterators { - it.Reset() - } - and.checkList = nil -} - -func (and *AndIterator) Clone() Iterator { - newAnd := NewAndIterator() - newAnd.AddSubIterator(and.primaryIt.Clone()) - newAnd.CopyTagsFrom(and) - for _, it := range and.internalIterators { - newAnd.AddSubIterator(it.Clone()) - } - if and.checkList != nil { - newAnd.optimizeCheck() - } - return newAnd -} - -// Returns a list.List of the subiterators, in order (primary iterator first). -func (and *AndIterator) GetSubIterators() *list.List { - l := list.New() - l.PushBack(and.primaryIt) - for _, it := range and.internalIterators { - l.PushBack(it) - } - return l -} - -// Overrides BaseIterator TagResults, as it needs to add it's own results and -// recurse down it's subiterators. -func (and *AndIterator) TagResults(out *map[string]TSVal) { - and.BaseIterator.TagResults(out) - if and.primaryIt != nil { - and.primaryIt.TagResults(out) - } - for _, it := range and.internalIterators { - it.TagResults(out) - } -} - -// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators. -func (and *AndIterator) GetResultTree() *ResultTree { - tree := NewResultTree(and.LastResult()) - tree.AddSubtree(and.primaryIt.GetResultTree()) - for _, it := range and.internalIterators { - tree.AddSubtree(it.GetResultTree()) - } - return tree -} - -// Prints information about this iterator. -func (and *AndIterator) DebugString(indent int) string { - var total string - for i, it := range and.internalIterators { - total += strings.Repeat(" ", indent+2) - total += fmt.Sprintf("%d:\n%s\n", i, it.DebugString(indent+4)) - } - var tags string - for _, k := range and.Tags() { - tags += fmt.Sprintf("%s;", k) - } - spaces := strings.Repeat(" ", indent+2) - - return fmt.Sprintf("%s(%s %d\n%stags:%s\n%sprimary_it:\n%s\n%sother_its:\n%s)", - strings.Repeat(" ", indent), - and.Type(), - and.GetUid(), - spaces, - tags, - spaces, - and.primaryIt.DebugString(indent+4), - spaces, - total) -} - -// Add a subiterator to this And iterator. -// -// The first iterator that is added becomes the primary iterator. This is -// important. Calling Optimize() is the way to change the order based on -// subiterator statistics. Without Optimize(), the order added is the order -// used. -func (and *AndIterator) AddSubIterator(sub Iterator) { - if and.itCount > 0 { - and.internalIterators = append(and.internalIterators, sub) - and.itCount++ - return - } - and.primaryIt = sub - and.itCount++ -} - -// Returns the Next value from the And iterator. Because the And is the -// intersection of its subiterators, it must choose one subiterator to produce a -// candidate, and check this value against the subiterators. A productive choice -// of primary iterator is therefore very important. -func (and *AndIterator) Next() (TSVal, bool) { - NextLogIn(and) - var curr TSVal - var exists bool - for { - - curr, exists = and.primaryIt.Next() - if !exists { - return NextLogOut(and, nil, false) - } - if and.checkSubIts(curr) { - and.Last = curr - return NextLogOut(and, curr, true) - } - } - panic("Somehow broke out of Next() loop in AndIterator") -} - -// Checks a value against the non-primary iterators, in order. -func (and *AndIterator) checkSubIts(val TSVal) bool { - var subIsGood = true - for _, it := range and.internalIterators { - subIsGood = it.Check(val) - if !subIsGood { - break - } - } - return subIsGood -} - -func (and *AndIterator) checkCheckList(val TSVal) bool { - var isGood = true - for e := and.checkList.Front(); e != nil; e = e.Next() { - isGood = e.Value.(Iterator).Check(val) - if !isGood { - break - } - } - return CheckLogOut(and, val, isGood) -} - -// Check a value against the entire iterator, in order. -func (and *AndIterator) Check(val TSVal) bool { - CheckLogIn(and, val) - if and.checkList != nil { - return and.checkCheckList(val) - } - mainGood := and.primaryIt.Check(val) - if !mainGood { - return CheckLogOut(and, val, false) - } - othersGood := and.checkSubIts(val) - if !othersGood { - return CheckLogOut(and, val, false) - } - and.Last = val - return CheckLogOut(and, val, true) -} - -// Returns the approximate size of the And iterator. Because we're dealing -// with an intersection, we know that the largest we can be is the size of the -// smallest iterator. This is the heuristic we shall follow. Better heuristics -// welcome. -func (and *AndIterator) Size() (int64, bool) { - val, b := and.primaryIt.Size() - for _, it := range and.internalIterators { - newval, newb := it.Size() - if val > newval { - val = newval - } - b = newb && b - } - return val, b -} - -// An And has no NextResult of its own -- that is, there are no other values -// which satisfy our previous result that are not the result itself. Our -// subiterators might, however, so just pass the call recursively. -func (and *AndIterator) NextResult() bool { - if and.primaryIt.NextResult() { - return true - } - for _, it := range and.internalIterators { - if it.NextResult() { - return true - } - } - return false -} - -// Perform and-specific cleanup, of which there currently is none. -func (and *AndIterator) cleanUp() { -} - -// Close this iterator, and, by extension, close the subiterators. -// Close should be idempotent, and it follows that if it's subiterators -// follow this contract, the And follows the contract. -func (and *AndIterator) Close() { - and.cleanUp() - and.primaryIt.Close() - for _, it := range and.internalIterators { - it.Close() - } -} - -// Register this as an "and" iterator. -func (and *AndIterator) Type() string { return "and" } diff --git a/graph/and-iterator_test.go b/graph/and-iterator_test.go deleted file mode 100644 index d0fbf2e..0000000 --- a/graph/and-iterator_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - "testing" -) - -// Make sure that tags work on the And. -func TestTag(t *testing.T) { - fix1 := newFixedIterator() - fix1.AddValue(234) - fix1.AddTag("foo") - and := NewAndIterator() - and.AddSubIterator(fix1) - and.AddTag("bar") - out := fix1.Tags() - if len(out) != 1 { - t.Errorf("Expected length 1, got %d", len(out)) - } - if out[0] != "foo" { - t.Errorf("Cannot get tag back, got %s", out[0]) - } - - val, ok := and.Next() - if !ok { - t.Errorf("And did not next") - } - if val != 234 { - t.Errorf("Unexpected value") - } - tags := make(map[string]TSVal) - and.TagResults(&tags) - if tags["bar"] != 234 { - t.Errorf("no bar tag") - } - if tags["foo"] != 234 { - t.Errorf("no foo tag") - } -} - -// Do a simple itersection of fixed values. -func TestAndAndFixedIterators(t *testing.T) { - fix1 := newFixedIterator() - fix1.AddValue(1) - fix1.AddValue(2) - fix1.AddValue(3) - fix1.AddValue(4) - fix2 := newFixedIterator() - fix2.AddValue(3) - fix2.AddValue(4) - fix2.AddValue(5) - and := NewAndIterator() - and.AddSubIterator(fix1) - and.AddSubIterator(fix2) - // Should be as big as smallest subiterator - size, accurate := and.Size() - if size != 3 { - t.Error("Incorrect size") - } - if !accurate { - t.Error("not accurate") - } - - val, ok := and.Next() - if val != 3 || ok == false { - t.Error("Incorrect first value") - } - - val, ok = and.Next() - if val != 4 || ok == false { - t.Error("Incorrect second value") - } - - val, ok = and.Next() - if ok { - t.Error("Too many values") - } - -} - -// If there's no intersection, the size should still report the same, -// but there should be nothing to Next() -func TestNonOverlappingFixedIterators(t *testing.T) { - fix1 := newFixedIterator() - fix1.AddValue(1) - fix1.AddValue(2) - fix1.AddValue(3) - fix1.AddValue(4) - fix2 := newFixedIterator() - fix2.AddValue(5) - fix2.AddValue(6) - fix2.AddValue(7) - and := NewAndIterator() - and.AddSubIterator(fix1) - and.AddSubIterator(fix2) - // Should be as big as smallest subiterator - size, accurate := and.Size() - if size != 3 { - t.Error("Incorrect size") - } - if !accurate { - t.Error("not accurate") - } - - _, ok := and.Next() - if ok { - t.Error("Too many values") - } - -} - -func TestAllIterators(t *testing.T) { - all1 := NewInt64AllIterator(1, 5) - all2 := NewInt64AllIterator(4, 10) - and := NewAndIterator() - and.AddSubIterator(all2) - and.AddSubIterator(all1) - - val, ok := and.Next() - if val.(int64) != 4 || ok == false { - t.Error("Incorrect first value") - } - - val, ok = and.Next() - if val.(int64) != 5 || ok == false { - t.Error("Incorrect second value") - } - - val, ok = and.Next() - if ok { - t.Error("Too many values") - } - -} diff --git a/graph/and_iterator.go b/graph/and_iterator.go new file mode 100644 index 0000000..a3458aa --- /dev/null +++ b/graph/and_iterator.go @@ -0,0 +1,248 @@ +// Defines the And iterator, one of the base iterators. And requires no +// knowledge of the constituent TripleStore; its sole purpose is to act as an +// intersection operator across the subiterators it is given. If one iterator +// contains [1,3,5] and another [2,3,4] -- then And is an iterator that +// 'contains' [3] +// +// It accomplishes this in one of two ways. If it is a Next()ed iterator (that +// is, it is a top level iterator, or on the "Next() path", then it will Next() +// it's primary iterator (helpfully, and.primary_it) and Check() the resultant +// value against it's other iterators. If it matches all of them, then it +// returns that value. Otherwise, it repeats the process. +// +// If it's on a Check() path, it merely Check()s every iterator, and returns the +// logical AND of each result. + +package graph + +import ( + "container/list" + "fmt" + "strings" +) + +// The And iterator. Consists of a BaseIterator and a number of subiterators, the primary of which will +// be Next()ed if next is called. +type AndIterator struct { + BaseIterator + internalIterators []Iterator + itCount int + primaryIt Iterator + checkList *list.List +} + +// Creates a new And iterator. +func NewAndIterator() *AndIterator { + var and AndIterator + BaseIteratorInit(&and.BaseIterator) + and.internalIterators = make([]Iterator, 0, 20) + and.checkList = nil + return &and +} + +// Reset all internal iterators +func (and *AndIterator) Reset() { + and.primaryIt.Reset() + for _, it := range and.internalIterators { + it.Reset() + } + and.checkList = nil +} + +func (and *AndIterator) Clone() Iterator { + newAnd := NewAndIterator() + newAnd.AddSubIterator(and.primaryIt.Clone()) + newAnd.CopyTagsFrom(and) + for _, it := range and.internalIterators { + newAnd.AddSubIterator(it.Clone()) + } + if and.checkList != nil { + newAnd.optimizeCheck() + } + return newAnd +} + +// Returns a list.List of the subiterators, in order (primary iterator first). +func (and *AndIterator) GetSubIterators() *list.List { + l := list.New() + l.PushBack(and.primaryIt) + for _, it := range and.internalIterators { + l.PushBack(it) + } + return l +} + +// Overrides BaseIterator TagResults, as it needs to add it's own results and +// recurse down it's subiterators. +func (and *AndIterator) TagResults(out *map[string]TSVal) { + and.BaseIterator.TagResults(out) + if and.primaryIt != nil { + and.primaryIt.TagResults(out) + } + for _, it := range and.internalIterators { + it.TagResults(out) + } +} + +// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators. +func (and *AndIterator) GetResultTree() *ResultTree { + tree := NewResultTree(and.LastResult()) + tree.AddSubtree(and.primaryIt.GetResultTree()) + for _, it := range and.internalIterators { + tree.AddSubtree(it.GetResultTree()) + } + return tree +} + +// Prints information about this iterator. +func (and *AndIterator) DebugString(indent int) string { + var total string + for i, it := range and.internalIterators { + total += strings.Repeat(" ", indent+2) + total += fmt.Sprintf("%d:\n%s\n", i, it.DebugString(indent+4)) + } + var tags string + for _, k := range and.Tags() { + tags += fmt.Sprintf("%s;", k) + } + spaces := strings.Repeat(" ", indent+2) + + return fmt.Sprintf("%s(%s %d\n%stags:%s\n%sprimary_it:\n%s\n%sother_its:\n%s)", + strings.Repeat(" ", indent), + and.Type(), + and.GetUid(), + spaces, + tags, + spaces, + and.primaryIt.DebugString(indent+4), + spaces, + total) +} + +// Add a subiterator to this And iterator. +// +// The first iterator that is added becomes the primary iterator. This is +// important. Calling Optimize() is the way to change the order based on +// subiterator statistics. Without Optimize(), the order added is the order +// used. +func (and *AndIterator) AddSubIterator(sub Iterator) { + if and.itCount > 0 { + and.internalIterators = append(and.internalIterators, sub) + and.itCount++ + return + } + and.primaryIt = sub + and.itCount++ +} + +// Returns the Next value from the And iterator. Because the And is the +// intersection of its subiterators, it must choose one subiterator to produce a +// candidate, and check this value against the subiterators. A productive choice +// of primary iterator is therefore very important. +func (and *AndIterator) Next() (TSVal, bool) { + NextLogIn(and) + var curr TSVal + var exists bool + for { + + curr, exists = and.primaryIt.Next() + if !exists { + return NextLogOut(and, nil, false) + } + if and.checkSubIts(curr) { + and.Last = curr + return NextLogOut(and, curr, true) + } + } + panic("Somehow broke out of Next() loop in AndIterator") +} + +// Checks a value against the non-primary iterators, in order. +func (and *AndIterator) checkSubIts(val TSVal) bool { + var subIsGood = true + for _, it := range and.internalIterators { + subIsGood = it.Check(val) + if !subIsGood { + break + } + } + return subIsGood +} + +func (and *AndIterator) checkCheckList(val TSVal) bool { + var isGood = true + for e := and.checkList.Front(); e != nil; e = e.Next() { + isGood = e.Value.(Iterator).Check(val) + if !isGood { + break + } + } + return CheckLogOut(and, val, isGood) +} + +// Check a value against the entire iterator, in order. +func (and *AndIterator) Check(val TSVal) bool { + CheckLogIn(and, val) + if and.checkList != nil { + return and.checkCheckList(val) + } + mainGood := and.primaryIt.Check(val) + if !mainGood { + return CheckLogOut(and, val, false) + } + othersGood := and.checkSubIts(val) + if !othersGood { + return CheckLogOut(and, val, false) + } + and.Last = val + return CheckLogOut(and, val, true) +} + +// Returns the approximate size of the And iterator. Because we're dealing +// with an intersection, we know that the largest we can be is the size of the +// smallest iterator. This is the heuristic we shall follow. Better heuristics +// welcome. +func (and *AndIterator) Size() (int64, bool) { + val, b := and.primaryIt.Size() + for _, it := range and.internalIterators { + newval, newb := it.Size() + if val > newval { + val = newval + } + b = newb && b + } + return val, b +} + +// An And has no NextResult of its own -- that is, there are no other values +// which satisfy our previous result that are not the result itself. Our +// subiterators might, however, so just pass the call recursively. +func (and *AndIterator) NextResult() bool { + if and.primaryIt.NextResult() { + return true + } + for _, it := range and.internalIterators { + if it.NextResult() { + return true + } + } + return false +} + +// Perform and-specific cleanup, of which there currently is none. +func (and *AndIterator) cleanUp() { +} + +// Close this iterator, and, by extension, close the subiterators. +// Close should be idempotent, and it follows that if it's subiterators +// follow this contract, the And follows the contract. +func (and *AndIterator) Close() { + and.cleanUp() + and.primaryIt.Close() + for _, it := range and.internalIterators { + it.Close() + } +} + +// Register this as an "and" iterator. +func (and *AndIterator) Type() string { return "and" } diff --git a/graph/and_iterator_optimize.go b/graph/and_iterator_optimize.go new file mode 100644 index 0000000..950d681 --- /dev/null +++ b/graph/and_iterator_optimize.go @@ -0,0 +1,330 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Perhaps the most tricky file in this entire module. Really a method on the +// AndIterator, but important enough to deserve its own file. +// +// Calling Optimize() on an And iterator, like any iterator, requires that we +// preserve the underlying meaning. However, the And has many choices, namely, +// which one of it's subiterators will be the branch that does the Next()ing, +// and which ordering of the remaining iterators is the most efficient. In +// short, this is where a lot of the query optimization happens, and there are +// many wins to be had here, as well as many bad bugs. The worst class of bug +// changes the meaning of the query. The second worst class makes things really +// slow. +// +// The good news is this: If Optimize() is never called (turned off, perhaps) we can +// be sure the results are as good as the query language called for. +// +// In short, tread lightly. + +import ( + "container/list" +) + +// Optimizes the AndIterator, by picking the most efficient way to Next() and +// Check() its subiterators. For SQL fans, this is equivalent to JOIN. +func (and *AndIterator) Optimize() (Iterator, bool) { + // First, let's get the list of iterators, in order (first one is Next()ed, + // the rest are Check()ed) + oldItList := and.GetSubIterators() + + // And call Optimize() on our subtree, replacing each one in the order we + // found them. it_list is the newly optimized versions of these, and changed + // is another list, of only the ones that have returned replacements and + // changed. + itList := optimizeSubIterators(oldItList) + + // Close the replaced iterators (they ought to close themselves, but Close() + // is idempotent, so this just protects against any machinations). + closeIteratorList(oldItList, nil) + + // If we can find only one subiterator which is equivalent to this whole and, + // we can replace the And... + out := and.optimizeReplacement(itList) + if out != nil { + // ...Move the tags to the replacement... + moveTagsTo(out, and) + // ...Close everyone except `out`, our replacement... + closeIteratorList(itList, out) + // ...And return it. + return out, true + } + + // And now, without changing any of the iterators, we reorder them. it_list is + // now a permutation of itself, but the contents are unchanged. + itList = optimizeOrder(itList) + + // Okay! At this point we have an optimized order. + + // The easiest thing to do at this point is merely to create a new And iterator + // and replace ourselves with our (reordered, optimized) clone. + newAnd := NewAndIterator() + + // Add the subiterators in order. + for e := itList.Front(); e != nil; e = e.Next() { + newAnd.AddSubIterator(e.Value.(Iterator)) + } + + // Move the tags hanging on us (like any good replacement). + newAnd.CopyTagsFrom(and) + + newAnd.optimizeCheck() + + // And close ourselves but not our subiterators -- some may still be alive in + // the new And (they were unchanged upon calling Optimize() on them, at the + // start). + and.cleanUp() + return newAnd, true +} + +// Closes a list of iterators, except the one passed in `except`. Closes all +// of the iterators in the list if `except` is nil. +func closeIteratorList(l *list.List, except Iterator) { + for e := l.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + if it != except { + e.Value.(Iterator).Close() + } + } +} + +// Find if there is a single subiterator which is a valid replacement for this +// AndIterator. +func (and *AndIterator) optimizeReplacement(itList *list.List) Iterator { + // If we were created with no SubIterators, we're as good as Null. + if itList.Len() == 0 { + return &NullIterator{} + } + if itList.Len() == 1 { + // When there's only one iterator, there's only one choice. + return itList.Front().Value.(Iterator) + } + // If any of our subiterators, post-optimization, are also Null, then + // there's no point in continuing the branch, we will have no results + // and we are null as well. + if hasAnyNullIterators(itList) { + return &NullIterator{} + } + + // If we have one useful iterator, use that. + it := hasOneUsefulIterator(itList) + if it != nil { + return it + } + return nil +} + +// optimizeOrder(l) takes a list and returns a list, containing the same contents +// but with a new ordering, however it wishes. +func optimizeOrder(l *list.List) *list.List { + out := list.New() + var bestIt Iterator + bestCost := int64(1 << 62) + // bad contains iterators that can't be (efficiently) nexted, such as + // "optional" or "not". Separate them out and tack them on at the end. + bad := list.New() + + // Find the iterator with the projected "best" total cost. + // Total cost is defined as The Next()ed iterator's cost to Next() out + // all of it's contents, and to Check() each of those against everyone + // else. + for e := l.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + if !it.Nextable() { + bad.PushBack(it) + continue + } + rootStats := e.Value.(Iterator).GetStats() + projectedCost := rootStats.NextCost + for f := l.Front(); f != nil; f = f.Next() { + if !f.Value.(Iterator).Nextable() { + continue + } + if f == e { + continue + } + stats := f.Value.(Iterator).GetStats() + projectedCost += stats.CheckCost + } + projectedCost = projectedCost * rootStats.Size + if projectedCost < bestCost { + bestIt = it + bestCost = projectedCost + } + } + + // TODO(barakmich): Optimization of order need not stop here. Picking a smart + // Check() order based on probability of getting a false Check() first is + // useful (fail faster). + + // Put the best iterator (the one we wish to Next()) at the front... + out.PushBack(bestIt) + // ...And push everyone else after... + for e := l.Front(); e != nil; e = e.Next() { + thisIt := e.Value.(Iterator) + if !thisIt.Nextable() { + continue + } + if thisIt != bestIt { + out.PushBack(thisIt) + } + } + // ...And finally, the difficult children on the end. + out.PushBackList(bad) + return out +} + +// optimizeCheck(l) creates an alternate check list, containing the same contents +// but with a new ordering, however it wishes. +func (and *AndIterator) optimizeCheck() { + subIts := and.GetSubIterators() + out := list.New() + + // Find the iterator with the lowest Check() cost, push it to the front, repeat. + for subIts.Len() != 0 { + var best *list.Element + bestCost := int64(1 << 62) + for e := subIts.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + rootStats := it.GetStats() + projectedCost := rootStats.CheckCost + if projectedCost < bestCost { + best = e + bestCost = projectedCost + } + } + out.PushBack(best.Value) + subIts.Remove(best) + } + + and.checkList = out +} + +// If we're replacing ourselves by a single iterator, we need to grab the +// result tags from the iterators that, while still valid and would hold +// the same values as this and, are not going to stay. +// getSubTags() returns a map of the tags for all the subiterators. +func (and *AndIterator) getSubTags() map[string]bool { + subs := and.GetSubIterators() + tags := make(map[string]bool) + for e := subs.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + for _, tag := range it.Tags() { + tags[tag] = true + } + } + for _, tag := range and.Tags() { + tags[tag] = true + } + return tags +} + +// moveTagsTo() gets the tags for all of the And's subiterators and the +// And itself, and moves them to `out`. +func moveTagsTo(out Iterator, and *AndIterator) { + tagmap := and.getSubTags() + for _, tag := range out.Tags() { + if tagmap[tag] { + delete(tagmap, tag) + } + } + for k, _ := range tagmap { + out.AddTag(k) + } +} + +// optimizeSubIterators(l) takes a list of iterators and calls Optimize() on all +// of them. It returns two lists -- the first contains the same list as l, where +// any replacements are made by Optimize() and the second contains the originals +// which were replaced. +func optimizeSubIterators(l *list.List) *list.List { + itList := list.New() + for e := l.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + newIt, change := it.Optimize() + if change { + itList.PushBack(newIt) + } else { + itList.PushBack(it.Clone()) + } + } + return itList +} + +// Check a list of iterators for any Null iterators. +func hasAnyNullIterators(l *list.List) bool { + for e := l.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + if it.Type() == "null" { + return true + } + } + return false +} + +// There are two "not-useful" iterators -- namely "null" which returns +// nothing, and "all" which returns everything. Particularly, we want +// to see if we're intersecting with a bunch of "all" iterators, and, +// if we are, then we have only one useful iterator. +func hasOneUsefulIterator(l *list.List) Iterator { + usefulCount := 0 + var usefulIt Iterator + for e := l.Front(); e != nil; e = e.Next() { + it := e.Value.(Iterator) + switch it.Type() { + case "null", "all": + continue + case "optional": + // Optional is weird -- it's not useful, but we can't optimize + // away from it. Therefore, we skip this optimization + // if we see one. + return nil + default: + usefulCount++ + usefulIt = it + } + } + + if usefulCount == 1 { + return usefulIt + } + return nil +} + +// and.GetStats() lives here in and-iterator-optimize.go because it may +// in the future return different statistics based on how it is optimized. +// For now, however, it's pretty static. +func (and *AndIterator) GetStats() *IteratorStats { + primaryStats := and.primaryIt.GetStats() + CheckCost := primaryStats.CheckCost + NextCost := primaryStats.NextCost + Size := primaryStats.Size + for _, it := range and.internalIterators { + stats := it.GetStats() + NextCost += stats.CheckCost + CheckCost += stats.CheckCost + if Size > stats.Size { + Size = stats.Size + } + } + return &IteratorStats{ + CheckCost: CheckCost, + NextCost: NextCost, + Size: Size, + } + +} diff --git a/graph/and_iterator_optimize_test.go b/graph/and_iterator_optimize_test.go new file mode 100644 index 0000000..4ecee81 --- /dev/null +++ b/graph/and_iterator_optimize_test.go @@ -0,0 +1,111 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Tests relating to methods in and-iterator-optimize. Many are pretty simplistic, but +// nonetheless cover a lot of basic cases. + +import ( + "reflect" + "sort" + "testing" +) + +func TestIteratorPromotion(t *testing.T) { + all := NewInt64AllIterator(1, 3) + fixed := newFixedIterator() + fixed.AddValue(3) + a := NewAndIterator() + a.AddSubIterator(all) + a.AddSubIterator(fixed) + all.AddTag("a") + fixed.AddTag("b") + a.AddTag("c") + newIt, changed := a.Optimize() + if !changed { + t.Error("Iterator didn't optimize") + } + if newIt.Type() != "fixed" { + t.Error("Expected fixed iterator") + } + tagsExpected := []string{"a", "b", "c"} + tags := newIt.Tags() + sort.Strings(tags) + if !reflect.DeepEqual(tags, tagsExpected) { + t.Fatal("Tags don't match") + } +} + +func TestNullIteratorAnd(t *testing.T) { + all := NewInt64AllIterator(1, 3) + null := NewNullIterator() + a := NewAndIterator() + a.AddSubIterator(all) + a.AddSubIterator(null) + newIt, changed := a.Optimize() + if !changed { + t.Error("Didn't change") + } + if newIt.Type() != "null" { + t.Error("Expected null iterator, got ", newIt.Type()) + } +} + +func TestReorderWithTag(t *testing.T) { + all := NewInt64AllIterator(100, 300) + all.AddTag("good") + all2 := NewInt64AllIterator(1, 30000) + all2.AddTag("slow") + a := NewAndIterator() + // Make all2 the default iterator + a.AddSubIterator(all2) + a.AddSubIterator(all) + + newIt, changed := a.Optimize() + if !changed { + t.Error("Expected new iterator") + } + expectedTags := []string{"good", "slow"} + tagsOut := make([]string, 0) + l := newIt.GetSubIterators() + for e := l.Front(); e != nil; e = e.Next() { + for _, x := range e.Value.(Iterator).Tags() { + tagsOut = append(tagsOut, x) + } + } + if !reflect.DeepEqual(expectedTags, tagsOut) { + t.Fatal("Tags don't match") + } +} + +func TestAndStatistics(t *testing.T) { + all := NewInt64AllIterator(100, 300) + all.AddTag("good") + all2 := NewInt64AllIterator(1, 30000) + all2.AddTag("slow") + a := NewAndIterator() + // Make all2 the default iterator + a.AddSubIterator(all2) + a.AddSubIterator(all) + stats1 := a.GetStats() + newIt, changed := a.Optimize() + if !changed { + t.Error("Didn't optimize") + } + stats2 := newIt.GetStats() + if stats2.NextCost > stats1.NextCost { + t.Error("And didn't optimize. Next cost old ", stats1.NextCost, "and new ", stats2.NextCost) + } +} diff --git a/graph/and_iterator_test.go b/graph/and_iterator_test.go new file mode 100644 index 0000000..d0fbf2e --- /dev/null +++ b/graph/and_iterator_test.go @@ -0,0 +1,147 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "testing" +) + +// Make sure that tags work on the And. +func TestTag(t *testing.T) { + fix1 := newFixedIterator() + fix1.AddValue(234) + fix1.AddTag("foo") + and := NewAndIterator() + and.AddSubIterator(fix1) + and.AddTag("bar") + out := fix1.Tags() + if len(out) != 1 { + t.Errorf("Expected length 1, got %d", len(out)) + } + if out[0] != "foo" { + t.Errorf("Cannot get tag back, got %s", out[0]) + } + + val, ok := and.Next() + if !ok { + t.Errorf("And did not next") + } + if val != 234 { + t.Errorf("Unexpected value") + } + tags := make(map[string]TSVal) + and.TagResults(&tags) + if tags["bar"] != 234 { + t.Errorf("no bar tag") + } + if tags["foo"] != 234 { + t.Errorf("no foo tag") + } +} + +// Do a simple itersection of fixed values. +func TestAndAndFixedIterators(t *testing.T) { + fix1 := newFixedIterator() + fix1.AddValue(1) + fix1.AddValue(2) + fix1.AddValue(3) + fix1.AddValue(4) + fix2 := newFixedIterator() + fix2.AddValue(3) + fix2.AddValue(4) + fix2.AddValue(5) + and := NewAndIterator() + and.AddSubIterator(fix1) + and.AddSubIterator(fix2) + // Should be as big as smallest subiterator + size, accurate := and.Size() + if size != 3 { + t.Error("Incorrect size") + } + if !accurate { + t.Error("not accurate") + } + + val, ok := and.Next() + if val != 3 || ok == false { + t.Error("Incorrect first value") + } + + val, ok = and.Next() + if val != 4 || ok == false { + t.Error("Incorrect second value") + } + + val, ok = and.Next() + if ok { + t.Error("Too many values") + } + +} + +// If there's no intersection, the size should still report the same, +// but there should be nothing to Next() +func TestNonOverlappingFixedIterators(t *testing.T) { + fix1 := newFixedIterator() + fix1.AddValue(1) + fix1.AddValue(2) + fix1.AddValue(3) + fix1.AddValue(4) + fix2 := newFixedIterator() + fix2.AddValue(5) + fix2.AddValue(6) + fix2.AddValue(7) + and := NewAndIterator() + and.AddSubIterator(fix1) + and.AddSubIterator(fix2) + // Should be as big as smallest subiterator + size, accurate := and.Size() + if size != 3 { + t.Error("Incorrect size") + } + if !accurate { + t.Error("not accurate") + } + + _, ok := and.Next() + if ok { + t.Error("Too many values") + } + +} + +func TestAllIterators(t *testing.T) { + all1 := NewInt64AllIterator(1, 5) + all2 := NewInt64AllIterator(4, 10) + and := NewAndIterator() + and.AddSubIterator(all2) + and.AddSubIterator(all1) + + val, ok := and.Next() + if val.(int64) != 4 || ok == false { + t.Error("Incorrect first value") + } + + val, ok = and.Next() + if val.(int64) != 5 || ok == false { + t.Error("Incorrect second value") + } + + val, ok = and.Next() + if ok { + t.Error("Too many values") + } + +} diff --git a/graph/fixed-iterator.go b/graph/fixed-iterator.go deleted file mode 100644 index 7578611..0000000 --- a/graph/fixed-iterator.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Defines one of the base iterators, the Fixed iterator. A fixed iterator is quite simple; it -// contains an explicit fixed array of values. -// -// A fixed iterator requires an Equality function to be passed to it, by reason that TSVal, the -// opaque Triple store value, may not answer to ==. - -import ( - "fmt" - "strings" -) - -// A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and -// an equality function. -type FixedIterator struct { - BaseIterator - values []TSVal - lastIndex int - cmp Equality -} - -// Define the signature of an equality function. -type Equality func(a, b TSVal) bool - -// Define an equality function of purely ==, which works for native types. -func BasicEquality(a, b TSVal) bool { - if a == b { - return true - } - return false -} - -// Creates a new Fixed iterator based around == equality. -func newFixedIterator() *FixedIterator { - return NewFixedIteratorWithCompare(BasicEquality) -} - -// Creates a new Fixed iterator with a custom comparitor. -func NewFixedIteratorWithCompare(compareFn Equality) *FixedIterator { - var it FixedIterator - BaseIteratorInit(&it.BaseIterator) - it.values = make([]TSVal, 0, 20) - it.lastIndex = 0 - it.cmp = compareFn - return &it -} - -func (f *FixedIterator) Reset() { - f.lastIndex = 0 -} - -func (f *FixedIterator) Close() { -} - -func (f *FixedIterator) Clone() Iterator { - out := NewFixedIteratorWithCompare(f.cmp) - for _, val := range f.values { - out.AddValue(val) - } - out.CopyTagsFrom(f) - return out -} - -// Add a value to the iterator. The array now contains this value. -// TODO(barakmich): This ought to be a set someday, disallowing repeated values. -func (f *FixedIterator) AddValue(v TSVal) { - f.values = append(f.values, v) -} - -// Print some information about the iterator. -func (f *FixedIterator) DebugString(indent int) string { - value := "" - if len(f.values) > 0 { - value = fmt.Sprint(f.values[0]) - } - return fmt.Sprintf("%s(%s tags: %s Size: %d id0: %d)", - strings.Repeat(" ", indent), - f.Type(), - f.FixedTags(), - len(f.values), - value, - ) -} - -// Register this iterator as a Fixed iterator. -func (f *FixedIterator) Type() string { - return "fixed" -} - -// Check if the passed value is equal to one of the values stored in the iterator. -func (f *FixedIterator) Check(v TSVal) bool { - // Could be optimized by keeping it sorted or using a better datastructure. - // However, for fixed iterators, which are by definition kind of tiny, this - // isn't a big issue. - CheckLogIn(f, v) - for _, x := range f.values { - if f.cmp(x, v) { - f.Last = x - return CheckLogOut(f, v, true) - } - } - return CheckLogOut(f, v, false) -} - -// Return the next stored value from the iterator. -func (f *FixedIterator) Next() (TSVal, bool) { - NextLogIn(f) - if f.lastIndex == len(f.values) { - return NextLogOut(f, nil, false) - } - out := f.values[f.lastIndex] - f.Last = out - f.lastIndex++ - return NextLogOut(f, out, true) -} - -// Optimize() for a Fixed iterator is simple. Returns a Null iterator if it's empty -// (so that other iterators upstream can treat this as null) or there is no -// optimization. -func (f *FixedIterator) Optimize() (Iterator, bool) { - - if len(f.values) == 1 && f.values[0] == nil { - return &NullIterator{}, true - } - - return f, false -} - -// Size is the number of values stored. -func (f *FixedIterator) Size() (int64, bool) { - return int64(len(f.values)), true -} - -// As we right now have to scan the entire list, Next and Check are linear with the -// size. However, a better data structure could remove these limits. -func (a *FixedIterator) GetStats() *IteratorStats { - return &IteratorStats{ - CheckCost: int64(len(a.values)), - NextCost: int64(len(a.values)), - Size: int64(len(a.values)), - } -} diff --git a/graph/fixed_iterator.go b/graph/fixed_iterator.go new file mode 100644 index 0000000..7578611 --- /dev/null +++ b/graph/fixed_iterator.go @@ -0,0 +1,157 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Defines one of the base iterators, the Fixed iterator. A fixed iterator is quite simple; it +// contains an explicit fixed array of values. +// +// A fixed iterator requires an Equality function to be passed to it, by reason that TSVal, the +// opaque Triple store value, may not answer to ==. + +import ( + "fmt" + "strings" +) + +// A Fixed iterator consists of it's values, an index (where it is in the process of Next()ing) and +// an equality function. +type FixedIterator struct { + BaseIterator + values []TSVal + lastIndex int + cmp Equality +} + +// Define the signature of an equality function. +type Equality func(a, b TSVal) bool + +// Define an equality function of purely ==, which works for native types. +func BasicEquality(a, b TSVal) bool { + if a == b { + return true + } + return false +} + +// Creates a new Fixed iterator based around == equality. +func newFixedIterator() *FixedIterator { + return NewFixedIteratorWithCompare(BasicEquality) +} + +// Creates a new Fixed iterator with a custom comparitor. +func NewFixedIteratorWithCompare(compareFn Equality) *FixedIterator { + var it FixedIterator + BaseIteratorInit(&it.BaseIterator) + it.values = make([]TSVal, 0, 20) + it.lastIndex = 0 + it.cmp = compareFn + return &it +} + +func (f *FixedIterator) Reset() { + f.lastIndex = 0 +} + +func (f *FixedIterator) Close() { +} + +func (f *FixedIterator) Clone() Iterator { + out := NewFixedIteratorWithCompare(f.cmp) + for _, val := range f.values { + out.AddValue(val) + } + out.CopyTagsFrom(f) + return out +} + +// Add a value to the iterator. The array now contains this value. +// TODO(barakmich): This ought to be a set someday, disallowing repeated values. +func (f *FixedIterator) AddValue(v TSVal) { + f.values = append(f.values, v) +} + +// Print some information about the iterator. +func (f *FixedIterator) DebugString(indent int) string { + value := "" + if len(f.values) > 0 { + value = fmt.Sprint(f.values[0]) + } + return fmt.Sprintf("%s(%s tags: %s Size: %d id0: %d)", + strings.Repeat(" ", indent), + f.Type(), + f.FixedTags(), + len(f.values), + value, + ) +} + +// Register this iterator as a Fixed iterator. +func (f *FixedIterator) Type() string { + return "fixed" +} + +// Check if the passed value is equal to one of the values stored in the iterator. +func (f *FixedIterator) Check(v TSVal) bool { + // Could be optimized by keeping it sorted or using a better datastructure. + // However, for fixed iterators, which are by definition kind of tiny, this + // isn't a big issue. + CheckLogIn(f, v) + for _, x := range f.values { + if f.cmp(x, v) { + f.Last = x + return CheckLogOut(f, v, true) + } + } + return CheckLogOut(f, v, false) +} + +// Return the next stored value from the iterator. +func (f *FixedIterator) Next() (TSVal, bool) { + NextLogIn(f) + if f.lastIndex == len(f.values) { + return NextLogOut(f, nil, false) + } + out := f.values[f.lastIndex] + f.Last = out + f.lastIndex++ + return NextLogOut(f, out, true) +} + +// Optimize() for a Fixed iterator is simple. Returns a Null iterator if it's empty +// (so that other iterators upstream can treat this as null) or there is no +// optimization. +func (f *FixedIterator) Optimize() (Iterator, bool) { + + if len(f.values) == 1 && f.values[0] == nil { + return &NullIterator{}, true + } + + return f, false +} + +// Size is the number of values stored. +func (f *FixedIterator) Size() (int64, bool) { + return int64(len(f.values)), true +} + +// As we right now have to scan the entire list, Next and Check are linear with the +// size. However, a better data structure could remove these limits. +func (a *FixedIterator) GetStats() *IteratorStats { + return &IteratorStats{ + CheckCost: int64(len(a.values)), + NextCost: int64(len(a.values)), + Size: int64(len(a.values)), + } +} diff --git a/graph/hasa-iterator.go b/graph/hasa-iterator.go deleted file mode 100644 index 362b96d..0000000 --- a/graph/hasa-iterator.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Defines one of the base iterators, the HasA iterator. The HasA takes a -// subiterator of links, and acts as an iterator of nodes in the given -// direction. The name comes from the idea that a "link HasA subject" or a "link -// HasA predicate". -// -// HasA is weird in that it may return the same value twice if on the Next() -// path. That's okay -- in reality, it can be viewed as returning the value for -// a new triple, but to make logic much simpler, here we have the HasA. -// -// Likewise, it's important to think about Check()ing a HasA. When given a -// value to check, it means "Check all predicates that have this value for your -// direction against the subiterator." This would imply that there's more than -// one possibility for the same Check()ed value. While we could return the -// number of options, it's simpler to return one, and then call NextResult() -// enough times to enumerate the options. (In fact, one could argue that the -// raison d'etre for NextResult() is this iterator). -// -// Alternatively, can be seen as the dual of the LinksTo iterator. - -import ( - "container/list" - "fmt" - "github.com/barakmich/glog" - "strings" -) - -// A HasaIterator consists of a reference back to the TripleStore that it references, -// a primary subiterator, a direction in which the triples for that subiterator point, -// and a temporary holder for the iterator generated on Check(). -type HasaIterator struct { - BaseIterator - ts TripleStore - primaryIt Iterator - direction string - resultIt Iterator -} - -// Construct a new HasA iterator, given the triple subiterator, and the triple -// direction for which it stands. -func NewHasaIterator(ts TripleStore, subIt Iterator, dir string) *HasaIterator { - var hasa HasaIterator - BaseIteratorInit(&hasa.BaseIterator) - hasa.ts = ts - hasa.primaryIt = subIt - hasa.direction = dir - return &hasa -} - -// Return our sole subiterator, in a list.List. -func (h *HasaIterator) GetSubIterators() *list.List { - l := list.New() - l.PushBack(h.primaryIt) - return l -} - -func (h *HasaIterator) Reset() { - h.primaryIt.Reset() - if h.resultIt != nil { - h.resultIt.Close() - } -} - -func (h *HasaIterator) Clone() Iterator { - out := NewHasaIterator(h.ts, h.primaryIt.Clone(), h.direction) - out.CopyTagsFrom(h) - return out -} - -// Direction accessor. -func (h *HasaIterator) Direction() string { return h.direction } - -// Pass the Optimize() call along to the subiterator. If it becomes Null, -// then the HasA becomes Null (there are no triples that have any directions). -func (h *HasaIterator) Optimize() (Iterator, bool) { - - newPrimary, changed := h.primaryIt.Optimize() - if changed { - h.primaryIt = newPrimary - if h.primaryIt.Type() == "null" { - return h.primaryIt, true - } - } - return h, false -} - -// Pass the TagResults down the chain. -func (h *HasaIterator) TagResults(out *map[string]TSVal) { - h.BaseIterator.TagResults(out) - h.primaryIt.TagResults(out) -} - -// DEPRECATED Return results in a ResultTree. -func (h *HasaIterator) GetResultTree() *ResultTree { - tree := NewResultTree(h.LastResult()) - tree.AddSubtree(h.primaryIt.GetResultTree()) - return tree -} - -// Print some information about this iterator. -func (h *HasaIterator) DebugString(indent int) string { - var tags string - for _, k := range h.Tags() { - tags += fmt.Sprintf("%s;", k) - } - return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), h.Type(), h.GetUid(), tags, h.direction, h.primaryIt.DebugString(indent+4)) -} - -// Check a value against our internal iterator. In order to do this, we must first open a new -// iterator of "triples that have `val` in our direction", given to us by the triple store, -// and then Next() values out of that iterator and Check() them against our subiterator. -func (h *HasaIterator) Check(val TSVal) bool { - CheckLogIn(h, val) - if glog.V(4) { - glog.V(4).Infoln("Id is", h.ts.GetNameFor(val)) - } - // TODO(barakmich): Optimize this - if h.resultIt != nil { - h.resultIt.Close() - } - h.resultIt = h.ts.GetTripleIterator(h.direction, val) - return CheckLogOut(h, val, h.GetCheckResult()) -} - -// GetCheckResult() is shared code between Check() and GetNextResult() -- calls next on the -// result iterator (a triple iterator based on the last checked value) and returns true if -// another match is made. -func (h *HasaIterator) GetCheckResult() bool { - for { - linkVal, ok := h.resultIt.Next() - if !ok { - break - } - if glog.V(4) { - glog.V(4).Infoln("Triple is", h.ts.GetTriple(linkVal).ToString()) - } - if h.primaryIt.Check(linkVal) { - h.Last = h.ts.GetTripleDirection(linkVal, h.direction) - return true - } - } - return false -} - -// Get the next result that matches this branch. -func (h *HasaIterator) NextResult() bool { - // Order here is important. If the subiterator has a NextResult, then we - // need do nothing -- there is a next result, and we shouldn't move forward. - // However, we then need to get the next result from our last Check(). - // - // The upshot is, the end of NextResult() bubbles up from the bottom of the - // iterator tree up, and we need to respect that. - if h.primaryIt.NextResult() { - return true - } - return h.GetCheckResult() -} - -// Get the next result from this iterator. This is simpler than Check. We have a -// subiterator we can get a value from, and we can take that resultant triple, -// pull our direction out of it, and return that. -func (h *HasaIterator) Next() (TSVal, bool) { - NextLogIn(h) - if h.resultIt != nil { - h.resultIt.Close() - } - h.resultIt = &NullIterator{} - - tID, ok := h.primaryIt.Next() - if !ok { - return NextLogOut(h, 0, false) - } - name := h.ts.GetTriple(tID).Get(h.direction) - val := h.ts.GetIdFor(name) - h.Last = val - return NextLogOut(h, val, true) -} - -// GetStats() returns the statistics on the HasA iterator. This is curious. Next -// cost is easy, it's an extra call or so on top of the subiterator Next cost. -// CheckCost involves going to the TripleStore, iterating out values, and hoping -// one sticks -- potentially expensive, depending on fanout. Size, however, is -// potentially smaller. we know at worst it's the size of the subiterator, but -// if there are many repeated values, it could be much smaller in totality. -func (h *HasaIterator) GetStats() *IteratorStats { - subitStats := h.primaryIt.GetStats() - // TODO(barakmich): These should really come from the triplestore itself - // and be optimized. - faninFactor := int64(1) - fanoutFactor := int64(30) - nextConstant := int64(2) - tripleConstant := int64(1) - return &IteratorStats{ - NextCost: tripleConstant + subitStats.NextCost, - CheckCost: (fanoutFactor * nextConstant) * subitStats.CheckCost, - Size: faninFactor * subitStats.Size, - } -} - -// Close the subiterator, the result iterator (if any) and the HasA. -func (h *HasaIterator) Close() { - if h.resultIt != nil { - h.resultIt.Close() - } - h.primaryIt.Close() -} - -// Register this iterator as a HasA. -func (h *HasaIterator) Type() string { return "hasa" } diff --git a/graph/hasa_iterator.go b/graph/hasa_iterator.go new file mode 100644 index 0000000..362b96d --- /dev/null +++ b/graph/hasa_iterator.go @@ -0,0 +1,224 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Defines one of the base iterators, the HasA iterator. The HasA takes a +// subiterator of links, and acts as an iterator of nodes in the given +// direction. The name comes from the idea that a "link HasA subject" or a "link +// HasA predicate". +// +// HasA is weird in that it may return the same value twice if on the Next() +// path. That's okay -- in reality, it can be viewed as returning the value for +// a new triple, but to make logic much simpler, here we have the HasA. +// +// Likewise, it's important to think about Check()ing a HasA. When given a +// value to check, it means "Check all predicates that have this value for your +// direction against the subiterator." This would imply that there's more than +// one possibility for the same Check()ed value. While we could return the +// number of options, it's simpler to return one, and then call NextResult() +// enough times to enumerate the options. (In fact, one could argue that the +// raison d'etre for NextResult() is this iterator). +// +// Alternatively, can be seen as the dual of the LinksTo iterator. + +import ( + "container/list" + "fmt" + "github.com/barakmich/glog" + "strings" +) + +// A HasaIterator consists of a reference back to the TripleStore that it references, +// a primary subiterator, a direction in which the triples for that subiterator point, +// and a temporary holder for the iterator generated on Check(). +type HasaIterator struct { + BaseIterator + ts TripleStore + primaryIt Iterator + direction string + resultIt Iterator +} + +// Construct a new HasA iterator, given the triple subiterator, and the triple +// direction for which it stands. +func NewHasaIterator(ts TripleStore, subIt Iterator, dir string) *HasaIterator { + var hasa HasaIterator + BaseIteratorInit(&hasa.BaseIterator) + hasa.ts = ts + hasa.primaryIt = subIt + hasa.direction = dir + return &hasa +} + +// Return our sole subiterator, in a list.List. +func (h *HasaIterator) GetSubIterators() *list.List { + l := list.New() + l.PushBack(h.primaryIt) + return l +} + +func (h *HasaIterator) Reset() { + h.primaryIt.Reset() + if h.resultIt != nil { + h.resultIt.Close() + } +} + +func (h *HasaIterator) Clone() Iterator { + out := NewHasaIterator(h.ts, h.primaryIt.Clone(), h.direction) + out.CopyTagsFrom(h) + return out +} + +// Direction accessor. +func (h *HasaIterator) Direction() string { return h.direction } + +// Pass the Optimize() call along to the subiterator. If it becomes Null, +// then the HasA becomes Null (there are no triples that have any directions). +func (h *HasaIterator) Optimize() (Iterator, bool) { + + newPrimary, changed := h.primaryIt.Optimize() + if changed { + h.primaryIt = newPrimary + if h.primaryIt.Type() == "null" { + return h.primaryIt, true + } + } + return h, false +} + +// Pass the TagResults down the chain. +func (h *HasaIterator) TagResults(out *map[string]TSVal) { + h.BaseIterator.TagResults(out) + h.primaryIt.TagResults(out) +} + +// DEPRECATED Return results in a ResultTree. +func (h *HasaIterator) GetResultTree() *ResultTree { + tree := NewResultTree(h.LastResult()) + tree.AddSubtree(h.primaryIt.GetResultTree()) + return tree +} + +// Print some information about this iterator. +func (h *HasaIterator) DebugString(indent int) string { + var tags string + for _, k := range h.Tags() { + tags += fmt.Sprintf("%s;", k) + } + return fmt.Sprintf("%s(%s %d tags:%s direction:%s\n%s)", strings.Repeat(" ", indent), h.Type(), h.GetUid(), tags, h.direction, h.primaryIt.DebugString(indent+4)) +} + +// Check a value against our internal iterator. In order to do this, we must first open a new +// iterator of "triples that have `val` in our direction", given to us by the triple store, +// and then Next() values out of that iterator and Check() them against our subiterator. +func (h *HasaIterator) Check(val TSVal) bool { + CheckLogIn(h, val) + if glog.V(4) { + glog.V(4).Infoln("Id is", h.ts.GetNameFor(val)) + } + // TODO(barakmich): Optimize this + if h.resultIt != nil { + h.resultIt.Close() + } + h.resultIt = h.ts.GetTripleIterator(h.direction, val) + return CheckLogOut(h, val, h.GetCheckResult()) +} + +// GetCheckResult() is shared code between Check() and GetNextResult() -- calls next on the +// result iterator (a triple iterator based on the last checked value) and returns true if +// another match is made. +func (h *HasaIterator) GetCheckResult() bool { + for { + linkVal, ok := h.resultIt.Next() + if !ok { + break + } + if glog.V(4) { + glog.V(4).Infoln("Triple is", h.ts.GetTriple(linkVal).ToString()) + } + if h.primaryIt.Check(linkVal) { + h.Last = h.ts.GetTripleDirection(linkVal, h.direction) + return true + } + } + return false +} + +// Get the next result that matches this branch. +func (h *HasaIterator) NextResult() bool { + // Order here is important. If the subiterator has a NextResult, then we + // need do nothing -- there is a next result, and we shouldn't move forward. + // However, we then need to get the next result from our last Check(). + // + // The upshot is, the end of NextResult() bubbles up from the bottom of the + // iterator tree up, and we need to respect that. + if h.primaryIt.NextResult() { + return true + } + return h.GetCheckResult() +} + +// Get the next result from this iterator. This is simpler than Check. We have a +// subiterator we can get a value from, and we can take that resultant triple, +// pull our direction out of it, and return that. +func (h *HasaIterator) Next() (TSVal, bool) { + NextLogIn(h) + if h.resultIt != nil { + h.resultIt.Close() + } + h.resultIt = &NullIterator{} + + tID, ok := h.primaryIt.Next() + if !ok { + return NextLogOut(h, 0, false) + } + name := h.ts.GetTriple(tID).Get(h.direction) + val := h.ts.GetIdFor(name) + h.Last = val + return NextLogOut(h, val, true) +} + +// GetStats() returns the statistics on the HasA iterator. This is curious. Next +// cost is easy, it's an extra call or so on top of the subiterator Next cost. +// CheckCost involves going to the TripleStore, iterating out values, and hoping +// one sticks -- potentially expensive, depending on fanout. Size, however, is +// potentially smaller. we know at worst it's the size of the subiterator, but +// if there are many repeated values, it could be much smaller in totality. +func (h *HasaIterator) GetStats() *IteratorStats { + subitStats := h.primaryIt.GetStats() + // TODO(barakmich): These should really come from the triplestore itself + // and be optimized. + faninFactor := int64(1) + fanoutFactor := int64(30) + nextConstant := int64(2) + tripleConstant := int64(1) + return &IteratorStats{ + NextCost: tripleConstant + subitStats.NextCost, + CheckCost: (fanoutFactor * nextConstant) * subitStats.CheckCost, + Size: faninFactor * subitStats.Size, + } +} + +// Close the subiterator, the result iterator (if any) and the HasA. +func (h *HasaIterator) Close() { + if h.resultIt != nil { + h.resultIt.Close() + } + h.primaryIt.Close() +} + +// Register this iterator as a HasA. +func (h *HasaIterator) Type() string { return "hasa" } diff --git a/graph/linksto-iterator.go b/graph/linksto-iterator.go deleted file mode 100644 index 45c5e2f..0000000 --- a/graph/linksto-iterator.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Defines one of the base iterators, the LinksTo iterator. A LinksTo takes a -// subiterator of nodes, and contains an iteration of links which "link to" -// those nodes in a given direction. -// -// Next()ing a LinksTo is straightforward -- iterate through all links to // -// things in the subiterator, and then advance the subiterator, and do it again. -// LinksTo is therefore sensitive to growing with a fanout. (A small-sized -// subiterator could cause LinksTo to be large). -// -// Check()ing a LinksTo means, given a link, take the direction we care about -// and check if it's in our subiterator. Checking is therefore fairly cheap, and -// similar to checking the subiterator alone. -// -// Can be seen as the dual of the HasA iterator. - -import ( - "container/list" - "fmt" - "strings" -) - -// A LinksTo has a reference back to the TripleStore (to create the iterators -// for each node) the subiterator, and the direction the iterator comes from. -// `next_it` is the tempoarary iterator held per result in `primary_it`. -type LinksToIterator struct { - BaseIterator - ts TripleStore - primaryIt Iterator - direction string - nextIt Iterator -} - -// Construct a new LinksTo iterator around a direction and a subiterator of -// nodes. -func NewLinksToIterator(ts TripleStore, it Iterator, dir string) *LinksToIterator { - var lto LinksToIterator - BaseIteratorInit(<o.BaseIterator) - lto.ts = ts - lto.primaryIt = it - lto.direction = dir - lto.nextIt = &NullIterator{} - return <o -} - -func (l *LinksToIterator) Reset() { - l.primaryIt.Reset() - if l.nextIt != nil { - l.nextIt.Close() - } - l.nextIt = &NullIterator{} -} - -func (l *LinksToIterator) Clone() Iterator { - out := NewLinksToIterator(l.ts, l.primaryIt.Clone(), l.direction) - out.CopyTagsFrom(l) - return out -} - -// Return the direction under consideration. -func (l *LinksToIterator) Direction() string { return l.direction } - -// Tag these results, and our subiterator's results. -func (l *LinksToIterator) TagResults(out *map[string]TSVal) { - l.BaseIterator.TagResults(out) - l.primaryIt.TagResults(out) -} - -// DEPRECATED -func (l *LinksToIterator) GetResultTree() *ResultTree { - tree := NewResultTree(l.LastResult()) - tree.AddSubtree(l.primaryIt.GetResultTree()) - return tree -} - -// Print the iterator. -func (l *LinksToIterator) DebugString(indent int) string { - return fmt.Sprintf("%s(%s %d direction:%s\n%s)", - strings.Repeat(" ", indent), - l.Type(), l.GetUid(), l.direction, l.primaryIt.DebugString(indent+4)) -} - -// If it checks in the right direction for the subiterator, it is a valid link -// for the LinksTo. -func (l *LinksToIterator) Check(val TSVal) bool { - CheckLogIn(l, val) - node := l.ts.GetTripleDirection(val, l.direction) - if l.primaryIt.Check(node) { - l.Last = val - return CheckLogOut(l, val, true) - } - return CheckLogOut(l, val, false) -} - -// Return a list containing only our subiterator. -func (lto *LinksToIterator) GetSubIterators() *list.List { - l := list.New() - l.PushBack(lto.primaryIt) - return l -} - -// Optimize the LinksTo, by replacing it if it can be. -func (lto *LinksToIterator) Optimize() (Iterator, bool) { - newPrimary, changed := lto.primaryIt.Optimize() - if changed { - lto.primaryIt = newPrimary - if lto.primaryIt.Type() == "null" { - lto.nextIt.Close() - return lto.primaryIt, true - } - } - // Ask the TripleStore if we can be replaced. Often times, this is a great - // optimization opportunity (there's a fixed iterator underneath us, for - // example). - newReplacement, hasOne := lto.ts.OptimizeIterator(lto) - if hasOne { - lto.Close() - return newReplacement, true - } - return lto, false -} - -// Next()ing a LinksTo operates as described above. -func (l *LinksToIterator) Next() (TSVal, bool) { - NextLogIn(l) - val, ok := l.nextIt.Next() - if !ok { - // Subiterator is empty, get another one - candidate, ok := l.primaryIt.Next() - if !ok { - // We're out of nodes in our subiterator, so we're done as well. - return NextLogOut(l, 0, false) - } - l.nextIt.Close() - l.nextIt = l.ts.GetTripleIterator(l.direction, candidate) - // Recurse -- return the first in the next set. - return l.Next() - } - l.Last = val - return NextLogOut(l, val, ok) -} - -// Close our subiterators. -func (l *LinksToIterator) Close() { - l.nextIt.Close() - l.primaryIt.Close() -} - -// We won't ever have a new result, but our subiterators might. -func (l *LinksToIterator) NextResult() bool { - return l.primaryIt.NextResult() -} - -// Register the LinksTo. -func (l *LinksToIterator) Type() string { return "linksto" } - -// Return a guess as to how big or costly it is to next the iterator. -func (l *LinksToIterator) GetStats() *IteratorStats { - subitStats := l.primaryIt.GetStats() - // TODO(barakmich): These should really come from the triplestore itself - fanoutFactor := int64(20) - checkConstant := int64(1) - nextConstant := int64(2) - return &IteratorStats{ - NextCost: nextConstant + subitStats.NextCost, - CheckCost: checkConstant + subitStats.CheckCost, - Size: fanoutFactor * subitStats.Size, - } -} diff --git a/graph/linksto-iterator_test.go b/graph/linksto-iterator_test.go deleted file mode 100644 index 06cdd2a..0000000 --- a/graph/linksto-iterator_test.go +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - "testing" -) - -func TestLinksTo(t *testing.T) { - ts := new(TestTripleStore) - tsFixed := newFixedIterator() - tsFixed.AddValue(2) - ts.On("GetIdFor", "cool").Return(1) - ts.On("GetTripleIterator", "o", 1).Return(tsFixed) - fixed := newFixedIterator() - fixed.AddValue(ts.GetIdFor("cool")) - lto := NewLinksToIterator(ts, fixed, "o") - val, ok := lto.Next() - if !ok { - t.Error("At least one triple matches the fixed object") - } - if val != 2 { - t.Errorf("Triple index 2, such as %s, should match %s", ts.GetTriple(2), ts.GetTriple(val)) - } -} diff --git a/graph/linksto_iterator.go b/graph/linksto_iterator.go new file mode 100644 index 0000000..45c5e2f --- /dev/null +++ b/graph/linksto_iterator.go @@ -0,0 +1,184 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Defines one of the base iterators, the LinksTo iterator. A LinksTo takes a +// subiterator of nodes, and contains an iteration of links which "link to" +// those nodes in a given direction. +// +// Next()ing a LinksTo is straightforward -- iterate through all links to // +// things in the subiterator, and then advance the subiterator, and do it again. +// LinksTo is therefore sensitive to growing with a fanout. (A small-sized +// subiterator could cause LinksTo to be large). +// +// Check()ing a LinksTo means, given a link, take the direction we care about +// and check if it's in our subiterator. Checking is therefore fairly cheap, and +// similar to checking the subiterator alone. +// +// Can be seen as the dual of the HasA iterator. + +import ( + "container/list" + "fmt" + "strings" +) + +// A LinksTo has a reference back to the TripleStore (to create the iterators +// for each node) the subiterator, and the direction the iterator comes from. +// `next_it` is the tempoarary iterator held per result in `primary_it`. +type LinksToIterator struct { + BaseIterator + ts TripleStore + primaryIt Iterator + direction string + nextIt Iterator +} + +// Construct a new LinksTo iterator around a direction and a subiterator of +// nodes. +func NewLinksToIterator(ts TripleStore, it Iterator, dir string) *LinksToIterator { + var lto LinksToIterator + BaseIteratorInit(<o.BaseIterator) + lto.ts = ts + lto.primaryIt = it + lto.direction = dir + lto.nextIt = &NullIterator{} + return <o +} + +func (l *LinksToIterator) Reset() { + l.primaryIt.Reset() + if l.nextIt != nil { + l.nextIt.Close() + } + l.nextIt = &NullIterator{} +} + +func (l *LinksToIterator) Clone() Iterator { + out := NewLinksToIterator(l.ts, l.primaryIt.Clone(), l.direction) + out.CopyTagsFrom(l) + return out +} + +// Return the direction under consideration. +func (l *LinksToIterator) Direction() string { return l.direction } + +// Tag these results, and our subiterator's results. +func (l *LinksToIterator) TagResults(out *map[string]TSVal) { + l.BaseIterator.TagResults(out) + l.primaryIt.TagResults(out) +} + +// DEPRECATED +func (l *LinksToIterator) GetResultTree() *ResultTree { + tree := NewResultTree(l.LastResult()) + tree.AddSubtree(l.primaryIt.GetResultTree()) + return tree +} + +// Print the iterator. +func (l *LinksToIterator) DebugString(indent int) string { + return fmt.Sprintf("%s(%s %d direction:%s\n%s)", + strings.Repeat(" ", indent), + l.Type(), l.GetUid(), l.direction, l.primaryIt.DebugString(indent+4)) +} + +// If it checks in the right direction for the subiterator, it is a valid link +// for the LinksTo. +func (l *LinksToIterator) Check(val TSVal) bool { + CheckLogIn(l, val) + node := l.ts.GetTripleDirection(val, l.direction) + if l.primaryIt.Check(node) { + l.Last = val + return CheckLogOut(l, val, true) + } + return CheckLogOut(l, val, false) +} + +// Return a list containing only our subiterator. +func (lto *LinksToIterator) GetSubIterators() *list.List { + l := list.New() + l.PushBack(lto.primaryIt) + return l +} + +// Optimize the LinksTo, by replacing it if it can be. +func (lto *LinksToIterator) Optimize() (Iterator, bool) { + newPrimary, changed := lto.primaryIt.Optimize() + if changed { + lto.primaryIt = newPrimary + if lto.primaryIt.Type() == "null" { + lto.nextIt.Close() + return lto.primaryIt, true + } + } + // Ask the TripleStore if we can be replaced. Often times, this is a great + // optimization opportunity (there's a fixed iterator underneath us, for + // example). + newReplacement, hasOne := lto.ts.OptimizeIterator(lto) + if hasOne { + lto.Close() + return newReplacement, true + } + return lto, false +} + +// Next()ing a LinksTo operates as described above. +func (l *LinksToIterator) Next() (TSVal, bool) { + NextLogIn(l) + val, ok := l.nextIt.Next() + if !ok { + // Subiterator is empty, get another one + candidate, ok := l.primaryIt.Next() + if !ok { + // We're out of nodes in our subiterator, so we're done as well. + return NextLogOut(l, 0, false) + } + l.nextIt.Close() + l.nextIt = l.ts.GetTripleIterator(l.direction, candidate) + // Recurse -- return the first in the next set. + return l.Next() + } + l.Last = val + return NextLogOut(l, val, ok) +} + +// Close our subiterators. +func (l *LinksToIterator) Close() { + l.nextIt.Close() + l.primaryIt.Close() +} + +// We won't ever have a new result, but our subiterators might. +func (l *LinksToIterator) NextResult() bool { + return l.primaryIt.NextResult() +} + +// Register the LinksTo. +func (l *LinksToIterator) Type() string { return "linksto" } + +// Return a guess as to how big or costly it is to next the iterator. +func (l *LinksToIterator) GetStats() *IteratorStats { + subitStats := l.primaryIt.GetStats() + // TODO(barakmich): These should really come from the triplestore itself + fanoutFactor := int64(20) + checkConstant := int64(1) + nextConstant := int64(2) + return &IteratorStats{ + NextCost: nextConstant + subitStats.NextCost, + CheckCost: checkConstant + subitStats.CheckCost, + Size: fanoutFactor * subitStats.Size, + } +} diff --git a/graph/linksto_iterator_test.go b/graph/linksto_iterator_test.go new file mode 100644 index 0000000..06cdd2a --- /dev/null +++ b/graph/linksto_iterator_test.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "testing" +) + +func TestLinksTo(t *testing.T) { + ts := new(TestTripleStore) + tsFixed := newFixedIterator() + tsFixed.AddValue(2) + ts.On("GetIdFor", "cool").Return(1) + ts.On("GetTripleIterator", "o", 1).Return(tsFixed) + fixed := newFixedIterator() + fixed.AddValue(ts.GetIdFor("cool")) + lto := NewLinksToIterator(ts, fixed, "o") + val, ok := lto.Next() + if !ok { + t.Error("At least one triple matches the fixed object") + } + if val != 2 { + t.Errorf("Triple index 2, such as %s, should match %s", ts.GetTriple(2), ts.GetTriple(val)) + } +} diff --git a/graph/memstore/llrb-iterator.go b/graph/memstore/llrb-iterator.go deleted file mode 100644 index 692a3c6..0000000 --- a/graph/memstore/llrb-iterator.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "fmt" - "math" - "strings" - - "github.com/petar/GoLLRB/llrb" - - "github.com/google/cayley/graph" -) - -type LlrbIterator struct { - graph.BaseIterator - tree *llrb.LLRB - data string - isRunning bool - iterLast Int64 -} - -type Int64 int64 - -func (i Int64) Less(than llrb.Item) bool { - return i < than.(Int64) -} - -func IterateOne(tree *llrb.LLRB, last Int64) Int64 { - var next Int64 - tree.AscendGreaterOrEqual(last, func(i llrb.Item) bool { - if i.(Int64) == last { - return true - } else { - next = i.(Int64) - return false - } - }) - return next -} - -func NewLlrbIterator(tree *llrb.LLRB, data string) *LlrbIterator { - var it LlrbIterator - graph.BaseIteratorInit(&it.BaseIterator) - it.tree = tree - it.iterLast = Int64(-1) - it.data = data - return &it -} - -func (it *LlrbIterator) Reset() { - it.iterLast = Int64(-1) -} - -func (it *LlrbIterator) Clone() graph.Iterator { - var new_it = NewLlrbIterator(it.tree, it.data) - new_it.CopyTagsFrom(it) - return new_it -} - -func (it *LlrbIterator) Close() {} - -func (it *LlrbIterator) Next() (graph.TSVal, bool) { - graph.NextLogIn(it) - if it.tree.Max() == nil || it.Last == int64(it.tree.Max().(Int64)) { - return graph.NextLogOut(it, nil, false) - } - it.iterLast = IterateOne(it.tree, it.iterLast) - it.Last = int64(it.iterLast) - return graph.NextLogOut(it, it.Last, true) -} - -func (it *LlrbIterator) Size() (int64, bool) { - return int64(it.tree.Len()), true -} - -func (it *LlrbIterator) Check(v graph.TSVal) bool { - graph.CheckLogIn(it, v) - if it.tree.Has(Int64(v.(int64))) { - it.Last = v - return graph.CheckLogOut(it, v, true) - } - return graph.CheckLogOut(it, v, false) -} - -func (it *LlrbIterator) DebugString(indent int) string { - size, _ := it.Size() - return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data) -} - -func (it *LlrbIterator) Type() string { - return "llrb" -} -func (it *LlrbIterator) Sorted() bool { - return true -} -func (it *LlrbIterator) Optimize() (graph.Iterator, bool) { - return it, false -} - -func (it *LlrbIterator) GetStats() *graph.IteratorStats { - return &graph.IteratorStats{ - CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1, - NextCost: 1, - Size: int64(it.tree.Len()), - } -} diff --git a/graph/memstore/llrb_iterator.go b/graph/memstore/llrb_iterator.go new file mode 100644 index 0000000..692a3c6 --- /dev/null +++ b/graph/memstore/llrb_iterator.go @@ -0,0 +1,119 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "fmt" + "math" + "strings" + + "github.com/petar/GoLLRB/llrb" + + "github.com/google/cayley/graph" +) + +type LlrbIterator struct { + graph.BaseIterator + tree *llrb.LLRB + data string + isRunning bool + iterLast Int64 +} + +type Int64 int64 + +func (i Int64) Less(than llrb.Item) bool { + return i < than.(Int64) +} + +func IterateOne(tree *llrb.LLRB, last Int64) Int64 { + var next Int64 + tree.AscendGreaterOrEqual(last, func(i llrb.Item) bool { + if i.(Int64) == last { + return true + } else { + next = i.(Int64) + return false + } + }) + return next +} + +func NewLlrbIterator(tree *llrb.LLRB, data string) *LlrbIterator { + var it LlrbIterator + graph.BaseIteratorInit(&it.BaseIterator) + it.tree = tree + it.iterLast = Int64(-1) + it.data = data + return &it +} + +func (it *LlrbIterator) Reset() { + it.iterLast = Int64(-1) +} + +func (it *LlrbIterator) Clone() graph.Iterator { + var new_it = NewLlrbIterator(it.tree, it.data) + new_it.CopyTagsFrom(it) + return new_it +} + +func (it *LlrbIterator) Close() {} + +func (it *LlrbIterator) Next() (graph.TSVal, bool) { + graph.NextLogIn(it) + if it.tree.Max() == nil || it.Last == int64(it.tree.Max().(Int64)) { + return graph.NextLogOut(it, nil, false) + } + it.iterLast = IterateOne(it.tree, it.iterLast) + it.Last = int64(it.iterLast) + return graph.NextLogOut(it, it.Last, true) +} + +func (it *LlrbIterator) Size() (int64, bool) { + return int64(it.tree.Len()), true +} + +func (it *LlrbIterator) Check(v graph.TSVal) bool { + graph.CheckLogIn(it, v) + if it.tree.Has(Int64(v.(int64))) { + it.Last = v + return graph.CheckLogOut(it, v, true) + } + return graph.CheckLogOut(it, v, false) +} + +func (it *LlrbIterator) DebugString(indent int) string { + size, _ := it.Size() + return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data) +} + +func (it *LlrbIterator) Type() string { + return "llrb" +} +func (it *LlrbIterator) Sorted() bool { + return true +} +func (it *LlrbIterator) Optimize() (graph.Iterator, bool) { + return it, false +} + +func (it *LlrbIterator) GetStats() *graph.IteratorStats { + return &graph.IteratorStats{ + CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1, + NextCost: 1, + Size: int64(it.tree.Len()), + } +} diff --git a/graph/memstore/memstore-all-iterator.go b/graph/memstore/memstore-all-iterator.go deleted file mode 100644 index 99cf734..0000000 --- a/graph/memstore/memstore-all-iterator.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "github.com/google/cayley/graph" -) - -type MemstoreAllIterator struct { - graph.Int64AllIterator - ts *MemTripleStore -} - -func NewMemstoreAllIterator(ts *MemTripleStore) *MemstoreAllIterator { - var out MemstoreAllIterator - out.Int64AllIterator = *graph.NewInt64AllIterator(1, ts.idCounter-1) - out.ts = ts - return &out -} - -func (memall *MemstoreAllIterator) Next() (graph.TSVal, bool) { - next, out := memall.Int64AllIterator.Next() - if !out { - return next, out - } - i64 := next.(int64) - _, ok := memall.ts.revIdMap[i64] - if !ok { - return memall.Next() - } - memall.Last = next - return next, out -} diff --git a/graph/memstore/memstore_all_iterator.go b/graph/memstore/memstore_all_iterator.go new file mode 100644 index 0000000..99cf734 --- /dev/null +++ b/graph/memstore/memstore_all_iterator.go @@ -0,0 +1,45 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "github.com/google/cayley/graph" +) + +type MemstoreAllIterator struct { + graph.Int64AllIterator + ts *MemTripleStore +} + +func NewMemstoreAllIterator(ts *MemTripleStore) *MemstoreAllIterator { + var out MemstoreAllIterator + out.Int64AllIterator = *graph.NewInt64AllIterator(1, ts.idCounter-1) + out.ts = ts + return &out +} + +func (memall *MemstoreAllIterator) Next() (graph.TSVal, bool) { + next, out := memall.Int64AllIterator.Next() + if !out { + return next, out + } + i64 := next.(int64) + _, ok := memall.ts.revIdMap[i64] + if !ok { + return memall.Next() + } + memall.Last = next + return next, out +} diff --git a/graph/memstore/memtriplestore-iterator-optimize.go b/graph/memstore/memtriplestore-iterator-optimize.go deleted file mode 100644 index 7c895fc..0000000 --- a/graph/memstore/memtriplestore-iterator-optimize.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "github.com/google/cayley/graph" -) - -func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { - switch it.Type() { - case "linksto": - return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) - - } - return it, false -} - -func (ts *MemTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { - l := it.GetSubIterators() - if l.Len() != 1 { - return it, false - } - primaryIt := l.Front().Value.(graph.Iterator) - if primaryIt.Type() == "fixed" { - size, _ := primaryIt.Size() - if size == 1 { - val, ok := primaryIt.Next() - if !ok { - panic("Sizes lie") - } - newIt := ts.GetTripleIterator(it.Direction(), val) - newIt.CopyTagsFrom(it) - for _, tag := range primaryIt.Tags() { - newIt.AddFixedTag(tag, val) - } - return newIt, true - } - } - it.Close() - return it, false -} diff --git a/graph/memstore/memtriplestore_iterator_optimize.go b/graph/memstore/memtriplestore_iterator_optimize.go new file mode 100644 index 0000000..7c895fc --- /dev/null +++ b/graph/memstore/memtriplestore_iterator_optimize.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "github.com/google/cayley/graph" +) + +func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { + switch it.Type() { + case "linksto": + return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) + + } + return it, false +} + +func (ts *MemTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { + l := it.GetSubIterators() + if l.Len() != 1 { + return it, false + } + primaryIt := l.Front().Value.(graph.Iterator) + if primaryIt.Type() == "fixed" { + size, _ := primaryIt.Size() + if size == 1 { + val, ok := primaryIt.Next() + if !ok { + panic("Sizes lie") + } + newIt := ts.GetTripleIterator(it.Direction(), val) + newIt.CopyTagsFrom(it) + for _, tag := range primaryIt.Tags() { + newIt.AddFixedTag(tag, val) + } + return newIt, true + } + } + it.Close() + return it, false +} diff --git a/graph/mongo/mongo-iterator.go b/graph/mongo/mongo-iterator.go deleted file mode 100644 index addcfb0..0000000 --- a/graph/mongo/mongo-iterator.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mongo - -import ( - "fmt" - "strings" - - "github.com/barakmich/glog" - "labix.org/v2/mgo" - "labix.org/v2/mgo/bson" - - "github.com/google/cayley/graph" -) - -type MongoIterator struct { - graph.BaseIterator - ts *MongoTripleStore - dir string - iter *mgo.Iter - hash string - name string - size int64 - isAll bool - constraint bson.M - collection string -} - -func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator { - var m MongoIterator - graph.BaseIteratorInit(&m.BaseIterator) - - m.name = ts.GetNameFor(val) - m.collection = collection - switch dir { - - case "s": - m.constraint = bson.M{"Sub": m.name} - case "p": - m.constraint = bson.M{"Pred": m.name} - case "o": - m.constraint = bson.M{"Obj": m.name} - case "c": - m.constraint = bson.M{"Provenance": m.name} - } - - m.ts = ts - m.dir = dir - m.iter = ts.db.C(collection).Find(m.constraint).Iter() - size, err := ts.db.C(collection).Find(m.constraint).Count() - if err != nil { - glog.Errorln("Trouble getting size for iterator! ", err) - return nil - } - m.size = int64(size) - m.hash = val.(string) - m.isAll = false - return &m -} - -func NewMongoAllIterator(ts *MongoTripleStore, collection string) *MongoIterator { - var m MongoIterator - m.ts = ts - m.dir = "all" - m.constraint = nil - m.collection = collection - m.iter = ts.db.C(collection).Find(nil).Iter() - size, err := ts.db.C(collection).Count() - if err != nil { - glog.Errorln("Trouble getting size for iterator! ", err) - return nil - } - m.size = int64(size) - m.hash = "" - m.isAll = true - return &m -} - -func (m *MongoIterator) Reset() { - m.iter.Close() - m.iter = m.ts.db.C(m.collection).Find(m.constraint).Iter() - -} - -func (m *MongoIterator) Close() { - m.iter.Close() -} - -func (m *MongoIterator) Clone() graph.Iterator { - var newM graph.Iterator - if m.isAll { - newM = NewMongoAllIterator(m.ts, m.collection) - } else { - newM = NewMongoIterator(m.ts, m.collection, m.dir, m.hash) - } - newM.CopyTagsFrom(m) - return newM -} - -func (m *MongoIterator) Next() (graph.TSVal, bool) { - var result struct { - Id string "_id" - //Sub string "Sub" - //Pred string "Pred" - //Obj string "Obj" - } - found := m.iter.Next(&result) - if !found { - err := m.iter.Err() - if err != nil { - glog.Errorln("Error Nexting MongoIterator: ", err) - } - return nil, false - } - m.Last = result.Id - return result.Id, true -} - -func (m *MongoIterator) Check(v graph.TSVal) bool { - graph.CheckLogIn(m, v) - if m.isAll { - m.Last = v - return graph.CheckLogOut(m, v, true) - } - var offset int - switch m.dir { - case "s": - offset = 0 - case "p": - offset = (m.ts.hasher.Size() * 2) - case "o": - offset = (m.ts.hasher.Size() * 2) * 2 - case "c": - offset = (m.ts.hasher.Size() * 2) * 3 - } - val := v.(string)[offset : m.ts.hasher.Size()*2+offset] - if val == m.hash { - m.Last = v - return graph.CheckLogOut(m, v, true) - } - return graph.CheckLogOut(m, v, false) -} - -func (m *MongoIterator) Size() (int64, bool) { - return m.size, true -} - -func (m *MongoIterator) Type() string { - if m.isAll { - return "all" - } - return "mongo" -} -func (m *MongoIterator) Sorted() bool { return true } -func (m *MongoIterator) Optimize() (graph.Iterator, bool) { return m, false } - -func (m *MongoIterator) DebugString(indent int) string { - size, _ := m.Size() - return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), m.Type(), size, m.hash, m.name) -} - -func (m *MongoIterator) GetStats() *graph.IteratorStats { - size, _ := m.Size() - return &graph.IteratorStats{ - CheckCost: 1, - NextCost: 5, - Size: size, - } -} diff --git a/graph/mongo/mongo-triplestore-iterator-optimize.go b/graph/mongo/mongo-triplestore-iterator-optimize.go deleted file mode 100644 index d10bc22..0000000 --- a/graph/mongo/mongo-triplestore-iterator-optimize.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mongo - -import ( - "github.com/google/cayley/graph" -) - -func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { - switch it.Type() { - case "linksto": - return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) - - } - return it, false -} - -func (ts *MongoTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { - l := it.GetSubIterators() - if l.Len() != 1 { - return it, false - } - primaryIt := l.Front().Value.(graph.Iterator) - if primaryIt.Type() == "fixed" { - size, _ := primaryIt.Size() - if size == 1 { - val, ok := primaryIt.Next() - if !ok { - panic("Sizes lie") - } - newIt := ts.GetTripleIterator(it.Direction(), val) - newIt.CopyTagsFrom(it) - for _, tag := range primaryIt.Tags() { - newIt.AddFixedTag(tag, val) - } - it.Close() - return newIt, true - } - } - return it, false -} diff --git a/graph/mongo/mongo-triplestore.go b/graph/mongo/mongo-triplestore.go deleted file mode 100644 index 917ab4d..0000000 --- a/graph/mongo/mongo-triplestore.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mongo - -import ( - "crypto/sha1" - "encoding/hex" - "hash" - "log" - - "labix.org/v2/mgo" - "labix.org/v2/mgo/bson" - - "github.com/barakmich/glog" - "github.com/google/cayley/graph" -) - -const DefaultDBName = "cayley" - -type MongoTripleStore struct { - session *mgo.Session - db *mgo.Database - hasher hash.Hash - idCache *IDLru -} - -func CreateNewMongoGraph(addr string, options graph.OptionsDict) bool { - conn, err := mgo.Dial(addr) - if err != nil { - glog.Fatal("Error connecting: ", err) - return false - } - conn.SetSafe(&mgo.Safe{}) - dbName := DefaultDBName - if val, ok := options.GetStringKey("database_name"); ok { - dbName = val - } - db := conn.DB(dbName) - indexOpts := mgo.Index{ - Key: []string{"Sub"}, - Unique: false, - DropDups: false, - Background: true, - Sparse: true, - } - db.C("triples").EnsureIndex(indexOpts) - indexOpts.Key = []string{"Pred"} - db.C("triples").EnsureIndex(indexOpts) - indexOpts.Key = []string{"Obj"} - db.C("triples").EnsureIndex(indexOpts) - indexOpts.Key = []string{"Provenance"} - db.C("triples").EnsureIndex(indexOpts) - return true -} - -func NewMongoTripleStore(addr string, options graph.OptionsDict) *MongoTripleStore { - var ts MongoTripleStore - conn, err := mgo.Dial(addr) - if err != nil { - glog.Fatal("Error connecting: ", err) - } - conn.SetSafe(&mgo.Safe{}) - dbName := DefaultDBName - if val, ok := options.GetStringKey("database_name"); ok { - dbName = val - } - ts.db = conn.DB(dbName) - ts.session = conn - ts.hasher = sha1.New() - ts.idCache = NewIDLru(1 << 16) - return &ts -} - -func (ts *MongoTripleStore) getIdForTriple(t *graph.Triple) string { - id := ts.ConvertStringToByteHash(t.Sub) - id += ts.ConvertStringToByteHash(t.Pred) - id += ts.ConvertStringToByteHash(t.Obj) - id += ts.ConvertStringToByteHash(t.Provenance) - return id -} - -func (ts *MongoTripleStore) ConvertStringToByteHash(s string) string { - ts.hasher.Reset() - key := make([]byte, 0, ts.hasher.Size()) - ts.hasher.Write([]byte(s)) - key = ts.hasher.Sum(key) - return hex.EncodeToString(key) -} - -type MongoNode struct { - Id string "_id" - Name string "Name" - Size int "Size" -} - -func (ts *MongoTripleStore) updateNodeBy(node_name string, inc int) { - var size MongoNode - node := ts.GetIdFor(node_name) - err := ts.db.C("nodes").FindId(node).One(&size) - if err != nil { - if err.Error() == "not found" { - // Not found. Okay. - size.Id = node.(string) - size.Name = node_name - size.Size = inc - } else { - glog.Error("Error:", err) - return - } - } else { - size.Id = node.(string) - size.Name = node_name - size.Size += inc - } - - // Removing something... - if inc < 0 { - if size.Size <= 0 { - err := ts.db.C("nodes").RemoveId(node) - if err != nil { - glog.Error("Error: ", err, " while removing node ", node_name) - return - } - } - } - - _, err2 := ts.db.C("nodes").UpsertId(node, size) - if err2 != nil { - glog.Error("Error: ", err) - } -} - -func (ts *MongoTripleStore) writeTriple(t *graph.Triple) bool { - tripledoc := bson.M{"_id": ts.getIdForTriple(t), "Sub": t.Sub, "Pred": t.Pred, "Obj": t.Obj, "Provenance": t.Provenance} - err := ts.db.C("triples").Insert(tripledoc) - if err != nil { - // Among the reasons I hate MongoDB. "Errors don't happen! Right guys?" - if err.(*mgo.LastError).Code == 11000 { - return false - } - glog.Error("Error: ", err) - return false - } - return true -} - -func (ts *MongoTripleStore) AddTriple(t *graph.Triple) { - _ = ts.writeTriple(t) - ts.updateNodeBy(t.Sub, 1) - ts.updateNodeBy(t.Pred, 1) - ts.updateNodeBy(t.Obj, 1) - if t.Provenance != "" { - ts.updateNodeBy(t.Provenance, 1) - } -} - -func (ts *MongoTripleStore) AddTripleSet(in []*graph.Triple) { - ts.session.SetSafe(nil) - idMap := make(map[string]int) - for _, t := range in { - wrote := ts.writeTriple(t) - if wrote { - idMap[t.Sub]++ - idMap[t.Obj]++ - idMap[t.Pred]++ - if t.Provenance != "" { - idMap[t.Provenance]++ - } - } - } - for k, v := range idMap { - ts.updateNodeBy(k, v) - } - ts.session.SetSafe(&mgo.Safe{}) -} - -func (ts *MongoTripleStore) RemoveTriple(t *graph.Triple) { - err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t)) - if err == mgo.ErrNotFound { - return - } else if err != nil { - log.Println("Error: ", err, " while removing triple ", t) - return - } - ts.updateNodeBy(t.Sub, -1) - ts.updateNodeBy(t.Pred, -1) - ts.updateNodeBy(t.Obj, -1) - if t.Provenance != "" { - ts.updateNodeBy(t.Provenance, -1) - } -} - -func (ts *MongoTripleStore) GetTriple(val graph.TSVal) *graph.Triple { - var bsonDoc bson.M - err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc) - if err != nil { - log.Println("Error: Couldn't retrieve triple", val.(string), err) - } - return graph.MakeTriple( - bsonDoc["Sub"].(string), - bsonDoc["Pred"].(string), - bsonDoc["Obj"].(string), - bsonDoc["Provenance"].(string)) -} - -func (ts *MongoTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { - return NewMongoIterator(ts, "triples", dir, val) -} - -func (ts *MongoTripleStore) GetNodesAllIterator() graph.Iterator { - return NewMongoAllIterator(ts, "nodes") -} - -func (ts *MongoTripleStore) GetTriplesAllIterator() graph.Iterator { - return NewMongoAllIterator(ts, "triples") -} - -func (ts *MongoTripleStore) GetIdFor(s string) graph.TSVal { - return ts.ConvertStringToByteHash(s) -} - -func (ts *MongoTripleStore) GetNameFor(v graph.TSVal) string { - val, ok := ts.idCache.Get(v.(string)) - if ok { - return val - } - var node MongoNode - err := ts.db.C("nodes").FindId(v.(string)).One(&node) - if err != nil { - log.Println("Error: Couldn't retrieve node", v.(string), err) - } - ts.idCache.Put(v.(string), node.Name) - return node.Name -} - -func (ts *MongoTripleStore) Size() int64 { - count, err := ts.db.C("triples").Count() - if err != nil { - glog.Error("Error: ", err) - return 0 - } - return int64(count) -} - -func compareStrings(a, b graph.TSVal) bool { - return a.(string) == b.(string) -} - -func (ts *MongoTripleStore) MakeFixed() *graph.FixedIterator { - return graph.NewFixedIteratorWithCompare(compareStrings) -} - -func (ts *MongoTripleStore) Close() { - ts.db.Session.Close() -} - -func (ts *MongoTripleStore) GetTripleDirection(in graph.TSVal, dir string) graph.TSVal { - // Maybe do the trick here - var offset int - switch dir { - case "s": - offset = 0 - case "p": - offset = (ts.hasher.Size() * 2) - case "o": - offset = (ts.hasher.Size() * 2) * 2 - case "c": - offset = (ts.hasher.Size() * 2) * 3 - } - val := in.(string)[offset : ts.hasher.Size()*2+offset] - return val -} - -func (ts *MongoTripleStore) BulkLoad(t_chan chan *graph.Triple) { - ts.session.SetSafe(nil) - for triple := range t_chan { - ts.writeTriple(triple) - } - outputTo := bson.M{"replace": "nodes", "sharded": true} - glog.Infoln("Mapreducing") - job := mgo.MapReduce{ - Map: `function() { - var len = this["_id"].length - var s_key = this["_id"].slice(0, len / 4) - var p_key = this["_id"].slice(len / 4, 2 * len / 4) - var o_key = this["_id"].slice(2 * len / 4, 3 * len / 4) - var c_key = this["_id"].slice(3 * len / 4) - emit(s_key, {"_id": s_key, "Name" : this.Sub, "Size" : 1}) - emit(p_key, {"_id": p_key, "Name" : this.Pred, "Size" : 1}) - emit(o_key, {"_id": o_key, "Name" : this.Obj, "Size" : 1}) - if (this.Provenance != "") { - emit(c_key, {"_id": c_key, "Name" : this.Provenance, "Size" : 1}) - } - } - `, - Reduce: ` - function(key, value_list) { - out = {"_id": key, "Name": value_list[0].Name} - count = 0 - for (var i = 0; i < value_list.length; i++) { - count = count + value_list[i].Size - - } - out["Size"] = count - return out - } - `, - Out: outputTo, - } - ts.db.C("triples").Find(nil).MapReduce(&job, nil) - glog.Infoln("Fixing") - ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) { - db.nodes.update({"_id": result._id}, result.value) - }) }`}, {"args", bson.D{}}}, nil) - - ts.session.SetSafe(&mgo.Safe{}) -} diff --git a/graph/mongo/mongo_iterator.go b/graph/mongo/mongo_iterator.go new file mode 100644 index 0000000..addcfb0 --- /dev/null +++ b/graph/mongo/mongo_iterator.go @@ -0,0 +1,181 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongo + +import ( + "fmt" + "strings" + + "github.com/barakmich/glog" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + + "github.com/google/cayley/graph" +) + +type MongoIterator struct { + graph.BaseIterator + ts *MongoTripleStore + dir string + iter *mgo.Iter + hash string + name string + size int64 + isAll bool + constraint bson.M + collection string +} + +func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator { + var m MongoIterator + graph.BaseIteratorInit(&m.BaseIterator) + + m.name = ts.GetNameFor(val) + m.collection = collection + switch dir { + + case "s": + m.constraint = bson.M{"Sub": m.name} + case "p": + m.constraint = bson.M{"Pred": m.name} + case "o": + m.constraint = bson.M{"Obj": m.name} + case "c": + m.constraint = bson.M{"Provenance": m.name} + } + + m.ts = ts + m.dir = dir + m.iter = ts.db.C(collection).Find(m.constraint).Iter() + size, err := ts.db.C(collection).Find(m.constraint).Count() + if err != nil { + glog.Errorln("Trouble getting size for iterator! ", err) + return nil + } + m.size = int64(size) + m.hash = val.(string) + m.isAll = false + return &m +} + +func NewMongoAllIterator(ts *MongoTripleStore, collection string) *MongoIterator { + var m MongoIterator + m.ts = ts + m.dir = "all" + m.constraint = nil + m.collection = collection + m.iter = ts.db.C(collection).Find(nil).Iter() + size, err := ts.db.C(collection).Count() + if err != nil { + glog.Errorln("Trouble getting size for iterator! ", err) + return nil + } + m.size = int64(size) + m.hash = "" + m.isAll = true + return &m +} + +func (m *MongoIterator) Reset() { + m.iter.Close() + m.iter = m.ts.db.C(m.collection).Find(m.constraint).Iter() + +} + +func (m *MongoIterator) Close() { + m.iter.Close() +} + +func (m *MongoIterator) Clone() graph.Iterator { + var newM graph.Iterator + if m.isAll { + newM = NewMongoAllIterator(m.ts, m.collection) + } else { + newM = NewMongoIterator(m.ts, m.collection, m.dir, m.hash) + } + newM.CopyTagsFrom(m) + return newM +} + +func (m *MongoIterator) Next() (graph.TSVal, bool) { + var result struct { + Id string "_id" + //Sub string "Sub" + //Pred string "Pred" + //Obj string "Obj" + } + found := m.iter.Next(&result) + if !found { + err := m.iter.Err() + if err != nil { + glog.Errorln("Error Nexting MongoIterator: ", err) + } + return nil, false + } + m.Last = result.Id + return result.Id, true +} + +func (m *MongoIterator) Check(v graph.TSVal) bool { + graph.CheckLogIn(m, v) + if m.isAll { + m.Last = v + return graph.CheckLogOut(m, v, true) + } + var offset int + switch m.dir { + case "s": + offset = 0 + case "p": + offset = (m.ts.hasher.Size() * 2) + case "o": + offset = (m.ts.hasher.Size() * 2) * 2 + case "c": + offset = (m.ts.hasher.Size() * 2) * 3 + } + val := v.(string)[offset : m.ts.hasher.Size()*2+offset] + if val == m.hash { + m.Last = v + return graph.CheckLogOut(m, v, true) + } + return graph.CheckLogOut(m, v, false) +} + +func (m *MongoIterator) Size() (int64, bool) { + return m.size, true +} + +func (m *MongoIterator) Type() string { + if m.isAll { + return "all" + } + return "mongo" +} +func (m *MongoIterator) Sorted() bool { return true } +func (m *MongoIterator) Optimize() (graph.Iterator, bool) { return m, false } + +func (m *MongoIterator) DebugString(indent int) string { + size, _ := m.Size() + return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), m.Type(), size, m.hash, m.name) +} + +func (m *MongoIterator) GetStats() *graph.IteratorStats { + size, _ := m.Size() + return &graph.IteratorStats{ + CheckCost: 1, + NextCost: 5, + Size: size, + } +} diff --git a/graph/mongo/mongo_triplestore.go b/graph/mongo/mongo_triplestore.go new file mode 100644 index 0000000..917ab4d --- /dev/null +++ b/graph/mongo/mongo_triplestore.go @@ -0,0 +1,329 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongo + +import ( + "crypto/sha1" + "encoding/hex" + "hash" + "log" + + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + + "github.com/barakmich/glog" + "github.com/google/cayley/graph" +) + +const DefaultDBName = "cayley" + +type MongoTripleStore struct { + session *mgo.Session + db *mgo.Database + hasher hash.Hash + idCache *IDLru +} + +func CreateNewMongoGraph(addr string, options graph.OptionsDict) bool { + conn, err := mgo.Dial(addr) + if err != nil { + glog.Fatal("Error connecting: ", err) + return false + } + conn.SetSafe(&mgo.Safe{}) + dbName := DefaultDBName + if val, ok := options.GetStringKey("database_name"); ok { + dbName = val + } + db := conn.DB(dbName) + indexOpts := mgo.Index{ + Key: []string{"Sub"}, + Unique: false, + DropDups: false, + Background: true, + Sparse: true, + } + db.C("triples").EnsureIndex(indexOpts) + indexOpts.Key = []string{"Pred"} + db.C("triples").EnsureIndex(indexOpts) + indexOpts.Key = []string{"Obj"} + db.C("triples").EnsureIndex(indexOpts) + indexOpts.Key = []string{"Provenance"} + db.C("triples").EnsureIndex(indexOpts) + return true +} + +func NewMongoTripleStore(addr string, options graph.OptionsDict) *MongoTripleStore { + var ts MongoTripleStore + conn, err := mgo.Dial(addr) + if err != nil { + glog.Fatal("Error connecting: ", err) + } + conn.SetSafe(&mgo.Safe{}) + dbName := DefaultDBName + if val, ok := options.GetStringKey("database_name"); ok { + dbName = val + } + ts.db = conn.DB(dbName) + ts.session = conn + ts.hasher = sha1.New() + ts.idCache = NewIDLru(1 << 16) + return &ts +} + +func (ts *MongoTripleStore) getIdForTriple(t *graph.Triple) string { + id := ts.ConvertStringToByteHash(t.Sub) + id += ts.ConvertStringToByteHash(t.Pred) + id += ts.ConvertStringToByteHash(t.Obj) + id += ts.ConvertStringToByteHash(t.Provenance) + return id +} + +func (ts *MongoTripleStore) ConvertStringToByteHash(s string) string { + ts.hasher.Reset() + key := make([]byte, 0, ts.hasher.Size()) + ts.hasher.Write([]byte(s)) + key = ts.hasher.Sum(key) + return hex.EncodeToString(key) +} + +type MongoNode struct { + Id string "_id" + Name string "Name" + Size int "Size" +} + +func (ts *MongoTripleStore) updateNodeBy(node_name string, inc int) { + var size MongoNode + node := ts.GetIdFor(node_name) + err := ts.db.C("nodes").FindId(node).One(&size) + if err != nil { + if err.Error() == "not found" { + // Not found. Okay. + size.Id = node.(string) + size.Name = node_name + size.Size = inc + } else { + glog.Error("Error:", err) + return + } + } else { + size.Id = node.(string) + size.Name = node_name + size.Size += inc + } + + // Removing something... + if inc < 0 { + if size.Size <= 0 { + err := ts.db.C("nodes").RemoveId(node) + if err != nil { + glog.Error("Error: ", err, " while removing node ", node_name) + return + } + } + } + + _, err2 := ts.db.C("nodes").UpsertId(node, size) + if err2 != nil { + glog.Error("Error: ", err) + } +} + +func (ts *MongoTripleStore) writeTriple(t *graph.Triple) bool { + tripledoc := bson.M{"_id": ts.getIdForTriple(t), "Sub": t.Sub, "Pred": t.Pred, "Obj": t.Obj, "Provenance": t.Provenance} + err := ts.db.C("triples").Insert(tripledoc) + if err != nil { + // Among the reasons I hate MongoDB. "Errors don't happen! Right guys?" + if err.(*mgo.LastError).Code == 11000 { + return false + } + glog.Error("Error: ", err) + return false + } + return true +} + +func (ts *MongoTripleStore) AddTriple(t *graph.Triple) { + _ = ts.writeTriple(t) + ts.updateNodeBy(t.Sub, 1) + ts.updateNodeBy(t.Pred, 1) + ts.updateNodeBy(t.Obj, 1) + if t.Provenance != "" { + ts.updateNodeBy(t.Provenance, 1) + } +} + +func (ts *MongoTripleStore) AddTripleSet(in []*graph.Triple) { + ts.session.SetSafe(nil) + idMap := make(map[string]int) + for _, t := range in { + wrote := ts.writeTriple(t) + if wrote { + idMap[t.Sub]++ + idMap[t.Obj]++ + idMap[t.Pred]++ + if t.Provenance != "" { + idMap[t.Provenance]++ + } + } + } + for k, v := range idMap { + ts.updateNodeBy(k, v) + } + ts.session.SetSafe(&mgo.Safe{}) +} + +func (ts *MongoTripleStore) RemoveTriple(t *graph.Triple) { + err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t)) + if err == mgo.ErrNotFound { + return + } else if err != nil { + log.Println("Error: ", err, " while removing triple ", t) + return + } + ts.updateNodeBy(t.Sub, -1) + ts.updateNodeBy(t.Pred, -1) + ts.updateNodeBy(t.Obj, -1) + if t.Provenance != "" { + ts.updateNodeBy(t.Provenance, -1) + } +} + +func (ts *MongoTripleStore) GetTriple(val graph.TSVal) *graph.Triple { + var bsonDoc bson.M + err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc) + if err != nil { + log.Println("Error: Couldn't retrieve triple", val.(string), err) + } + return graph.MakeTriple( + bsonDoc["Sub"].(string), + bsonDoc["Pred"].(string), + bsonDoc["Obj"].(string), + bsonDoc["Provenance"].(string)) +} + +func (ts *MongoTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { + return NewMongoIterator(ts, "triples", dir, val) +} + +func (ts *MongoTripleStore) GetNodesAllIterator() graph.Iterator { + return NewMongoAllIterator(ts, "nodes") +} + +func (ts *MongoTripleStore) GetTriplesAllIterator() graph.Iterator { + return NewMongoAllIterator(ts, "triples") +} + +func (ts *MongoTripleStore) GetIdFor(s string) graph.TSVal { + return ts.ConvertStringToByteHash(s) +} + +func (ts *MongoTripleStore) GetNameFor(v graph.TSVal) string { + val, ok := ts.idCache.Get(v.(string)) + if ok { + return val + } + var node MongoNode + err := ts.db.C("nodes").FindId(v.(string)).One(&node) + if err != nil { + log.Println("Error: Couldn't retrieve node", v.(string), err) + } + ts.idCache.Put(v.(string), node.Name) + return node.Name +} + +func (ts *MongoTripleStore) Size() int64 { + count, err := ts.db.C("triples").Count() + if err != nil { + glog.Error("Error: ", err) + return 0 + } + return int64(count) +} + +func compareStrings(a, b graph.TSVal) bool { + return a.(string) == b.(string) +} + +func (ts *MongoTripleStore) MakeFixed() *graph.FixedIterator { + return graph.NewFixedIteratorWithCompare(compareStrings) +} + +func (ts *MongoTripleStore) Close() { + ts.db.Session.Close() +} + +func (ts *MongoTripleStore) GetTripleDirection(in graph.TSVal, dir string) graph.TSVal { + // Maybe do the trick here + var offset int + switch dir { + case "s": + offset = 0 + case "p": + offset = (ts.hasher.Size() * 2) + case "o": + offset = (ts.hasher.Size() * 2) * 2 + case "c": + offset = (ts.hasher.Size() * 2) * 3 + } + val := in.(string)[offset : ts.hasher.Size()*2+offset] + return val +} + +func (ts *MongoTripleStore) BulkLoad(t_chan chan *graph.Triple) { + ts.session.SetSafe(nil) + for triple := range t_chan { + ts.writeTriple(triple) + } + outputTo := bson.M{"replace": "nodes", "sharded": true} + glog.Infoln("Mapreducing") + job := mgo.MapReduce{ + Map: `function() { + var len = this["_id"].length + var s_key = this["_id"].slice(0, len / 4) + var p_key = this["_id"].slice(len / 4, 2 * len / 4) + var o_key = this["_id"].slice(2 * len / 4, 3 * len / 4) + var c_key = this["_id"].slice(3 * len / 4) + emit(s_key, {"_id": s_key, "Name" : this.Sub, "Size" : 1}) + emit(p_key, {"_id": p_key, "Name" : this.Pred, "Size" : 1}) + emit(o_key, {"_id": o_key, "Name" : this.Obj, "Size" : 1}) + if (this.Provenance != "") { + emit(c_key, {"_id": c_key, "Name" : this.Provenance, "Size" : 1}) + } + } + `, + Reduce: ` + function(key, value_list) { + out = {"_id": key, "Name": value_list[0].Name} + count = 0 + for (var i = 0; i < value_list.length; i++) { + count = count + value_list[i].Size + + } + out["Size"] = count + return out + } + `, + Out: outputTo, + } + ts.db.C("triples").Find(nil).MapReduce(&job, nil) + glog.Infoln("Fixing") + ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) { + db.nodes.update({"_id": result._id}, result.value) + }) }`}, {"args", bson.D{}}}, nil) + + ts.session.SetSafe(&mgo.Safe{}) +} diff --git a/graph/mongo/mongo_triplestore_iterator_optimize.go b/graph/mongo/mongo_triplestore_iterator_optimize.go new file mode 100644 index 0000000..d10bc22 --- /dev/null +++ b/graph/mongo/mongo_triplestore_iterator_optimize.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongo + +import ( + "github.com/google/cayley/graph" +) + +func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { + switch it.Type() { + case "linksto": + return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) + + } + return it, false +} + +func (ts *MongoTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { + l := it.GetSubIterators() + if l.Len() != 1 { + return it, false + } + primaryIt := l.Front().Value.(graph.Iterator) + if primaryIt.Type() == "fixed" { + size, _ := primaryIt.Size() + if size == 1 { + val, ok := primaryIt.Next() + if !ok { + panic("Sizes lie") + } + newIt := ts.GetTripleIterator(it.Direction(), val) + newIt.CopyTagsFrom(it) + for _, tag := range primaryIt.Tags() { + newIt.AddFixedTag(tag, val) + } + it.Close() + return newIt, true + } + } + return it, false +} diff --git a/graph/optional-iterator.go b/graph/optional-iterator.go deleted file mode 100644 index 8050ba8..0000000 --- a/graph/optional-iterator.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// "Optional" is kind of odd. It's not an iterator in the strictest sense, but -// it's easier to implement as an iterator. -// -// Consider what it means. It means that we have a subconstraint which we do -// not want to constrain the query -- we just want it to return the matching -// subgraph if one matches at all. By analogy to regular expressions, it is the -// '?' operator. -// -// If it were a proper iterator of its own (and indeed, a reasonable refactor -// of this iterator would be to make it such) it would contain an all iterator -// -- all things in the graph. It matches everything (as does the regex "(a)?") - -import ( - "fmt" - "github.com/barakmich/glog" - "strings" -) - -// An optional iterator has the subconstraint iterator we wish to be optional -// and whether the last check we received was true or false. -type OptionalIterator struct { - BaseIterator - subIt Iterator - lastCheck bool -} - -// Creates a new optional iterator. -func NewOptionalIterator(it Iterator) *OptionalIterator { - var o OptionalIterator - BaseIteratorInit(&o.BaseIterator) - o.nextable = false - o.subIt = it - return &o -} - -func (o *OptionalIterator) Reset() { - o.subIt.Reset() - o.lastCheck = false -} - -func (o *OptionalIterator) Close() { - o.subIt.Close() -} - -func (o *OptionalIterator) Clone() Iterator { - out := NewOptionalIterator(o.subIt.Clone()) - out.CopyTagsFrom(o) - return out -} - -// Nexting the iterator is unsupported -- error and return an empty set. -// (As above, a reasonable alternative would be to Next() an all iterator) -func (o *OptionalIterator) Next() (TSVal, bool) { - glog.Errorln("Nexting an un-nextable iterator") - return nil, false -} - -// An optional iterator only has a next result if, (a) last time we checked -// we had any results whatsoever, and (b) there was another subresult in our -// optional subbranch. -func (o *OptionalIterator) NextResult() bool { - if o.lastCheck { - return o.subIt.NextResult() - } - return false -} - -// Check() is the real hack of this iterator. It always returns true, regardless -// of whether the subiterator matched. But we keep track of whether the subiterator -// matched for results purposes. -func (o *OptionalIterator) Check(val TSVal) bool { - checked := o.subIt.Check(val) - o.lastCheck = checked - o.Last = val - return true -} - -// If we failed the check, then the subiterator should not contribute to the result -// set. Otherwise, go ahead and tag it. -func (o *OptionalIterator) TagResults(out *map[string]TSVal) { - if o.lastCheck == false { - return - } - o.subIt.TagResults(out) -} - -// Registers the optional iterator. -func (o *OptionalIterator) Type() string { return "optional" } - -// Prints the optional and it's subiterator. -func (o *OptionalIterator) DebugString(indent int) string { - return fmt.Sprintf("%s(%s tags:%s\n%s)", - strings.Repeat(" ", indent), - o.Type(), - o.Tags(), - o.subIt.DebugString(indent+4)) -} - -// There's nothing to optimize for an optional. Optimize the subiterator and -// potentially replace it. -func (o *OptionalIterator) Optimize() (Iterator, bool) { - newSub, changed := o.subIt.Optimize() - if changed { - o.subIt.Close() - o.subIt = newSub - } - return o, false -} - -// We're only as expensive as our subiterator. Except, we can't be nexted. -func (o *OptionalIterator) GetStats() *IteratorStats { - subStats := o.subIt.GetStats() - return &IteratorStats{ - CheckCost: subStats.CheckCost, - NextCost: int64(1 << 62), - Size: subStats.Size, - } -} diff --git a/graph/optional_iterator.go b/graph/optional_iterator.go new file mode 100644 index 0000000..8050ba8 --- /dev/null +++ b/graph/optional_iterator.go @@ -0,0 +1,134 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// "Optional" is kind of odd. It's not an iterator in the strictest sense, but +// it's easier to implement as an iterator. +// +// Consider what it means. It means that we have a subconstraint which we do +// not want to constrain the query -- we just want it to return the matching +// subgraph if one matches at all. By analogy to regular expressions, it is the +// '?' operator. +// +// If it were a proper iterator of its own (and indeed, a reasonable refactor +// of this iterator would be to make it such) it would contain an all iterator +// -- all things in the graph. It matches everything (as does the regex "(a)?") + +import ( + "fmt" + "github.com/barakmich/glog" + "strings" +) + +// An optional iterator has the subconstraint iterator we wish to be optional +// and whether the last check we received was true or false. +type OptionalIterator struct { + BaseIterator + subIt Iterator + lastCheck bool +} + +// Creates a new optional iterator. +func NewOptionalIterator(it Iterator) *OptionalIterator { + var o OptionalIterator + BaseIteratorInit(&o.BaseIterator) + o.nextable = false + o.subIt = it + return &o +} + +func (o *OptionalIterator) Reset() { + o.subIt.Reset() + o.lastCheck = false +} + +func (o *OptionalIterator) Close() { + o.subIt.Close() +} + +func (o *OptionalIterator) Clone() Iterator { + out := NewOptionalIterator(o.subIt.Clone()) + out.CopyTagsFrom(o) + return out +} + +// Nexting the iterator is unsupported -- error and return an empty set. +// (As above, a reasonable alternative would be to Next() an all iterator) +func (o *OptionalIterator) Next() (TSVal, bool) { + glog.Errorln("Nexting an un-nextable iterator") + return nil, false +} + +// An optional iterator only has a next result if, (a) last time we checked +// we had any results whatsoever, and (b) there was another subresult in our +// optional subbranch. +func (o *OptionalIterator) NextResult() bool { + if o.lastCheck { + return o.subIt.NextResult() + } + return false +} + +// Check() is the real hack of this iterator. It always returns true, regardless +// of whether the subiterator matched. But we keep track of whether the subiterator +// matched for results purposes. +func (o *OptionalIterator) Check(val TSVal) bool { + checked := o.subIt.Check(val) + o.lastCheck = checked + o.Last = val + return true +} + +// If we failed the check, then the subiterator should not contribute to the result +// set. Otherwise, go ahead and tag it. +func (o *OptionalIterator) TagResults(out *map[string]TSVal) { + if o.lastCheck == false { + return + } + o.subIt.TagResults(out) +} + +// Registers the optional iterator. +func (o *OptionalIterator) Type() string { return "optional" } + +// Prints the optional and it's subiterator. +func (o *OptionalIterator) DebugString(indent int) string { + return fmt.Sprintf("%s(%s tags:%s\n%s)", + strings.Repeat(" ", indent), + o.Type(), + o.Tags(), + o.subIt.DebugString(indent+4)) +} + +// There's nothing to optimize for an optional. Optimize the subiterator and +// potentially replace it. +func (o *OptionalIterator) Optimize() (Iterator, bool) { + newSub, changed := o.subIt.Optimize() + if changed { + o.subIt.Close() + o.subIt = newSub + } + return o, false +} + +// We're only as expensive as our subiterator. Except, we can't be nexted. +func (o *OptionalIterator) GetStats() *IteratorStats { + subStats := o.subIt.GetStats() + return &IteratorStats{ + CheckCost: subStats.CheckCost, + NextCost: int64(1 << 62), + Size: subStats.Size, + } +} diff --git a/graph/or-iterator.go b/graph/or-iterator.go deleted file mode 100644 index a0de623..0000000 --- a/graph/or-iterator.go +++ /dev/null @@ -1,287 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// Defines the or and short-circuiting or iterator. Or is the union operator for it's subiterators. -// Short-circuiting-or is a little different. It will return values from the first iterator that returns -// values at all, and then stops. -// -// Never reorders the iterators from the order they arrive. It is either the union or the first one. -// May return the same value twice -- once for each branch. - -import ( - "container/list" - "fmt" - "strings" -) - -type OrIterator struct { - BaseIterator - isShortCircuiting bool - internalIterators []Iterator - itCount int - currentIterator int -} - -func NewOrIterator() *OrIterator { - var or OrIterator - BaseIteratorInit(&or.BaseIterator) - or.internalIterators = make([]Iterator, 0, 20) - or.isShortCircuiting = false - or.currentIterator = -1 - return &or -} - -func NewShortCircuitOrIterator() *OrIterator { - var or OrIterator - BaseIteratorInit(&or.BaseIterator) - or.internalIterators = make([]Iterator, 0, 20) - or.isShortCircuiting = true - or.currentIterator = -1 - return &or -} - -// Reset all internal iterators -func (or *OrIterator) Reset() { - for _, it := range or.internalIterators { - it.Reset() - } - or.currentIterator = -1 -} - -func (or *OrIterator) Clone() Iterator { - var newOr *OrIterator - if or.isShortCircuiting { - newOr = NewShortCircuitOrIterator() - } else { - newOr = NewOrIterator() - } - for _, it := range or.internalIterators { - newOr.AddSubIterator(it.Clone()) - } - or.CopyTagsFrom(or) - return newOr -} - -// Returns a list.List of the subiterators, in order. -func (or *OrIterator) GetSubIterators() *list.List { - l := list.New() - for _, it := range or.internalIterators { - l.PushBack(it) - } - return l -} - -// Overrides BaseIterator TagResults, as it needs to add it's own results and -// recurse down it's subiterators. -func (or *OrIterator) TagResults(out *map[string]TSVal) { - or.BaseIterator.TagResults(out) - or.internalIterators[or.currentIterator].TagResults(out) -} - -// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators. -func (or *OrIterator) GetResultTree() *ResultTree { - tree := NewResultTree(or.LastResult()) - for _, it := range or.internalIterators { - tree.AddSubtree(it.GetResultTree()) - } - return tree -} - -// Prints information about this iterator. -func (or *OrIterator) DebugString(indent int) string { - var total string - for i, it := range or.internalIterators { - total += strings.Repeat(" ", indent+2) - total += fmt.Sprintf("%d:\n%s\n", i, it.DebugString(indent+4)) - } - var tags string - for _, k := range or.Tags() { - tags += fmt.Sprintf("%s;", k) - } - spaces := strings.Repeat(" ", indent+2) - - return fmt.Sprintf("%s(%s\n%stags:%s\n%sits:\n%s)", - strings.Repeat(" ", indent), - or.Type(), - spaces, - tags, - spaces, - total) -} - -// Add a subiterator to this Or iterator. Order matters. -func (or *OrIterator) AddSubIterator(sub Iterator) { - or.internalIterators = append(or.internalIterators, sub) - or.itCount++ -} - -// Returns the Next value from the Or iterator. Because the Or is the -// union of its subiterators, it must produce from all subiterators -- unless -// it's shortcircuiting, in which case, it's the first one that returns anything. -func (or *OrIterator) Next() (TSVal, bool) { - NextLogIn(or) - var curr TSVal - var exists bool - firstTime := false - for { - if or.currentIterator == -1 { - or.currentIterator = 0 - firstTime = true - } - curIt := or.internalIterators[or.currentIterator] - curr, exists = curIt.Next() - if !exists { - if or.isShortCircuiting && !firstTime { - return NextLogOut(or, nil, false) - } - or.currentIterator++ - if or.currentIterator == or.itCount { - return NextLogOut(or, nil, false) - } - } else { - or.Last = curr - return NextLogOut(or, curr, true) - } - } - panic("Somehow broke out of Next() loop in OrIterator") -} - -// Checks a value against the iterators, in order. -func (or *OrIterator) checkSubIts(val TSVal) bool { - var subIsGood = false - for i, it := range or.internalIterators { - subIsGood = it.Check(val) - if subIsGood { - or.currentIterator = i - break - } - } - return subIsGood -} - -// Check a value against the entire iterator, in order. -func (or *OrIterator) Check(val TSVal) bool { - CheckLogIn(or, val) - anyGood := or.checkSubIts(val) - if !anyGood { - return CheckLogOut(or, val, false) - } - or.Last = val - return CheckLogOut(or, val, true) -} - -// Returns the approximate size of the Or iterator. Because we're dealing -// with a union, we know that the largest we can be is the sum of all the iterators, -// or in the case of short-circuiting, the longest. -func (or *OrIterator) Size() (int64, bool) { - var val int64 - var b bool - if or.isShortCircuiting { - val = 0 - b = true - for _, it := range or.internalIterators { - newval, newb := it.Size() - if val < newval { - val = newval - } - b = newb && b - } - } else { - val = 0 - b = true - for _, it := range or.internalIterators { - newval, newb := it.Size() - val += newval - b = newb && b - } - } - return val, b -} - -// An Or has no NextResult of its own -- that is, there are no other values -// which satisfy our previous result that are not the result itself. Our -// subiterators might, however, so just pass the call recursively. In the case of -// shortcircuiting, only allow new results from the currently checked iterator -func (or *OrIterator) NextResult() bool { - if or.currentIterator != -1 { - return or.internalIterators[or.currentIterator].NextResult() - } - return false -} - -// Perform or-specific cleanup, of which there currently is none. -func (or *OrIterator) cleanUp() {} - -// Close this iterator, and, by extension, close the subiterators. -// Close should be idempotent, and it follows that if it's subiterators -// follow this contract, the And follows the contract. -func (or *OrIterator) Close() { - or.cleanUp() - for _, it := range or.internalIterators { - it.Close() - } -} - -func (or *OrIterator) Optimize() (Iterator, bool) { - oldItList := or.GetSubIterators() - itList := optimizeSubIterators(oldItList) - // Close the replaced iterators (they ought to close themselves, but Close() - // is idempotent, so this just protects against any machinations). - closeIteratorList(oldItList, nil) - newOr := NewOrIterator() - newOr.isShortCircuiting = or.isShortCircuiting - - // Add the subiterators in order. - for e := itList.Front(); e != nil; e = e.Next() { - newOr.AddSubIterator(e.Value.(Iterator)) - } - - // Move the tags hanging on us (like any good replacement). - newOr.CopyTagsFrom(or) - - // And close ourselves but not our subiterators -- some may still be alive in - // the new And (they were unchanged upon calling Optimize() on them, at the - // start). - or.cleanUp() - return newOr, true -} - -func (or *OrIterator) GetStats() *IteratorStats { - CheckCost := int64(0) - NextCost := int64(0) - Size := int64(0) - for _, it := range or.internalIterators { - stats := it.GetStats() - NextCost += stats.NextCost - CheckCost += stats.CheckCost - if or.isShortCircuiting { - if Size < stats.Size { - Size = stats.Size - } - } else { - Size += stats.Size - } - } - return &IteratorStats{ - CheckCost: CheckCost, - NextCost: NextCost, - Size: Size, - } - -} - -// Register this as an "or" iterator. -func (or *OrIterator) Type() string { return "or" } diff --git a/graph/or-iterator_test.go b/graph/or-iterator_test.go deleted file mode 100644 index 9450094..0000000 --- a/graph/or-iterator_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - . "github.com/smartystreets/goconvey/convey" - "testing" -) - -func extractNumbersFromIterator(it Iterator) []int { - var outputNumbers []int - for { - val, ok := it.Next() - if !ok { - break - } - outputNumbers = append(outputNumbers, val.(int)) - } - return outputNumbers -} - -func TestOrIteratorBasics(t *testing.T) { - var orIt *OrIterator - - Convey("Given an Or Iterator of two fixed iterators", t, func() { - orIt = NewOrIterator() - fixed1 := newFixedIterator() - fixed1.AddValue(1) - fixed1.AddValue(2) - fixed1.AddValue(3) - fixed2 := newFixedIterator() - fixed2.AddValue(3) - fixed2.AddValue(9) - fixed2.AddValue(20) - fixed2.AddValue(21) - orIt.AddSubIterator(fixed1) - orIt.AddSubIterator(fixed2) - - Convey("It should guess its size.", func() { - v, _ := orIt.Size() - So(v, ShouldEqual, 7) - }) - - Convey("It should extract all the numbers, potentially twice.", func() { - allNumbers := []int{1, 2, 3, 3, 9, 20, 21} - So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) - orIt.Reset() - So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) - // Optimization works - newOr, _ := orIt.Optimize() - So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers) - }) - - Convey("It should check that numbers in either iterator exist.", func() { - So(orIt.Check(2), ShouldEqual, true) - So(orIt.Check(3), ShouldEqual, true) - So(orIt.Check(21), ShouldEqual, true) - }) - - Convey("It should check that numbers not in either iterator are false.", func() { - So(orIt.Check(22), ShouldEqual, false) - So(orIt.Check(5), ShouldEqual, false) - So(orIt.Check(0), ShouldEqual, false) - }) - - }) - -} - -func TestShortCircuitingOrBasics(t *testing.T) { - var orIt *OrIterator - - Convey("Given a short-circuiting Or of two fixed iterators", t, func() { - orIt = NewShortCircuitOrIterator() - fixed1 := newFixedIterator() - fixed1.AddValue(1) - fixed1.AddValue(2) - fixed1.AddValue(3) - fixed2 := newFixedIterator() - fixed2.AddValue(3) - fixed2.AddValue(9) - fixed2.AddValue(20) - fixed2.AddValue(21) - - Convey("It should guess its size.", func() { - orIt.AddSubIterator(fixed1) - orIt.AddSubIterator(fixed2) - v, _ := orIt.Size() - So(v, ShouldEqual, 4) - }) - - Convey("It should extract the first iterators' numbers.", func() { - orIt.AddSubIterator(fixed1) - orIt.AddSubIterator(fixed2) - allNumbers := []int{1, 2, 3} - So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) - orIt.Reset() - So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) - // Optimization works - newOr, _ := orIt.Optimize() - So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers) - }) - - Convey("It should check that numbers in either iterator exist.", func() { - orIt.AddSubIterator(fixed1) - orIt.AddSubIterator(fixed2) - So(orIt.Check(2), ShouldEqual, true) - So(orIt.Check(3), ShouldEqual, true) - So(orIt.Check(21), ShouldEqual, true) - So(orIt.Check(22), ShouldEqual, false) - So(orIt.Check(5), ShouldEqual, false) - So(orIt.Check(0), ShouldEqual, false) - - }) - - Convey("It should check that it pulls the second iterator's numbers if the first is empty.", func() { - orIt.AddSubIterator(newFixedIterator()) - orIt.AddSubIterator(fixed2) - allNumbers := []int{3, 9, 20, 21} - So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) - orIt.Reset() - So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) - // Optimization works - newOr, _ := orIt.Optimize() - So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers) - }) - - }) - -} diff --git a/graph/or_iterator.go b/graph/or_iterator.go new file mode 100644 index 0000000..a0de623 --- /dev/null +++ b/graph/or_iterator.go @@ -0,0 +1,287 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// Defines the or and short-circuiting or iterator. Or is the union operator for it's subiterators. +// Short-circuiting-or is a little different. It will return values from the first iterator that returns +// values at all, and then stops. +// +// Never reorders the iterators from the order they arrive. It is either the union or the first one. +// May return the same value twice -- once for each branch. + +import ( + "container/list" + "fmt" + "strings" +) + +type OrIterator struct { + BaseIterator + isShortCircuiting bool + internalIterators []Iterator + itCount int + currentIterator int +} + +func NewOrIterator() *OrIterator { + var or OrIterator + BaseIteratorInit(&or.BaseIterator) + or.internalIterators = make([]Iterator, 0, 20) + or.isShortCircuiting = false + or.currentIterator = -1 + return &or +} + +func NewShortCircuitOrIterator() *OrIterator { + var or OrIterator + BaseIteratorInit(&or.BaseIterator) + or.internalIterators = make([]Iterator, 0, 20) + or.isShortCircuiting = true + or.currentIterator = -1 + return &or +} + +// Reset all internal iterators +func (or *OrIterator) Reset() { + for _, it := range or.internalIterators { + it.Reset() + } + or.currentIterator = -1 +} + +func (or *OrIterator) Clone() Iterator { + var newOr *OrIterator + if or.isShortCircuiting { + newOr = NewShortCircuitOrIterator() + } else { + newOr = NewOrIterator() + } + for _, it := range or.internalIterators { + newOr.AddSubIterator(it.Clone()) + } + or.CopyTagsFrom(or) + return newOr +} + +// Returns a list.List of the subiterators, in order. +func (or *OrIterator) GetSubIterators() *list.List { + l := list.New() + for _, it := range or.internalIterators { + l.PushBack(it) + } + return l +} + +// Overrides BaseIterator TagResults, as it needs to add it's own results and +// recurse down it's subiterators. +func (or *OrIterator) TagResults(out *map[string]TSVal) { + or.BaseIterator.TagResults(out) + or.internalIterators[or.currentIterator].TagResults(out) +} + +// DEPRECATED Returns the ResultTree for this iterator, recurses to it's subiterators. +func (or *OrIterator) GetResultTree() *ResultTree { + tree := NewResultTree(or.LastResult()) + for _, it := range or.internalIterators { + tree.AddSubtree(it.GetResultTree()) + } + return tree +} + +// Prints information about this iterator. +func (or *OrIterator) DebugString(indent int) string { + var total string + for i, it := range or.internalIterators { + total += strings.Repeat(" ", indent+2) + total += fmt.Sprintf("%d:\n%s\n", i, it.DebugString(indent+4)) + } + var tags string + for _, k := range or.Tags() { + tags += fmt.Sprintf("%s;", k) + } + spaces := strings.Repeat(" ", indent+2) + + return fmt.Sprintf("%s(%s\n%stags:%s\n%sits:\n%s)", + strings.Repeat(" ", indent), + or.Type(), + spaces, + tags, + spaces, + total) +} + +// Add a subiterator to this Or iterator. Order matters. +func (or *OrIterator) AddSubIterator(sub Iterator) { + or.internalIterators = append(or.internalIterators, sub) + or.itCount++ +} + +// Returns the Next value from the Or iterator. Because the Or is the +// union of its subiterators, it must produce from all subiterators -- unless +// it's shortcircuiting, in which case, it's the first one that returns anything. +func (or *OrIterator) Next() (TSVal, bool) { + NextLogIn(or) + var curr TSVal + var exists bool + firstTime := false + for { + if or.currentIterator == -1 { + or.currentIterator = 0 + firstTime = true + } + curIt := or.internalIterators[or.currentIterator] + curr, exists = curIt.Next() + if !exists { + if or.isShortCircuiting && !firstTime { + return NextLogOut(or, nil, false) + } + or.currentIterator++ + if or.currentIterator == or.itCount { + return NextLogOut(or, nil, false) + } + } else { + or.Last = curr + return NextLogOut(or, curr, true) + } + } + panic("Somehow broke out of Next() loop in OrIterator") +} + +// Checks a value against the iterators, in order. +func (or *OrIterator) checkSubIts(val TSVal) bool { + var subIsGood = false + for i, it := range or.internalIterators { + subIsGood = it.Check(val) + if subIsGood { + or.currentIterator = i + break + } + } + return subIsGood +} + +// Check a value against the entire iterator, in order. +func (or *OrIterator) Check(val TSVal) bool { + CheckLogIn(or, val) + anyGood := or.checkSubIts(val) + if !anyGood { + return CheckLogOut(or, val, false) + } + or.Last = val + return CheckLogOut(or, val, true) +} + +// Returns the approximate size of the Or iterator. Because we're dealing +// with a union, we know that the largest we can be is the sum of all the iterators, +// or in the case of short-circuiting, the longest. +func (or *OrIterator) Size() (int64, bool) { + var val int64 + var b bool + if or.isShortCircuiting { + val = 0 + b = true + for _, it := range or.internalIterators { + newval, newb := it.Size() + if val < newval { + val = newval + } + b = newb && b + } + } else { + val = 0 + b = true + for _, it := range or.internalIterators { + newval, newb := it.Size() + val += newval + b = newb && b + } + } + return val, b +} + +// An Or has no NextResult of its own -- that is, there are no other values +// which satisfy our previous result that are not the result itself. Our +// subiterators might, however, so just pass the call recursively. In the case of +// shortcircuiting, only allow new results from the currently checked iterator +func (or *OrIterator) NextResult() bool { + if or.currentIterator != -1 { + return or.internalIterators[or.currentIterator].NextResult() + } + return false +} + +// Perform or-specific cleanup, of which there currently is none. +func (or *OrIterator) cleanUp() {} + +// Close this iterator, and, by extension, close the subiterators. +// Close should be idempotent, and it follows that if it's subiterators +// follow this contract, the And follows the contract. +func (or *OrIterator) Close() { + or.cleanUp() + for _, it := range or.internalIterators { + it.Close() + } +} + +func (or *OrIterator) Optimize() (Iterator, bool) { + oldItList := or.GetSubIterators() + itList := optimizeSubIterators(oldItList) + // Close the replaced iterators (they ought to close themselves, but Close() + // is idempotent, so this just protects against any machinations). + closeIteratorList(oldItList, nil) + newOr := NewOrIterator() + newOr.isShortCircuiting = or.isShortCircuiting + + // Add the subiterators in order. + for e := itList.Front(); e != nil; e = e.Next() { + newOr.AddSubIterator(e.Value.(Iterator)) + } + + // Move the tags hanging on us (like any good replacement). + newOr.CopyTagsFrom(or) + + // And close ourselves but not our subiterators -- some may still be alive in + // the new And (they were unchanged upon calling Optimize() on them, at the + // start). + or.cleanUp() + return newOr, true +} + +func (or *OrIterator) GetStats() *IteratorStats { + CheckCost := int64(0) + NextCost := int64(0) + Size := int64(0) + for _, it := range or.internalIterators { + stats := it.GetStats() + NextCost += stats.NextCost + CheckCost += stats.CheckCost + if or.isShortCircuiting { + if Size < stats.Size { + Size = stats.Size + } + } else { + Size += stats.Size + } + } + return &IteratorStats{ + CheckCost: CheckCost, + NextCost: NextCost, + Size: Size, + } + +} + +// Register this as an "or" iterator. +func (or *OrIterator) Type() string { return "or" } diff --git a/graph/or_iterator_test.go b/graph/or_iterator_test.go new file mode 100644 index 0000000..9450094 --- /dev/null +++ b/graph/or_iterator_test.go @@ -0,0 +1,142 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func extractNumbersFromIterator(it Iterator) []int { + var outputNumbers []int + for { + val, ok := it.Next() + if !ok { + break + } + outputNumbers = append(outputNumbers, val.(int)) + } + return outputNumbers +} + +func TestOrIteratorBasics(t *testing.T) { + var orIt *OrIterator + + Convey("Given an Or Iterator of two fixed iterators", t, func() { + orIt = NewOrIterator() + fixed1 := newFixedIterator() + fixed1.AddValue(1) + fixed1.AddValue(2) + fixed1.AddValue(3) + fixed2 := newFixedIterator() + fixed2.AddValue(3) + fixed2.AddValue(9) + fixed2.AddValue(20) + fixed2.AddValue(21) + orIt.AddSubIterator(fixed1) + orIt.AddSubIterator(fixed2) + + Convey("It should guess its size.", func() { + v, _ := orIt.Size() + So(v, ShouldEqual, 7) + }) + + Convey("It should extract all the numbers, potentially twice.", func() { + allNumbers := []int{1, 2, 3, 3, 9, 20, 21} + So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) + orIt.Reset() + So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) + // Optimization works + newOr, _ := orIt.Optimize() + So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers) + }) + + Convey("It should check that numbers in either iterator exist.", func() { + So(orIt.Check(2), ShouldEqual, true) + So(orIt.Check(3), ShouldEqual, true) + So(orIt.Check(21), ShouldEqual, true) + }) + + Convey("It should check that numbers not in either iterator are false.", func() { + So(orIt.Check(22), ShouldEqual, false) + So(orIt.Check(5), ShouldEqual, false) + So(orIt.Check(0), ShouldEqual, false) + }) + + }) + +} + +func TestShortCircuitingOrBasics(t *testing.T) { + var orIt *OrIterator + + Convey("Given a short-circuiting Or of two fixed iterators", t, func() { + orIt = NewShortCircuitOrIterator() + fixed1 := newFixedIterator() + fixed1.AddValue(1) + fixed1.AddValue(2) + fixed1.AddValue(3) + fixed2 := newFixedIterator() + fixed2.AddValue(3) + fixed2.AddValue(9) + fixed2.AddValue(20) + fixed2.AddValue(21) + + Convey("It should guess its size.", func() { + orIt.AddSubIterator(fixed1) + orIt.AddSubIterator(fixed2) + v, _ := orIt.Size() + So(v, ShouldEqual, 4) + }) + + Convey("It should extract the first iterators' numbers.", func() { + orIt.AddSubIterator(fixed1) + orIt.AddSubIterator(fixed2) + allNumbers := []int{1, 2, 3} + So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) + orIt.Reset() + So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) + // Optimization works + newOr, _ := orIt.Optimize() + So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers) + }) + + Convey("It should check that numbers in either iterator exist.", func() { + orIt.AddSubIterator(fixed1) + orIt.AddSubIterator(fixed2) + So(orIt.Check(2), ShouldEqual, true) + So(orIt.Check(3), ShouldEqual, true) + So(orIt.Check(21), ShouldEqual, true) + So(orIt.Check(22), ShouldEqual, false) + So(orIt.Check(5), ShouldEqual, false) + So(orIt.Check(0), ShouldEqual, false) + + }) + + Convey("It should check that it pulls the second iterator's numbers if the first is empty.", func() { + orIt.AddSubIterator(newFixedIterator()) + orIt.AddSubIterator(fixed2) + allNumbers := []int{3, 9, 20, 21} + So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) + orIt.Reset() + So(extractNumbersFromIterator(orIt), ShouldResemble, allNumbers) + // Optimization works + newOr, _ := orIt.Optimize() + So(extractNumbersFromIterator(newOr), ShouldResemble, allNumbers) + }) + + }) + +} diff --git a/graph/query-shape.go b/graph/query-shape.go deleted file mode 100644 index dece079..0000000 --- a/graph/query-shape.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -type Node struct { - Id int `json:"id"` - Tags []string `json:"tags,omitempty"` - Values []string `json:"values,omitempty"` - IsLinkNode bool `json:"is_link_node"` - IsFixed bool `json:"is_fixed"` -} - -type Link struct { - Source int `json:"source"` - Target int `json:"target"` - Pred int `json:"type"` - LinkNode int `json:"link_node"` -} - -type queryShape struct { - nodes []Node - links []Link - ts TripleStore - nodeId int - hasaIds []int - hasaDirs []string -} - -func OutputQueryShapeForIterator(it Iterator, ts TripleStore, outputMap *map[string]interface{}) { - qs := &queryShape{} - qs.nodes = make([]Node, 0) - qs.links = make([]Link, 0) - qs.hasaIds = make([]int, 0) - qs.hasaDirs = make([]string, 0) - qs.ts = ts - qs.nodeId = 1 - - node := qs.MakeNode(it.Clone()) - qs.AddNode(node) - (*outputMap)["nodes"] = qs.nodes - (*outputMap)["links"] = qs.links -} - -func (qs *queryShape) AddNode(n *Node) { - qs.nodes = append(qs.nodes, *n) -} - -func (qs *queryShape) AddLink(l *Link) { - qs.links = append(qs.links, *l) -} - -func (qs *queryShape) LastHasa() (int, string) { - return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1] -} - -func (qs *queryShape) PushHasa(i int, s string) { - qs.hasaIds = append(qs.hasaIds, i) - qs.hasaDirs = append(qs.hasaDirs, s) -} - -func (qs *queryShape) RemoveHasa() { - qs.hasaIds = qs.hasaIds[:len(qs.hasaIds)-1] - qs.hasaDirs = qs.hasaDirs[:len(qs.hasaDirs)-1] -} - -func (qs *queryShape) StealNode(left *Node, right *Node) { - for _, v := range right.Values { - left.Values = append(left.Values, v) - } - for _, v := range right.Tags { - left.Tags = append(left.Tags, v) - } - left.IsLinkNode = left.IsLinkNode || right.IsLinkNode - left.IsFixed = left.IsFixed || right.IsFixed - for i, link := range qs.links { - rewrite := false - if link.LinkNode == right.Id { - link.LinkNode = left.Id - rewrite = true - } - if link.Source == right.Id { - link.Source = left.Id - rewrite = true - } - if link.Target == right.Id { - link.Target = left.Id - rewrite = true - } - if rewrite { - qs.links = append(append(qs.links[:i], qs.links[i+1:]...), link) - } - } -} - -func (qs *queryShape) MakeNode(it Iterator) *Node { - var n Node - n.IsLinkNode = false - n.IsFixed = false - n.Id = qs.nodeId - n.Tags = make([]string, 0) - n.Values = make([]string, 0) - for _, tag := range it.Tags() { - n.Tags = append(n.Tags, tag) - } - for k, _ := range it.FixedTags() { - n.Tags = append(n.Tags, k) - } - - switch it.Type() { - case "and": - list := it.GetSubIterators() - for e := list.Front(); e != nil; e = e.Next() { - subit := e.Value.(Iterator) - qs.nodeId++ - newNode := qs.MakeNode(subit) - if subit.Type() != "or" { - qs.StealNode(&n, newNode) - } else { - qs.AddNode(newNode) - qs.AddLink(&Link{n.Id, newNode.Id, 0, 0}) - } - } - case "fixed": - n.IsFixed = true - for { - val, more := it.Next() - if !more { - break - } - n.Values = append(n.Values, qs.ts.GetNameFor(val)) - } - case "hasa": - hasa := it.(*HasaIterator) - qs.PushHasa(n.Id, hasa.direction) - qs.nodeId++ - newNode := qs.MakeNode(hasa.primaryIt) - qs.AddNode(newNode) - qs.RemoveHasa() - case "or": - list := it.GetSubIterators() - for e := list.Front(); e != nil; e = e.Next() { - subit := e.Value.(Iterator) - qs.nodeId++ - newNode := qs.MakeNode(subit) - if subit.Type() == "or" { - qs.StealNode(&n, newNode) - } else { - qs.AddNode(newNode) - qs.AddLink(&Link{n.Id, newNode.Id, 0, 0}) - } - } - case "linksto": - n.IsLinkNode = true - lto := it.(*LinksToIterator) - qs.nodeId++ - newNode := qs.MakeNode(lto.primaryIt) - hasaID, hasaDir := qs.LastHasa() - if (hasaDir == "s" && lto.direction == "o") || - (hasaDir == "o" && lto.direction == "s") { - qs.AddNode(newNode) - if hasaDir == "s" { - qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id}) - } else { - qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id}) - } - } else if lto.primaryIt.Type() == "fixed" { - qs.StealNode(&n, newNode) - } else { - qs.AddNode(newNode) - } - case "optional": - // Unsupported, for the moment - fallthrough - case "all": - } - return &n -} diff --git a/graph/query-shape_test.go b/graph/query-shape_test.go deleted file mode 100644 index b0d6950..0000000 --- a/graph/query-shape_test.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - . "github.com/smartystreets/goconvey/convey" - "testing" -) - -func buildHasaWithTag(ts TripleStore, tag string, target string) *HasaIterator { - fixed_obj := ts.MakeFixed() - fixed_pred := ts.MakeFixed() - fixed_obj.AddValue(ts.GetIdFor(target)) - fixed_pred.AddValue(ts.GetIdFor("status")) - fixed_obj.AddTag(tag) - lto1 := NewLinksToIterator(ts, fixed_obj, "o") - lto2 := NewLinksToIterator(ts, fixed_pred, "p") - and := NewAndIterator() - and.AddSubIterator(lto1) - and.AddSubIterator(lto2) - hasa := NewHasaIterator(ts, and, "s") - return hasa -} - -func TestQueryShape(t *testing.T) { - var queryShape map[string]interface{} - var ts *TestTripleStore - ts = new(TestTripleStore) - ts.On("GetIdFor", "cool").Return(1) - ts.On("GetNameFor", 1).Return("cool") - ts.On("GetIdFor", "status").Return(2) - ts.On("GetNameFor", 2).Return("status") - ts.On("GetIdFor", "fun").Return(3) - ts.On("GetNameFor", 3).Return("fun") - ts.On("GetIdFor", "name").Return(4) - ts.On("GetNameFor", 4).Return("name") - - Convey("Given a single linkage iterator's shape", t, func() { - queryShape = make(map[string]interface{}) - hasa := buildHasaWithTag(ts, "tag", "cool") - hasa.AddTag("top") - OutputQueryShapeForIterator(hasa, ts, &queryShape) - - Convey("It should have three nodes and one link", func() { - nodes := queryShape["nodes"].([]Node) - links := queryShape["links"].([]Link) - So(len(nodes), ShouldEqual, 3) - So(len(links), ShouldEqual, 1) - }) - - Convey("These nodes should be correctly tagged", func() { - nodes := queryShape["nodes"].([]Node) - So(nodes[0].Tags, ShouldResemble, []string{"tag"}) - So(nodes[1].IsLinkNode, ShouldEqual, true) - So(nodes[2].Tags, ShouldResemble, []string{"top"}) - - }) - - Convey("The link should be correctly typed", func() { - nodes := queryShape["nodes"].([]Node) - links := queryShape["links"].([]Link) - So(links[0].Source, ShouldEqual, nodes[2].Id) - So(links[0].Target, ShouldEqual, nodes[0].Id) - So(links[0].LinkNode, ShouldEqual, nodes[1].Id) - So(links[0].Pred, ShouldEqual, 0) - - }) - - }) - - Convey("Given a name-of-an-and-iterator's shape", t, func() { - queryShape = make(map[string]interface{}) - hasa1 := buildHasaWithTag(ts, "tag1", "cool") - hasa1.AddTag("hasa1") - hasa2 := buildHasaWithTag(ts, "tag2", "fun") - hasa1.AddTag("hasa2") - andInternal := NewAndIterator() - andInternal.AddSubIterator(hasa1) - andInternal.AddSubIterator(hasa2) - fixed_pred := ts.MakeFixed() - fixed_pred.AddValue(ts.GetIdFor("name")) - lto1 := NewLinksToIterator(ts, andInternal, "s") - lto2 := NewLinksToIterator(ts, fixed_pred, "p") - and := NewAndIterator() - and.AddSubIterator(lto1) - and.AddSubIterator(lto2) - hasa := NewHasaIterator(ts, and, "o") - OutputQueryShapeForIterator(hasa, ts, &queryShape) - - Convey("It should have seven nodes and three links", func() { - nodes := queryShape["nodes"].([]Node) - links := queryShape["links"].([]Link) - So(len(nodes), ShouldEqual, 7) - So(len(links), ShouldEqual, 3) - }) - - Convey("Three of the nodes are link nodes, four aren't", func() { - nodes := queryShape["nodes"].([]Node) - count := 0 - for _, node := range nodes { - if node.IsLinkNode { - count++ - } - } - So(count, ShouldEqual, 3) - }) - - Convey("These nodes should be correctly tagged", nil) - - }) - -} diff --git a/graph/query_shape.go b/graph/query_shape.go new file mode 100644 index 0000000..dece079 --- /dev/null +++ b/graph/query_shape.go @@ -0,0 +1,189 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +type Node struct { + Id int `json:"id"` + Tags []string `json:"tags,omitempty"` + Values []string `json:"values,omitempty"` + IsLinkNode bool `json:"is_link_node"` + IsFixed bool `json:"is_fixed"` +} + +type Link struct { + Source int `json:"source"` + Target int `json:"target"` + Pred int `json:"type"` + LinkNode int `json:"link_node"` +} + +type queryShape struct { + nodes []Node + links []Link + ts TripleStore + nodeId int + hasaIds []int + hasaDirs []string +} + +func OutputQueryShapeForIterator(it Iterator, ts TripleStore, outputMap *map[string]interface{}) { + qs := &queryShape{} + qs.nodes = make([]Node, 0) + qs.links = make([]Link, 0) + qs.hasaIds = make([]int, 0) + qs.hasaDirs = make([]string, 0) + qs.ts = ts + qs.nodeId = 1 + + node := qs.MakeNode(it.Clone()) + qs.AddNode(node) + (*outputMap)["nodes"] = qs.nodes + (*outputMap)["links"] = qs.links +} + +func (qs *queryShape) AddNode(n *Node) { + qs.nodes = append(qs.nodes, *n) +} + +func (qs *queryShape) AddLink(l *Link) { + qs.links = append(qs.links, *l) +} + +func (qs *queryShape) LastHasa() (int, string) { + return qs.hasaIds[len(qs.hasaIds)-1], qs.hasaDirs[len(qs.hasaDirs)-1] +} + +func (qs *queryShape) PushHasa(i int, s string) { + qs.hasaIds = append(qs.hasaIds, i) + qs.hasaDirs = append(qs.hasaDirs, s) +} + +func (qs *queryShape) RemoveHasa() { + qs.hasaIds = qs.hasaIds[:len(qs.hasaIds)-1] + qs.hasaDirs = qs.hasaDirs[:len(qs.hasaDirs)-1] +} + +func (qs *queryShape) StealNode(left *Node, right *Node) { + for _, v := range right.Values { + left.Values = append(left.Values, v) + } + for _, v := range right.Tags { + left.Tags = append(left.Tags, v) + } + left.IsLinkNode = left.IsLinkNode || right.IsLinkNode + left.IsFixed = left.IsFixed || right.IsFixed + for i, link := range qs.links { + rewrite := false + if link.LinkNode == right.Id { + link.LinkNode = left.Id + rewrite = true + } + if link.Source == right.Id { + link.Source = left.Id + rewrite = true + } + if link.Target == right.Id { + link.Target = left.Id + rewrite = true + } + if rewrite { + qs.links = append(append(qs.links[:i], qs.links[i+1:]...), link) + } + } +} + +func (qs *queryShape) MakeNode(it Iterator) *Node { + var n Node + n.IsLinkNode = false + n.IsFixed = false + n.Id = qs.nodeId + n.Tags = make([]string, 0) + n.Values = make([]string, 0) + for _, tag := range it.Tags() { + n.Tags = append(n.Tags, tag) + } + for k, _ := range it.FixedTags() { + n.Tags = append(n.Tags, k) + } + + switch it.Type() { + case "and": + list := it.GetSubIterators() + for e := list.Front(); e != nil; e = e.Next() { + subit := e.Value.(Iterator) + qs.nodeId++ + newNode := qs.MakeNode(subit) + if subit.Type() != "or" { + qs.StealNode(&n, newNode) + } else { + qs.AddNode(newNode) + qs.AddLink(&Link{n.Id, newNode.Id, 0, 0}) + } + } + case "fixed": + n.IsFixed = true + for { + val, more := it.Next() + if !more { + break + } + n.Values = append(n.Values, qs.ts.GetNameFor(val)) + } + case "hasa": + hasa := it.(*HasaIterator) + qs.PushHasa(n.Id, hasa.direction) + qs.nodeId++ + newNode := qs.MakeNode(hasa.primaryIt) + qs.AddNode(newNode) + qs.RemoveHasa() + case "or": + list := it.GetSubIterators() + for e := list.Front(); e != nil; e = e.Next() { + subit := e.Value.(Iterator) + qs.nodeId++ + newNode := qs.MakeNode(subit) + if subit.Type() == "or" { + qs.StealNode(&n, newNode) + } else { + qs.AddNode(newNode) + qs.AddLink(&Link{n.Id, newNode.Id, 0, 0}) + } + } + case "linksto": + n.IsLinkNode = true + lto := it.(*LinksToIterator) + qs.nodeId++ + newNode := qs.MakeNode(lto.primaryIt) + hasaID, hasaDir := qs.LastHasa() + if (hasaDir == "s" && lto.direction == "o") || + (hasaDir == "o" && lto.direction == "s") { + qs.AddNode(newNode) + if hasaDir == "s" { + qs.AddLink(&Link{hasaID, newNode.Id, 0, n.Id}) + } else { + qs.AddLink(&Link{newNode.Id, hasaID, 0, n.Id}) + } + } else if lto.primaryIt.Type() == "fixed" { + qs.StealNode(&n, newNode) + } else { + qs.AddNode(newNode) + } + case "optional": + // Unsupported, for the moment + fallthrough + case "all": + } + return &n +} diff --git a/graph/query_shape_test.go b/graph/query_shape_test.go new file mode 100644 index 0000000..b0d6950 --- /dev/null +++ b/graph/query_shape_test.go @@ -0,0 +1,124 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func buildHasaWithTag(ts TripleStore, tag string, target string) *HasaIterator { + fixed_obj := ts.MakeFixed() + fixed_pred := ts.MakeFixed() + fixed_obj.AddValue(ts.GetIdFor(target)) + fixed_pred.AddValue(ts.GetIdFor("status")) + fixed_obj.AddTag(tag) + lto1 := NewLinksToIterator(ts, fixed_obj, "o") + lto2 := NewLinksToIterator(ts, fixed_pred, "p") + and := NewAndIterator() + and.AddSubIterator(lto1) + and.AddSubIterator(lto2) + hasa := NewHasaIterator(ts, and, "s") + return hasa +} + +func TestQueryShape(t *testing.T) { + var queryShape map[string]interface{} + var ts *TestTripleStore + ts = new(TestTripleStore) + ts.On("GetIdFor", "cool").Return(1) + ts.On("GetNameFor", 1).Return("cool") + ts.On("GetIdFor", "status").Return(2) + ts.On("GetNameFor", 2).Return("status") + ts.On("GetIdFor", "fun").Return(3) + ts.On("GetNameFor", 3).Return("fun") + ts.On("GetIdFor", "name").Return(4) + ts.On("GetNameFor", 4).Return("name") + + Convey("Given a single linkage iterator's shape", t, func() { + queryShape = make(map[string]interface{}) + hasa := buildHasaWithTag(ts, "tag", "cool") + hasa.AddTag("top") + OutputQueryShapeForIterator(hasa, ts, &queryShape) + + Convey("It should have three nodes and one link", func() { + nodes := queryShape["nodes"].([]Node) + links := queryShape["links"].([]Link) + So(len(nodes), ShouldEqual, 3) + So(len(links), ShouldEqual, 1) + }) + + Convey("These nodes should be correctly tagged", func() { + nodes := queryShape["nodes"].([]Node) + So(nodes[0].Tags, ShouldResemble, []string{"tag"}) + So(nodes[1].IsLinkNode, ShouldEqual, true) + So(nodes[2].Tags, ShouldResemble, []string{"top"}) + + }) + + Convey("The link should be correctly typed", func() { + nodes := queryShape["nodes"].([]Node) + links := queryShape["links"].([]Link) + So(links[0].Source, ShouldEqual, nodes[2].Id) + So(links[0].Target, ShouldEqual, nodes[0].Id) + So(links[0].LinkNode, ShouldEqual, nodes[1].Id) + So(links[0].Pred, ShouldEqual, 0) + + }) + + }) + + Convey("Given a name-of-an-and-iterator's shape", t, func() { + queryShape = make(map[string]interface{}) + hasa1 := buildHasaWithTag(ts, "tag1", "cool") + hasa1.AddTag("hasa1") + hasa2 := buildHasaWithTag(ts, "tag2", "fun") + hasa1.AddTag("hasa2") + andInternal := NewAndIterator() + andInternal.AddSubIterator(hasa1) + andInternal.AddSubIterator(hasa2) + fixed_pred := ts.MakeFixed() + fixed_pred.AddValue(ts.GetIdFor("name")) + lto1 := NewLinksToIterator(ts, andInternal, "s") + lto2 := NewLinksToIterator(ts, fixed_pred, "p") + and := NewAndIterator() + and.AddSubIterator(lto1) + and.AddSubIterator(lto2) + hasa := NewHasaIterator(ts, and, "o") + OutputQueryShapeForIterator(hasa, ts, &queryShape) + + Convey("It should have seven nodes and three links", func() { + nodes := queryShape["nodes"].([]Node) + links := queryShape["links"].([]Link) + So(len(nodes), ShouldEqual, 7) + So(len(links), ShouldEqual, 3) + }) + + Convey("Three of the nodes are link nodes, four aren't", func() { + nodes := queryShape["nodes"].([]Node) + count := 0 + for _, node := range nodes { + if node.IsLinkNode { + count++ + } + } + So(count, ShouldEqual, 3) + }) + + Convey("These nodes should be correctly tagged", nil) + + }) + +} diff --git a/graph/result-tree-evaluator.go b/graph/result-tree-evaluator.go deleted file mode 100644 index e75cf56..0000000 --- a/graph/result-tree-evaluator.go +++ /dev/null @@ -1,70 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - "container/list" - "fmt" -) - -type ResultTree struct { - result TSVal - subtrees *list.List -} - -func NewResultTree(result TSVal) *ResultTree { - var tree ResultTree - tree.subtrees = list.New() - tree.result = result - return &tree -} - -func (tree *ResultTree) ToString() string { - base := fmt.Sprintf("(%d", tree.result) - if tree.subtrees.Len() != 0 { - for e := tree.subtrees.Front(); e != nil; e = e.Next() { - base += fmt.Sprintf(" %s", (e.Value.(*ResultTree)).ToString()) - } - } - base += ")" - return base -} - -func (tree *ResultTree) AddSubtree(sub *ResultTree) { - tree.subtrees.PushBack(sub) -} - -func StringResultTreeEvaluator(it Iterator) string { - ok := true - out := "" - for { - _, ok = it.Next() - if !ok { - break - } - out += it.GetResultTree().ToString() - out += "\n" - for it.NextResult() == true { - out += " " - out += it.GetResultTree().ToString() - out += "\n" - } - } - return out -} - -func PrintResultTreeEvaluator(it Iterator) { - fmt.Print(StringResultTreeEvaluator(it)) -} diff --git a/graph/result-tree-evaluator_test.go b/graph/result-tree-evaluator_test.go deleted file mode 100644 index 349bc08..0000000 --- a/graph/result-tree-evaluator_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - "testing" -) - -func TestSingleIterator(t *testing.T) { - all := NewInt64AllIterator(1, 3) - result := StringResultTreeEvaluator(all) - expected := "(1)\n(2)\n(3)\n" - if expected != result { - t.Errorf("Expected \"%s\" got \"%s\"", expected, result) - } -} - -func TestAndIterator(t *testing.T) { - all1 := NewInt64AllIterator(1, 3) - all2 := NewInt64AllIterator(3, 5) - and := NewAndIterator() - and.AddSubIterator(all1) - and.AddSubIterator(all2) - - result := StringResultTreeEvaluator(and) - expected := "(3 (3) (3))\n" - if expected != result { - t.Errorf("Expected \"%s\" got \"%s\"", expected, result) - } -} diff --git a/graph/result_tree_evaluator.go b/graph/result_tree_evaluator.go new file mode 100644 index 0000000..e75cf56 --- /dev/null +++ b/graph/result_tree_evaluator.go @@ -0,0 +1,70 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "container/list" + "fmt" +) + +type ResultTree struct { + result TSVal + subtrees *list.List +} + +func NewResultTree(result TSVal) *ResultTree { + var tree ResultTree + tree.subtrees = list.New() + tree.result = result + return &tree +} + +func (tree *ResultTree) ToString() string { + base := fmt.Sprintf("(%d", tree.result) + if tree.subtrees.Len() != 0 { + for e := tree.subtrees.Front(); e != nil; e = e.Next() { + base += fmt.Sprintf(" %s", (e.Value.(*ResultTree)).ToString()) + } + } + base += ")" + return base +} + +func (tree *ResultTree) AddSubtree(sub *ResultTree) { + tree.subtrees.PushBack(sub) +} + +func StringResultTreeEvaluator(it Iterator) string { + ok := true + out := "" + for { + _, ok = it.Next() + if !ok { + break + } + out += it.GetResultTree().ToString() + out += "\n" + for it.NextResult() == true { + out += " " + out += it.GetResultTree().ToString() + out += "\n" + } + } + return out +} + +func PrintResultTreeEvaluator(it Iterator) { + fmt.Print(StringResultTreeEvaluator(it)) +} diff --git a/graph/result_tree_evaluator_test.go b/graph/result_tree_evaluator_test.go new file mode 100644 index 0000000..349bc08 --- /dev/null +++ b/graph/result_tree_evaluator_test.go @@ -0,0 +1,42 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "testing" +) + +func TestSingleIterator(t *testing.T) { + all := NewInt64AllIterator(1, 3) + result := StringResultTreeEvaluator(all) + expected := "(1)\n(2)\n(3)\n" + if expected != result { + t.Errorf("Expected \"%s\" got \"%s\"", expected, result) + } +} + +func TestAndIterator(t *testing.T) { + all1 := NewInt64AllIterator(1, 3) + all2 := NewInt64AllIterator(3, 5) + and := NewAndIterator() + and.AddSubIterator(all1) + and.AddSubIterator(all2) + + result := StringResultTreeEvaluator(and) + expected := "(3 (3) (3))\n" + if expected != result { + t.Errorf("Expected \"%s\" got \"%s\"", expected, result) + } +} diff --git a/graph/sexp/sexp-session.go b/graph/sexp/sexp-session.go deleted file mode 100644 index 7065e02..0000000 --- a/graph/sexp/sexp-session.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sexp - -// Defines a running session of the sexp query language. - -import ( - "errors" - "fmt" - "sort" - - "github.com/google/cayley/graph" -) - -type SexpSession struct { - ts graph.TripleStore - debug bool -} - -func NewSexpSession(inputTripleStore graph.TripleStore) *SexpSession { - var s SexpSession - s.ts = inputTripleStore - return &s -} - -func (s *SexpSession) ToggleDebug() { - s.debug = !s.debug -} - -func (s *SexpSession) InputParses(input string) (graph.ParseResult, error) { - var parenDepth int - for i, x := range input { - if x == '(' { - parenDepth++ - } - if x == ')' { - parenDepth-- - if parenDepth < 0 { - min := 0 - if (i - 10) > min { - min = i - 10 - } - return graph.ParseFail, errors.New(fmt.Sprintf("Too many close parens at char %d: %s", i, input[min:i])) - } - } - } - if parenDepth > 0 { - return graph.ParseMore, nil - } - if len(ParseString(input)) > 0 { - return graph.Parsed, nil - } - return graph.ParseFail, errors.New("Invalid Syntax") -} - -func (s *SexpSession) ExecInput(input string, out chan interface{}, limit int) { - it := BuildIteratorTreeForQuery(s.ts, input) - newIt, changed := it.Optimize() - if changed { - it = newIt - } - - if s.debug { - fmt.Println(it.DebugString(0)) - } - nResults := 0 - for { - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - out <- &tags - nResults++ - if nResults > limit && limit != -1 { - break - } - for it.NextResult() == true { - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - out <- &tags - nResults++ - if nResults > limit && limit != -1 { - break - } - } - } - close(out) -} - -func (s *SexpSession) ToText(result interface{}) string { - out := fmt.Sprintln("****") - tags := result.(*map[string]graph.TSVal) - tagKeys := make([]string, len(*tags)) - i := 0 - for k, _ := range *tags { - tagKeys[i] = k - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - if k == "$_" { - continue - } - out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) - } - return out -} diff --git a/graph/sexp/sexp_session.go b/graph/sexp/sexp_session.go new file mode 100644 index 0000000..7065e02 --- /dev/null +++ b/graph/sexp/sexp_session.go @@ -0,0 +1,121 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sexp + +// Defines a running session of the sexp query language. + +import ( + "errors" + "fmt" + "sort" + + "github.com/google/cayley/graph" +) + +type SexpSession struct { + ts graph.TripleStore + debug bool +} + +func NewSexpSession(inputTripleStore graph.TripleStore) *SexpSession { + var s SexpSession + s.ts = inputTripleStore + return &s +} + +func (s *SexpSession) ToggleDebug() { + s.debug = !s.debug +} + +func (s *SexpSession) InputParses(input string) (graph.ParseResult, error) { + var parenDepth int + for i, x := range input { + if x == '(' { + parenDepth++ + } + if x == ')' { + parenDepth-- + if parenDepth < 0 { + min := 0 + if (i - 10) > min { + min = i - 10 + } + return graph.ParseFail, errors.New(fmt.Sprintf("Too many close parens at char %d: %s", i, input[min:i])) + } + } + } + if parenDepth > 0 { + return graph.ParseMore, nil + } + if len(ParseString(input)) > 0 { + return graph.Parsed, nil + } + return graph.ParseFail, errors.New("Invalid Syntax") +} + +func (s *SexpSession) ExecInput(input string, out chan interface{}, limit int) { + it := BuildIteratorTreeForQuery(s.ts, input) + newIt, changed := it.Optimize() + if changed { + it = newIt + } + + if s.debug { + fmt.Println(it.DebugString(0)) + } + nResults := 0 + for { + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + out <- &tags + nResults++ + if nResults > limit && limit != -1 { + break + } + for it.NextResult() == true { + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + out <- &tags + nResults++ + if nResults > limit && limit != -1 { + break + } + } + } + close(out) +} + +func (s *SexpSession) ToText(result interface{}) string { + out := fmt.Sprintln("****") + tags := result.(*map[string]graph.TSVal) + tagKeys := make([]string, len(*tags)) + i := 0 + for k, _ := range *tags { + tagKeys[i] = k + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + if k == "$_" { + continue + } + out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) + } + return out +} diff --git a/graph/value-comparison-iterator.go b/graph/value-comparison-iterator.go deleted file mode 100644 index ecfe86e..0000000 --- a/graph/value-comparison-iterator.go +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -// "Value Comparison" is a unary operator -- a filter across the values in the -// relevant subiterator. -// -// This is hugely useful for things like provenance, but value ranges in general -// come up from time to time. At *worst* we're as big as our underlying iterator. -// At best, we're the null iterator. -// -// This is ripe for backend-side optimization. If you can run a value iterator, -// from a sorted set -- some sort of value index, then go for it. -// -// In MQL terms, this is the [{"age>=": 21}] concept. - -import ( - "fmt" - "log" - "strconv" - "strings" -) - -type ComparisonOperator int - -const ( - kCompareLT ComparisonOperator = iota - kCompareLTE - kCompareGT - kCompareGTE - // Why no Equals? Because that's usually an AndIterator. -) - -type ValueComparisonIterator struct { - BaseIterator - subIt Iterator - op ComparisonOperator - comparisonValue interface{} - ts TripleStore -} - -func NewValueComparisonIterator( - subIt Iterator, - operator ComparisonOperator, - value interface{}, - ts TripleStore) *ValueComparisonIterator { - - var vc ValueComparisonIterator - BaseIteratorInit(&vc.BaseIterator) - vc.subIt = subIt - vc.op = operator - vc.comparisonValue = value - vc.ts = ts - return &vc -} - -// Here's the non-boilerplate part of the ValueComparison iterator. Given a value -// and our operator, determine whether or not we meet the requirement. -func (vc *ValueComparisonIterator) doComparison(val TSVal) bool { - //TODO(barakmich): Implement string comparison. - nodeStr := vc.ts.GetNameFor(val) - switch cVal := vc.comparisonValue.(type) { - case int: - cInt := int64(cVal) - intVal, err := strconv.ParseInt(nodeStr, 10, 64) - if err != nil { - return false - } - return RunIntOp(intVal, vc.op, cInt) - case int64: - intVal, err := strconv.ParseInt(nodeStr, 10, 64) - if err != nil { - return false - } - return RunIntOp(intVal, vc.op, cVal) - default: - return true - } -} - -func (vc *ValueComparisonIterator) Close() { - vc.subIt.Close() -} - -func RunIntOp(a int64, op ComparisonOperator, b int64) bool { - switch op { - case kCompareLT: - return a < b - case kCompareLTE: - return a <= b - case kCompareGT: - return a > b - case kCompareGTE: - return a >= b - default: - log.Fatal("Unknown operator type") - return false - } -} - -func (vc *ValueComparisonIterator) Reset() { - vc.subIt.Reset() -} - -func (vc *ValueComparisonIterator) Clone() Iterator { - out := NewValueComparisonIterator(vc.subIt.Clone(), vc.op, vc.comparisonValue, vc.ts) - out.CopyTagsFrom(vc) - return out -} - -func (vc *ValueComparisonIterator) Next() (TSVal, bool) { - var val TSVal - var ok bool - for { - val, ok = vc.subIt.Next() - if !ok { - return nil, false - } - if vc.doComparison(val) { - break - } - } - vc.Last = val - return val, ok -} - -func (vc *ValueComparisonIterator) NextResult() bool { - for { - hasNext := vc.subIt.NextResult() - if !hasNext { - return false - } - if vc.doComparison(vc.subIt.LastResult()) { - return true - } - } - vc.Last = vc.subIt.LastResult() - return true -} - -func (vc *ValueComparisonIterator) Check(val TSVal) bool { - if !vc.doComparison(val) { - return false - } - return vc.subIt.Check(val) -} - -// If we failed the check, then the subiterator should not contribute to the result -// set. Otherwise, go ahead and tag it. -func (vc *ValueComparisonIterator) TagResults(out *map[string]TSVal) { - vc.BaseIterator.TagResults(out) - vc.subIt.TagResults(out) -} - -// Registers the value-comparison iterator. -func (vc *ValueComparisonIterator) Type() string { return "value-comparison" } - -// Prints the value-comparison and its subiterator. -func (vc *ValueComparisonIterator) DebugString(indent int) string { - return fmt.Sprintf("%s(%s\n%s)", - strings.Repeat(" ", indent), - vc.Type(), vc.subIt.DebugString(indent+4)) -} - -// There's nothing to optimize, locally, for a value-comparison iterator. -// Replace the underlying iterator if need be. -// potentially replace it. -func (vc *ValueComparisonIterator) Optimize() (Iterator, bool) { - newSub, changed := vc.subIt.Optimize() - if changed { - vc.subIt.Close() - vc.subIt = newSub - } - return vc, false -} - -// We're only as expensive as our subiterator. -// Again, optimized value comparison iterators should do better. -func (vc *ValueComparisonIterator) GetStats() *IteratorStats { - return vc.subIt.GetStats() -} diff --git a/graph/value-comparison-iterator_test.go b/graph/value-comparison-iterator_test.go deleted file mode 100644 index 23c795d..0000000 --- a/graph/value-comparison-iterator_test.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package graph - -import ( - "testing" -) - -func SetupMockTripleStore(nameMap map[string]int) *TestTripleStore { - ts := new(TestTripleStore) - for k, v := range nameMap { - ts.On("GetIdFor", k).Return(v) - ts.On("GetNameFor", v).Return(k) - } - return ts -} - -func SimpleValueTripleStore() *TestTripleStore { - ts := SetupMockTripleStore(map[string]int{ - "0": 0, - "1": 1, - "2": 2, - "3": 3, - "4": 4, - "5": 5, - }) - return ts -} - -func BuildFixedIterator() *FixedIterator { - fixed := newFixedIterator() - fixed.AddValue(0) - fixed.AddValue(1) - fixed.AddValue(2) - fixed.AddValue(3) - fixed.AddValue(4) - return fixed -} - -func checkIteratorContains(ts TripleStore, it Iterator, expected []string, t *testing.T) { - var actual []string - actual = nil - for { - val, ok := it.Next() - if !ok { - break - } - actual = append(actual, ts.GetNameFor(val)) - } - actualSet := actual[:] - for _, a := range expected { - found := false - for j, b := range actualSet { - if a == b { - actualSet = append(actualSet[:j], actualSet[j+1:]...) - found = true - break - } - } - if !found { - t.Error("Couldn't find", a, "in actual output.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet) - return - } - } - if len(actualSet) != 0 { - t.Error("Actual output has more than expected.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet) - } -} - -func TestWorkingIntValueComparison(t *testing.T) { - ts := SimpleValueTripleStore() - fixed := BuildFixedIterator() - vc := NewValueComparisonIterator(fixed, kCompareLT, int64(3), ts) - checkIteratorContains(ts, vc, []string{"0", "1", "2"}, t) -} - -func TestFailingIntValueComparison(t *testing.T) { - ts := SimpleValueTripleStore() - fixed := BuildFixedIterator() - vc := NewValueComparisonIterator(fixed, kCompareLT, int64(0), ts) - checkIteratorContains(ts, vc, []string{}, t) -} - -func TestWorkingGT(t *testing.T) { - ts := SimpleValueTripleStore() - fixed := BuildFixedIterator() - vc := NewValueComparisonIterator(fixed, kCompareGT, int64(2), ts) - checkIteratorContains(ts, vc, []string{"3", "4"}, t) -} - -func TestWorkingGTE(t *testing.T) { - ts := SimpleValueTripleStore() - fixed := BuildFixedIterator() - vc := NewValueComparisonIterator(fixed, kCompareGTE, int64(2), ts) - checkIteratorContains(ts, vc, []string{"2", "3", "4"}, t) -} - -func TestVCICheck(t *testing.T) { - ts := SimpleValueTripleStore() - fixed := BuildFixedIterator() - vc := NewValueComparisonIterator(fixed, kCompareGTE, int64(2), ts) - if vc.Check(1) { - t.Error("1 is less than 2, should be GTE") - } - if !vc.Check(2) { - t.Error("2 is GTE 2") - } - if !vc.Check(3) { - t.Error("3 is GTE 2") - } - if vc.Check(5) { - t.Error("5 is not in the underlying iterator") - } -} diff --git a/graph/value_comparison_iterator.go b/graph/value_comparison_iterator.go new file mode 100644 index 0000000..ecfe86e --- /dev/null +++ b/graph/value_comparison_iterator.go @@ -0,0 +1,193 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +// "Value Comparison" is a unary operator -- a filter across the values in the +// relevant subiterator. +// +// This is hugely useful for things like provenance, but value ranges in general +// come up from time to time. At *worst* we're as big as our underlying iterator. +// At best, we're the null iterator. +// +// This is ripe for backend-side optimization. If you can run a value iterator, +// from a sorted set -- some sort of value index, then go for it. +// +// In MQL terms, this is the [{"age>=": 21}] concept. + +import ( + "fmt" + "log" + "strconv" + "strings" +) + +type ComparisonOperator int + +const ( + kCompareLT ComparisonOperator = iota + kCompareLTE + kCompareGT + kCompareGTE + // Why no Equals? Because that's usually an AndIterator. +) + +type ValueComparisonIterator struct { + BaseIterator + subIt Iterator + op ComparisonOperator + comparisonValue interface{} + ts TripleStore +} + +func NewValueComparisonIterator( + subIt Iterator, + operator ComparisonOperator, + value interface{}, + ts TripleStore) *ValueComparisonIterator { + + var vc ValueComparisonIterator + BaseIteratorInit(&vc.BaseIterator) + vc.subIt = subIt + vc.op = operator + vc.comparisonValue = value + vc.ts = ts + return &vc +} + +// Here's the non-boilerplate part of the ValueComparison iterator. Given a value +// and our operator, determine whether or not we meet the requirement. +func (vc *ValueComparisonIterator) doComparison(val TSVal) bool { + //TODO(barakmich): Implement string comparison. + nodeStr := vc.ts.GetNameFor(val) + switch cVal := vc.comparisonValue.(type) { + case int: + cInt := int64(cVal) + intVal, err := strconv.ParseInt(nodeStr, 10, 64) + if err != nil { + return false + } + return RunIntOp(intVal, vc.op, cInt) + case int64: + intVal, err := strconv.ParseInt(nodeStr, 10, 64) + if err != nil { + return false + } + return RunIntOp(intVal, vc.op, cVal) + default: + return true + } +} + +func (vc *ValueComparisonIterator) Close() { + vc.subIt.Close() +} + +func RunIntOp(a int64, op ComparisonOperator, b int64) bool { + switch op { + case kCompareLT: + return a < b + case kCompareLTE: + return a <= b + case kCompareGT: + return a > b + case kCompareGTE: + return a >= b + default: + log.Fatal("Unknown operator type") + return false + } +} + +func (vc *ValueComparisonIterator) Reset() { + vc.subIt.Reset() +} + +func (vc *ValueComparisonIterator) Clone() Iterator { + out := NewValueComparisonIterator(vc.subIt.Clone(), vc.op, vc.comparisonValue, vc.ts) + out.CopyTagsFrom(vc) + return out +} + +func (vc *ValueComparisonIterator) Next() (TSVal, bool) { + var val TSVal + var ok bool + for { + val, ok = vc.subIt.Next() + if !ok { + return nil, false + } + if vc.doComparison(val) { + break + } + } + vc.Last = val + return val, ok +} + +func (vc *ValueComparisonIterator) NextResult() bool { + for { + hasNext := vc.subIt.NextResult() + if !hasNext { + return false + } + if vc.doComparison(vc.subIt.LastResult()) { + return true + } + } + vc.Last = vc.subIt.LastResult() + return true +} + +func (vc *ValueComparisonIterator) Check(val TSVal) bool { + if !vc.doComparison(val) { + return false + } + return vc.subIt.Check(val) +} + +// If we failed the check, then the subiterator should not contribute to the result +// set. Otherwise, go ahead and tag it. +func (vc *ValueComparisonIterator) TagResults(out *map[string]TSVal) { + vc.BaseIterator.TagResults(out) + vc.subIt.TagResults(out) +} + +// Registers the value-comparison iterator. +func (vc *ValueComparisonIterator) Type() string { return "value-comparison" } + +// Prints the value-comparison and its subiterator. +func (vc *ValueComparisonIterator) DebugString(indent int) string { + return fmt.Sprintf("%s(%s\n%s)", + strings.Repeat(" ", indent), + vc.Type(), vc.subIt.DebugString(indent+4)) +} + +// There's nothing to optimize, locally, for a value-comparison iterator. +// Replace the underlying iterator if need be. +// potentially replace it. +func (vc *ValueComparisonIterator) Optimize() (Iterator, bool) { + newSub, changed := vc.subIt.Optimize() + if changed { + vc.subIt.Close() + vc.subIt = newSub + } + return vc, false +} + +// We're only as expensive as our subiterator. +// Again, optimized value comparison iterators should do better. +func (vc *ValueComparisonIterator) GetStats() *IteratorStats { + return vc.subIt.GetStats() +} diff --git a/graph/value_comparison_iterator_test.go b/graph/value_comparison_iterator_test.go new file mode 100644 index 0000000..23c795d --- /dev/null +++ b/graph/value_comparison_iterator_test.go @@ -0,0 +1,126 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "testing" +) + +func SetupMockTripleStore(nameMap map[string]int) *TestTripleStore { + ts := new(TestTripleStore) + for k, v := range nameMap { + ts.On("GetIdFor", k).Return(v) + ts.On("GetNameFor", v).Return(k) + } + return ts +} + +func SimpleValueTripleStore() *TestTripleStore { + ts := SetupMockTripleStore(map[string]int{ + "0": 0, + "1": 1, + "2": 2, + "3": 3, + "4": 4, + "5": 5, + }) + return ts +} + +func BuildFixedIterator() *FixedIterator { + fixed := newFixedIterator() + fixed.AddValue(0) + fixed.AddValue(1) + fixed.AddValue(2) + fixed.AddValue(3) + fixed.AddValue(4) + return fixed +} + +func checkIteratorContains(ts TripleStore, it Iterator, expected []string, t *testing.T) { + var actual []string + actual = nil + for { + val, ok := it.Next() + if !ok { + break + } + actual = append(actual, ts.GetNameFor(val)) + } + actualSet := actual[:] + for _, a := range expected { + found := false + for j, b := range actualSet { + if a == b { + actualSet = append(actualSet[:j], actualSet[j+1:]...) + found = true + break + } + } + if !found { + t.Error("Couldn't find", a, "in actual output.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet) + return + } + } + if len(actualSet) != 0 { + t.Error("Actual output has more than expected.\nActual:", actual, "\nExpected: ", expected, "\nRemainder: ", actualSet) + } +} + +func TestWorkingIntValueComparison(t *testing.T) { + ts := SimpleValueTripleStore() + fixed := BuildFixedIterator() + vc := NewValueComparisonIterator(fixed, kCompareLT, int64(3), ts) + checkIteratorContains(ts, vc, []string{"0", "1", "2"}, t) +} + +func TestFailingIntValueComparison(t *testing.T) { + ts := SimpleValueTripleStore() + fixed := BuildFixedIterator() + vc := NewValueComparisonIterator(fixed, kCompareLT, int64(0), ts) + checkIteratorContains(ts, vc, []string{}, t) +} + +func TestWorkingGT(t *testing.T) { + ts := SimpleValueTripleStore() + fixed := BuildFixedIterator() + vc := NewValueComparisonIterator(fixed, kCompareGT, int64(2), ts) + checkIteratorContains(ts, vc, []string{"3", "4"}, t) +} + +func TestWorkingGTE(t *testing.T) { + ts := SimpleValueTripleStore() + fixed := BuildFixedIterator() + vc := NewValueComparisonIterator(fixed, kCompareGTE, int64(2), ts) + checkIteratorContains(ts, vc, []string{"2", "3", "4"}, t) +} + +func TestVCICheck(t *testing.T) { + ts := SimpleValueTripleStore() + fixed := BuildFixedIterator() + vc := NewValueComparisonIterator(fixed, kCompareGTE, int64(2), ts) + if vc.Check(1) { + t.Error("1 is less than 2, should be GTE") + } + if !vc.Check(2) { + t.Error("2 is GTE 2") + } + if !vc.Check(3) { + t.Error("3 is GTE 2") + } + if vc.Check(5) { + t.Error("5 is not in the underlying iterator") + } +} diff --git a/gremlin/gremlin-build-iterator.go b/gremlin/gremlin-build-iterator.go deleted file mode 100644 index b6e9a45..0000000 --- a/gremlin/gremlin-build-iterator.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "strconv" - - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" - - "github.com/google/cayley/graph" -) - -func getStrings(obj *otto.Object, field string) []string { - strings := make([]string, 0) - val, _ := obj.Get(field) - if !val.IsUndefined() { - export, _ := val.Export() - array := export.([]interface{}) - for _, arg := range array { - strings = append(strings, arg.(string)) - } - } - return strings -} - -func getStringArgs(obj *otto.Object) []string { return getStrings(obj, "string_args") } - -func buildIteratorTree(obj *otto.Object, ts graph.TripleStore) graph.Iterator { - if !isVertexChain(obj) { - return graph.NewNullIterator() - } - return buildIteratorTreeHelper(obj, ts, graph.NewNullIterator()) -} - -func makeListOfStringsFromArrayValue(obj *otto.Object) []string { - var output []string - lengthValue, _ := obj.Get("length") - length, _ := lengthValue.ToInteger() - ulength := uint32(length) - for index := uint32(0); index < ulength; index += 1 { - name := strconv.FormatInt(int64(index), 10) - value, err := obj.Get(name) - if err != nil { - continue - } - if !value.IsString() { - continue - } - s, _ := value.ToString() - output = append(output, s) - } - return output -} - -func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator { - if val.IsNull() || val.IsUndefined() { - return ts.GetNodesAllIterator() - } - if val.IsPrimitive() { - thing, _ := val.Export() - switch v := thing.(type) { - case string: - it := ts.MakeFixed() - it.AddValue(ts.GetIdFor(v)) - return it - default: - glog.Errorln("Trying to build unknown primitive value.") - } - } - switch val.Class() { - case "Object": - return buildIteratorTree(val.Object(), ts) - case "Array": - // Had better be an array of strings - strings := makeListOfStringsFromArrayValue(val.Object()) - it := ts.MakeFixed() - for _, x := range strings { - it.AddValue(ts.GetIdFor(x)) - } - return it - case "Number": - fallthrough - case "Boolean": - fallthrough - case "Date": - fallthrough - case "String": - it := ts.MakeFixed() - str, _ := val.ToString() - it.AddValue(ts.GetIdFor(str)) - return it - default: - glog.Errorln("Trying to handle unsupported Javascript value.") - return graph.NewNullIterator() - } -} - -func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Iterator, isReverse bool) graph.Iterator { - argList, _ := obj.Get("_gremlin_values") - if argList.Class() != "GoArray" { - glog.Errorln("How is arglist not an array? Return nothing.", argList.Class()) - return graph.NewNullIterator() - } - argArray := argList.Object() - lengthVal, _ := argArray.Get("length") - length, _ := lengthVal.ToInteger() - var predicateNodeIterator graph.Iterator - if length == 0 { - predicateNodeIterator = ts.GetNodesAllIterator() - } else { - zero, _ := argArray.Get("0") - predicateNodeIterator = buildIteratorFromValue(zero, ts) - } - if length >= 2 { - var tags []string - one, _ := argArray.Get("1") - if one.IsString() { - s, _ := one.ToString() - tags = append(tags, s) - } else if one.Class() == "Array" { - tags = makeListOfStringsFromArrayValue(one.Object()) - } - for _, tag := range tags { - predicateNodeIterator.AddTag(tag) - } - } - - in, out := "s", "o" - if isReverse { - in, out = out, in - } - lto := graph.NewLinksToIterator(ts, base, in) - and := graph.NewAndIterator() - and.AddSubIterator(graph.NewLinksToIterator(ts, predicateNodeIterator, "p")) - and.AddSubIterator(lto) - return graph.NewHasaIterator(ts, and, out) -} - -func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.Iterator) graph.Iterator { - var it graph.Iterator - it = base - // TODO: Better error handling - kindVal, _ := obj.Get("_gremlin_type") - stringArgs := getStringArgs(obj) - var subIt graph.Iterator - prevVal, _ := obj.Get("_gremlin_prev") - if !prevVal.IsObject() { - subIt = base - } else { - subIt = buildIteratorTreeHelper(prevVal.Object(), ts, base) - } - - kind, _ := kindVal.ToString() - switch kind { - case "vertex": - if len(stringArgs) == 0 { - it = ts.GetNodesAllIterator() - } else { - fixed := ts.MakeFixed() - for _, name := range stringArgs { - fixed.AddValue(ts.GetIdFor(name)) - } - it = fixed - } - case "tag": - it = subIt - for _, tag := range stringArgs { - it.AddTag(tag) - } - case "save": - all := ts.GetNodesAllIterator() - if len(stringArgs) > 2 || len(stringArgs) == 0 { - return graph.NewNullIterator() - } - if len(stringArgs) == 2 { - all.AddTag(stringArgs[1]) - } else { - all.AddTag(stringArgs[0]) - } - predFixed := ts.MakeFixed() - predFixed.AddValue(ts.GetIdFor(stringArgs[0])) - subAnd := graph.NewAndIterator() - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "o")) - hasa := graph.NewHasaIterator(ts, subAnd, "s") - and := graph.NewAndIterator() - and.AddSubIterator(hasa) - and.AddSubIterator(subIt) - it = and - case "saver": - all := ts.GetNodesAllIterator() - if len(stringArgs) > 2 || len(stringArgs) == 0 { - return graph.NewNullIterator() - } - if len(stringArgs) == 2 { - all.AddTag(stringArgs[1]) - } else { - all.AddTag(stringArgs[0]) - } - predFixed := ts.MakeFixed() - predFixed.AddValue(ts.GetIdFor(stringArgs[0])) - subAnd := graph.NewAndIterator() - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "s")) - hasa := graph.NewHasaIterator(ts, subAnd, "o") - and := graph.NewAndIterator() - and.AddSubIterator(hasa) - and.AddSubIterator(subIt) - it = and - case "has": - fixed := ts.MakeFixed() - if len(stringArgs) < 2 { - return graph.NewNullIterator() - } - for _, name := range stringArgs[1:] { - fixed.AddValue(ts.GetIdFor(name)) - } - predFixed := ts.MakeFixed() - predFixed.AddValue(ts.GetIdFor(stringArgs[0])) - subAnd := graph.NewAndIterator() - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, fixed, "o")) - hasa := graph.NewHasaIterator(ts, subAnd, "s") - and := graph.NewAndIterator() - and.AddSubIterator(hasa) - and.AddSubIterator(subIt) - it = and - case "morphism": - it = base - case "and": - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if !isVertexChain(firstArg.Object()) { - return graph.NewNullIterator() - } - argIt := buildIteratorTree(firstArg.Object(), ts) - - and := graph.NewAndIterator() - and.AddSubIterator(subIt) - and.AddSubIterator(argIt) - it = and - case "back": - arg, _ := obj.Get("_gremlin_back_chain") - argIt := buildIteratorTree(arg.Object(), ts) - and := graph.NewAndIterator() - and.AddSubIterator(subIt) - and.AddSubIterator(argIt) - it = and - case "is": - fixed := ts.MakeFixed() - for _, name := range stringArgs { - fixed.AddValue(ts.GetIdFor(name)) - } - and := graph.NewAndIterator() - and.AddSubIterator(fixed) - and.AddSubIterator(subIt) - it = and - case "or": - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if !isVertexChain(firstArg.Object()) { - return graph.NewNullIterator() - } - argIt := buildIteratorTree(firstArg.Object(), ts) - - or := graph.NewOrIterator() - or.AddSubIterator(subIt) - or.AddSubIterator(argIt) - it = or - case "both": - // Hardly the most efficient pattern, but the most general. - // Worth looking into an Optimize() optimization here. - clone := subIt.Clone() - it1 := buildInOutIterator(obj, ts, subIt, false) - it2 := buildInOutIterator(obj, ts, clone, true) - - or := graph.NewOrIterator() - or.AddSubIterator(it1) - or.AddSubIterator(it2) - it = or - case "out": - it = buildInOutIterator(obj, ts, subIt, false) - case "follow": - // Follow a morphism - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if isVertexChain(firstArg.Object()) { - return graph.NewNullIterator() - } - it = buildIteratorTreeHelper(firstArg.Object(), ts, subIt) - case "followr": - // Follow a morphism - arg, _ := obj.Get("_gremlin_followr") - if isVertexChain(arg.Object()) { - return graph.NewNullIterator() - } - it = buildIteratorTreeHelper(arg.Object(), ts, subIt) - case "in": - it = buildInOutIterator(obj, ts, subIt, true) - } - return it -} diff --git a/gremlin/gremlin-env.go b/gremlin/gremlin-env.go deleted file mode 100644 index 4e7f332..0000000 --- a/gremlin/gremlin-env.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -// Builds a new Gremlin environment pointing at a session. - -import ( - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" -) - -func BuildGremlinEnv(ses *GremlinSession) *otto.Otto { - env := otto.New() - setupGremlin(env, ses) - return env -} - -func concatStringArgs(call otto.FunctionCall) *[]interface{} { - outStrings := make([]interface{}, 0) - for _, arg := range call.ArgumentList { - if arg.IsString() { - outStrings = append(outStrings, arg.String()) - } - if arg.IsObject() && arg.Class() == "Array" { - obj, _ := arg.Export() - for _, x := range obj.([]interface{}) { - outStrings = append(outStrings, x.(string)) - } - } - } - return &outStrings -} - -func isVertexChain(obj *otto.Object) bool { - val, _ := obj.Get("_gremlin_type") - if x, _ := val.ToString(); x == "vertex" { - return true - } - val, _ = obj.Get("_gremlin_prev") - if val.IsObject() { - return isVertexChain(val.Object()) - } - return false -} - -func setupGremlin(env *otto.Otto, ses *GremlinSession) { - graph, _ := env.Object("graph = {}") - graph.Set("Vertex", func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, err := call.Otto.Object("out") - if err != nil { - glog.Error(err.Error()) - return otto.TrueValue() - } - out.Set("_gremlin_type", "vertex") - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - embedTraversals(env, ses, out) - embedFinals(env, ses, out) - return out.Value() - }) - - graph.Set("Morphism", func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", "morphism") - embedTraversals(env, ses, out) - return out.Value() - }) - graph.Set("Emit", func(call otto.FunctionCall) otto.Value { - value := call.Argument(0) - if value.IsDefined() { - ses.SendResult(&GremlinResult{metaresult: false, err: "", val: &value, actualResults: nil}) - } - return otto.NullValue() - }) - env.Run("graph.V = graph.Vertex") - env.Run("graph.M = graph.Morphism") - env.Run("g = graph") - -} diff --git a/gremlin/gremlin-finals.go b/gremlin/gremlin-finals.go deleted file mode 100644 index f8c978d..0000000 --- a/gremlin/gremlin-finals.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" - - "github.com/google/cayley/graph" -) - -const GremlinTopResultTag = "id" - -func embedFinals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { - obj.Set("All", allFunc(env, ses, obj)) - obj.Set("GetLimit", limitFunc(env, ses, obj)) - obj.Set("ToArray", toArrayFunc(env, ses, obj, false)) - obj.Set("ToValue", toValueFunc(env, ses, obj, false)) - obj.Set("TagArray", toArrayFunc(env, ses, obj, true)) - obj.Set("TagValue", toValueFunc(env, ses, obj, true)) - obj.Set("Map", mapFunc(env, ses, obj)) - obj.Set("ForEach", mapFunc(env, ses, obj)) -} - -func allFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - ses.limit = -1 - ses.count = 0 - runIteratorOnSession(it, ses) - return otto.NullValue() - } -} - -func limitFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - if len(call.ArgumentList) > 0 { - limitVal, _ := call.Argument(0).ToInteger() - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - ses.limit = int(limitVal) - ses.count = 0 - runIteratorOnSession(it, ses) - } - return otto.NullValue() - } -} - -func toArrayFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - limit := -1 - if len(call.ArgumentList) > 0 { - limitParsed, _ := call.Argument(0).ToInteger() - limit = int(limitParsed) - } - var val otto.Value - var err error - if !withTags { - array := runIteratorToArrayNoTags(it, ses, limit) - val, err = call.Otto.ToValue(array) - } else { - array := runIteratorToArray(it, ses, limit) - val, err = call.Otto.ToValue(array) - } - - if err != nil { - glog.Error(err) - return otto.NullValue() - } - return val - } -} - -func toValueFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - limit := 1 - var val otto.Value - var err error - if !withTags { - array := runIteratorToArrayNoTags(it, ses, limit) - if len(array) < 1 { - return otto.NullValue() - } - val, err = call.Otto.ToValue(array[0]) - } else { - array := runIteratorToArray(it, ses, limit) - if len(array) < 1 { - return otto.NullValue() - } - val, err = call.Otto.ToValue(array[0]) - } - if err != nil { - glog.Error(err) - return otto.NullValue() - } else { - return val - } - - } -} - -func mapFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - limit := -1 - if len(call.ArgumentList) == 0 { - return otto.NullValue() - } - callback := call.Argument(len(call.ArgumentList) - 1) - if len(call.ArgumentList) > 1 { - limitParsed, _ := call.Argument(0).ToInteger() - limit = int(limitParsed) - } - runIteratorWithCallback(it, ses, callback, call, limit) - return otto.NullValue() - } -} - -func tagsToValueMap(m map[string]graph.TSVal, ses *GremlinSession) map[string]string { - outputMap := make(map[string]string) - for k, v := range m { - outputMap[k] = ses.ts.GetNameFor(v) - } - return outputMap -} - -func runIteratorToArray(it graph.Iterator, ses *GremlinSession, limit int) []map[string]string { - output := make([]map[string]string, 0) - count := 0 - it, _ = it.Optimize() - for { - if ses.doHalt { - return nil - } - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - output = append(output, tagsToValueMap(tags, ses)) - count++ - if limit >= 0 && count >= limit { - break - } - for it.NextResult() == true { - if ses.doHalt { - return nil - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - output = append(output, tagsToValueMap(tags, ses)) - count++ - if limit >= 0 && count >= limit { - break - } - } - } - it.Close() - return output -} - -func runIteratorToArrayNoTags(it graph.Iterator, ses *GremlinSession, limit int) []string { - output := make([]string, 0) - count := 0 - it, _ = it.Optimize() - for { - if ses.doHalt { - return nil - } - val, ok := it.Next() - if !ok { - break - } - output = append(output, ses.ts.GetNameFor(val)) - count++ - if limit >= 0 && count >= limit { - break - } - } - it.Close() - return output -} - -func runIteratorWithCallback(it graph.Iterator, ses *GremlinSession, callback otto.Value, this otto.FunctionCall, limit int) { - count := 0 - it, _ = it.Optimize() - for { - if ses.doHalt { - return - } - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) - val, _ = callback.Call(this.This, val) - count++ - if limit >= 0 && count >= limit { - break - } - for it.NextResult() == true { - if ses.doHalt { - return - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) - val, _ = callback.Call(this.This, val) - count++ - if limit >= 0 && count >= limit { - break - } - } - } - it.Close() -} - -func runIteratorOnSession(it graph.Iterator, ses *GremlinSession) { - if ses.lookingForQueryShape { - graph.OutputQueryShapeForIterator(it, ses.ts, &(ses.queryShape)) - return - } - it, _ = it.Optimize() - glog.V(2).Infoln(it.DebugString(0)) - for { - // TODO(barakmich): Better halting. - if ses.doHalt { - return - } - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) - if !cont { - break - } - for it.NextResult() == true { - if ses.doHalt { - return - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) - if !cont { - break - } - } - } - it.Close() -} diff --git a/gremlin/gremlin-functional_test.go b/gremlin/gremlin-functional_test.go deleted file mode 100644 index f6c65fb..0000000 --- a/gremlin/gremlin-functional_test.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "sort" - "testing" - - . "github.com/smartystreets/goconvey/convey" - - "github.com/google/cayley/graph/memstore" -) - -// +---+ +---+ -// | A |------- ->| F |<-- -// +---+ \------>+---+-/ +---+ \--+---+ -// ------>|#B#| | | E | -// +---+-------/ >+---+ | +---+ -// | C | / v -// +---+ -/ +---+ -// ---- +---+/ |#G#| -// \-->|#D#|------------->+---+ -// +---+ -// - -func buildTripleStore() *GremlinSession { - ts := memstore.MakeTestingMemstore() - return NewGremlinSession(ts, -1, false) -} - -func shouldBeUnordered(actual interface{}, expected ...interface{}) string { - if len(expected) != 1 { - return "Only one list supported" - } - actualStr := actual.([]string) - expectedStr := expected[0].([]string) - sort.Strings(actualStr) - sort.Strings(expectedStr) - return ShouldResemble(actualStr, expectedStr) -} - -func runQueryGetTag(query string, tag string) ([]string, int) { - js := buildTripleStore() - output := make([]string, 0) - c := make(chan interface{}, 5) - js.ExecInput(query, c, -1) - count := 0 - for result := range c { - count++ - data := result.(*GremlinResult) - if data.val == nil { - val := (*data.actualResults)[tag] - if val != nil { - output = append(output, js.ts.GetNameFor(val)) - } - } - } - return output, count -} - -func ConveyQuery(doc string, query string, expected []string) { - ConveyQueryTag(doc, query, GremlinTopResultTag, expected) -} - -func ConveyQueryTag(doc string, query string, tag string, expected []string) { - Convey(doc, func() { - actual, _ := runQueryGetTag(query, tag) - So(actual, shouldBeUnordered, expected) - }) -} - -func TestGremlin(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - - ConveyQuery("Can get a single vertex", - `g.V("A").All()`, - []string{"A"}) - - ConveyQuery("Can use .Out()", - `g.V("A").Out("follows").All()`, - []string{"B"}) - - ConveyQuery("Can use .In()", - `g.V("B").In("follows").All()`, - []string{"A", "C", "D"}) - - ConveyQuery("Can use .Both()", - `g.V("F").Both("follows").All()`, - []string{"B", "G", "E"}) - - ConveyQuery("Can use .Tag()-.Is()-.Back()", - `g.V("B").In("follows").Tag("foo").Out("status").Is("cool").Back("foo").All()`, - []string{"D"}) - - ConveyQuery("Can separate .Tag()-.Is()-.Back()", - ` - x = g.V("C").Out("follows").Tag("foo").Out("status").Is("cool").Back("foo") - x.In("follows").Is("D").Back("foo").All() - `, - []string{"B"}) - - Convey("Can do multiple .Back()s", func() { - query := ` - g.V("E").Out("follows").As("f").Out("follows").Out("status").Is("cool").Back("f").In("follows").In("follows").As("acd").Out("status").Is("cool").Back("f").All() - ` - expected := []string{"D"} - actual, _ := runQueryGetTag(query, "acd") - So(actual, shouldBeUnordered, expected) - }) - - }) -} - -func TestGremlinMorphism(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - - ConveyQuery("Simple morphism works", - ` - grandfollows = g.M().Out("follows").Out("follows") - g.V("C").Follow(grandfollows).All() - `, - []string{"G", "F", "B"}) - - ConveyQuery("Reverse morphism works", - ` - grandfollows = g.M().Out("follows").Out("follows") - g.V("F").FollowR(grandfollows).All() - `, []string{"A", "C", "D"}) - - }) -} - -func TestGremlinIntersection(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - ConveyQuery("Simple intersection", - ` - function follows(x) { return g.V(x).Out("follows") } - - follows("D").And(follows("C")).All() - `, []string{"B"}) - - ConveyQuery("Simple Morphism Intersection", - ` - grandfollows = g.M().Out("follows").Out("follows") - function gfollows(x) { return g.V(x).Follow(grandfollows) } - - gfollows("A").And(gfollows("C")).All() - `, []string{"F"}) - - ConveyQuery("Double Morphism Intersection", - ` - grandfollows = g.M().Out("follows").Out("follows") - function gfollows(x) { return g.V(x).Follow(grandfollows) } - - gfollows("E").And(gfollows("C")).And(gfollows("B")).All() - `, []string{"G"}) - - ConveyQuery("Reverse Intersection", - ` - grandfollows = g.M().Out("follows").Out("follows") - - g.V("G").FollowR(grandfollows).Intersect(g.V("F").FollowR(grandfollows)).All() - `, []string{"C"}) - - ConveyQuery("Standard sort of morphism intersection, continue follow", - ` - gfollowers = g.M().In("follows").In("follows") - function cool(x) { return g.V(x).As("a").Out("status").Is("cool").Back("a") } - cool("G").Follow(gfollowers).Intersect(cool("B").Follow(gfollowers)).All() - `, []string{"C"}) - - }) -} - -func TestGremlinHas(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - ConveyQuery("Test a simple Has", - `g.V().Has("status", "cool").All()`, - []string{"G", "D", "B"}) - - ConveyQuery("Test a double Has", - `g.V().Has("status", "cool").Has("follows", "F").All()`, - []string{"B"}) - - }) -} - -func TestGremlinTag(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - ConveyQueryTag("Test a simple save", - `g.V().Save("status", "somecool").All()`, - "somecool", - []string{"cool", "cool", "cool"}) - - ConveyQueryTag("Test a simple saveR", - `g.V("cool").SaveR("status", "who").All()`, - "who", - []string{"G", "D", "B"}) - - ConveyQueryTag("Test an out save", - `g.V("D").Out(null, "pred").All()`, - "pred", - []string{"follows", "follows", "status"}) - - ConveyQueryTag("Test a tag list", - `g.V("D").Out(null, ["pred", "foo", "bar"]).All()`, - "foo", - []string{"follows", "follows", "status"}) - - ConveyQuery("Test a pred list", - `g.V("D").Out(["follows", "status"]).All()`, - []string{"B", "G", "cool"}) - - ConveyQuery("Test a predicate path", - `g.V("D").Out(g.V("follows"), "pred").All()`, - []string{"B", "G"}) - }) -} diff --git a/gremlin/gremlin-session.go b/gremlin/gremlin-session.go deleted file mode 100644 index a0b0483..0000000 --- a/gremlin/gremlin-session.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "errors" - "fmt" - "sort" - "time" - - "github.com/robertkrimen/otto" - - "github.com/google/cayley/graph" -) - -type GremlinSession struct { - ts graph.TripleStore - currentChannel chan interface{} - env *otto.Otto - debug bool - limit int - count int - dataOutput []interface{} - lookingForQueryShape bool - queryShape map[string]interface{} - err error - script *otto.Script - doHalt bool - timeoutSec time.Duration - emptyEnv *otto.Otto -} - -func NewGremlinSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool) *GremlinSession { - var g GremlinSession - g.ts = inputTripleStore - g.env = BuildGremlinEnv(&g) - g.limit = -1 - g.count = 0 - g.lookingForQueryShape = false - if persist { - g.emptyEnv = g.env - } - if timeoutSec < 0 { - g.timeoutSec = time.Duration(-1) - } else { - g.timeoutSec = time.Duration(timeoutSec) - } - g.ClearJson() - return &g -} - -type GremlinResult struct { - metaresult bool - err string - val *otto.Value - actualResults *map[string]graph.TSVal -} - -func (g *GremlinSession) ToggleDebug() { - g.debug = !g.debug -} - -func (g *GremlinSession) GetQuery(input string, output_struct chan map[string]interface{}) { - defer close(output_struct) - g.queryShape = make(map[string]interface{}) - g.lookingForQueryShape = true - g.env.Run(input) - output_struct <- g.queryShape - g.queryShape = nil -} - -func (g *GremlinSession) InputParses(input string) (graph.ParseResult, error) { - script, err := g.env.Compile("", input) - if err != nil { - return graph.ParseFail, err - } - g.script = script - return graph.Parsed, nil -} - -func (g *GremlinSession) SendResult(result *GremlinResult) bool { - if g.limit >= 0 && g.limit == g.count { - return false - } - if g.doHalt { - return false - } - if g.currentChannel != nil { - g.currentChannel <- result - g.count++ - if g.limit >= 0 && g.limit == g.count { - return false - } else { - return true - } - } - return false -} - -var halt = errors.New("Query Timeout") - -func (g *GremlinSession) runUnsafe(input interface{}) (otto.Value, error) { - g.doHalt = false - defer func() { - if caught := recover(); caught != nil { - if caught == halt { - g.err = halt - return - } - panic(caught) // Something else happened, repanic! - } - }() - - g.env.Interrupt = make(chan func(), 1) // The buffer prevents blocking - - if g.timeoutSec != -1 { - go func() { - time.Sleep(g.timeoutSec * time.Second) // Stop after two seconds - g.doHalt = true - if g.env != nil { - g.env.Interrupt <- func() { - panic(halt) - } - g.env = g.emptyEnv - } - }() - } - - return g.env.Run(input) // Here be dragons (risky code) -} - -func (g *GremlinSession) ExecInput(input string, out chan interface{}, limit int) { - defer close(out) - g.err = nil - g.currentChannel = out - var err error - var value otto.Value - if g.script == nil { - value, err = g.runUnsafe(input) - } else { - value, err = g.runUnsafe(g.script) - } - if err != nil { - out <- &GremlinResult{metaresult: true, - err: err.Error(), - val: &value, - actualResults: nil} - } else { - out <- &GremlinResult{metaresult: true, - err: "", - val: &value, - actualResults: nil} - } - g.currentChannel = nil - g.script = nil - g.env = g.emptyEnv - return -} - -func (s *GremlinSession) ToText(result interface{}) string { - data := result.(*GremlinResult) - if data.metaresult { - if data.err != "" { - return fmt.Sprintln("Error: ", data.err) - } - if data.val != nil { - s, _ := data.val.Export() - if data.val.IsObject() { - typeVal, _ := data.val.Object().Get("_gremlin_type") - if !typeVal.IsUndefined() { - s = "[internal Iterator]" - } - } - return fmt.Sprintln("=>", s) - } - return "" - } - var out string - out = fmt.Sprintln("****") - if data.val == nil { - tags := data.actualResults - tagKeys := make([]string, len(*tags)) - i := 0 - for k, _ := range *tags { - tagKeys[i] = k - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - if k == "$_" { - continue - } - out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) - } - } else { - if data.val.IsObject() { - export, _ := data.val.Export() - mapExport := export.(map[string]string) - for k, v := range mapExport { - out += fmt.Sprintf("%s : %v\n", k, v) - } - } else { - strVersion, _ := data.val.ToString() - out += fmt.Sprintf("%s\n", strVersion) - } - } - return out -} - -// Web stuff -func (ses *GremlinSession) BuildJson(result interface{}) { - data := result.(*GremlinResult) - if !data.metaresult { - if data.val == nil { - obj := make(map[string]string) - tags := data.actualResults - tagKeys := make([]string, len(*tags)) - i := 0 - for k, _ := range *tags { - tagKeys[i] = k - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - obj[k] = ses.ts.GetNameFor((*tags)[k]) - } - ses.dataOutput = append(ses.dataOutput, obj) - } else { - if data.val.IsObject() { - export, _ := data.val.Export() - ses.dataOutput = append(ses.dataOutput, export) - } else { - strVersion, _ := data.val.ToString() - ses.dataOutput = append(ses.dataOutput, strVersion) - } - } - } - -} - -func (ses *GremlinSession) GetJson() (interface{}, error) { - defer ses.ClearJson() - if ses.err != nil { - return nil, ses.err - } - if ses.doHalt { - return nil, halt - } - return ses.dataOutput, nil -} - -func (ses *GremlinSession) ClearJson() { - ses.dataOutput = nil -} diff --git a/gremlin/gremlin-traversals.go b/gremlin/gremlin-traversals.go deleted file mode 100644 index c0f4704..0000000 --- a/gremlin/gremlin-traversals.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -// Adds special traversal functions to JS Gremlin objects. Most of these just build the chain of objects, and won't often need the session. - -import ( - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" -) - -func embedTraversals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { - obj.Set("In", gremlinFunc("in", obj, env, ses)) - obj.Set("Out", gremlinFunc("out", obj, env, ses)) - obj.Set("Is", gremlinFunc("is", obj, env, ses)) - obj.Set("Both", gremlinFunc("both", obj, env, ses)) - obj.Set("Follow", gremlinFunc("follow", obj, env, ses)) - obj.Set("FollowR", gremlinFollowR("followr", obj, env, ses)) - obj.Set("And", gremlinFunc("and", obj, env, ses)) - obj.Set("Intersect", gremlinFunc("and", obj, env, ses)) - obj.Set("Union", gremlinFunc("or", obj, env, ses)) - obj.Set("Or", gremlinFunc("or", obj, env, ses)) - obj.Set("Back", gremlinBack("back", obj, env, ses)) - obj.Set("Tag", gremlinFunc("tag", obj, env, ses)) - obj.Set("As", gremlinFunc("tag", obj, env, ses)) - obj.Set("Has", gremlinFunc("has", obj, env, ses)) - obj.Set("Save", gremlinFunc("save", obj, env, ses)) - obj.Set("SaveR", gremlinFunc("saver", obj, env, ses)) -} - -func gremlinFunc(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", kind) - out.Set("_gremlin_values", call.ArgumentList) - out.Set("_gremlin_prev", prevObj) - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - embedTraversals(env, ses, out) - if isVertexChain(call.This.Object()) { - embedFinals(env, ses, out) - } - return out.Value() - } -} - -func gremlinBack(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", kind) - out.Set("_gremlin_values", call.ArgumentList) - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - var otherChain *otto.Object - var thisObj *otto.Object - if len(*outStrings) != 0 { - otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, (*outStrings)[0].(string)) - } else { - otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, "") - } - out.Set("_gremlin_prev", thisObj) - out.Set("_gremlin_back_chain", otherChain) - embedTraversals(env, ses, out) - if isVertexChain(call.This.Object()) { - embedFinals(env, ses, out) - } - return out.Value() - - } -} - -func gremlinFollowR(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", kind) - out.Set("_gremlin_values", call.ArgumentList) - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - if len(call.ArgumentList) == 0 { - return prevObj.Value() - } - arg := call.Argument(0) - if isVertexChain(arg.Object()) { - return prevObj.Value() - } - newChain, _ := reverseGremlinChainTo(call.Otto, arg.Object(), "") - out.Set("_gremlin_prev", prevObj) - out.Set("_gremlin_followr", newChain) - embedTraversals(env, ses, out) - if isVertexChain(call.This.Object()) { - embedFinals(env, ses, out) - } - return out.Value() - - } -} - -func reverseGremlinChainTo(env *otto.Otto, prevObj *otto.Object, tag string) (*otto.Object, *otto.Object) { - env.Run("var _base_object = {}") - base, err := env.Object("_base_object") - if err != nil { - glog.Error(err) - return otto.NullValue().Object(), otto.NullValue().Object() - } - if isVertexChain(prevObj) { - base.Set("_gremlin_type", "vertex") - } else { - base.Set("_gremlin_type", "morphism") - } - return reverseGremlinChainHelper(env, prevObj, base, tag) -} - -func reverseGremlinChainHelper(env *otto.Otto, chain *otto.Object, newBase *otto.Object, tag string) (*otto.Object, *otto.Object) { - kindVal, _ := chain.Get("_gremlin_type") - kind, _ := kindVal.ToString() - - if tag != "" { - if kind == "tag" { - tags := getStringArgs(chain) - for _, t := range tags { - if t == tag { - return newBase, chain - } - } - } - } - - if kind == "morphism" || kind == "vertex" { - return newBase, chain - } - var newKind string - switch kind { - case "in": - newKind = "out" - case "out": - newKind = "in" - default: - newKind = kind - } - prev, _ := chain.Get("_gremlin_prev") - env.Run("var out = {}") - out, _ := env.Object("out") - out.Set("_gremlin_type", newKind) - values, _ := chain.Get("_gremlin_values") - out.Set("_gremlin_values", values) - back, _ := chain.Get("_gremlin_back_chain") - out.Set("_gremlin_back_chain", back) - out.Set("_gremlin_prev", newBase) - strings, _ := chain.Get("string_args") - out.Set("string_args", strings) - return reverseGremlinChainHelper(env, prev.Object(), out, tag) -} - -func debugChain(obj *otto.Object) bool { - val, _ := obj.Get("_gremlin_type") - x, _ := val.ToString() - glog.V(2).Infoln(x) - val, _ = obj.Get("_gremlin_prev") - if val.IsObject() { - return debugChain(val.Object()) - } - return false -} diff --git a/gremlin/gremlin_build_iterator.go b/gremlin/gremlin_build_iterator.go new file mode 100644 index 0000000..b6e9a45 --- /dev/null +++ b/gremlin/gremlin_build_iterator.go @@ -0,0 +1,315 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "strconv" + + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" +) + +func getStrings(obj *otto.Object, field string) []string { + strings := make([]string, 0) + val, _ := obj.Get(field) + if !val.IsUndefined() { + export, _ := val.Export() + array := export.([]interface{}) + for _, arg := range array { + strings = append(strings, arg.(string)) + } + } + return strings +} + +func getStringArgs(obj *otto.Object) []string { return getStrings(obj, "string_args") } + +func buildIteratorTree(obj *otto.Object, ts graph.TripleStore) graph.Iterator { + if !isVertexChain(obj) { + return graph.NewNullIterator() + } + return buildIteratorTreeHelper(obj, ts, graph.NewNullIterator()) +} + +func makeListOfStringsFromArrayValue(obj *otto.Object) []string { + var output []string + lengthValue, _ := obj.Get("length") + length, _ := lengthValue.ToInteger() + ulength := uint32(length) + for index := uint32(0); index < ulength; index += 1 { + name := strconv.FormatInt(int64(index), 10) + value, err := obj.Get(name) + if err != nil { + continue + } + if !value.IsString() { + continue + } + s, _ := value.ToString() + output = append(output, s) + } + return output +} + +func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator { + if val.IsNull() || val.IsUndefined() { + return ts.GetNodesAllIterator() + } + if val.IsPrimitive() { + thing, _ := val.Export() + switch v := thing.(type) { + case string: + it := ts.MakeFixed() + it.AddValue(ts.GetIdFor(v)) + return it + default: + glog.Errorln("Trying to build unknown primitive value.") + } + } + switch val.Class() { + case "Object": + return buildIteratorTree(val.Object(), ts) + case "Array": + // Had better be an array of strings + strings := makeListOfStringsFromArrayValue(val.Object()) + it := ts.MakeFixed() + for _, x := range strings { + it.AddValue(ts.GetIdFor(x)) + } + return it + case "Number": + fallthrough + case "Boolean": + fallthrough + case "Date": + fallthrough + case "String": + it := ts.MakeFixed() + str, _ := val.ToString() + it.AddValue(ts.GetIdFor(str)) + return it + default: + glog.Errorln("Trying to handle unsupported Javascript value.") + return graph.NewNullIterator() + } +} + +func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Iterator, isReverse bool) graph.Iterator { + argList, _ := obj.Get("_gremlin_values") + if argList.Class() != "GoArray" { + glog.Errorln("How is arglist not an array? Return nothing.", argList.Class()) + return graph.NewNullIterator() + } + argArray := argList.Object() + lengthVal, _ := argArray.Get("length") + length, _ := lengthVal.ToInteger() + var predicateNodeIterator graph.Iterator + if length == 0 { + predicateNodeIterator = ts.GetNodesAllIterator() + } else { + zero, _ := argArray.Get("0") + predicateNodeIterator = buildIteratorFromValue(zero, ts) + } + if length >= 2 { + var tags []string + one, _ := argArray.Get("1") + if one.IsString() { + s, _ := one.ToString() + tags = append(tags, s) + } else if one.Class() == "Array" { + tags = makeListOfStringsFromArrayValue(one.Object()) + } + for _, tag := range tags { + predicateNodeIterator.AddTag(tag) + } + } + + in, out := "s", "o" + if isReverse { + in, out = out, in + } + lto := graph.NewLinksToIterator(ts, base, in) + and := graph.NewAndIterator() + and.AddSubIterator(graph.NewLinksToIterator(ts, predicateNodeIterator, "p")) + and.AddSubIterator(lto) + return graph.NewHasaIterator(ts, and, out) +} + +func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.Iterator) graph.Iterator { + var it graph.Iterator + it = base + // TODO: Better error handling + kindVal, _ := obj.Get("_gremlin_type") + stringArgs := getStringArgs(obj) + var subIt graph.Iterator + prevVal, _ := obj.Get("_gremlin_prev") + if !prevVal.IsObject() { + subIt = base + } else { + subIt = buildIteratorTreeHelper(prevVal.Object(), ts, base) + } + + kind, _ := kindVal.ToString() + switch kind { + case "vertex": + if len(stringArgs) == 0 { + it = ts.GetNodesAllIterator() + } else { + fixed := ts.MakeFixed() + for _, name := range stringArgs { + fixed.AddValue(ts.GetIdFor(name)) + } + it = fixed + } + case "tag": + it = subIt + for _, tag := range stringArgs { + it.AddTag(tag) + } + case "save": + all := ts.GetNodesAllIterator() + if len(stringArgs) > 2 || len(stringArgs) == 0 { + return graph.NewNullIterator() + } + if len(stringArgs) == 2 { + all.AddTag(stringArgs[1]) + } else { + all.AddTag(stringArgs[0]) + } + predFixed := ts.MakeFixed() + predFixed.AddValue(ts.GetIdFor(stringArgs[0])) + subAnd := graph.NewAndIterator() + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "o")) + hasa := graph.NewHasaIterator(ts, subAnd, "s") + and := graph.NewAndIterator() + and.AddSubIterator(hasa) + and.AddSubIterator(subIt) + it = and + case "saver": + all := ts.GetNodesAllIterator() + if len(stringArgs) > 2 || len(stringArgs) == 0 { + return graph.NewNullIterator() + } + if len(stringArgs) == 2 { + all.AddTag(stringArgs[1]) + } else { + all.AddTag(stringArgs[0]) + } + predFixed := ts.MakeFixed() + predFixed.AddValue(ts.GetIdFor(stringArgs[0])) + subAnd := graph.NewAndIterator() + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "s")) + hasa := graph.NewHasaIterator(ts, subAnd, "o") + and := graph.NewAndIterator() + and.AddSubIterator(hasa) + and.AddSubIterator(subIt) + it = and + case "has": + fixed := ts.MakeFixed() + if len(stringArgs) < 2 { + return graph.NewNullIterator() + } + for _, name := range stringArgs[1:] { + fixed.AddValue(ts.GetIdFor(name)) + } + predFixed := ts.MakeFixed() + predFixed.AddValue(ts.GetIdFor(stringArgs[0])) + subAnd := graph.NewAndIterator() + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, fixed, "o")) + hasa := graph.NewHasaIterator(ts, subAnd, "s") + and := graph.NewAndIterator() + and.AddSubIterator(hasa) + and.AddSubIterator(subIt) + it = and + case "morphism": + it = base + case "and": + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if !isVertexChain(firstArg.Object()) { + return graph.NewNullIterator() + } + argIt := buildIteratorTree(firstArg.Object(), ts) + + and := graph.NewAndIterator() + and.AddSubIterator(subIt) + and.AddSubIterator(argIt) + it = and + case "back": + arg, _ := obj.Get("_gremlin_back_chain") + argIt := buildIteratorTree(arg.Object(), ts) + and := graph.NewAndIterator() + and.AddSubIterator(subIt) + and.AddSubIterator(argIt) + it = and + case "is": + fixed := ts.MakeFixed() + for _, name := range stringArgs { + fixed.AddValue(ts.GetIdFor(name)) + } + and := graph.NewAndIterator() + and.AddSubIterator(fixed) + and.AddSubIterator(subIt) + it = and + case "or": + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if !isVertexChain(firstArg.Object()) { + return graph.NewNullIterator() + } + argIt := buildIteratorTree(firstArg.Object(), ts) + + or := graph.NewOrIterator() + or.AddSubIterator(subIt) + or.AddSubIterator(argIt) + it = or + case "both": + // Hardly the most efficient pattern, but the most general. + // Worth looking into an Optimize() optimization here. + clone := subIt.Clone() + it1 := buildInOutIterator(obj, ts, subIt, false) + it2 := buildInOutIterator(obj, ts, clone, true) + + or := graph.NewOrIterator() + or.AddSubIterator(it1) + or.AddSubIterator(it2) + it = or + case "out": + it = buildInOutIterator(obj, ts, subIt, false) + case "follow": + // Follow a morphism + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if isVertexChain(firstArg.Object()) { + return graph.NewNullIterator() + } + it = buildIteratorTreeHelper(firstArg.Object(), ts, subIt) + case "followr": + // Follow a morphism + arg, _ := obj.Get("_gremlin_followr") + if isVertexChain(arg.Object()) { + return graph.NewNullIterator() + } + it = buildIteratorTreeHelper(arg.Object(), ts, subIt) + case "in": + it = buildInOutIterator(obj, ts, subIt, true) + } + return it +} diff --git a/gremlin/gremlin_env.go b/gremlin/gremlin_env.go new file mode 100644 index 0000000..4e7f332 --- /dev/null +++ b/gremlin/gremlin_env.go @@ -0,0 +1,95 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +// Builds a new Gremlin environment pointing at a session. + +import ( + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" +) + +func BuildGremlinEnv(ses *GremlinSession) *otto.Otto { + env := otto.New() + setupGremlin(env, ses) + return env +} + +func concatStringArgs(call otto.FunctionCall) *[]interface{} { + outStrings := make([]interface{}, 0) + for _, arg := range call.ArgumentList { + if arg.IsString() { + outStrings = append(outStrings, arg.String()) + } + if arg.IsObject() && arg.Class() == "Array" { + obj, _ := arg.Export() + for _, x := range obj.([]interface{}) { + outStrings = append(outStrings, x.(string)) + } + } + } + return &outStrings +} + +func isVertexChain(obj *otto.Object) bool { + val, _ := obj.Get("_gremlin_type") + if x, _ := val.ToString(); x == "vertex" { + return true + } + val, _ = obj.Get("_gremlin_prev") + if val.IsObject() { + return isVertexChain(val.Object()) + } + return false +} + +func setupGremlin(env *otto.Otto, ses *GremlinSession) { + graph, _ := env.Object("graph = {}") + graph.Set("Vertex", func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, err := call.Otto.Object("out") + if err != nil { + glog.Error(err.Error()) + return otto.TrueValue() + } + out.Set("_gremlin_type", "vertex") + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + embedTraversals(env, ses, out) + embedFinals(env, ses, out) + return out.Value() + }) + + graph.Set("Morphism", func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", "morphism") + embedTraversals(env, ses, out) + return out.Value() + }) + graph.Set("Emit", func(call otto.FunctionCall) otto.Value { + value := call.Argument(0) + if value.IsDefined() { + ses.SendResult(&GremlinResult{metaresult: false, err: "", val: &value, actualResults: nil}) + } + return otto.NullValue() + }) + env.Run("graph.V = graph.Vertex") + env.Run("graph.M = graph.Morphism") + env.Run("g = graph") + +} diff --git a/gremlin/gremlin_finals.go b/gremlin/gremlin_finals.go new file mode 100644 index 0000000..f8c978d --- /dev/null +++ b/gremlin/gremlin_finals.go @@ -0,0 +1,274 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" +) + +const GremlinTopResultTag = "id" + +func embedFinals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { + obj.Set("All", allFunc(env, ses, obj)) + obj.Set("GetLimit", limitFunc(env, ses, obj)) + obj.Set("ToArray", toArrayFunc(env, ses, obj, false)) + obj.Set("ToValue", toValueFunc(env, ses, obj, false)) + obj.Set("TagArray", toArrayFunc(env, ses, obj, true)) + obj.Set("TagValue", toValueFunc(env, ses, obj, true)) + obj.Set("Map", mapFunc(env, ses, obj)) + obj.Set("ForEach", mapFunc(env, ses, obj)) +} + +func allFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + ses.limit = -1 + ses.count = 0 + runIteratorOnSession(it, ses) + return otto.NullValue() + } +} + +func limitFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) > 0 { + limitVal, _ := call.Argument(0).ToInteger() + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + ses.limit = int(limitVal) + ses.count = 0 + runIteratorOnSession(it, ses) + } + return otto.NullValue() + } +} + +func toArrayFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + limit := -1 + if len(call.ArgumentList) > 0 { + limitParsed, _ := call.Argument(0).ToInteger() + limit = int(limitParsed) + } + var val otto.Value + var err error + if !withTags { + array := runIteratorToArrayNoTags(it, ses, limit) + val, err = call.Otto.ToValue(array) + } else { + array := runIteratorToArray(it, ses, limit) + val, err = call.Otto.ToValue(array) + } + + if err != nil { + glog.Error(err) + return otto.NullValue() + } + return val + } +} + +func toValueFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + limit := 1 + var val otto.Value + var err error + if !withTags { + array := runIteratorToArrayNoTags(it, ses, limit) + if len(array) < 1 { + return otto.NullValue() + } + val, err = call.Otto.ToValue(array[0]) + } else { + array := runIteratorToArray(it, ses, limit) + if len(array) < 1 { + return otto.NullValue() + } + val, err = call.Otto.ToValue(array[0]) + } + if err != nil { + glog.Error(err) + return otto.NullValue() + } else { + return val + } + + } +} + +func mapFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + limit := -1 + if len(call.ArgumentList) == 0 { + return otto.NullValue() + } + callback := call.Argument(len(call.ArgumentList) - 1) + if len(call.ArgumentList) > 1 { + limitParsed, _ := call.Argument(0).ToInteger() + limit = int(limitParsed) + } + runIteratorWithCallback(it, ses, callback, call, limit) + return otto.NullValue() + } +} + +func tagsToValueMap(m map[string]graph.TSVal, ses *GremlinSession) map[string]string { + outputMap := make(map[string]string) + for k, v := range m { + outputMap[k] = ses.ts.GetNameFor(v) + } + return outputMap +} + +func runIteratorToArray(it graph.Iterator, ses *GremlinSession, limit int) []map[string]string { + output := make([]map[string]string, 0) + count := 0 + it, _ = it.Optimize() + for { + if ses.doHalt { + return nil + } + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + output = append(output, tagsToValueMap(tags, ses)) + count++ + if limit >= 0 && count >= limit { + break + } + for it.NextResult() == true { + if ses.doHalt { + return nil + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + output = append(output, tagsToValueMap(tags, ses)) + count++ + if limit >= 0 && count >= limit { + break + } + } + } + it.Close() + return output +} + +func runIteratorToArrayNoTags(it graph.Iterator, ses *GremlinSession, limit int) []string { + output := make([]string, 0) + count := 0 + it, _ = it.Optimize() + for { + if ses.doHalt { + return nil + } + val, ok := it.Next() + if !ok { + break + } + output = append(output, ses.ts.GetNameFor(val)) + count++ + if limit >= 0 && count >= limit { + break + } + } + it.Close() + return output +} + +func runIteratorWithCallback(it graph.Iterator, ses *GremlinSession, callback otto.Value, this otto.FunctionCall, limit int) { + count := 0 + it, _ = it.Optimize() + for { + if ses.doHalt { + return + } + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) + val, _ = callback.Call(this.This, val) + count++ + if limit >= 0 && count >= limit { + break + } + for it.NextResult() == true { + if ses.doHalt { + return + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) + val, _ = callback.Call(this.This, val) + count++ + if limit >= 0 && count >= limit { + break + } + } + } + it.Close() +} + +func runIteratorOnSession(it graph.Iterator, ses *GremlinSession) { + if ses.lookingForQueryShape { + graph.OutputQueryShapeForIterator(it, ses.ts, &(ses.queryShape)) + return + } + it, _ = it.Optimize() + glog.V(2).Infoln(it.DebugString(0)) + for { + // TODO(barakmich): Better halting. + if ses.doHalt { + return + } + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) + if !cont { + break + } + for it.NextResult() == true { + if ses.doHalt { + return + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) + if !cont { + break + } + } + } + it.Close() +} diff --git a/gremlin/gremlin_functional_test.go b/gremlin/gremlin_functional_test.go new file mode 100644 index 0000000..f6c65fb --- /dev/null +++ b/gremlin/gremlin_functional_test.go @@ -0,0 +1,230 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "sort" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph/memstore" +) + +// +---+ +---+ +// | A |------- ->| F |<-- +// +---+ \------>+---+-/ +---+ \--+---+ +// ------>|#B#| | | E | +// +---+-------/ >+---+ | +---+ +// | C | / v +// +---+ -/ +---+ +// ---- +---+/ |#G#| +// \-->|#D#|------------->+---+ +// +---+ +// + +func buildTripleStore() *GremlinSession { + ts := memstore.MakeTestingMemstore() + return NewGremlinSession(ts, -1, false) +} + +func shouldBeUnordered(actual interface{}, expected ...interface{}) string { + if len(expected) != 1 { + return "Only one list supported" + } + actualStr := actual.([]string) + expectedStr := expected[0].([]string) + sort.Strings(actualStr) + sort.Strings(expectedStr) + return ShouldResemble(actualStr, expectedStr) +} + +func runQueryGetTag(query string, tag string) ([]string, int) { + js := buildTripleStore() + output := make([]string, 0) + c := make(chan interface{}, 5) + js.ExecInput(query, c, -1) + count := 0 + for result := range c { + count++ + data := result.(*GremlinResult) + if data.val == nil { + val := (*data.actualResults)[tag] + if val != nil { + output = append(output, js.ts.GetNameFor(val)) + } + } + } + return output, count +} + +func ConveyQuery(doc string, query string, expected []string) { + ConveyQueryTag(doc, query, GremlinTopResultTag, expected) +} + +func ConveyQueryTag(doc string, query string, tag string, expected []string) { + Convey(doc, func() { + actual, _ := runQueryGetTag(query, tag) + So(actual, shouldBeUnordered, expected) + }) +} + +func TestGremlin(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + + ConveyQuery("Can get a single vertex", + `g.V("A").All()`, + []string{"A"}) + + ConveyQuery("Can use .Out()", + `g.V("A").Out("follows").All()`, + []string{"B"}) + + ConveyQuery("Can use .In()", + `g.V("B").In("follows").All()`, + []string{"A", "C", "D"}) + + ConveyQuery("Can use .Both()", + `g.V("F").Both("follows").All()`, + []string{"B", "G", "E"}) + + ConveyQuery("Can use .Tag()-.Is()-.Back()", + `g.V("B").In("follows").Tag("foo").Out("status").Is("cool").Back("foo").All()`, + []string{"D"}) + + ConveyQuery("Can separate .Tag()-.Is()-.Back()", + ` + x = g.V("C").Out("follows").Tag("foo").Out("status").Is("cool").Back("foo") + x.In("follows").Is("D").Back("foo").All() + `, + []string{"B"}) + + Convey("Can do multiple .Back()s", func() { + query := ` + g.V("E").Out("follows").As("f").Out("follows").Out("status").Is("cool").Back("f").In("follows").In("follows").As("acd").Out("status").Is("cool").Back("f").All() + ` + expected := []string{"D"} + actual, _ := runQueryGetTag(query, "acd") + So(actual, shouldBeUnordered, expected) + }) + + }) +} + +func TestGremlinMorphism(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + + ConveyQuery("Simple morphism works", + ` + grandfollows = g.M().Out("follows").Out("follows") + g.V("C").Follow(grandfollows).All() + `, + []string{"G", "F", "B"}) + + ConveyQuery("Reverse morphism works", + ` + grandfollows = g.M().Out("follows").Out("follows") + g.V("F").FollowR(grandfollows).All() + `, []string{"A", "C", "D"}) + + }) +} + +func TestGremlinIntersection(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + ConveyQuery("Simple intersection", + ` + function follows(x) { return g.V(x).Out("follows") } + + follows("D").And(follows("C")).All() + `, []string{"B"}) + + ConveyQuery("Simple Morphism Intersection", + ` + grandfollows = g.M().Out("follows").Out("follows") + function gfollows(x) { return g.V(x).Follow(grandfollows) } + + gfollows("A").And(gfollows("C")).All() + `, []string{"F"}) + + ConveyQuery("Double Morphism Intersection", + ` + grandfollows = g.M().Out("follows").Out("follows") + function gfollows(x) { return g.V(x).Follow(grandfollows) } + + gfollows("E").And(gfollows("C")).And(gfollows("B")).All() + `, []string{"G"}) + + ConveyQuery("Reverse Intersection", + ` + grandfollows = g.M().Out("follows").Out("follows") + + g.V("G").FollowR(grandfollows).Intersect(g.V("F").FollowR(grandfollows)).All() + `, []string{"C"}) + + ConveyQuery("Standard sort of morphism intersection, continue follow", + ` + gfollowers = g.M().In("follows").In("follows") + function cool(x) { return g.V(x).As("a").Out("status").Is("cool").Back("a") } + cool("G").Follow(gfollowers).Intersect(cool("B").Follow(gfollowers)).All() + `, []string{"C"}) + + }) +} + +func TestGremlinHas(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + ConveyQuery("Test a simple Has", + `g.V().Has("status", "cool").All()`, + []string{"G", "D", "B"}) + + ConveyQuery("Test a double Has", + `g.V().Has("status", "cool").Has("follows", "F").All()`, + []string{"B"}) + + }) +} + +func TestGremlinTag(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + ConveyQueryTag("Test a simple save", + `g.V().Save("status", "somecool").All()`, + "somecool", + []string{"cool", "cool", "cool"}) + + ConveyQueryTag("Test a simple saveR", + `g.V("cool").SaveR("status", "who").All()`, + "who", + []string{"G", "D", "B"}) + + ConveyQueryTag("Test an out save", + `g.V("D").Out(null, "pred").All()`, + "pred", + []string{"follows", "follows", "status"}) + + ConveyQueryTag("Test a tag list", + `g.V("D").Out(null, ["pred", "foo", "bar"]).All()`, + "foo", + []string{"follows", "follows", "status"}) + + ConveyQuery("Test a pred list", + `g.V("D").Out(["follows", "status"]).All()`, + []string{"B", "G", "cool"}) + + ConveyQuery("Test a predicate path", + `g.V("D").Out(g.V("follows"), "pred").All()`, + []string{"B", "G"}) + }) +} diff --git a/gremlin/gremlin_session.go b/gremlin/gremlin_session.go new file mode 100644 index 0000000..a0b0483 --- /dev/null +++ b/gremlin/gremlin_session.go @@ -0,0 +1,266 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "errors" + "fmt" + "sort" + "time" + + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" +) + +type GremlinSession struct { + ts graph.TripleStore + currentChannel chan interface{} + env *otto.Otto + debug bool + limit int + count int + dataOutput []interface{} + lookingForQueryShape bool + queryShape map[string]interface{} + err error + script *otto.Script + doHalt bool + timeoutSec time.Duration + emptyEnv *otto.Otto +} + +func NewGremlinSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool) *GremlinSession { + var g GremlinSession + g.ts = inputTripleStore + g.env = BuildGremlinEnv(&g) + g.limit = -1 + g.count = 0 + g.lookingForQueryShape = false + if persist { + g.emptyEnv = g.env + } + if timeoutSec < 0 { + g.timeoutSec = time.Duration(-1) + } else { + g.timeoutSec = time.Duration(timeoutSec) + } + g.ClearJson() + return &g +} + +type GremlinResult struct { + metaresult bool + err string + val *otto.Value + actualResults *map[string]graph.TSVal +} + +func (g *GremlinSession) ToggleDebug() { + g.debug = !g.debug +} + +func (g *GremlinSession) GetQuery(input string, output_struct chan map[string]interface{}) { + defer close(output_struct) + g.queryShape = make(map[string]interface{}) + g.lookingForQueryShape = true + g.env.Run(input) + output_struct <- g.queryShape + g.queryShape = nil +} + +func (g *GremlinSession) InputParses(input string) (graph.ParseResult, error) { + script, err := g.env.Compile("", input) + if err != nil { + return graph.ParseFail, err + } + g.script = script + return graph.Parsed, nil +} + +func (g *GremlinSession) SendResult(result *GremlinResult) bool { + if g.limit >= 0 && g.limit == g.count { + return false + } + if g.doHalt { + return false + } + if g.currentChannel != nil { + g.currentChannel <- result + g.count++ + if g.limit >= 0 && g.limit == g.count { + return false + } else { + return true + } + } + return false +} + +var halt = errors.New("Query Timeout") + +func (g *GremlinSession) runUnsafe(input interface{}) (otto.Value, error) { + g.doHalt = false + defer func() { + if caught := recover(); caught != nil { + if caught == halt { + g.err = halt + return + } + panic(caught) // Something else happened, repanic! + } + }() + + g.env.Interrupt = make(chan func(), 1) // The buffer prevents blocking + + if g.timeoutSec != -1 { + go func() { + time.Sleep(g.timeoutSec * time.Second) // Stop after two seconds + g.doHalt = true + if g.env != nil { + g.env.Interrupt <- func() { + panic(halt) + } + g.env = g.emptyEnv + } + }() + } + + return g.env.Run(input) // Here be dragons (risky code) +} + +func (g *GremlinSession) ExecInput(input string, out chan interface{}, limit int) { + defer close(out) + g.err = nil + g.currentChannel = out + var err error + var value otto.Value + if g.script == nil { + value, err = g.runUnsafe(input) + } else { + value, err = g.runUnsafe(g.script) + } + if err != nil { + out <- &GremlinResult{metaresult: true, + err: err.Error(), + val: &value, + actualResults: nil} + } else { + out <- &GremlinResult{metaresult: true, + err: "", + val: &value, + actualResults: nil} + } + g.currentChannel = nil + g.script = nil + g.env = g.emptyEnv + return +} + +func (s *GremlinSession) ToText(result interface{}) string { + data := result.(*GremlinResult) + if data.metaresult { + if data.err != "" { + return fmt.Sprintln("Error: ", data.err) + } + if data.val != nil { + s, _ := data.val.Export() + if data.val.IsObject() { + typeVal, _ := data.val.Object().Get("_gremlin_type") + if !typeVal.IsUndefined() { + s = "[internal Iterator]" + } + } + return fmt.Sprintln("=>", s) + } + return "" + } + var out string + out = fmt.Sprintln("****") + if data.val == nil { + tags := data.actualResults + tagKeys := make([]string, len(*tags)) + i := 0 + for k, _ := range *tags { + tagKeys[i] = k + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + if k == "$_" { + continue + } + out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) + } + } else { + if data.val.IsObject() { + export, _ := data.val.Export() + mapExport := export.(map[string]string) + for k, v := range mapExport { + out += fmt.Sprintf("%s : %v\n", k, v) + } + } else { + strVersion, _ := data.val.ToString() + out += fmt.Sprintf("%s\n", strVersion) + } + } + return out +} + +// Web stuff +func (ses *GremlinSession) BuildJson(result interface{}) { + data := result.(*GremlinResult) + if !data.metaresult { + if data.val == nil { + obj := make(map[string]string) + tags := data.actualResults + tagKeys := make([]string, len(*tags)) + i := 0 + for k, _ := range *tags { + tagKeys[i] = k + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + obj[k] = ses.ts.GetNameFor((*tags)[k]) + } + ses.dataOutput = append(ses.dataOutput, obj) + } else { + if data.val.IsObject() { + export, _ := data.val.Export() + ses.dataOutput = append(ses.dataOutput, export) + } else { + strVersion, _ := data.val.ToString() + ses.dataOutput = append(ses.dataOutput, strVersion) + } + } + } + +} + +func (ses *GremlinSession) GetJson() (interface{}, error) { + defer ses.ClearJson() + if ses.err != nil { + return nil, ses.err + } + if ses.doHalt { + return nil, halt + } + return ses.dataOutput, nil +} + +func (ses *GremlinSession) ClearJson() { + ses.dataOutput = nil +} diff --git a/gremlin/gremlin_traversals.go b/gremlin/gremlin_traversals.go new file mode 100644 index 0000000..c0f4704 --- /dev/null +++ b/gremlin/gremlin_traversals.go @@ -0,0 +1,184 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +// Adds special traversal functions to JS Gremlin objects. Most of these just build the chain of objects, and won't often need the session. + +import ( + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" +) + +func embedTraversals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { + obj.Set("In", gremlinFunc("in", obj, env, ses)) + obj.Set("Out", gremlinFunc("out", obj, env, ses)) + obj.Set("Is", gremlinFunc("is", obj, env, ses)) + obj.Set("Both", gremlinFunc("both", obj, env, ses)) + obj.Set("Follow", gremlinFunc("follow", obj, env, ses)) + obj.Set("FollowR", gremlinFollowR("followr", obj, env, ses)) + obj.Set("And", gremlinFunc("and", obj, env, ses)) + obj.Set("Intersect", gremlinFunc("and", obj, env, ses)) + obj.Set("Union", gremlinFunc("or", obj, env, ses)) + obj.Set("Or", gremlinFunc("or", obj, env, ses)) + obj.Set("Back", gremlinBack("back", obj, env, ses)) + obj.Set("Tag", gremlinFunc("tag", obj, env, ses)) + obj.Set("As", gremlinFunc("tag", obj, env, ses)) + obj.Set("Has", gremlinFunc("has", obj, env, ses)) + obj.Set("Save", gremlinFunc("save", obj, env, ses)) + obj.Set("SaveR", gremlinFunc("saver", obj, env, ses)) +} + +func gremlinFunc(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", kind) + out.Set("_gremlin_values", call.ArgumentList) + out.Set("_gremlin_prev", prevObj) + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + embedTraversals(env, ses, out) + if isVertexChain(call.This.Object()) { + embedFinals(env, ses, out) + } + return out.Value() + } +} + +func gremlinBack(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", kind) + out.Set("_gremlin_values", call.ArgumentList) + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + var otherChain *otto.Object + var thisObj *otto.Object + if len(*outStrings) != 0 { + otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, (*outStrings)[0].(string)) + } else { + otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, "") + } + out.Set("_gremlin_prev", thisObj) + out.Set("_gremlin_back_chain", otherChain) + embedTraversals(env, ses, out) + if isVertexChain(call.This.Object()) { + embedFinals(env, ses, out) + } + return out.Value() + + } +} + +func gremlinFollowR(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", kind) + out.Set("_gremlin_values", call.ArgumentList) + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + if len(call.ArgumentList) == 0 { + return prevObj.Value() + } + arg := call.Argument(0) + if isVertexChain(arg.Object()) { + return prevObj.Value() + } + newChain, _ := reverseGremlinChainTo(call.Otto, arg.Object(), "") + out.Set("_gremlin_prev", prevObj) + out.Set("_gremlin_followr", newChain) + embedTraversals(env, ses, out) + if isVertexChain(call.This.Object()) { + embedFinals(env, ses, out) + } + return out.Value() + + } +} + +func reverseGremlinChainTo(env *otto.Otto, prevObj *otto.Object, tag string) (*otto.Object, *otto.Object) { + env.Run("var _base_object = {}") + base, err := env.Object("_base_object") + if err != nil { + glog.Error(err) + return otto.NullValue().Object(), otto.NullValue().Object() + } + if isVertexChain(prevObj) { + base.Set("_gremlin_type", "vertex") + } else { + base.Set("_gremlin_type", "morphism") + } + return reverseGremlinChainHelper(env, prevObj, base, tag) +} + +func reverseGremlinChainHelper(env *otto.Otto, chain *otto.Object, newBase *otto.Object, tag string) (*otto.Object, *otto.Object) { + kindVal, _ := chain.Get("_gremlin_type") + kind, _ := kindVal.ToString() + + if tag != "" { + if kind == "tag" { + tags := getStringArgs(chain) + for _, t := range tags { + if t == tag { + return newBase, chain + } + } + } + } + + if kind == "morphism" || kind == "vertex" { + return newBase, chain + } + var newKind string + switch kind { + case "in": + newKind = "out" + case "out": + newKind = "in" + default: + newKind = kind + } + prev, _ := chain.Get("_gremlin_prev") + env.Run("var out = {}") + out, _ := env.Object("out") + out.Set("_gremlin_type", newKind) + values, _ := chain.Get("_gremlin_values") + out.Set("_gremlin_values", values) + back, _ := chain.Get("_gremlin_back_chain") + out.Set("_gremlin_back_chain", back) + out.Set("_gremlin_prev", newBase) + strings, _ := chain.Get("string_args") + out.Set("string_args", strings) + return reverseGremlinChainHelper(env, prev.Object(), out, tag) +} + +func debugChain(obj *otto.Object) bool { + val, _ := obj.Get("_gremlin_type") + x, _ := val.ToString() + glog.V(2).Infoln(x) + val, _ = obj.Get("_gremlin_prev") + if val.IsObject() { + return debugChain(val.Object()) + } + return false +} diff --git a/http/cayley-http-docs.go b/http/cayley-http-docs.go deleted file mode 100644 index 45e747d..0000000 --- a/http/cayley-http-docs.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "fmt" - "io/ioutil" - "net/http" - "os" - - "github.com/julienschmidt/httprouter" - "github.com/russross/blackfriday" -) - -type DocRequestHandler struct { -} - -func MarkdownWithCSS(input []byte, title string) []byte { - // set up the HTML renderer - htmlFlags := 0 - htmlFlags |= blackfriday.HTML_USE_XHTML - htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS - htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS - htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES - htmlFlags |= blackfriday.HTML_COMPLETE_PAGE - renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS) - - // set up the parser - extensions := 0 - //extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS - extensions |= blackfriday.EXTENSION_TABLES - extensions |= blackfriday.EXTENSION_FENCED_CODE - extensions |= blackfriday.EXTENSION_AUTOLINK - extensions |= blackfriday.EXTENSION_STRIKETHROUGH - //extensions |= blackfriday.EXTENSION_SPACE_HEADERS - extensions |= blackfriday.EXTENSION_HEADER_IDS - extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS - - return blackfriday.Markdown(input, renderer, extensions) -} - -func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { - docpage := params.ByName("docpage") - if docpage == "" { - docpage = "Index" - } - file, err := os.Open(fmt.Sprintf("docs/%s.md", docpage)) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - data, err := ioutil.ReadAll(file) - if err != nil { - http.Error(w, err.Error(), http.StatusNoContent) - return - } - output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage)) - fmt.Fprint(w, string(output)) -} - -var markdownCSS = "/static/css/docs.css" diff --git a/http/cayley-http-query.go b/http/cayley-http-query.go deleted file mode 100644 index 4a99dd0..0000000 --- a/http/cayley-http-query.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/graph" - "github.com/google/cayley/gremlin" - "github.com/google/cayley/mql" -) - -type SuccessQueryWrapper struct { - Result interface{} `json:"result"` -} - -type ErrorQueryWrapper struct { - Error string `json:"error"` -} - -func WrapErrResult(err error) ([]byte, error) { - var wrap ErrorQueryWrapper - wrap.Error = err.Error() - return json.MarshalIndent(wrap, "", " ") -} - -func WrapResult(result interface{}) ([]byte, error) { - var wrap SuccessQueryWrapper - wrap.Result = result - return json.MarshalIndent(wrap, "", " ") -} - -func RunJsonQuery(query string, ses graph.HttpSession) (interface{}, error) { - c := make(chan interface{}, 5) - go ses.ExecInput(query, c, 100) - for res := range c { - ses.BuildJson(res) - } - return ses.GetJson() -} - -func GetQueryShape(query string, ses graph.HttpSession) ([]byte, error) { - c := make(chan map[string]interface{}, 5) - go ses.GetQuery(query, c) - var data map[string]interface{} - for res := range c { - data = res - } - return json.Marshal(data) -} - -// TODO(barakmich): Turn this into proper middleware. -func (api *Api) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - var ses graph.HttpSession - switch params.ByName("query_lang") { - case "gremlin": - ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) - case "mql": - ses = mql.NewMqlSession(api.ts) - default: - return FormatJson400(w, "Need a query language.") - } - var err error - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - code := string(bodyBytes) - result, err := ses.InputParses(code) - switch result { - case graph.Parsed: - var output interface{} - var bytes []byte - var err error - output, err = RunJsonQuery(code, ses) - if err != nil { - bytes, err = WrapErrResult(err) - http.Error(w, string(bytes), 400) - ses = nil - return 400 - } - bytes, err = WrapResult(output) - if err != nil { - ses = nil - return FormatJson400(w, err) - } - fmt.Fprint(w, string(bytes)) - ses = nil - return 200 - case graph.ParseFail: - ses = nil - return FormatJson400(w, err) - default: - ses = nil - return FormatJsonError(w, 500, "Incomplete data?") - } - http.Error(w, "", http.StatusNotFound) - ses = nil - return http.StatusNotFound -} - -func (api *Api) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - var ses graph.HttpSession - switch params.ByName("query_lang") { - case "gremlin": - ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) - case "mql": - ses = mql.NewMqlSession(api.ts) - default: - return FormatJson400(w, "Need a query language.") - } - var err error - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - code := string(bodyBytes) - result, err := ses.InputParses(code) - switch result { - case graph.Parsed: - var output []byte - var err error - output, err = GetQueryShape(code, ses) - if err != nil { - return FormatJson400(w, err) - } - fmt.Fprint(w, string(output)) - return 200 - case graph.ParseFail: - return FormatJson400(w, err) - default: - return FormatJsonError(w, 500, "Incomplete data?") - } - http.Error(w, "", http.StatusNotFound) - return http.StatusNotFound -} diff --git a/http/cayley-http-write.go b/http/cayley-http-write.go deleted file mode 100644 index 20fb60d..0000000 --- a/http/cayley-http-write.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "strconv" - - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/graph" - "github.com/google/cayley/nquads" -) - -func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) { - var tripleList []*graph.Triple - err := json.Unmarshal(jsonBody, &tripleList) - if err != nil { - return nil, err - } - for i, t := range tripleList { - if !t.IsValid() { - return nil, errors.New(fmt.Sprintf("Invalid triple at index %d. %s", i, t.ToString())) - } - } - return tripleList, nil -} - -func (api *Api) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int { - if api.config.ReadOnly { - return FormatJson400(w, "Database is read-only.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - tripleList, terr := ParseJsonToTripleList(bodyBytes) - if terr != nil { - return FormatJson400(w, terr) - } - api.ts.AddTripleSet(tripleList) - fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", len(tripleList)) - return 200 -} - -func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - if api.config.ReadOnly { - return FormatJson400(w, "Database is read-only.") - } - - formFile, _, err := r.FormFile("NQuadFile") - if err != nil { - glog.Errorln(err) - return FormatJsonError(w, 500, "Couldn't read file: "+err.Error()) - } - - defer formFile.Close() - - blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) - if blockErr != nil { - blockSize = int64(api.config.LoadSize) - } - - tChan := make(chan *graph.Triple) - go nquads.ReadNQuadsFromReader(tChan, formFile) - tripleblock := make([]*graph.Triple, blockSize) - nTriples := 0 - i := int64(0) - for t := range tChan { - tripleblock[i] = t - i++ - nTriples++ - if i == blockSize { - api.ts.AddTripleSet(tripleblock) - i = 0 - } - } - api.ts.AddTripleSet(tripleblock[0:i]) - fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", nTriples) - return 200 -} - -func (api *Api) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - if api.config.ReadOnly { - return FormatJson400(w, "Database is read-only.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - tripleList, terr := ParseJsonToTripleList(bodyBytes) - if terr != nil { - return FormatJson400(w, terr) - } - count := 0 - for _, triple := range tripleList { - api.ts.RemoveTriple(triple) - count++ - } - fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d triples.\"}", count) - return 200 -} diff --git a/http/cayley-http.go b/http/cayley-http.go deleted file mode 100644 index 68d50bb..0000000 --- a/http/cayley-http.go +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "fmt" - "html/template" - "net/http" - "time" - - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - - cfg "github.com/google/cayley/config" - "github.com/google/cayley/graph" -) - -type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int - -func LogRequest(handler ResponseHandler) httprouter.Handle { - return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { - start := time.Now() - addr := req.Header.Get("X-Real-IP") - if addr == "" { - addr = req.Header.Get("X-Forwarded-For") - if addr == "" { - addr = req.RemoteAddr - } - } - glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr) - code := handler(w, req, params) - glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start)) - - } -} - -func FormatJson400(w http.ResponseWriter, err interface{}) int { - return FormatJsonError(w, 400, err) -} - -func FormatJsonError(w http.ResponseWriter, code int, err interface{}) int { - http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code) - return code -} - -type TemplateRequestHandler struct { - templates *template.Template -} - -func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { - uiType := params.ByName("ui_type") - if r.URL.Path == "/" { - uiType = "query" - } - err := h.templates.ExecuteTemplate(w, uiType+".html", h) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -type Api struct { - config *cfg.CayleyConfig - ts graph.TripleStore -} - -func (api *Api) ApiV1(r *httprouter.Router) { - r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) - r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) - r.POST("/api/v1/write", LogRequest(api.ServeV1Write)) - r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad)) - //TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form? - r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete)) -} - -func SetupRoutes(ts graph.TripleStore, config *cfg.CayleyConfig) { - r := httprouter.New() - var templates = template.Must(template.ParseGlob("templates/*.tmpl")) - templates.ParseGlob("templates/*.html") - root := &TemplateRequestHandler{templates: templates} - docs := &DocRequestHandler{} - api := &Api{config: config, ts: ts} - api.ApiV1(r) - - //m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true})) - //r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) - r.GET("/docs/:docpage", docs.ServeHTTP) - r.GET("/ui/:ui_type", root.ServeHTTP) - r.GET("/", root.ServeHTTP) - http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) - http.Handle("/", r) -} - -func CayleyHTTP(ts graph.TripleStore, config *cfg.CayleyConfig) { - SetupRoutes(ts, config) - glog.Infof("Cayley now listening on %s:%s\n", config.ListenHost, config.ListenPort) - fmt.Printf("Cayley now listening on %s:%s\n", config.ListenHost, config.ListenPort) - err := http.ListenAndServe(fmt.Sprintf("%s:%s", config.ListenHost, config.ListenPort), nil) - if err != nil { - glog.Fatal("ListenAndServe: ", err) - } -} diff --git a/http/cayley-http_test.go b/http/cayley-http_test.go deleted file mode 100644 index d59749f..0000000 --- a/http/cayley-http_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "testing" - . "github.com/smartystreets/goconvey/convey" -) - -func TestParseJSONOkay(t *testing.T) { - Convey("Parse JSON", t, func() { - bytelist := []byte(`[ - {"subject": "foo", "predicate": "bar", "object": "baz"}, - {"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"} - ]`) - x, err := ParseJsonToTripleList(bytelist) - So(err, ShouldBeNil) - So(len(x), ShouldEqual, 2) - So(x[0].Sub, ShouldEqual, "foo") - So(x[0].Provenance, ShouldEqual, "") - So(x[1].Provenance, ShouldEqual, "graph") - }) - - Convey("Parse JSON extra field", t, func() { - bytelist := []byte(`[ - {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} - ]`) - _, err := ParseJsonToTripleList(bytelist) - So(err, ShouldBeNil) - }) -} - -func TestParseJSONFail(t *testing.T) { - Convey("Parse JSON Fail", t, func() { - bytelist := []byte(`[ - {"subject": "foo", "predicate": "bar"} - ]`) - _, err := ParseJsonToTripleList(bytelist) - So(err, ShouldNotBeNil) - }) -} diff --git a/http/cayley_http.go b/http/cayley_http.go new file mode 100644 index 0000000..68d50bb --- /dev/null +++ b/http/cayley_http.go @@ -0,0 +1,113 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "html/template" + "net/http" + "time" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + cfg "github.com/google/cayley/config" + "github.com/google/cayley/graph" +) + +type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int + +func LogRequest(handler ResponseHandler) httprouter.Handle { + return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + start := time.Now() + addr := req.Header.Get("X-Real-IP") + if addr == "" { + addr = req.Header.Get("X-Forwarded-For") + if addr == "" { + addr = req.RemoteAddr + } + } + glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr) + code := handler(w, req, params) + glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start)) + + } +} + +func FormatJson400(w http.ResponseWriter, err interface{}) int { + return FormatJsonError(w, 400, err) +} + +func FormatJsonError(w http.ResponseWriter, code int, err interface{}) int { + http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code) + return code +} + +type TemplateRequestHandler struct { + templates *template.Template +} + +func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + uiType := params.ByName("ui_type") + if r.URL.Path == "/" { + uiType = "query" + } + err := h.templates.ExecuteTemplate(w, uiType+".html", h) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +type Api struct { + config *cfg.CayleyConfig + ts graph.TripleStore +} + +func (api *Api) ApiV1(r *httprouter.Router) { + r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) + r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) + r.POST("/api/v1/write", LogRequest(api.ServeV1Write)) + r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad)) + //TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form? + r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete)) +} + +func SetupRoutes(ts graph.TripleStore, config *cfg.CayleyConfig) { + r := httprouter.New() + var templates = template.Must(template.ParseGlob("templates/*.tmpl")) + templates.ParseGlob("templates/*.html") + root := &TemplateRequestHandler{templates: templates} + docs := &DocRequestHandler{} + api := &Api{config: config, ts: ts} + api.ApiV1(r) + + //m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true})) + //r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) + r.GET("/docs/:docpage", docs.ServeHTTP) + r.GET("/ui/:ui_type", root.ServeHTTP) + r.GET("/", root.ServeHTTP) + http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) + http.Handle("/", r) +} + +func CayleyHTTP(ts graph.TripleStore, config *cfg.CayleyConfig) { + SetupRoutes(ts, config) + glog.Infof("Cayley now listening on %s:%s\n", config.ListenHost, config.ListenPort) + fmt.Printf("Cayley now listening on %s:%s\n", config.ListenHost, config.ListenPort) + err := http.ListenAndServe(fmt.Sprintf("%s:%s", config.ListenHost, config.ListenPort), nil) + if err != nil { + glog.Fatal("ListenAndServe: ", err) + } +} diff --git a/http/cayley_http_docs.go b/http/cayley_http_docs.go new file mode 100644 index 0000000..45e747d --- /dev/null +++ b/http/cayley_http_docs.go @@ -0,0 +1,73 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + + "github.com/julienschmidt/httprouter" + "github.com/russross/blackfriday" +) + +type DocRequestHandler struct { +} + +func MarkdownWithCSS(input []byte, title string) []byte { + // set up the HTML renderer + htmlFlags := 0 + htmlFlags |= blackfriday.HTML_USE_XHTML + htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS + htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS + htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES + htmlFlags |= blackfriday.HTML_COMPLETE_PAGE + renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS) + + // set up the parser + extensions := 0 + //extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS + extensions |= blackfriday.EXTENSION_TABLES + extensions |= blackfriday.EXTENSION_FENCED_CODE + extensions |= blackfriday.EXTENSION_AUTOLINK + extensions |= blackfriday.EXTENSION_STRIKETHROUGH + //extensions |= blackfriday.EXTENSION_SPACE_HEADERS + extensions |= blackfriday.EXTENSION_HEADER_IDS + extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS + + return blackfriday.Markdown(input, renderer, extensions) +} + +func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + docpage := params.ByName("docpage") + if docpage == "" { + docpage = "Index" + } + file, err := os.Open(fmt.Sprintf("docs/%s.md", docpage)) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + data, err := ioutil.ReadAll(file) + if err != nil { + http.Error(w, err.Error(), http.StatusNoContent) + return + } + output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage)) + fmt.Fprint(w, string(output)) +} + +var markdownCSS = "/static/css/docs.css" diff --git a/http/cayley_http_query.go b/http/cayley_http_query.go new file mode 100644 index 0000000..4a99dd0 --- /dev/null +++ b/http/cayley_http_query.go @@ -0,0 +1,153 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/gremlin" + "github.com/google/cayley/mql" +) + +type SuccessQueryWrapper struct { + Result interface{} `json:"result"` +} + +type ErrorQueryWrapper struct { + Error string `json:"error"` +} + +func WrapErrResult(err error) ([]byte, error) { + var wrap ErrorQueryWrapper + wrap.Error = err.Error() + return json.MarshalIndent(wrap, "", " ") +} + +func WrapResult(result interface{}) ([]byte, error) { + var wrap SuccessQueryWrapper + wrap.Result = result + return json.MarshalIndent(wrap, "", " ") +} + +func RunJsonQuery(query string, ses graph.HttpSession) (interface{}, error) { + c := make(chan interface{}, 5) + go ses.ExecInput(query, c, 100) + for res := range c { + ses.BuildJson(res) + } + return ses.GetJson() +} + +func GetQueryShape(query string, ses graph.HttpSession) ([]byte, error) { + c := make(chan map[string]interface{}, 5) + go ses.GetQuery(query, c) + var data map[string]interface{} + for res := range c { + data = res + } + return json.Marshal(data) +} + +// TODO(barakmich): Turn this into proper middleware. +func (api *Api) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + var ses graph.HttpSession + switch params.ByName("query_lang") { + case "gremlin": + ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) + case "mql": + ses = mql.NewMqlSession(api.ts) + default: + return FormatJson400(w, "Need a query language.") + } + var err error + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + code := string(bodyBytes) + result, err := ses.InputParses(code) + switch result { + case graph.Parsed: + var output interface{} + var bytes []byte + var err error + output, err = RunJsonQuery(code, ses) + if err != nil { + bytes, err = WrapErrResult(err) + http.Error(w, string(bytes), 400) + ses = nil + return 400 + } + bytes, err = WrapResult(output) + if err != nil { + ses = nil + return FormatJson400(w, err) + } + fmt.Fprint(w, string(bytes)) + ses = nil + return 200 + case graph.ParseFail: + ses = nil + return FormatJson400(w, err) + default: + ses = nil + return FormatJsonError(w, 500, "Incomplete data?") + } + http.Error(w, "", http.StatusNotFound) + ses = nil + return http.StatusNotFound +} + +func (api *Api) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + var ses graph.HttpSession + switch params.ByName("query_lang") { + case "gremlin": + ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) + case "mql": + ses = mql.NewMqlSession(api.ts) + default: + return FormatJson400(w, "Need a query language.") + } + var err error + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + code := string(bodyBytes) + result, err := ses.InputParses(code) + switch result { + case graph.Parsed: + var output []byte + var err error + output, err = GetQueryShape(code, ses) + if err != nil { + return FormatJson400(w, err) + } + fmt.Fprint(w, string(output)) + return 200 + case graph.ParseFail: + return FormatJson400(w, err) + default: + return FormatJsonError(w, 500, "Incomplete data?") + } + http.Error(w, "", http.StatusNotFound) + return http.StatusNotFound +} diff --git a/http/cayley_http_test.go b/http/cayley_http_test.go new file mode 100644 index 0000000..d59749f --- /dev/null +++ b/http/cayley_http_test.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "testing" + . "github.com/smartystreets/goconvey/convey" +) + +func TestParseJSONOkay(t *testing.T) { + Convey("Parse JSON", t, func() { + bytelist := []byte(`[ + {"subject": "foo", "predicate": "bar", "object": "baz"}, + {"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"} + ]`) + x, err := ParseJsonToTripleList(bytelist) + So(err, ShouldBeNil) + So(len(x), ShouldEqual, 2) + So(x[0].Sub, ShouldEqual, "foo") + So(x[0].Provenance, ShouldEqual, "") + So(x[1].Provenance, ShouldEqual, "graph") + }) + + Convey("Parse JSON extra field", t, func() { + bytelist := []byte(`[ + {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} + ]`) + _, err := ParseJsonToTripleList(bytelist) + So(err, ShouldBeNil) + }) +} + +func TestParseJSONFail(t *testing.T) { + Convey("Parse JSON Fail", t, func() { + bytelist := []byte(`[ + {"subject": "foo", "predicate": "bar"} + ]`) + _, err := ParseJsonToTripleList(bytelist) + So(err, ShouldNotBeNil) + }) +} diff --git a/http/cayley_http_write.go b/http/cayley_http_write.go new file mode 100644 index 0000000..20fb60d --- /dev/null +++ b/http/cayley_http_write.go @@ -0,0 +1,119 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strconv" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/nquads" +) + +func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) { + var tripleList []*graph.Triple + err := json.Unmarshal(jsonBody, &tripleList) + if err != nil { + return nil, err + } + for i, t := range tripleList { + if !t.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid triple at index %d. %s", i, t.ToString())) + } + } + return tripleList, nil +} + +func (api *Api) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int { + if api.config.ReadOnly { + return FormatJson400(w, "Database is read-only.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + tripleList, terr := ParseJsonToTripleList(bodyBytes) + if terr != nil { + return FormatJson400(w, terr) + } + api.ts.AddTripleSet(tripleList) + fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", len(tripleList)) + return 200 +} + +func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + if api.config.ReadOnly { + return FormatJson400(w, "Database is read-only.") + } + + formFile, _, err := r.FormFile("NQuadFile") + if err != nil { + glog.Errorln(err) + return FormatJsonError(w, 500, "Couldn't read file: "+err.Error()) + } + + defer formFile.Close() + + blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) + if blockErr != nil { + blockSize = int64(api.config.LoadSize) + } + + tChan := make(chan *graph.Triple) + go nquads.ReadNQuadsFromReader(tChan, formFile) + tripleblock := make([]*graph.Triple, blockSize) + nTriples := 0 + i := int64(0) + for t := range tChan { + tripleblock[i] = t + i++ + nTriples++ + if i == blockSize { + api.ts.AddTripleSet(tripleblock) + i = 0 + } + } + api.ts.AddTripleSet(tripleblock[0:i]) + fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", nTriples) + return 200 +} + +func (api *Api) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + if api.config.ReadOnly { + return FormatJson400(w, "Database is read-only.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + tripleList, terr := ParseJsonToTripleList(bodyBytes) + if terr != nil { + return FormatJson400(w, terr) + } + count := 0 + for _, triple := range tripleList { + api.ts.RemoveTriple(triple) + count++ + } + fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d triples.\"}", count) + return 200 +} diff --git a/mql/mql-build-iterator.go b/mql/mql-build-iterator.go deleted file mode 100644 index 6273696..0000000 --- a/mql/mql-build-iterator.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "errors" - "fmt" - "log" - "math" - "strings" - - "github.com/google/cayley/graph" -) - -func (m *MqlQuery) buildFixed(s string) graph.Iterator { - f := m.ses.ts.MakeFixed() - f.AddValue(m.ses.ts.GetIdFor(s)) - return f -} - -func (m *MqlQuery) buildResultIterator(path MqlPath) graph.Iterator { - all := m.ses.ts.GetNodesAllIterator() - all.AddTag(string(path)) - return graph.NewOptionalIterator(all) -} - -func (m *MqlQuery) BuildIteratorTree(query interface{}) { - m.isRepeated = make(map[MqlPath]bool) - m.queryStructure = make(map[MqlPath]map[string]interface{}) - m.queryResult = make(map[MqlResultPath]map[string]interface{}) - m.queryResult[""] = make(map[string]interface{}) - - m.it, m.err = m.buildIteratorTreeInternal(query, NewMqlPath()) - if m.err != nil { - m.isError = true - } -} - -func (m *MqlQuery) buildIteratorTreeInternal(query interface{}, path MqlPath) (graph.Iterator, error) { - var it graph.Iterator - var err error - err = nil - switch t := query.(type) { - case bool: - // for JSON booleans - // Treat the bool as a string and call it a day. - // Things which are really bool-like are special cases and will be dealt with separately. - if t { - it = m.buildFixed("true") - } - it = m.buildFixed("false") - case float64: - // for JSON numbers - // Damn you, Javascript, and your lack of integer values. - if math.Floor(t) == t { - // Treat it like an integer. - it = m.buildFixed(fmt.Sprintf("%d", t)) - } else { - it = m.buildFixed(fmt.Sprintf("%f", t)) - } - case string: - // for JSON strings - it = m.buildFixed(t) - case []interface{}: - // for JSON arrays - m.isRepeated[path] = true - if len(t) == 0 { - it = m.buildResultIterator(path) - } else if len(t) == 1 { - it, err = m.buildIteratorTreeInternal(t[0], path) - } else { - err = errors.New(fmt.Sprintf("Multiple fields at location root%s", path.DisplayString())) - } - case map[string]interface{}: - // for JSON objects - it, err = m.buildIteratorTreeMapInternal(t, path) - case nil: - it = m.buildResultIterator(path) - default: - log.Fatal("Unknown JSON type?", query) - } - if err != nil { - return nil, err - } - it.AddTag(string(path)) - return it, nil -} - -func (m *MqlQuery) buildIteratorTreeMapInternal(query map[string]interface{}, path MqlPath) (graph.Iterator, error) { - it := graph.NewAndIterator() - it.AddSubIterator(m.ses.ts.GetNodesAllIterator()) - var err error - err = nil - outputStructure := make(map[string]interface{}) - for key, subquery := range query { - outputStructure[key] = nil - reverse := false - pred := key - if strings.HasPrefix(pred, "@") { - i := strings.Index(pred, ":") - if i != -1 { - pred = pred[(i + 1):] - } - } - if strings.HasPrefix(pred, "!") { - reverse = true - pred = strings.TrimPrefix(pred, "!") - } - - // Other special constructs here - var subit graph.Iterator - if key == "id" { - subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) - if err != nil { - return nil, err - } - it.AddSubIterator(subit) - } else { - subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) - if err != nil { - return nil, err - } - subAnd := graph.NewAndIterator() - predFixed := m.ses.ts.MakeFixed() - predFixed.AddValue(m.ses.ts.GetIdFor(pred)) - subAnd.AddSubIterator(graph.NewLinksToIterator(m.ses.ts, predFixed, "p")) - if reverse { - lto := graph.NewLinksToIterator(m.ses.ts, subit, "s") - subAnd.AddSubIterator(lto) - hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "o") - it.AddSubIterator(hasa) - } else { - lto := graph.NewLinksToIterator(m.ses.ts, subit, "o") - subAnd.AddSubIterator(lto) - hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "s") - it.AddSubIterator(hasa) - } - } - } - if err != nil { - return nil, err - } - m.queryStructure[path] = outputStructure - return it, nil -} - -type MqlResultPathSlice []MqlResultPath - -func (sl MqlResultPathSlice) Len() int { - return len(sl) -} - -func (sl MqlResultPathSlice) Less(i, j int) bool { - iLen := len(strings.Split(string(sl[i]), "\x30")) - jLen := len(strings.Split(string(sl[j]), "\x30")) - if iLen < jLen { - return true - } - if iLen == jLen { - if len(string(sl[i])) < len(string(sl[j])) { - return true - } - } - return false -} - -func (sl MqlResultPathSlice) Swap(i, j int) { - sl[i], sl[j] = sl[j], sl[i] -} diff --git a/mql/mql-fill.go b/mql/mql-fill.go deleted file mode 100644 index 26de32a..0000000 --- a/mql/mql-fill.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "sort" - - "github.com/google/cayley/graph" -) - -func (m *MqlQuery) treeifyResult(tags map[string]graph.TSVal) map[MqlResultPath]string { - // Transform the map into something a little more interesting. - results := make(map[MqlPath]string) - for k, v := range tags { - results[MqlPath(k)] = m.ses.ts.GetNameFor(v) - } - resultPaths := make(map[MqlResultPath]string) - for k, v := range results { - resultPaths[k.ToResultPathFromMap(results)] = v - } - - var paths MqlResultPathSlice - - for path, _ := range resultPaths { - paths = append(paths, path) - } - - sort.Sort(paths) - - // Build Structure - for _, path := range paths { - currentPath := path.getPath() - value := resultPaths[path] - namePath := path.AppendValue(value) - if _, ok := m.queryResult[namePath]; !ok { - targetPath, key := path.splitLastPath() - if path == "" { - targetPath, key = "", value - if _, ok := m.queryResult[""][value]; !ok { - m.resultOrder = append(m.resultOrder, value) - } - } - if _, ok := m.queryStructure[currentPath]; ok { - // If there's substructure, then copy that in. - newStruct := m.copyPathStructure(currentPath) - if m.isRepeated[currentPath] && currentPath != "" { - switch t := m.queryResult[targetPath][key].(type) { - case nil: - x := make([]interface{}, 0) - x = append(x, newStruct) - m.queryResult[targetPath][key] = x - m.queryResult[namePath] = newStruct - case []interface{}: - m.queryResult[targetPath][key] = append(t, newStruct) - m.queryResult[namePath] = newStruct - } - - } else { - m.queryResult[namePath] = newStruct - m.queryResult[targetPath][key] = newStruct - } - } - } - } - - // Fill values - for _, path := range paths { - currentPath := path.getPath() - value := resultPaths[path] - namePath := path.AppendValue(value) - if _, ok := m.queryStructure[currentPath]; ok { - // We're dealing with ids. - if _, ok := m.queryResult[namePath]["id"]; ok { - m.queryResult[namePath]["id"] = value - } - } else { - // Just a value. - targetPath, key := path.splitLastPath() - if m.isRepeated[currentPath] { - switch t := m.queryResult[targetPath][key].(type) { - case nil: - x := make([]interface{}, 0) - x = append(x, value) - m.queryResult[targetPath][key] = x - case []interface{}: - m.queryResult[targetPath][key] = append(t, value) - } - - } else { - m.queryResult[targetPath][key] = value - } - } - } - - return resultPaths -} - -func (m *MqlQuery) buildResults() { - for _, v := range m.resultOrder { - m.results = append(m.results, m.queryResult[""][v]) - } -} diff --git a/mql/mql-functional_test.go b/mql/mql-functional_test.go deleted file mode 100644 index 97c2eac..0000000 --- a/mql/mql-functional_test.go +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "encoding/json" - "testing" - - . "github.com/smartystreets/goconvey/convey" - - "github.com/google/cayley/graph/memstore" -) - -// +---+ +---+ -// | A |------- ->| F |<-- -// +---+ \------>+---+-/ +---+ \--+---+ -// ------>|#B#| | | E | -// +---+-------/ >+---+ | +---+ -// | C | / v -// +---+ -/ +---+ -// ---- +---+/ |#G#| -// \-->|#D#|------------->+---+ -// +---+ -// - -func buildTripleStore() *MqlSession { - ts := memstore.MakeTestingMemstore() - return NewMqlSession(ts) -} - -func compareJsonInterfaces(actual interface{}, expected interface{}, path MqlPath, t *testing.T) { - isError := false - switch ex := expected.(type) { - case bool: - switch ac := actual.(type) { - case bool: - if ac != ex { - isError = true - } - default: - t.Log("Mismatched type") - isError = true - } - case float64: - switch ac := actual.(type) { - case float64: - if ac != ex { - isError = true - } - default: - t.Log("Mismatched type") - isError = true - } - case string: - switch ac := actual.(type) { - case string: - if ac != ex { - isError = true - } - default: - isError = true - } - case []interface{}: - switch ac := actual.(type) { - case []interface{}: - if len(ac) != len(ex) { - t.Log("Different lengths") - isError = true - } else { - for i, elem := range ex { - compareJsonInterfaces(ac[i], elem, path.Follow(string(i)), t) - } - } - default: - t.Log("Mismatched type") - isError = true - } - case map[string]interface{}: - switch ac := actual.(type) { - case map[string]interface{}: - for k, v := range ex { - actual_value, ok := ac[k] - if !ok { - t.Log("Key", k, "not in actual output.") - isError = true - } else { - compareJsonInterfaces(actual_value, v, path.Follow(string(k)), t) - } - } - default: - t.Log("Mismatched type") - isError = true - } - case nil: - switch ac := actual.(type) { - case nil: - if ac != ex { - isError = true - } - default: - t.Log("Mismatched type") - isError = true - } - default: - t.Error("Unknown JSON type?", expected) - } - - if isError { - actual_bytes, _ := json.MarshalIndent(actual, "", " ") - expected_bytes, _ := json.MarshalIndent(expected, "", " ") - t.Error(path.DisplayString(), ":\n", string(actual_bytes), "\nexpected", string(expected_bytes)) - } -} - -func runAndTestQuery(query string, expected string, t *testing.T) { - ses := buildTripleStore() - c := make(chan interface{}, 5) - go ses.ExecInput(query, c, -1) - for result := range c { - ses.BuildJson(result) - } - actual_struct, _ := ses.GetJson() - var expected_struct interface{} - json.Unmarshal([]byte(expected), &expected_struct) - compareJsonInterfaces(actual_struct, expected_struct, NewMqlPath(), t) - ses.ClearJson() -} - -func TestGetAllIds(t *testing.T) { - Convey("Should get all IDs in the database", t, func() { - query := ` - [{"id": null}] - ` - expected := ` - [ - {"id": "A"}, - {"id": "follows"}, - {"id": "B"}, - {"id": "C"}, - {"id": "D"}, - {"id": "F"}, - {"id": "G"}, - {"id": "E"}, - {"id": "status"}, - {"id": "cool"}, - {"id": "status_graph"} - ] - ` - runAndTestQuery(query, expected, t) - }) -} - -func TestGetCool(t *testing.T) { - query := ` - [{"id": null, "status": "cool"}] - ` - expected := ` - [ - {"id": "B", "status": "cool"}, - {"id": "D", "status": "cool"}, - {"id": "G", "status": "cool"} - ] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetFollowsList(t *testing.T) { - query := ` - [{"id": "C", "follows": []}] - ` - expected := ` - [{ - "id": "C", - "follows": [ - "B", "D" - ] - }] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetFollowsStruct(t *testing.T) { - query := ` - [{"id": null, "follows": {"id": null, "status": "cool"}}] - ` - expected := ` - [ - {"id": "A", "follows": {"id": "B", "status": "cool"}}, - {"id": "C", "follows": {"id": "D", "status": "cool"}}, - {"id": "D", "follows": {"id": "G", "status": "cool"}}, - {"id": "F", "follows": {"id": "G", "status": "cool"}} - ] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetFollowsReverseStructList(t *testing.T) { - query := ` - [{"id": null, "!follows": [{"id": null, "status" : "cool"}]}] - ` - expected := ` - [ - {"id": "F", "!follows": [{"id": "B", "status": "cool"}]}, - {"id": "B", "!follows": [{"id": "D", "status": "cool"}]}, - {"id": "G", "!follows": [{"id": "D", "status": "cool"}]} - ] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetRevFollowsList(t *testing.T) { - query := ` - [{"id": "F", "!follows": []}] - ` - expected := ` - [{ - "id": "F", - "!follows": [ - "B", "E" - ] - }] - ` - runAndTestQuery(query, expected, t) -} - -func TestCoFollows(t *testing.T) { - query := ` - [{"id": null, "@A:follows": "B", "@B:follows": "D"}] - ` - expected := ` - [{ - "id": "C", - "@A:follows": "B", - "@B:follows": "D" - }] - ` - runAndTestQuery(query, expected, t) -} - -func TestRevCoFollows(t *testing.T) { - query := ` - [{"id": null, "!follows": {"id": "C"}, "@a:!follows": "D"}] - ` - expected := ` - [{ - "id": "B", - "!follows": {"id": "C"}, - "@a:!follows": "D" - }] - ` - runAndTestQuery(query, expected, t) -} diff --git a/mql/mql-query.go b/mql/mql-query.go deleted file mode 100644 index 66d8179..0000000 --- a/mql/mql-query.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "fmt" - "strings" - - "github.com/google/cayley/graph" -) - -type MqlPath string -type MqlResultPath string - -type MqlQuery struct { - ses *MqlSession - it graph.Iterator - isRepeated map[MqlPath]bool - queryStructure map[MqlPath]map[string]interface{} - queryResult map[MqlResultPath]map[string]interface{} - results []interface{} - resultOrder []string - isError bool - err error -} - -func (mqlQuery *MqlQuery) copyPathStructure(path MqlPath) map[string]interface{} { - output := make(map[string]interface{}) - for k, v := range mqlQuery.queryStructure[path] { - output[k] = v - } - return output -} - -func NewMqlPath() MqlPath { - return "" -} -func (p MqlPath) Follow(s string) MqlPath { - return MqlPath(fmt.Sprintf("%s\x1E%s", p, s)) -} - -func (p MqlPath) DisplayString() string { - return strings.Replace(string(p), "\x1E", ".", -1) -} - -func NewMqlResultPath() MqlResultPath { - return "" -} - -func (p MqlResultPath) FollowPath(followPiece string, value string) MqlResultPath { - if string(p) == "" { - return MqlResultPath(fmt.Sprintf("%s\x1E%s", value, followPiece)) - } - return MqlResultPath(fmt.Sprintf("%s\x1E%s\x1E%s", p, value, followPiece)) -} - -func (p MqlResultPath) getPath() MqlPath { - out := NewMqlPath() - pathPieces := strings.Split(string(p), "\x1E") - for len(pathPieces) > 1 { - a := pathPieces[1] - pathPieces = pathPieces[2:] - out = out.Follow(a) - } - return out -} - -func (p MqlResultPath) splitLastPath() (MqlResultPath, string) { - pathPieces := strings.Split(string(p), "\x1E") - return MqlResultPath(strings.Join(pathPieces[:len(pathPieces)-1], "\x1E")), pathPieces[len(pathPieces)-1] -} - -func (p MqlResultPath) AppendValue(value string) MqlResultPath { - if string(p) == "" { - return MqlResultPath(value) - } - return MqlResultPath(fmt.Sprintf("%s\x1E%s", p, value)) -} - -func (p MqlPath) ToResultPathFromMap(resultMap map[MqlPath]string) MqlResultPath { - output := NewMqlResultPath() - pathPieces := strings.Split(string(p), "\x1E")[1:] - pathSoFar := NewMqlPath() - for _, piece := range pathPieces { - output = output.FollowPath(piece, resultMap[pathSoFar]) - pathSoFar = pathSoFar.Follow(piece) - } - return output -} - -func NewMqlQuery(ses *MqlSession) *MqlQuery { - var q MqlQuery - q.ses = ses - q.results = make([]interface{}, 0) - q.resultOrder = make([]string, 0) - q.err = nil - q.isError = false - return &q -} diff --git a/mql/mql-session.go b/mql/mql-session.go deleted file mode 100644 index 96d88bf..0000000 --- a/mql/mql-session.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "encoding/json" - "fmt" - "sort" - - "github.com/barakmich/glog" - - "github.com/google/cayley/graph" -) - -type MqlSession struct { - ts graph.TripleStore - currentQuery *MqlQuery - debug bool -} - -func NewMqlSession(ts graph.TripleStore) *MqlSession { - var m MqlSession - m.ts = ts - return &m -} - -func (m *MqlSession) ToggleDebug() { - m.debug = !m.debug -} - -func (m *MqlSession) GetQuery(input string, output_struct chan map[string]interface{}) { - defer close(output_struct) - var mqlQuery interface{} - err := json.Unmarshal([]byte(input), &mqlQuery) - if err != nil { - return - } - m.currentQuery = NewMqlQuery(m) - m.currentQuery.BuildIteratorTree(mqlQuery) - output := make(map[string]interface{}) - graph.OutputQueryShapeForIterator(m.currentQuery.it, m.ts, &output) - nodes := output["nodes"].([]graph.Node) - new_nodes := make([]graph.Node, 0) - for _, n := range nodes { - n.Tags = nil - new_nodes = append(new_nodes, n) - } - output["nodes"] = new_nodes - output_struct <- output -} - -func (m *MqlSession) InputParses(input string) (graph.ParseResult, error) { - var x interface{} - err := json.Unmarshal([]byte(input), &x) - if err != nil { - return graph.ParseFail, err - } - return graph.Parsed, nil -} - -func (m *MqlSession) ExecInput(input string, c chan interface{}, limit int) { - defer close(c) - var mqlQuery interface{} - err := json.Unmarshal([]byte(input), &mqlQuery) - if err != nil { - return - } - m.currentQuery = NewMqlQuery(m) - m.currentQuery.BuildIteratorTree(mqlQuery) - if m.currentQuery.isError { - return - } - it, _ := m.currentQuery.it.Optimize() - if glog.V(2) { - glog.V(2).Infoln(it.DebugString(0)) - } - for { - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - c <- &tags - for it.NextResult() == true { - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - c <- &tags - } - } -} - -func (m *MqlSession) ToText(result interface{}) string { - tags := *(result.(*map[string]graph.TSVal)) - out := fmt.Sprintln("****") - tagKeys := make([]string, len(tags)) - m.currentQuery.treeifyResult(tags) - m.currentQuery.buildResults() - r, _ := json.MarshalIndent(m.currentQuery.results, "", " ") - fmt.Println(string(r)) - i := 0 - for k, _ := range tags { - tagKeys[i] = string(k) - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - if k == "$_" { - continue - } - out += fmt.Sprintf("%s : %s\n", k, m.ts.GetNameFor(tags[k])) - } - return out -} - -func (m *MqlSession) BuildJson(result interface{}) { - m.currentQuery.treeifyResult(*(result.(*map[string]graph.TSVal))) -} - -func (m *MqlSession) GetJson() (interface{}, error) { - m.currentQuery.buildResults() - if m.currentQuery.isError { - return nil, m.currentQuery.err - } else { - return m.currentQuery.results, nil - } -} - -func (m *MqlSession) ClearJson() { - // Since we create a new MqlQuery underneath every query, clearing isn't necessary. - return -} diff --git a/mql/mql_build_iterator.go b/mql/mql_build_iterator.go new file mode 100644 index 0000000..6273696 --- /dev/null +++ b/mql/mql_build_iterator.go @@ -0,0 +1,181 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "errors" + "fmt" + "log" + "math" + "strings" + + "github.com/google/cayley/graph" +) + +func (m *MqlQuery) buildFixed(s string) graph.Iterator { + f := m.ses.ts.MakeFixed() + f.AddValue(m.ses.ts.GetIdFor(s)) + return f +} + +func (m *MqlQuery) buildResultIterator(path MqlPath) graph.Iterator { + all := m.ses.ts.GetNodesAllIterator() + all.AddTag(string(path)) + return graph.NewOptionalIterator(all) +} + +func (m *MqlQuery) BuildIteratorTree(query interface{}) { + m.isRepeated = make(map[MqlPath]bool) + m.queryStructure = make(map[MqlPath]map[string]interface{}) + m.queryResult = make(map[MqlResultPath]map[string]interface{}) + m.queryResult[""] = make(map[string]interface{}) + + m.it, m.err = m.buildIteratorTreeInternal(query, NewMqlPath()) + if m.err != nil { + m.isError = true + } +} + +func (m *MqlQuery) buildIteratorTreeInternal(query interface{}, path MqlPath) (graph.Iterator, error) { + var it graph.Iterator + var err error + err = nil + switch t := query.(type) { + case bool: + // for JSON booleans + // Treat the bool as a string and call it a day. + // Things which are really bool-like are special cases and will be dealt with separately. + if t { + it = m.buildFixed("true") + } + it = m.buildFixed("false") + case float64: + // for JSON numbers + // Damn you, Javascript, and your lack of integer values. + if math.Floor(t) == t { + // Treat it like an integer. + it = m.buildFixed(fmt.Sprintf("%d", t)) + } else { + it = m.buildFixed(fmt.Sprintf("%f", t)) + } + case string: + // for JSON strings + it = m.buildFixed(t) + case []interface{}: + // for JSON arrays + m.isRepeated[path] = true + if len(t) == 0 { + it = m.buildResultIterator(path) + } else if len(t) == 1 { + it, err = m.buildIteratorTreeInternal(t[0], path) + } else { + err = errors.New(fmt.Sprintf("Multiple fields at location root%s", path.DisplayString())) + } + case map[string]interface{}: + // for JSON objects + it, err = m.buildIteratorTreeMapInternal(t, path) + case nil: + it = m.buildResultIterator(path) + default: + log.Fatal("Unknown JSON type?", query) + } + if err != nil { + return nil, err + } + it.AddTag(string(path)) + return it, nil +} + +func (m *MqlQuery) buildIteratorTreeMapInternal(query map[string]interface{}, path MqlPath) (graph.Iterator, error) { + it := graph.NewAndIterator() + it.AddSubIterator(m.ses.ts.GetNodesAllIterator()) + var err error + err = nil + outputStructure := make(map[string]interface{}) + for key, subquery := range query { + outputStructure[key] = nil + reverse := false + pred := key + if strings.HasPrefix(pred, "@") { + i := strings.Index(pred, ":") + if i != -1 { + pred = pred[(i + 1):] + } + } + if strings.HasPrefix(pred, "!") { + reverse = true + pred = strings.TrimPrefix(pred, "!") + } + + // Other special constructs here + var subit graph.Iterator + if key == "id" { + subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) + if err != nil { + return nil, err + } + it.AddSubIterator(subit) + } else { + subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) + if err != nil { + return nil, err + } + subAnd := graph.NewAndIterator() + predFixed := m.ses.ts.MakeFixed() + predFixed.AddValue(m.ses.ts.GetIdFor(pred)) + subAnd.AddSubIterator(graph.NewLinksToIterator(m.ses.ts, predFixed, "p")) + if reverse { + lto := graph.NewLinksToIterator(m.ses.ts, subit, "s") + subAnd.AddSubIterator(lto) + hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "o") + it.AddSubIterator(hasa) + } else { + lto := graph.NewLinksToIterator(m.ses.ts, subit, "o") + subAnd.AddSubIterator(lto) + hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "s") + it.AddSubIterator(hasa) + } + } + } + if err != nil { + return nil, err + } + m.queryStructure[path] = outputStructure + return it, nil +} + +type MqlResultPathSlice []MqlResultPath + +func (sl MqlResultPathSlice) Len() int { + return len(sl) +} + +func (sl MqlResultPathSlice) Less(i, j int) bool { + iLen := len(strings.Split(string(sl[i]), "\x30")) + jLen := len(strings.Split(string(sl[j]), "\x30")) + if iLen < jLen { + return true + } + if iLen == jLen { + if len(string(sl[i])) < len(string(sl[j])) { + return true + } + } + return false +} + +func (sl MqlResultPathSlice) Swap(i, j int) { + sl[i], sl[j] = sl[j], sl[i] +} diff --git a/mql/mql_fill.go b/mql/mql_fill.go new file mode 100644 index 0000000..26de32a --- /dev/null +++ b/mql/mql_fill.go @@ -0,0 +1,114 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "sort" + + "github.com/google/cayley/graph" +) + +func (m *MqlQuery) treeifyResult(tags map[string]graph.TSVal) map[MqlResultPath]string { + // Transform the map into something a little more interesting. + results := make(map[MqlPath]string) + for k, v := range tags { + results[MqlPath(k)] = m.ses.ts.GetNameFor(v) + } + resultPaths := make(map[MqlResultPath]string) + for k, v := range results { + resultPaths[k.ToResultPathFromMap(results)] = v + } + + var paths MqlResultPathSlice + + for path, _ := range resultPaths { + paths = append(paths, path) + } + + sort.Sort(paths) + + // Build Structure + for _, path := range paths { + currentPath := path.getPath() + value := resultPaths[path] + namePath := path.AppendValue(value) + if _, ok := m.queryResult[namePath]; !ok { + targetPath, key := path.splitLastPath() + if path == "" { + targetPath, key = "", value + if _, ok := m.queryResult[""][value]; !ok { + m.resultOrder = append(m.resultOrder, value) + } + } + if _, ok := m.queryStructure[currentPath]; ok { + // If there's substructure, then copy that in. + newStruct := m.copyPathStructure(currentPath) + if m.isRepeated[currentPath] && currentPath != "" { + switch t := m.queryResult[targetPath][key].(type) { + case nil: + x := make([]interface{}, 0) + x = append(x, newStruct) + m.queryResult[targetPath][key] = x + m.queryResult[namePath] = newStruct + case []interface{}: + m.queryResult[targetPath][key] = append(t, newStruct) + m.queryResult[namePath] = newStruct + } + + } else { + m.queryResult[namePath] = newStruct + m.queryResult[targetPath][key] = newStruct + } + } + } + } + + // Fill values + for _, path := range paths { + currentPath := path.getPath() + value := resultPaths[path] + namePath := path.AppendValue(value) + if _, ok := m.queryStructure[currentPath]; ok { + // We're dealing with ids. + if _, ok := m.queryResult[namePath]["id"]; ok { + m.queryResult[namePath]["id"] = value + } + } else { + // Just a value. + targetPath, key := path.splitLastPath() + if m.isRepeated[currentPath] { + switch t := m.queryResult[targetPath][key].(type) { + case nil: + x := make([]interface{}, 0) + x = append(x, value) + m.queryResult[targetPath][key] = x + case []interface{}: + m.queryResult[targetPath][key] = append(t, value) + } + + } else { + m.queryResult[targetPath][key] = value + } + } + } + + return resultPaths +} + +func (m *MqlQuery) buildResults() { + for _, v := range m.resultOrder { + m.results = append(m.results, m.queryResult[""][v]) + } +} diff --git a/mql/mql_functional_test.go b/mql/mql_functional_test.go new file mode 100644 index 0000000..97c2eac --- /dev/null +++ b/mql/mql_functional_test.go @@ -0,0 +1,264 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "encoding/json" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph/memstore" +) + +// +---+ +---+ +// | A |------- ->| F |<-- +// +---+ \------>+---+-/ +---+ \--+---+ +// ------>|#B#| | | E | +// +---+-------/ >+---+ | +---+ +// | C | / v +// +---+ -/ +---+ +// ---- +---+/ |#G#| +// \-->|#D#|------------->+---+ +// +---+ +// + +func buildTripleStore() *MqlSession { + ts := memstore.MakeTestingMemstore() + return NewMqlSession(ts) +} + +func compareJsonInterfaces(actual interface{}, expected interface{}, path MqlPath, t *testing.T) { + isError := false + switch ex := expected.(type) { + case bool: + switch ac := actual.(type) { + case bool: + if ac != ex { + isError = true + } + default: + t.Log("Mismatched type") + isError = true + } + case float64: + switch ac := actual.(type) { + case float64: + if ac != ex { + isError = true + } + default: + t.Log("Mismatched type") + isError = true + } + case string: + switch ac := actual.(type) { + case string: + if ac != ex { + isError = true + } + default: + isError = true + } + case []interface{}: + switch ac := actual.(type) { + case []interface{}: + if len(ac) != len(ex) { + t.Log("Different lengths") + isError = true + } else { + for i, elem := range ex { + compareJsonInterfaces(ac[i], elem, path.Follow(string(i)), t) + } + } + default: + t.Log("Mismatched type") + isError = true + } + case map[string]interface{}: + switch ac := actual.(type) { + case map[string]interface{}: + for k, v := range ex { + actual_value, ok := ac[k] + if !ok { + t.Log("Key", k, "not in actual output.") + isError = true + } else { + compareJsonInterfaces(actual_value, v, path.Follow(string(k)), t) + } + } + default: + t.Log("Mismatched type") + isError = true + } + case nil: + switch ac := actual.(type) { + case nil: + if ac != ex { + isError = true + } + default: + t.Log("Mismatched type") + isError = true + } + default: + t.Error("Unknown JSON type?", expected) + } + + if isError { + actual_bytes, _ := json.MarshalIndent(actual, "", " ") + expected_bytes, _ := json.MarshalIndent(expected, "", " ") + t.Error(path.DisplayString(), ":\n", string(actual_bytes), "\nexpected", string(expected_bytes)) + } +} + +func runAndTestQuery(query string, expected string, t *testing.T) { + ses := buildTripleStore() + c := make(chan interface{}, 5) + go ses.ExecInput(query, c, -1) + for result := range c { + ses.BuildJson(result) + } + actual_struct, _ := ses.GetJson() + var expected_struct interface{} + json.Unmarshal([]byte(expected), &expected_struct) + compareJsonInterfaces(actual_struct, expected_struct, NewMqlPath(), t) + ses.ClearJson() +} + +func TestGetAllIds(t *testing.T) { + Convey("Should get all IDs in the database", t, func() { + query := ` + [{"id": null}] + ` + expected := ` + [ + {"id": "A"}, + {"id": "follows"}, + {"id": "B"}, + {"id": "C"}, + {"id": "D"}, + {"id": "F"}, + {"id": "G"}, + {"id": "E"}, + {"id": "status"}, + {"id": "cool"}, + {"id": "status_graph"} + ] + ` + runAndTestQuery(query, expected, t) + }) +} + +func TestGetCool(t *testing.T) { + query := ` + [{"id": null, "status": "cool"}] + ` + expected := ` + [ + {"id": "B", "status": "cool"}, + {"id": "D", "status": "cool"}, + {"id": "G", "status": "cool"} + ] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetFollowsList(t *testing.T) { + query := ` + [{"id": "C", "follows": []}] + ` + expected := ` + [{ + "id": "C", + "follows": [ + "B", "D" + ] + }] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetFollowsStruct(t *testing.T) { + query := ` + [{"id": null, "follows": {"id": null, "status": "cool"}}] + ` + expected := ` + [ + {"id": "A", "follows": {"id": "B", "status": "cool"}}, + {"id": "C", "follows": {"id": "D", "status": "cool"}}, + {"id": "D", "follows": {"id": "G", "status": "cool"}}, + {"id": "F", "follows": {"id": "G", "status": "cool"}} + ] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetFollowsReverseStructList(t *testing.T) { + query := ` + [{"id": null, "!follows": [{"id": null, "status" : "cool"}]}] + ` + expected := ` + [ + {"id": "F", "!follows": [{"id": "B", "status": "cool"}]}, + {"id": "B", "!follows": [{"id": "D", "status": "cool"}]}, + {"id": "G", "!follows": [{"id": "D", "status": "cool"}]} + ] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetRevFollowsList(t *testing.T) { + query := ` + [{"id": "F", "!follows": []}] + ` + expected := ` + [{ + "id": "F", + "!follows": [ + "B", "E" + ] + }] + ` + runAndTestQuery(query, expected, t) +} + +func TestCoFollows(t *testing.T) { + query := ` + [{"id": null, "@A:follows": "B", "@B:follows": "D"}] + ` + expected := ` + [{ + "id": "C", + "@A:follows": "B", + "@B:follows": "D" + }] + ` + runAndTestQuery(query, expected, t) +} + +func TestRevCoFollows(t *testing.T) { + query := ` + [{"id": null, "!follows": {"id": "C"}, "@a:!follows": "D"}] + ` + expected := ` + [{ + "id": "B", + "!follows": {"id": "C"}, + "@a:!follows": "D" + }] + ` + runAndTestQuery(query, expected, t) +} diff --git a/mql/mql_query.go b/mql/mql_query.go new file mode 100644 index 0000000..66d8179 --- /dev/null +++ b/mql/mql_query.go @@ -0,0 +1,111 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "fmt" + "strings" + + "github.com/google/cayley/graph" +) + +type MqlPath string +type MqlResultPath string + +type MqlQuery struct { + ses *MqlSession + it graph.Iterator + isRepeated map[MqlPath]bool + queryStructure map[MqlPath]map[string]interface{} + queryResult map[MqlResultPath]map[string]interface{} + results []interface{} + resultOrder []string + isError bool + err error +} + +func (mqlQuery *MqlQuery) copyPathStructure(path MqlPath) map[string]interface{} { + output := make(map[string]interface{}) + for k, v := range mqlQuery.queryStructure[path] { + output[k] = v + } + return output +} + +func NewMqlPath() MqlPath { + return "" +} +func (p MqlPath) Follow(s string) MqlPath { + return MqlPath(fmt.Sprintf("%s\x1E%s", p, s)) +} + +func (p MqlPath) DisplayString() string { + return strings.Replace(string(p), "\x1E", ".", -1) +} + +func NewMqlResultPath() MqlResultPath { + return "" +} + +func (p MqlResultPath) FollowPath(followPiece string, value string) MqlResultPath { + if string(p) == "" { + return MqlResultPath(fmt.Sprintf("%s\x1E%s", value, followPiece)) + } + return MqlResultPath(fmt.Sprintf("%s\x1E%s\x1E%s", p, value, followPiece)) +} + +func (p MqlResultPath) getPath() MqlPath { + out := NewMqlPath() + pathPieces := strings.Split(string(p), "\x1E") + for len(pathPieces) > 1 { + a := pathPieces[1] + pathPieces = pathPieces[2:] + out = out.Follow(a) + } + return out +} + +func (p MqlResultPath) splitLastPath() (MqlResultPath, string) { + pathPieces := strings.Split(string(p), "\x1E") + return MqlResultPath(strings.Join(pathPieces[:len(pathPieces)-1], "\x1E")), pathPieces[len(pathPieces)-1] +} + +func (p MqlResultPath) AppendValue(value string) MqlResultPath { + if string(p) == "" { + return MqlResultPath(value) + } + return MqlResultPath(fmt.Sprintf("%s\x1E%s", p, value)) +} + +func (p MqlPath) ToResultPathFromMap(resultMap map[MqlPath]string) MqlResultPath { + output := NewMqlResultPath() + pathPieces := strings.Split(string(p), "\x1E")[1:] + pathSoFar := NewMqlPath() + for _, piece := range pathPieces { + output = output.FollowPath(piece, resultMap[pathSoFar]) + pathSoFar = pathSoFar.Follow(piece) + } + return output +} + +func NewMqlQuery(ses *MqlSession) *MqlQuery { + var q MqlQuery + q.ses = ses + q.results = make([]interface{}, 0) + q.resultOrder = make([]string, 0) + q.err = nil + q.isError = false + return &q +} diff --git a/mql/mql_session.go b/mql/mql_session.go new file mode 100644 index 0000000..96d88bf --- /dev/null +++ b/mql/mql_session.go @@ -0,0 +1,144 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/barakmich/glog" + + "github.com/google/cayley/graph" +) + +type MqlSession struct { + ts graph.TripleStore + currentQuery *MqlQuery + debug bool +} + +func NewMqlSession(ts graph.TripleStore) *MqlSession { + var m MqlSession + m.ts = ts + return &m +} + +func (m *MqlSession) ToggleDebug() { + m.debug = !m.debug +} + +func (m *MqlSession) GetQuery(input string, output_struct chan map[string]interface{}) { + defer close(output_struct) + var mqlQuery interface{} + err := json.Unmarshal([]byte(input), &mqlQuery) + if err != nil { + return + } + m.currentQuery = NewMqlQuery(m) + m.currentQuery.BuildIteratorTree(mqlQuery) + output := make(map[string]interface{}) + graph.OutputQueryShapeForIterator(m.currentQuery.it, m.ts, &output) + nodes := output["nodes"].([]graph.Node) + new_nodes := make([]graph.Node, 0) + for _, n := range nodes { + n.Tags = nil + new_nodes = append(new_nodes, n) + } + output["nodes"] = new_nodes + output_struct <- output +} + +func (m *MqlSession) InputParses(input string) (graph.ParseResult, error) { + var x interface{} + err := json.Unmarshal([]byte(input), &x) + if err != nil { + return graph.ParseFail, err + } + return graph.Parsed, nil +} + +func (m *MqlSession) ExecInput(input string, c chan interface{}, limit int) { + defer close(c) + var mqlQuery interface{} + err := json.Unmarshal([]byte(input), &mqlQuery) + if err != nil { + return + } + m.currentQuery = NewMqlQuery(m) + m.currentQuery.BuildIteratorTree(mqlQuery) + if m.currentQuery.isError { + return + } + it, _ := m.currentQuery.it.Optimize() + if glog.V(2) { + glog.V(2).Infoln(it.DebugString(0)) + } + for { + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + c <- &tags + for it.NextResult() == true { + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + c <- &tags + } + } +} + +func (m *MqlSession) ToText(result interface{}) string { + tags := *(result.(*map[string]graph.TSVal)) + out := fmt.Sprintln("****") + tagKeys := make([]string, len(tags)) + m.currentQuery.treeifyResult(tags) + m.currentQuery.buildResults() + r, _ := json.MarshalIndent(m.currentQuery.results, "", " ") + fmt.Println(string(r)) + i := 0 + for k, _ := range tags { + tagKeys[i] = string(k) + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + if k == "$_" { + continue + } + out += fmt.Sprintf("%s : %s\n", k, m.ts.GetNameFor(tags[k])) + } + return out +} + +func (m *MqlSession) BuildJson(result interface{}) { + m.currentQuery.treeifyResult(*(result.(*map[string]graph.TSVal))) +} + +func (m *MqlSession) GetJson() (interface{}, error) { + m.currentQuery.buildResults() + if m.currentQuery.isError { + return nil, m.currentQuery.err + } else { + return m.currentQuery.results, nil + } +} + +func (m *MqlSession) ClearJson() { + // Since we create a new MqlQuery underneath every query, clearing isn't necessary. + return +}