From 47c9752e5ed367eabf5e3fbd4cbf33f5187f565a Mon Sep 17 00:00:00 2001 From: kortschak Date: Sat, 28 Jun 2014 12:31:33 +0930 Subject: [PATCH] Destutter filenames --- graph/leveldb/all_iterator.go | 134 +++++++ graph/leveldb/iterator.go | 212 ++++++++++ graph/leveldb/leveldb_all_iterator.go | 134 ------- graph/leveldb/leveldb_iterator.go | 212 ---------- graph/leveldb/leveldb_triplestore.go | 429 --------------------- .../leveldb_triplestore_iterator_optimize.go | 53 --- graph/leveldb/triplestore.go | 429 +++++++++++++++++++++ graph/leveldb/triplestore_iterator_optimize.go | 53 +++ graph/memstore/all_iterator.go | 45 +++ graph/memstore/iterator.go | 119 ++++++ graph/memstore/llrb_iterator.go | 119 ------ graph/memstore/memstore_all_iterator.go | 45 --- graph/memstore/memtriplestore.go | 268 ------------- graph/memstore/memtriplestore_iterator_optimize.go | 53 --- graph/memstore/memtriplestore_test.go | 138 ------- graph/memstore/triplestore.go | 268 +++++++++++++ graph/memstore/triplestore_iterator_optimize.go | 53 +++ graph/memstore/triplestore_test.go | 138 +++++++ graph/mongo/iterator.go | 181 +++++++++ graph/mongo/mongo_iterator.go | 181 --------- graph/mongo/mongo_triplestore.go | 329 ---------------- graph/mongo/mongo_triplestore_iterator_optimize.go | 53 --- graph/mongo/triplestore.go | 329 ++++++++++++++++ graph/mongo/triplestore_iterator_optimize.go | 53 +++ graph/sexp/session.go | 121 ++++++ graph/sexp/sexp_session.go | 121 ------ http/cayley_http.go | 152 -------- http/cayley_http_docs.go | 74 ---- http/cayley_http_query.go | 153 -------- http/cayley_http_test.go | 53 --- http/cayley_http_write.go | 119 ------ http/docs.go | 74 ++++ http/http.go | 152 ++++++++ http/http_test.go | 53 +++ http/query.go | 153 ++++++++ http/write.go | 119 ++++++ query/gremlin/build_iterator.go | 315 +++++++++++++++ query/gremlin/environ.go | 95 +++++ query/gremlin/finals.go | 274 +++++++++++++ query/gremlin/functional_test.go | 230 +++++++++++ query/gremlin/gremlin_build_iterator.go | 315 --------------- query/gremlin/gremlin_env.go | 95 ----- query/gremlin/gremlin_finals.go | 274 ------------- query/gremlin/gremlin_functional_test.go | 230 ----------- query/gremlin/gremlin_session.go | 266 ------------- query/gremlin/gremlin_traversals.go | 184 --------- query/gremlin/session.go | 266 +++++++++++++ query/gremlin/traversals.go | 184 +++++++++ query/mql/build_iterator.go | 181 +++++++++ query/mql/fill.go | 114 ++++++ query/mql/functional_test.go | 264 +++++++++++++ query/mql/mql_build_iterator.go | 181 --------- query/mql/mql_fill.go | 114 ------ query/mql/mql_functional_test.go | 264 ------------- query/mql/mql_query.go | 111 ------ query/mql/mql_session.go | 144 ------- query/mql/query.go | 111 ++++++ query/mql/session.go | 144 +++++++ 58 files changed, 4864 insertions(+), 4864 deletions(-) create mode 100644 graph/leveldb/all_iterator.go create mode 100644 graph/leveldb/iterator.go delete mode 100644 graph/leveldb/leveldb_all_iterator.go delete mode 100644 graph/leveldb/leveldb_iterator.go delete mode 100644 graph/leveldb/leveldb_triplestore.go delete mode 100644 graph/leveldb/leveldb_triplestore_iterator_optimize.go create mode 100644 graph/leveldb/triplestore.go create mode 100644 graph/leveldb/triplestore_iterator_optimize.go create mode 100644 graph/memstore/all_iterator.go create mode 100644 graph/memstore/iterator.go delete mode 100644 graph/memstore/llrb_iterator.go delete mode 100644 graph/memstore/memstore_all_iterator.go delete mode 100644 graph/memstore/memtriplestore.go delete mode 100644 graph/memstore/memtriplestore_iterator_optimize.go delete mode 100644 graph/memstore/memtriplestore_test.go create mode 100644 graph/memstore/triplestore.go create mode 100644 graph/memstore/triplestore_iterator_optimize.go create mode 100644 graph/memstore/triplestore_test.go create mode 100644 graph/mongo/iterator.go delete mode 100644 graph/mongo/mongo_iterator.go delete mode 100644 graph/mongo/mongo_triplestore.go delete mode 100644 graph/mongo/mongo_triplestore_iterator_optimize.go create mode 100644 graph/mongo/triplestore.go create mode 100644 graph/mongo/triplestore_iterator_optimize.go create mode 100644 graph/sexp/session.go delete mode 100644 graph/sexp/sexp_session.go delete mode 100644 http/cayley_http.go delete mode 100644 http/cayley_http_docs.go delete mode 100644 http/cayley_http_query.go delete mode 100644 http/cayley_http_test.go delete mode 100644 http/cayley_http_write.go create mode 100644 http/docs.go create mode 100644 http/http.go create mode 100644 http/http_test.go create mode 100644 http/query.go create mode 100644 http/write.go create mode 100644 query/gremlin/build_iterator.go create mode 100644 query/gremlin/environ.go create mode 100644 query/gremlin/finals.go create mode 100644 query/gremlin/functional_test.go delete mode 100644 query/gremlin/gremlin_build_iterator.go delete mode 100644 query/gremlin/gremlin_env.go delete mode 100644 query/gremlin/gremlin_finals.go delete mode 100644 query/gremlin/gremlin_functional_test.go delete mode 100644 query/gremlin/gremlin_session.go delete mode 100644 query/gremlin/gremlin_traversals.go create mode 100644 query/gremlin/session.go create mode 100644 query/gremlin/traversals.go create mode 100644 query/mql/build_iterator.go create mode 100644 query/mql/fill.go create mode 100644 query/mql/functional_test.go delete mode 100644 query/mql/mql_build_iterator.go delete mode 100644 query/mql/mql_fill.go delete mode 100644 query/mql/mql_functional_test.go delete mode 100644 query/mql/mql_query.go delete mode 100644 query/mql/mql_session.go create mode 100644 query/mql/query.go create mode 100644 query/mql/session.go diff --git a/graph/leveldb/all_iterator.go b/graph/leveldb/all_iterator.go new file mode 100644 index 0000000..4fad9d8 --- /dev/null +++ b/graph/leveldb/all_iterator.go @@ -0,0 +1,134 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package leveldb + +import ( + "bytes" + "fmt" + "strings" + + leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator" + leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" + + "github.com/google/cayley/graph" +) + +type LevelDBAllIterator struct { + graph.BaseIterator + prefix []byte + dir string + open bool + it leveldb_it.Iterator + ts *LevelDBTripleStore + ro *leveldb_opt.ReadOptions +} + +func NewLevelDBAllIterator(prefix, dir string, ts *LevelDBTripleStore) *LevelDBAllIterator { + var it LevelDBAllIterator + graph.BaseIteratorInit(&it.BaseIterator) + it.ro = &leveldb_opt.ReadOptions{} + it.ro.DontFillCache = true + it.it = ts.db.NewIterator(nil, it.ro) + it.prefix = []byte(prefix) + it.dir = dir + it.open = true + it.ts = ts + it.it.Seek(it.prefix) + if !it.it.Valid() { + it.open = false + it.it.Release() + } + return &it +} + +func (a *LevelDBAllIterator) Reset() { + if !a.open { + a.it = a.ts.db.NewIterator(nil, a.ro) + a.open = true + } + a.it.Seek(a.prefix) + if !a.it.Valid() { + a.open = false + a.it.Release() + } +} + +func (a *LevelDBAllIterator) Clone() graph.Iterator { + out := NewLevelDBAllIterator(string(a.prefix), a.dir, a.ts) + out.CopyTagsFrom(a) + return out +} + +func (a *LevelDBAllIterator) Next() (graph.TSVal, bool) { + if !a.open { + a.Last = nil + return nil, false + } + var out []byte + out = make([]byte, len(a.it.Key())) + copy(out, a.it.Key()) + a.it.Next() + if !a.it.Valid() { + a.Close() + } + if !bytes.HasPrefix(out, a.prefix) { + a.Close() + return nil, false + } + a.Last = out + return out, true +} + +func (a *LevelDBAllIterator) Check(v graph.TSVal) bool { + a.Last = v + return true +} + +func (lit *LevelDBAllIterator) Close() { + if lit.open { + lit.it.Release() + lit.open = false + } +} + +func (a *LevelDBAllIterator) Size() (int64, bool) { + size, err := a.ts.GetApproximateSizeForPrefix(a.prefix) + if err == nil { + return size, false + } + // INT64_MAX + return int64(^uint64(0) >> 1), false +} + +func (lit *LevelDBAllIterator) DebugString(indent int) string { + size, _ := lit.Size() + return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), lit.Type(), lit.Tags(), size, lit.dir, lit) +} + +func (lit *LevelDBAllIterator) Type() string { return "all" } +func (lit *LevelDBAllIterator) Sorted() bool { return false } + +func (lit *LevelDBAllIterator) Optimize() (graph.Iterator, bool) { + return lit, false +} + +func (lit *LevelDBAllIterator) GetStats() *graph.IteratorStats { + s, _ := lit.Size() + return &graph.IteratorStats{ + CheckCost: 1, + NextCost: 2, + Size: s, + } +} diff --git a/graph/leveldb/iterator.go b/graph/leveldb/iterator.go new file mode 100644 index 0000000..b657073 --- /dev/null +++ b/graph/leveldb/iterator.go @@ -0,0 +1,212 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package leveldb + +import ( + "bytes" + _ "encoding/binary" + "fmt" + "strings" + + leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator" + leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" + + "github.com/google/cayley/graph" +) + +type LevelDBIterator struct { + graph.BaseIterator + nextPrefix []byte + checkId []byte + dir string + open bool + it leveldb_it.Iterator + ts *LevelDBTripleStore + ro *leveldb_opt.ReadOptions + originalPrefix string +} + +func NewLevelDBIterator(prefix, dir string, value graph.TSVal, ts *LevelDBTripleStore) *LevelDBIterator { + var it LevelDBIterator + graph.BaseIteratorInit(&it.BaseIterator) + it.checkId = value.([]byte) + it.dir = dir + it.originalPrefix = prefix + it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size()) + it.nextPrefix = append(it.nextPrefix, []byte(prefix)...) + it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...) + it.ro = &leveldb_opt.ReadOptions{} + it.ro.DontFillCache = true + it.it = ts.db.NewIterator(nil, it.ro) + it.open = true + it.ts = ts + ok := it.it.Seek(it.nextPrefix) + if !ok { + it.open = false + it.it.Release() + } + return &it +} + +func (lit *LevelDBIterator) Reset() { + if !lit.open { + lit.it = lit.ts.db.NewIterator(nil, lit.ro) + lit.open = true + } + ok := lit.it.Seek(lit.nextPrefix) + if !ok { + lit.open = false + lit.it.Release() + } +} + +func (lit *LevelDBIterator) Clone() graph.Iterator { + out := NewLevelDBIterator(lit.originalPrefix, lit.dir, lit.checkId, lit.ts) + out.CopyTagsFrom(lit) + return out +} + +func (lit *LevelDBIterator) Close() { + if lit.open { + lit.it.Release() + lit.open = false + } +} + +func (lit *LevelDBIterator) Next() (graph.TSVal, bool) { + if lit.it == nil { + lit.Last = nil + return nil, false + } + if !lit.open { + lit.Last = nil + return nil, false + } + if !lit.it.Valid() { + lit.Last = nil + lit.Close() + return nil, false + } + if bytes.HasPrefix(lit.it.Key(), lit.nextPrefix) { + out := make([]byte, len(lit.it.Key())) + copy(out, lit.it.Key()) + lit.Last = out + ok := lit.it.Next() + if !ok { + lit.Close() + } + return out, true + } + lit.Close() + lit.Last = nil + return nil, false +} + +func GetPositionFromPrefix(prefix []byte, dir string, ts *LevelDBTripleStore) int { + if bytes.Equal(prefix, []byte("sp")) { + switch dir { + case "s": + return 2 + case "p": + return ts.hasher.Size() + 2 + case "o": + return 2*ts.hasher.Size() + 2 + case "c": + return -1 + } + } + if bytes.Equal(prefix, []byte("po")) { + switch dir { + case "s": + return 2*ts.hasher.Size() + 2 + case "p": + return 2 + case "o": + return ts.hasher.Size() + 2 + case "c": + return -1 + } + } + if bytes.Equal(prefix, []byte("os")) { + switch dir { + case "s": + return ts.hasher.Size() + 2 + case "p": + return 2*ts.hasher.Size() + 2 + case "o": + return 2 + case "c": + return -1 + } + } + if bytes.Equal(prefix, []byte("cp")) { + switch dir { + case "s": + return 2*ts.hasher.Size() + 2 + case "p": + return ts.hasher.Size() + 2 + case "o": + return 3*ts.hasher.Size() + 2 + case "c": + return 2 + } + } + panic("Notreached") +} + +func (lit *LevelDBIterator) Check(v graph.TSVal) bool { + val := v.([]byte) + if val[0] == 'z' { + return false + } + offset := GetPositionFromPrefix(val[0:2], lit.dir, lit.ts) + if offset != -1 { + if bytes.HasPrefix(val[offset:], lit.checkId[1:]) { + return true + } + } else { + nameForDir := lit.ts.GetTriple(v).Get(lit.dir) + hashForDir := lit.ts.GetIdFor(nameForDir).([]byte) + if bytes.Equal(hashForDir, lit.checkId) { + return true + } + } + return false +} + +func (lit *LevelDBIterator) Size() (int64, bool) { + return lit.ts.GetSizeFor(lit.checkId), true +} + +func (lit *LevelDBIterator) DebugString(indent int) string { + size, _ := lit.Size() + return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), lit.Type(), lit.GetUid(), lit.Tags(), lit.dir, size, lit.ts.GetNameFor(lit.checkId)) +} + +func (lit *LevelDBIterator) Type() string { return "leveldb" } +func (lit *LevelDBIterator) Sorted() bool { return false } + +func (lit *LevelDBIterator) Optimize() (graph.Iterator, bool) { + return lit, false +} + +func (lit *LevelDBIterator) GetStats() *graph.IteratorStats { + s, _ := lit.Size() + return &graph.IteratorStats{ + CheckCost: 1, + NextCost: 2, + Size: s, + } +} diff --git a/graph/leveldb/leveldb_all_iterator.go b/graph/leveldb/leveldb_all_iterator.go deleted file mode 100644 index 4fad9d8..0000000 --- a/graph/leveldb/leveldb_all_iterator.go +++ /dev/null @@ -1,134 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "bytes" - "fmt" - "strings" - - leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator" - leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" - - "github.com/google/cayley/graph" -) - -type LevelDBAllIterator struct { - graph.BaseIterator - prefix []byte - dir string - open bool - it leveldb_it.Iterator - ts *LevelDBTripleStore - ro *leveldb_opt.ReadOptions -} - -func NewLevelDBAllIterator(prefix, dir string, ts *LevelDBTripleStore) *LevelDBAllIterator { - var it LevelDBAllIterator - graph.BaseIteratorInit(&it.BaseIterator) - it.ro = &leveldb_opt.ReadOptions{} - it.ro.DontFillCache = true - it.it = ts.db.NewIterator(nil, it.ro) - it.prefix = []byte(prefix) - it.dir = dir - it.open = true - it.ts = ts - it.it.Seek(it.prefix) - if !it.it.Valid() { - it.open = false - it.it.Release() - } - return &it -} - -func (a *LevelDBAllIterator) Reset() { - if !a.open { - a.it = a.ts.db.NewIterator(nil, a.ro) - a.open = true - } - a.it.Seek(a.prefix) - if !a.it.Valid() { - a.open = false - a.it.Release() - } -} - -func (a *LevelDBAllIterator) Clone() graph.Iterator { - out := NewLevelDBAllIterator(string(a.prefix), a.dir, a.ts) - out.CopyTagsFrom(a) - return out -} - -func (a *LevelDBAllIterator) Next() (graph.TSVal, bool) { - if !a.open { - a.Last = nil - return nil, false - } - var out []byte - out = make([]byte, len(a.it.Key())) - copy(out, a.it.Key()) - a.it.Next() - if !a.it.Valid() { - a.Close() - } - if !bytes.HasPrefix(out, a.prefix) { - a.Close() - return nil, false - } - a.Last = out - return out, true -} - -func (a *LevelDBAllIterator) Check(v graph.TSVal) bool { - a.Last = v - return true -} - -func (lit *LevelDBAllIterator) Close() { - if lit.open { - lit.it.Release() - lit.open = false - } -} - -func (a *LevelDBAllIterator) Size() (int64, bool) { - size, err := a.ts.GetApproximateSizeForPrefix(a.prefix) - if err == nil { - return size, false - } - // INT64_MAX - return int64(^uint64(0) >> 1), false -} - -func (lit *LevelDBAllIterator) DebugString(indent int) string { - size, _ := lit.Size() - return fmt.Sprintf("%s(%s tags: %v leveldb size:%d %s %p)", strings.Repeat(" ", indent), lit.Type(), lit.Tags(), size, lit.dir, lit) -} - -func (lit *LevelDBAllIterator) Type() string { return "all" } -func (lit *LevelDBAllIterator) Sorted() bool { return false } - -func (lit *LevelDBAllIterator) Optimize() (graph.Iterator, bool) { - return lit, false -} - -func (lit *LevelDBAllIterator) GetStats() *graph.IteratorStats { - s, _ := lit.Size() - return &graph.IteratorStats{ - CheckCost: 1, - NextCost: 2, - Size: s, - } -} diff --git a/graph/leveldb/leveldb_iterator.go b/graph/leveldb/leveldb_iterator.go deleted file mode 100644 index b657073..0000000 --- a/graph/leveldb/leveldb_iterator.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "bytes" - _ "encoding/binary" - "fmt" - "strings" - - leveldb_it "github.com/syndtr/goleveldb/leveldb/iterator" - leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" - - "github.com/google/cayley/graph" -) - -type LevelDBIterator struct { - graph.BaseIterator - nextPrefix []byte - checkId []byte - dir string - open bool - it leveldb_it.Iterator - ts *LevelDBTripleStore - ro *leveldb_opt.ReadOptions - originalPrefix string -} - -func NewLevelDBIterator(prefix, dir string, value graph.TSVal, ts *LevelDBTripleStore) *LevelDBIterator { - var it LevelDBIterator - graph.BaseIteratorInit(&it.BaseIterator) - it.checkId = value.([]byte) - it.dir = dir - it.originalPrefix = prefix - it.nextPrefix = make([]byte, 0, 2+ts.hasher.Size()) - it.nextPrefix = append(it.nextPrefix, []byte(prefix)...) - it.nextPrefix = append(it.nextPrefix, []byte(it.checkId[1:])...) - it.ro = &leveldb_opt.ReadOptions{} - it.ro.DontFillCache = true - it.it = ts.db.NewIterator(nil, it.ro) - it.open = true - it.ts = ts - ok := it.it.Seek(it.nextPrefix) - if !ok { - it.open = false - it.it.Release() - } - return &it -} - -func (lit *LevelDBIterator) Reset() { - if !lit.open { - lit.it = lit.ts.db.NewIterator(nil, lit.ro) - lit.open = true - } - ok := lit.it.Seek(lit.nextPrefix) - if !ok { - lit.open = false - lit.it.Release() - } -} - -func (lit *LevelDBIterator) Clone() graph.Iterator { - out := NewLevelDBIterator(lit.originalPrefix, lit.dir, lit.checkId, lit.ts) - out.CopyTagsFrom(lit) - return out -} - -func (lit *LevelDBIterator) Close() { - if lit.open { - lit.it.Release() - lit.open = false - } -} - -func (lit *LevelDBIterator) Next() (graph.TSVal, bool) { - if lit.it == nil { - lit.Last = nil - return nil, false - } - if !lit.open { - lit.Last = nil - return nil, false - } - if !lit.it.Valid() { - lit.Last = nil - lit.Close() - return nil, false - } - if bytes.HasPrefix(lit.it.Key(), lit.nextPrefix) { - out := make([]byte, len(lit.it.Key())) - copy(out, lit.it.Key()) - lit.Last = out - ok := lit.it.Next() - if !ok { - lit.Close() - } - return out, true - } - lit.Close() - lit.Last = nil - return nil, false -} - -func GetPositionFromPrefix(prefix []byte, dir string, ts *LevelDBTripleStore) int { - if bytes.Equal(prefix, []byte("sp")) { - switch dir { - case "s": - return 2 - case "p": - return ts.hasher.Size() + 2 - case "o": - return 2*ts.hasher.Size() + 2 - case "c": - return -1 - } - } - if bytes.Equal(prefix, []byte("po")) { - switch dir { - case "s": - return 2*ts.hasher.Size() + 2 - case "p": - return 2 - case "o": - return ts.hasher.Size() + 2 - case "c": - return -1 - } - } - if bytes.Equal(prefix, []byte("os")) { - switch dir { - case "s": - return ts.hasher.Size() + 2 - case "p": - return 2*ts.hasher.Size() + 2 - case "o": - return 2 - case "c": - return -1 - } - } - if bytes.Equal(prefix, []byte("cp")) { - switch dir { - case "s": - return 2*ts.hasher.Size() + 2 - case "p": - return ts.hasher.Size() + 2 - case "o": - return 3*ts.hasher.Size() + 2 - case "c": - return 2 - } - } - panic("Notreached") -} - -func (lit *LevelDBIterator) Check(v graph.TSVal) bool { - val := v.([]byte) - if val[0] == 'z' { - return false - } - offset := GetPositionFromPrefix(val[0:2], lit.dir, lit.ts) - if offset != -1 { - if bytes.HasPrefix(val[offset:], lit.checkId[1:]) { - return true - } - } else { - nameForDir := lit.ts.GetTriple(v).Get(lit.dir) - hashForDir := lit.ts.GetIdFor(nameForDir).([]byte) - if bytes.Equal(hashForDir, lit.checkId) { - return true - } - } - return false -} - -func (lit *LevelDBIterator) Size() (int64, bool) { - return lit.ts.GetSizeFor(lit.checkId), true -} - -func (lit *LevelDBIterator) DebugString(indent int) string { - size, _ := lit.Size() - return fmt.Sprintf("%s(%s %d tags: %v dir: %s size:%d %s)", strings.Repeat(" ", indent), lit.Type(), lit.GetUid(), lit.Tags(), lit.dir, size, lit.ts.GetNameFor(lit.checkId)) -} - -func (lit *LevelDBIterator) Type() string { return "leveldb" } -func (lit *LevelDBIterator) Sorted() bool { return false } - -func (lit *LevelDBIterator) Optimize() (graph.Iterator, bool) { - return lit, false -} - -func (lit *LevelDBIterator) GetStats() *graph.IteratorStats { - s, _ := lit.Size() - return &graph.IteratorStats{ - CheckCost: 1, - NextCost: 2, - Size: s, - } -} diff --git a/graph/leveldb/leveldb_triplestore.go b/graph/leveldb/leveldb_triplestore.go deleted file mode 100644 index edc2e10..0000000 --- a/graph/leveldb/leveldb_triplestore.go +++ /dev/null @@ -1,429 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "bytes" - "crypto/sha1" - "encoding/binary" - "encoding/json" - "fmt" - "hash" - - "github.com/barakmich/glog" - "github.com/syndtr/goleveldb/leveldb" - leveldb_cache "github.com/syndtr/goleveldb/leveldb/cache" - leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" - leveldb_util "github.com/syndtr/goleveldb/leveldb/util" - - "github.com/google/cayley/graph" -) - -const DefaultCacheSize = 2 -const DefaultWriteBufferSize = 20 - -type LevelDBTripleStore struct { - dbOpts *leveldb_opt.Options - db *leveldb.DB - path string - open bool - size int64 - hasher hash.Hash - writeopts *leveldb_opt.WriteOptions - readopts *leveldb_opt.ReadOptions -} - -func CreateNewLevelDB(path string) bool { - opts := &leveldb_opt.Options{} - db, err := leveldb.OpenFile(path, opts) - if err != nil { - glog.Errorln("Error: couldn't create database", err) - return false - } - defer db.Close() - ts := &LevelDBTripleStore{} - ts.db = db - ts.writeopts = &leveldb_opt.WriteOptions{ - Sync: true, - } - ts.Close() - return true -} - -func NewDefaultLevelDBTripleStore(path string, options graph.OptionsDict) *LevelDBTripleStore { - var ts LevelDBTripleStore - ts.path = path - cache_size := DefaultCacheSize - if val, ok := options.GetIntKey("cache_size_mb"); ok { - cache_size = val - } - ts.dbOpts = &leveldb_opt.Options{ - BlockCache: leveldb_cache.NewLRUCache(cache_size * leveldb_opt.MiB), - } - ts.dbOpts.ErrorIfMissing = true - - write_buffer_mb := DefaultWriteBufferSize - if val, ok := options.GetIntKey("write_buffer_mb"); ok { - write_buffer_mb = val - } - ts.dbOpts.WriteBuffer = write_buffer_mb * leveldb_opt.MiB - ts.hasher = sha1.New() - ts.writeopts = &leveldb_opt.WriteOptions{ - Sync: false, - } - ts.readopts = &leveldb_opt.ReadOptions{} - db, err := leveldb.OpenFile(ts.path, ts.dbOpts) - if err != nil { - panic("Error, couldn't open! " + err.Error()) - } - ts.db = db - glog.Infoln(ts.GetStats()) - ts.getSize() - return &ts -} - -func (ts *LevelDBTripleStore) GetStats() string { - out := "" - stats, err := ts.db.GetProperty("leveldb.stats") - if err == nil { - out += fmt.Sprintln("Stats: ", stats) - } - out += fmt.Sprintln("Size: ", ts.size) - return out -} - -func (ts *LevelDBTripleStore) Size() int64 { - return ts.size -} - -func (ts *LevelDBTripleStore) createKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte { - key := make([]byte, 0, 2+(ts.hasher.Size()*3)) - key = append(key, []byte(dir1+dir2)...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...) - return key -} - -func (ts *LevelDBTripleStore) createProvKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte { - key := make([]byte, 0, 2+(ts.hasher.Size()*4)) - key = append(key, []byte("c"+dir1)...) - key = append(key, ts.convertStringToByteHash(triple.Get("c"))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...) - key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...) - return key -} - -func (ts *LevelDBTripleStore) createValueKeyFor(s string) []byte { - key := make([]byte, 0, 1+ts.hasher.Size()) - key = append(key, []byte("z")...) - key = append(key, ts.convertStringToByteHash(s)...) - return key -} - -func (ts *LevelDBTripleStore) AddTriple(t *graph.Triple) { - batch := &leveldb.Batch{} - ts.buildWrite(batch, t) - err := ts.db.Write(batch, ts.writeopts) - if err != nil { - glog.Errorf("Couldn't write to DB for triple %s", t.ToString()) - return - } - ts.size++ -} - -func (ts *LevelDBTripleStore) RemoveTriple(t *graph.Triple) { - _, err := ts.db.Get(ts.createKeyFor("s", "p", "o", t), ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - glog.Errorf("Couldn't access DB to confirm deletion") - return - } - if err == leveldb.ErrNotFound { - // No such triple in the database, forget about it. - return - } - batch := &leveldb.Batch{} - batch.Delete(ts.createKeyFor("s", "p", "o", t)) - batch.Delete(ts.createKeyFor("o", "s", "p", t)) - batch.Delete(ts.createKeyFor("p", "o", "s", t)) - ts.UpdateValueKeyBy(t.Get("s"), -1, batch) - ts.UpdateValueKeyBy(t.Get("p"), -1, batch) - ts.UpdateValueKeyBy(t.Get("o"), -1, batch) - if t.Get("c") != "" { - batch.Delete(ts.createProvKeyFor("p", "s", "o", t)) - ts.UpdateValueKeyBy(t.Get("c"), -1, batch) - } - err = ts.db.Write(batch, nil) - if err != nil { - glog.Errorf("Couldn't delete triple %s", t.ToString()) - return - } - ts.size-- -} - -func (ts *LevelDBTripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) { - bytes, err := json.Marshal(*t) - if err != nil { - glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t.ToString(), err) - return - } - batch.Put(ts.createKeyFor("s", "p", "o", t), bytes) - batch.Put(ts.createKeyFor("o", "s", "p", t), bytes) - batch.Put(ts.createKeyFor("p", "o", "s", t), bytes) - if t.Get("c") != "" { - batch.Put(ts.createProvKeyFor("p", "s", "o", t), bytes) - } -} - -func (ts *LevelDBTripleStore) buildWrite(batch *leveldb.Batch, t *graph.Triple) { - ts.buildTripleWrite(batch, t) - ts.UpdateValueKeyBy(t.Get("s"), 1, nil) - ts.UpdateValueKeyBy(t.Get("p"), 1, nil) - ts.UpdateValueKeyBy(t.Get("o"), 1, nil) - if t.Get("c") != "" { - ts.UpdateValueKeyBy(t.Get("c"), 1, nil) - } -} - -type ValueData struct { - Name string - Size int64 -} - -func (ts *LevelDBTripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) { - value := &ValueData{name, int64(amount)} - key := ts.createValueKeyFor(name) - b, err := ts.db.Get(key, ts.readopts) - - // Error getting the node from the database. - if err != nil && err != leveldb.ErrNotFound { - glog.Errorf("Error reading Value %s from the DB\n", name) - return - } - - // Node exists in the database -- unmarshal and update. - if b != nil && err != leveldb.ErrNotFound { - err = json.Unmarshal(b, value) - if err != nil { - glog.Errorln("Error: couldn't reconstruct value ", err) - return - } - value.Size += int64(amount) - } - - // Are we deleting something? - if amount < 0 { - if value.Size <= 0 { - if batch == nil { - ts.db.Delete(key, ts.writeopts) - } else { - batch.Delete(key) - } - return - } - } - - // Repackage and rewrite. - bytes, err := json.Marshal(&value) - if err != nil { - glog.Errorf("Couldn't write to buffer for value %s\n %s", name, err) - return - } - if batch == nil { - ts.db.Put(key, bytes, ts.writeopts) - } else { - batch.Put(key, bytes) - } -} - -func (ts *LevelDBTripleStore) AddTripleSet(t_s []*graph.Triple) { - batch := &leveldb.Batch{} - newTs := len(t_s) - resizeMap := make(map[string]int) - for _, t := range t_s { - ts.buildTripleWrite(batch, t) - resizeMap[t.Sub]++ - resizeMap[t.Pred]++ - resizeMap[t.Obj]++ - if t.Provenance != "" { - resizeMap[t.Provenance]++ - } - } - for k, v := range resizeMap { - ts.UpdateValueKeyBy(k, v, batch) - } - err := ts.db.Write(batch, ts.writeopts) - if err != nil { - glog.Errorf("Couldn't write to DB for tripleset") - return - } - ts.size += int64(newTs) -} - -func (ldbts *LevelDBTripleStore) Close() { - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, ldbts.size) - if err == nil { - werr := ldbts.db.Put([]byte("__size"), buf.Bytes(), ldbts.writeopts) - if werr != nil { - glog.Errorf("Couldn't write size before closing!") - } - } else { - glog.Errorf("Couldn't convert size before closing!") - } - ldbts.db.Close() - ldbts.open = false -} - -func (ts *LevelDBTripleStore) GetTriple(k graph.TSVal) *graph.Triple { - var triple graph.Triple - b, err := ts.db.Get(k.([]byte), ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - glog.Errorln("Error: couldn't get triple from DB") - return &graph.Triple{} - } - if err == leveldb.ErrNotFound { - // No harm, no foul. - return &graph.Triple{} - } - err = json.Unmarshal(b, &triple) - if err != nil { - glog.Errorln("Error: couldn't reconstruct triple") - return &graph.Triple{} - } - return &triple -} - -func (ts *LevelDBTripleStore) convertStringToByteHash(s string) []byte { - ts.hasher.Reset() - key := make([]byte, 0, ts.hasher.Size()) - ts.hasher.Write([]byte(s)) - key = ts.hasher.Sum(key) - return key -} - -func (ts *LevelDBTripleStore) GetIdFor(s string) graph.TSVal { - return ts.createValueKeyFor(s) -} - -func (ts *LevelDBTripleStore) getValueData(value_key []byte) ValueData { - var out ValueData - if glog.V(3) { - glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key) - } - b, err := ts.db.Get(value_key, ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - glog.Errorln("Error: couldn't get value from DB") - return out - } - if b != nil && err != leveldb.ErrNotFound { - err = json.Unmarshal(b, &out) - if err != nil { - glog.Errorln("Error: couldn't reconstruct value") - return ValueData{} - } - } - return out -} - -func (ts *LevelDBTripleStore) GetNameFor(k graph.TSVal) string { - if k == nil { - glog.V(2).Infoln("k was nil") - return "" - } - return ts.getValueData(k.([]byte)).Name -} - -func (ts *LevelDBTripleStore) GetSizeFor(k graph.TSVal) int64 { - if k == nil { - return 0 - } - return int64(ts.getValueData(k.([]byte)).Size) -} - -func (ts *LevelDBTripleStore) getSize() { - var size int64 - b, err := ts.db.Get([]byte("__size"), ts.readopts) - if err != nil && err != leveldb.ErrNotFound { - panic("Couldn't read size " + err.Error()) - } - if err == leveldb.ErrNotFound { - // Must be a new database. Cool - ts.size = 0 - return - } - buf := bytes.NewBuffer(b) - err = binary.Read(buf, binary.LittleEndian, &size) - if err != nil { - glog.Errorln("Error: couldn't parse size") - } - ts.size = size -} - -func (ts *LevelDBTripleStore) GetApproximateSizeForPrefix(pre []byte) (int64, error) { - limit := make([]byte, len(pre)) - copy(limit, pre) - end := len(limit) - 1 - limit[end]++ - ranges := make([]leveldb_util.Range, 1) - ranges[0].Start = pre - ranges[0].Limit = limit - sizes, err := ts.db.GetApproximateSizes(ranges) - if err == nil { - return (int64(sizes[0]) >> 6) + 1, nil - } - return 0, nil -} - -func (ts *LevelDBTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { - switch dir { - case "s": - return NewLevelDBIterator("sp", "s", val, ts) - case "p": - return NewLevelDBIterator("po", "p", val, ts) - case "o": - return NewLevelDBIterator("os", "o", val, ts) - case "c": - return NewLevelDBIterator("cp", "c", val, ts) - } - panic("Notreached " + dir) -} - -func (ts *LevelDBTripleStore) GetNodesAllIterator() graph.Iterator { - return NewLevelDBAllIterator("z", "v", ts) -} - -func (ts *LevelDBTripleStore) GetTriplesAllIterator() graph.Iterator { - return NewLevelDBAllIterator("po", "p", ts) -} - -func (ts *LevelDBTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal { - v := val.([]uint8) - offset := GetPositionFromPrefix(v[0:2], direction, ts) - if offset != -1 { - return append([]byte("z"), v[offset:offset+ts.hasher.Size()]...) - } else { - return ts.GetTriple(val).Get(direction) - } -} - -func compareBytes(a, b graph.TSVal) bool { - return bytes.Equal(a.([]uint8), b.([]uint8)) -} - -func (ts *LevelDBTripleStore) MakeFixed() *graph.FixedIterator { - return graph.NewFixedIteratorWithCompare(compareBytes) -} diff --git a/graph/leveldb/leveldb_triplestore_iterator_optimize.go b/graph/leveldb/leveldb_triplestore_iterator_optimize.go deleted file mode 100644 index 243afba..0000000 --- a/graph/leveldb/leveldb_triplestore_iterator_optimize.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package leveldb - -import ( - "github.com/google/cayley/graph" -) - -func (ts *LevelDBTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { - switch it.Type() { - case "linksto": - return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) - - } - return it, false -} - -func (ts *LevelDBTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { - l := it.GetSubIterators() - if l.Len() != 1 { - return it, false - } - primaryIt := l.Front().Value.(graph.Iterator) - if primaryIt.Type() == "fixed" { - size, _ := primaryIt.Size() - if size == 1 { - val, ok := primaryIt.Next() - if !ok { - panic("Sizes lie") - } - newIt := ts.GetTripleIterator(it.Direction(), val) - newIt.CopyTagsFrom(it) - for _, tag := range primaryIt.Tags() { - newIt.AddFixedTag(tag, val) - } - it.Close() - return newIt, true - } - } - return it, false -} diff --git a/graph/leveldb/triplestore.go b/graph/leveldb/triplestore.go new file mode 100644 index 0000000..edc2e10 --- /dev/null +++ b/graph/leveldb/triplestore.go @@ -0,0 +1,429 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package leveldb + +import ( + "bytes" + "crypto/sha1" + "encoding/binary" + "encoding/json" + "fmt" + "hash" + + "github.com/barakmich/glog" + "github.com/syndtr/goleveldb/leveldb" + leveldb_cache "github.com/syndtr/goleveldb/leveldb/cache" + leveldb_opt "github.com/syndtr/goleveldb/leveldb/opt" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + + "github.com/google/cayley/graph" +) + +const DefaultCacheSize = 2 +const DefaultWriteBufferSize = 20 + +type LevelDBTripleStore struct { + dbOpts *leveldb_opt.Options + db *leveldb.DB + path string + open bool + size int64 + hasher hash.Hash + writeopts *leveldb_opt.WriteOptions + readopts *leveldb_opt.ReadOptions +} + +func CreateNewLevelDB(path string) bool { + opts := &leveldb_opt.Options{} + db, err := leveldb.OpenFile(path, opts) + if err != nil { + glog.Errorln("Error: couldn't create database", err) + return false + } + defer db.Close() + ts := &LevelDBTripleStore{} + ts.db = db + ts.writeopts = &leveldb_opt.WriteOptions{ + Sync: true, + } + ts.Close() + return true +} + +func NewDefaultLevelDBTripleStore(path string, options graph.OptionsDict) *LevelDBTripleStore { + var ts LevelDBTripleStore + ts.path = path + cache_size := DefaultCacheSize + if val, ok := options.GetIntKey("cache_size_mb"); ok { + cache_size = val + } + ts.dbOpts = &leveldb_opt.Options{ + BlockCache: leveldb_cache.NewLRUCache(cache_size * leveldb_opt.MiB), + } + ts.dbOpts.ErrorIfMissing = true + + write_buffer_mb := DefaultWriteBufferSize + if val, ok := options.GetIntKey("write_buffer_mb"); ok { + write_buffer_mb = val + } + ts.dbOpts.WriteBuffer = write_buffer_mb * leveldb_opt.MiB + ts.hasher = sha1.New() + ts.writeopts = &leveldb_opt.WriteOptions{ + Sync: false, + } + ts.readopts = &leveldb_opt.ReadOptions{} + db, err := leveldb.OpenFile(ts.path, ts.dbOpts) + if err != nil { + panic("Error, couldn't open! " + err.Error()) + } + ts.db = db + glog.Infoln(ts.GetStats()) + ts.getSize() + return &ts +} + +func (ts *LevelDBTripleStore) GetStats() string { + out := "" + stats, err := ts.db.GetProperty("leveldb.stats") + if err == nil { + out += fmt.Sprintln("Stats: ", stats) + } + out += fmt.Sprintln("Size: ", ts.size) + return out +} + +func (ts *LevelDBTripleStore) Size() int64 { + return ts.size +} + +func (ts *LevelDBTripleStore) createKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte { + key := make([]byte, 0, 2+(ts.hasher.Size()*3)) + key = append(key, []byte(dir1+dir2)...) + key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...) + key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...) + key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...) + return key +} + +func (ts *LevelDBTripleStore) createProvKeyFor(dir1, dir2, dir3 string, triple *graph.Triple) []byte { + key := make([]byte, 0, 2+(ts.hasher.Size()*4)) + key = append(key, []byte("c"+dir1)...) + key = append(key, ts.convertStringToByteHash(triple.Get("c"))...) + key = append(key, ts.convertStringToByteHash(triple.Get(dir1))...) + key = append(key, ts.convertStringToByteHash(triple.Get(dir2))...) + key = append(key, ts.convertStringToByteHash(triple.Get(dir3))...) + return key +} + +func (ts *LevelDBTripleStore) createValueKeyFor(s string) []byte { + key := make([]byte, 0, 1+ts.hasher.Size()) + key = append(key, []byte("z")...) + key = append(key, ts.convertStringToByteHash(s)...) + return key +} + +func (ts *LevelDBTripleStore) AddTriple(t *graph.Triple) { + batch := &leveldb.Batch{} + ts.buildWrite(batch, t) + err := ts.db.Write(batch, ts.writeopts) + if err != nil { + glog.Errorf("Couldn't write to DB for triple %s", t.ToString()) + return + } + ts.size++ +} + +func (ts *LevelDBTripleStore) RemoveTriple(t *graph.Triple) { + _, err := ts.db.Get(ts.createKeyFor("s", "p", "o", t), ts.readopts) + if err != nil && err != leveldb.ErrNotFound { + glog.Errorf("Couldn't access DB to confirm deletion") + return + } + if err == leveldb.ErrNotFound { + // No such triple in the database, forget about it. + return + } + batch := &leveldb.Batch{} + batch.Delete(ts.createKeyFor("s", "p", "o", t)) + batch.Delete(ts.createKeyFor("o", "s", "p", t)) + batch.Delete(ts.createKeyFor("p", "o", "s", t)) + ts.UpdateValueKeyBy(t.Get("s"), -1, batch) + ts.UpdateValueKeyBy(t.Get("p"), -1, batch) + ts.UpdateValueKeyBy(t.Get("o"), -1, batch) + if t.Get("c") != "" { + batch.Delete(ts.createProvKeyFor("p", "s", "o", t)) + ts.UpdateValueKeyBy(t.Get("c"), -1, batch) + } + err = ts.db.Write(batch, nil) + if err != nil { + glog.Errorf("Couldn't delete triple %s", t.ToString()) + return + } + ts.size-- +} + +func (ts *LevelDBTripleStore) buildTripleWrite(batch *leveldb.Batch, t *graph.Triple) { + bytes, err := json.Marshal(*t) + if err != nil { + glog.Errorf("Couldn't write to buffer for triple %s\n %s\n", t.ToString(), err) + return + } + batch.Put(ts.createKeyFor("s", "p", "o", t), bytes) + batch.Put(ts.createKeyFor("o", "s", "p", t), bytes) + batch.Put(ts.createKeyFor("p", "o", "s", t), bytes) + if t.Get("c") != "" { + batch.Put(ts.createProvKeyFor("p", "s", "o", t), bytes) + } +} + +func (ts *LevelDBTripleStore) buildWrite(batch *leveldb.Batch, t *graph.Triple) { + ts.buildTripleWrite(batch, t) + ts.UpdateValueKeyBy(t.Get("s"), 1, nil) + ts.UpdateValueKeyBy(t.Get("p"), 1, nil) + ts.UpdateValueKeyBy(t.Get("o"), 1, nil) + if t.Get("c") != "" { + ts.UpdateValueKeyBy(t.Get("c"), 1, nil) + } +} + +type ValueData struct { + Name string + Size int64 +} + +func (ts *LevelDBTripleStore) UpdateValueKeyBy(name string, amount int, batch *leveldb.Batch) { + value := &ValueData{name, int64(amount)} + key := ts.createValueKeyFor(name) + b, err := ts.db.Get(key, ts.readopts) + + // Error getting the node from the database. + if err != nil && err != leveldb.ErrNotFound { + glog.Errorf("Error reading Value %s from the DB\n", name) + return + } + + // Node exists in the database -- unmarshal and update. + if b != nil && err != leveldb.ErrNotFound { + err = json.Unmarshal(b, value) + if err != nil { + glog.Errorln("Error: couldn't reconstruct value ", err) + return + } + value.Size += int64(amount) + } + + // Are we deleting something? + if amount < 0 { + if value.Size <= 0 { + if batch == nil { + ts.db.Delete(key, ts.writeopts) + } else { + batch.Delete(key) + } + return + } + } + + // Repackage and rewrite. + bytes, err := json.Marshal(&value) + if err != nil { + glog.Errorf("Couldn't write to buffer for value %s\n %s", name, err) + return + } + if batch == nil { + ts.db.Put(key, bytes, ts.writeopts) + } else { + batch.Put(key, bytes) + } +} + +func (ts *LevelDBTripleStore) AddTripleSet(t_s []*graph.Triple) { + batch := &leveldb.Batch{} + newTs := len(t_s) + resizeMap := make(map[string]int) + for _, t := range t_s { + ts.buildTripleWrite(batch, t) + resizeMap[t.Sub]++ + resizeMap[t.Pred]++ + resizeMap[t.Obj]++ + if t.Provenance != "" { + resizeMap[t.Provenance]++ + } + } + for k, v := range resizeMap { + ts.UpdateValueKeyBy(k, v, batch) + } + err := ts.db.Write(batch, ts.writeopts) + if err != nil { + glog.Errorf("Couldn't write to DB for tripleset") + return + } + ts.size += int64(newTs) +} + +func (ldbts *LevelDBTripleStore) Close() { + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.LittleEndian, ldbts.size) + if err == nil { + werr := ldbts.db.Put([]byte("__size"), buf.Bytes(), ldbts.writeopts) + if werr != nil { + glog.Errorf("Couldn't write size before closing!") + } + } else { + glog.Errorf("Couldn't convert size before closing!") + } + ldbts.db.Close() + ldbts.open = false +} + +func (ts *LevelDBTripleStore) GetTriple(k graph.TSVal) *graph.Triple { + var triple graph.Triple + b, err := ts.db.Get(k.([]byte), ts.readopts) + if err != nil && err != leveldb.ErrNotFound { + glog.Errorln("Error: couldn't get triple from DB") + return &graph.Triple{} + } + if err == leveldb.ErrNotFound { + // No harm, no foul. + return &graph.Triple{} + } + err = json.Unmarshal(b, &triple) + if err != nil { + glog.Errorln("Error: couldn't reconstruct triple") + return &graph.Triple{} + } + return &triple +} + +func (ts *LevelDBTripleStore) convertStringToByteHash(s string) []byte { + ts.hasher.Reset() + key := make([]byte, 0, ts.hasher.Size()) + ts.hasher.Write([]byte(s)) + key = ts.hasher.Sum(key) + return key +} + +func (ts *LevelDBTripleStore) GetIdFor(s string) graph.TSVal { + return ts.createValueKeyFor(s) +} + +func (ts *LevelDBTripleStore) getValueData(value_key []byte) ValueData { + var out ValueData + if glog.V(3) { + glog.V(3).Infof("%s %v\n", string(value_key[0]), value_key) + } + b, err := ts.db.Get(value_key, ts.readopts) + if err != nil && err != leveldb.ErrNotFound { + glog.Errorln("Error: couldn't get value from DB") + return out + } + if b != nil && err != leveldb.ErrNotFound { + err = json.Unmarshal(b, &out) + if err != nil { + glog.Errorln("Error: couldn't reconstruct value") + return ValueData{} + } + } + return out +} + +func (ts *LevelDBTripleStore) GetNameFor(k graph.TSVal) string { + if k == nil { + glog.V(2).Infoln("k was nil") + return "" + } + return ts.getValueData(k.([]byte)).Name +} + +func (ts *LevelDBTripleStore) GetSizeFor(k graph.TSVal) int64 { + if k == nil { + return 0 + } + return int64(ts.getValueData(k.([]byte)).Size) +} + +func (ts *LevelDBTripleStore) getSize() { + var size int64 + b, err := ts.db.Get([]byte("__size"), ts.readopts) + if err != nil && err != leveldb.ErrNotFound { + panic("Couldn't read size " + err.Error()) + } + if err == leveldb.ErrNotFound { + // Must be a new database. Cool + ts.size = 0 + return + } + buf := bytes.NewBuffer(b) + err = binary.Read(buf, binary.LittleEndian, &size) + if err != nil { + glog.Errorln("Error: couldn't parse size") + } + ts.size = size +} + +func (ts *LevelDBTripleStore) GetApproximateSizeForPrefix(pre []byte) (int64, error) { + limit := make([]byte, len(pre)) + copy(limit, pre) + end := len(limit) - 1 + limit[end]++ + ranges := make([]leveldb_util.Range, 1) + ranges[0].Start = pre + ranges[0].Limit = limit + sizes, err := ts.db.GetApproximateSizes(ranges) + if err == nil { + return (int64(sizes[0]) >> 6) + 1, nil + } + return 0, nil +} + +func (ts *LevelDBTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { + switch dir { + case "s": + return NewLevelDBIterator("sp", "s", val, ts) + case "p": + return NewLevelDBIterator("po", "p", val, ts) + case "o": + return NewLevelDBIterator("os", "o", val, ts) + case "c": + return NewLevelDBIterator("cp", "c", val, ts) + } + panic("Notreached " + dir) +} + +func (ts *LevelDBTripleStore) GetNodesAllIterator() graph.Iterator { + return NewLevelDBAllIterator("z", "v", ts) +} + +func (ts *LevelDBTripleStore) GetTriplesAllIterator() graph.Iterator { + return NewLevelDBAllIterator("po", "p", ts) +} + +func (ts *LevelDBTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal { + v := val.([]uint8) + offset := GetPositionFromPrefix(v[0:2], direction, ts) + if offset != -1 { + return append([]byte("z"), v[offset:offset+ts.hasher.Size()]...) + } else { + return ts.GetTriple(val).Get(direction) + } +} + +func compareBytes(a, b graph.TSVal) bool { + return bytes.Equal(a.([]uint8), b.([]uint8)) +} + +func (ts *LevelDBTripleStore) MakeFixed() *graph.FixedIterator { + return graph.NewFixedIteratorWithCompare(compareBytes) +} diff --git a/graph/leveldb/triplestore_iterator_optimize.go b/graph/leveldb/triplestore_iterator_optimize.go new file mode 100644 index 0000000..243afba --- /dev/null +++ b/graph/leveldb/triplestore_iterator_optimize.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package leveldb + +import ( + "github.com/google/cayley/graph" +) + +func (ts *LevelDBTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { + switch it.Type() { + case "linksto": + return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) + + } + return it, false +} + +func (ts *LevelDBTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { + l := it.GetSubIterators() + if l.Len() != 1 { + return it, false + } + primaryIt := l.Front().Value.(graph.Iterator) + if primaryIt.Type() == "fixed" { + size, _ := primaryIt.Size() + if size == 1 { + val, ok := primaryIt.Next() + if !ok { + panic("Sizes lie") + } + newIt := ts.GetTripleIterator(it.Direction(), val) + newIt.CopyTagsFrom(it) + for _, tag := range primaryIt.Tags() { + newIt.AddFixedTag(tag, val) + } + it.Close() + return newIt, true + } + } + return it, false +} diff --git a/graph/memstore/all_iterator.go b/graph/memstore/all_iterator.go new file mode 100644 index 0000000..99cf734 --- /dev/null +++ b/graph/memstore/all_iterator.go @@ -0,0 +1,45 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "github.com/google/cayley/graph" +) + +type MemstoreAllIterator struct { + graph.Int64AllIterator + ts *MemTripleStore +} + +func NewMemstoreAllIterator(ts *MemTripleStore) *MemstoreAllIterator { + var out MemstoreAllIterator + out.Int64AllIterator = *graph.NewInt64AllIterator(1, ts.idCounter-1) + out.ts = ts + return &out +} + +func (memall *MemstoreAllIterator) Next() (graph.TSVal, bool) { + next, out := memall.Int64AllIterator.Next() + if !out { + return next, out + } + i64 := next.(int64) + _, ok := memall.ts.revIdMap[i64] + if !ok { + return memall.Next() + } + memall.Last = next + return next, out +} diff --git a/graph/memstore/iterator.go b/graph/memstore/iterator.go new file mode 100644 index 0000000..692a3c6 --- /dev/null +++ b/graph/memstore/iterator.go @@ -0,0 +1,119 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "fmt" + "math" + "strings" + + "github.com/petar/GoLLRB/llrb" + + "github.com/google/cayley/graph" +) + +type LlrbIterator struct { + graph.BaseIterator + tree *llrb.LLRB + data string + isRunning bool + iterLast Int64 +} + +type Int64 int64 + +func (i Int64) Less(than llrb.Item) bool { + return i < than.(Int64) +} + +func IterateOne(tree *llrb.LLRB, last Int64) Int64 { + var next Int64 + tree.AscendGreaterOrEqual(last, func(i llrb.Item) bool { + if i.(Int64) == last { + return true + } else { + next = i.(Int64) + return false + } + }) + return next +} + +func NewLlrbIterator(tree *llrb.LLRB, data string) *LlrbIterator { + var it LlrbIterator + graph.BaseIteratorInit(&it.BaseIterator) + it.tree = tree + it.iterLast = Int64(-1) + it.data = data + return &it +} + +func (it *LlrbIterator) Reset() { + it.iterLast = Int64(-1) +} + +func (it *LlrbIterator) Clone() graph.Iterator { + var new_it = NewLlrbIterator(it.tree, it.data) + new_it.CopyTagsFrom(it) + return new_it +} + +func (it *LlrbIterator) Close() {} + +func (it *LlrbIterator) Next() (graph.TSVal, bool) { + graph.NextLogIn(it) + if it.tree.Max() == nil || it.Last == int64(it.tree.Max().(Int64)) { + return graph.NextLogOut(it, nil, false) + } + it.iterLast = IterateOne(it.tree, it.iterLast) + it.Last = int64(it.iterLast) + return graph.NextLogOut(it, it.Last, true) +} + +func (it *LlrbIterator) Size() (int64, bool) { + return int64(it.tree.Len()), true +} + +func (it *LlrbIterator) Check(v graph.TSVal) bool { + graph.CheckLogIn(it, v) + if it.tree.Has(Int64(v.(int64))) { + it.Last = v + return graph.CheckLogOut(it, v, true) + } + return graph.CheckLogOut(it, v, false) +} + +func (it *LlrbIterator) DebugString(indent int) string { + size, _ := it.Size() + return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data) +} + +func (it *LlrbIterator) Type() string { + return "llrb" +} +func (it *LlrbIterator) Sorted() bool { + return true +} +func (it *LlrbIterator) Optimize() (graph.Iterator, bool) { + return it, false +} + +func (it *LlrbIterator) GetStats() *graph.IteratorStats { + return &graph.IteratorStats{ + CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1, + NextCost: 1, + Size: int64(it.tree.Len()), + } +} diff --git a/graph/memstore/llrb_iterator.go b/graph/memstore/llrb_iterator.go deleted file mode 100644 index 692a3c6..0000000 --- a/graph/memstore/llrb_iterator.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "fmt" - "math" - "strings" - - "github.com/petar/GoLLRB/llrb" - - "github.com/google/cayley/graph" -) - -type LlrbIterator struct { - graph.BaseIterator - tree *llrb.LLRB - data string - isRunning bool - iterLast Int64 -} - -type Int64 int64 - -func (i Int64) Less(than llrb.Item) bool { - return i < than.(Int64) -} - -func IterateOne(tree *llrb.LLRB, last Int64) Int64 { - var next Int64 - tree.AscendGreaterOrEqual(last, func(i llrb.Item) bool { - if i.(Int64) == last { - return true - } else { - next = i.(Int64) - return false - } - }) - return next -} - -func NewLlrbIterator(tree *llrb.LLRB, data string) *LlrbIterator { - var it LlrbIterator - graph.BaseIteratorInit(&it.BaseIterator) - it.tree = tree - it.iterLast = Int64(-1) - it.data = data - return &it -} - -func (it *LlrbIterator) Reset() { - it.iterLast = Int64(-1) -} - -func (it *LlrbIterator) Clone() graph.Iterator { - var new_it = NewLlrbIterator(it.tree, it.data) - new_it.CopyTagsFrom(it) - return new_it -} - -func (it *LlrbIterator) Close() {} - -func (it *LlrbIterator) Next() (graph.TSVal, bool) { - graph.NextLogIn(it) - if it.tree.Max() == nil || it.Last == int64(it.tree.Max().(Int64)) { - return graph.NextLogOut(it, nil, false) - } - it.iterLast = IterateOne(it.tree, it.iterLast) - it.Last = int64(it.iterLast) - return graph.NextLogOut(it, it.Last, true) -} - -func (it *LlrbIterator) Size() (int64, bool) { - return int64(it.tree.Len()), true -} - -func (it *LlrbIterator) Check(v graph.TSVal) bool { - graph.CheckLogIn(it, v) - if it.tree.Has(Int64(v.(int64))) { - it.Last = v - return graph.CheckLogOut(it, v, true) - } - return graph.CheckLogOut(it, v, false) -} - -func (it *LlrbIterator) DebugString(indent int) string { - size, _ := it.Size() - return fmt.Sprintf("%s(%s tags:%s size:%d %s)", strings.Repeat(" ", indent), it.Type(), it.Tags(), size, it.data) -} - -func (it *LlrbIterator) Type() string { - return "llrb" -} -func (it *LlrbIterator) Sorted() bool { - return true -} -func (it *LlrbIterator) Optimize() (graph.Iterator, bool) { - return it, false -} - -func (it *LlrbIterator) GetStats() *graph.IteratorStats { - return &graph.IteratorStats{ - CheckCost: int64(math.Log(float64(it.tree.Len()))) + 1, - NextCost: 1, - Size: int64(it.tree.Len()), - } -} diff --git a/graph/memstore/memstore_all_iterator.go b/graph/memstore/memstore_all_iterator.go deleted file mode 100644 index 99cf734..0000000 --- a/graph/memstore/memstore_all_iterator.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "github.com/google/cayley/graph" -) - -type MemstoreAllIterator struct { - graph.Int64AllIterator - ts *MemTripleStore -} - -func NewMemstoreAllIterator(ts *MemTripleStore) *MemstoreAllIterator { - var out MemstoreAllIterator - out.Int64AllIterator = *graph.NewInt64AllIterator(1, ts.idCounter-1) - out.ts = ts - return &out -} - -func (memall *MemstoreAllIterator) Next() (graph.TSVal, bool) { - next, out := memall.Int64AllIterator.Next() - if !out { - return next, out - } - i64 := next.(int64) - _, ok := memall.ts.revIdMap[i64] - if !ok { - return memall.Next() - } - memall.Last = next - return next, out -} diff --git a/graph/memstore/memtriplestore.go b/graph/memstore/memtriplestore.go deleted file mode 100644 index 7c7882e..0000000 --- a/graph/memstore/memtriplestore.go +++ /dev/null @@ -1,268 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "fmt" - - "github.com/barakmich/glog" - "github.com/google/cayley/graph" - - "github.com/petar/GoLLRB/llrb" -) - -type TripleDirectionIndex struct { - subject map[int64]*llrb.LLRB - predicate map[int64]*llrb.LLRB - object map[int64]*llrb.LLRB - provenance map[int64]*llrb.LLRB -} - -func NewTripleDirectionIndex() *TripleDirectionIndex { - var tdi TripleDirectionIndex - tdi.subject = make(map[int64]*llrb.LLRB) - tdi.predicate = make(map[int64]*llrb.LLRB) - tdi.object = make(map[int64]*llrb.LLRB) - tdi.provenance = make(map[int64]*llrb.LLRB) - return &tdi -} - -func (tdi *TripleDirectionIndex) GetForDir(s string) map[int64]*llrb.LLRB { - if s == "s" { - return tdi.subject - } else if s == "o" { - return tdi.object - } else if s == "p" { - return tdi.predicate - } else if s == "c" { - return tdi.provenance - } - panic("Bad direction") -} - -func (tdi *TripleDirectionIndex) GetOrCreate(dir string, id int64) *llrb.LLRB { - directionIndex := tdi.GetForDir(dir) - if _, ok := directionIndex[id]; !ok { - directionIndex[id] = llrb.New() - } - return directionIndex[id] -} - -func (tdi *TripleDirectionIndex) Get(dir string, id int64) (*llrb.LLRB, bool) { - directionIndex := tdi.GetForDir(dir) - tree, exists := directionIndex[id] - return tree, exists -} - -type MemTripleStore struct { - idCounter int64 - tripleIdCounter int64 - idMap map[string]int64 - revIdMap map[int64]string - triples []graph.Triple - size int64 - index TripleDirectionIndex - // vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree -} - -func NewMemTripleStore() *MemTripleStore { - var ts MemTripleStore - ts.idMap = make(map[string]int64) - ts.revIdMap = make(map[int64]string) - ts.triples = make([]graph.Triple, 1, 200) - - // Sentinel null triple so triple indices start at 1 - ts.triples[0] = graph.Triple{} - ts.size = 1 - ts.index = *NewTripleDirectionIndex() - ts.idCounter = 1 - ts.tripleIdCounter = 1 - return &ts -} - -func (ts *MemTripleStore) AddTripleSet(triples []*graph.Triple) { - for _, t := range triples { - ts.AddTriple(t) - } -} - -func (ts *MemTripleStore) tripleExists(t *graph.Triple) (bool, int64) { - smallest := -1 - var smallest_tree *llrb.LLRB - for _, dir := range graph.TripleDirections { - sid := t.Get(dir) - if dir == "c" && sid == "" { - continue - } - id, ok := ts.idMap[sid] - // If we've never heard about a node, it most not exist - if !ok { - return false, 0 - } - index, exists := ts.index.Get(dir, id) - if !exists { - // If it's never been indexed in this direction, it can't exist. - return false, 0 - } - if smallest == -1 || index.Len() < smallest { - smallest = index.Len() - smallest_tree = index - } - } - it := NewLlrbIterator(smallest_tree, "") - - for { - val, ok := it.Next() - if !ok { - break - } - if t.Equals(&ts.triples[val.(int64)]) { - return true, val.(int64) - } - } - return false, 0 -} - -func (ts *MemTripleStore) AddTriple(t *graph.Triple) { - if exists, _ := ts.tripleExists(t); exists { - return - } - var tripleID int64 - ts.triples = append(ts.triples, *t) - tripleID = ts.tripleIdCounter - ts.size++ - ts.tripleIdCounter++ - - for _, dir := range graph.TripleDirections { - sid := t.Get(dir) - if dir == "c" && sid == "" { - continue - } - if _, ok := ts.idMap[sid]; !ok { - ts.idMap[sid] = ts.idCounter - ts.revIdMap[ts.idCounter] = sid - ts.idCounter++ - } - } - - for _, dir := range graph.TripleDirections { - if dir == "c" && t.Get(dir) == "" { - continue - } - id := ts.idMap[t.Get(dir)] - tree := ts.index.GetOrCreate(dir, id) - tree.ReplaceOrInsert(Int64(tripleID)) - } - - // TODO(barakmich): Add VIP indexing -} - -func (ts *MemTripleStore) RemoveTriple(t *graph.Triple) { - var tripleID int64 - var exists bool - tripleID = 0 - if exists, tripleID = ts.tripleExists(t); !exists { - return - } - - ts.triples[tripleID] = graph.Triple{} - ts.size-- - - for _, dir := range graph.TripleDirections { - if dir == "c" && t.Get(dir) == "" { - continue - } - id := ts.idMap[t.Get(dir)] - tree := ts.index.GetOrCreate(dir, id) - tree.Delete(Int64(tripleID)) - } - - for _, dir := range graph.TripleDirections { - if dir == "c" && t.Get(dir) == "" { - continue - } - id, ok := ts.idMap[t.Get(dir)] - if !ok { - continue - } - stillExists := false - for _, dir := range graph.TripleDirections { - if dir == "c" && t.Get(dir) == "" { - continue - } - nodeTree := ts.index.GetOrCreate(dir, id) - if nodeTree.Len() != 0 { - stillExists = true - break - } - } - if !stillExists { - delete(ts.idMap, t.Get(dir)) - delete(ts.revIdMap, id) - } - } -} - -func (ts *MemTripleStore) GetTriple(index graph.TSVal) *graph.Triple { - return &ts.triples[index.(int64)] -} - -func (ts *MemTripleStore) GetTripleIterator(direction string, value graph.TSVal) graph.Iterator { - index, ok := ts.index.Get(direction, value.(int64)) - data := fmt.Sprintf("dir:%s val:%d", direction, value.(int64)) - if ok { - return NewLlrbIterator(index, data) - } - return &graph.NullIterator{} -} - -func (ts *MemTripleStore) Size() int64 { - return ts.size - 1 // Don't count the sentinel -} - -func (ts *MemTripleStore) DebugPrint() { - for i, t := range ts.triples { - if i == 0 { - continue - } - glog.V(2).Infoln("%d: %s", i, t.ToString()) - } -} - -func (ts *MemTripleStore) GetIdFor(name string) graph.TSVal { - return ts.idMap[name] -} - -func (ts *MemTripleStore) GetNameFor(id graph.TSVal) string { - return ts.revIdMap[id.(int64)] -} - -func (ts *MemTripleStore) GetTriplesAllIterator() graph.Iterator { - return graph.NewInt64AllIterator(0, ts.Size()) -} - -func (ts *MemTripleStore) MakeFixed() *graph.FixedIterator { - return graph.NewFixedIteratorWithCompare(graph.BasicEquality) -} - -func (ts *MemTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal { - name := ts.GetTriple(val).Get(direction) - return ts.GetIdFor(name) -} - -func (ts *MemTripleStore) GetNodesAllIterator() graph.Iterator { - return NewMemstoreAllIterator(ts) -} -func (ts *MemTripleStore) Close() {} diff --git a/graph/memstore/memtriplestore_iterator_optimize.go b/graph/memstore/memtriplestore_iterator_optimize.go deleted file mode 100644 index 7c895fc..0000000 --- a/graph/memstore/memtriplestore_iterator_optimize.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "github.com/google/cayley/graph" -) - -func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { - switch it.Type() { - case "linksto": - return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) - - } - return it, false -} - -func (ts *MemTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { - l := it.GetSubIterators() - if l.Len() != 1 { - return it, false - } - primaryIt := l.Front().Value.(graph.Iterator) - if primaryIt.Type() == "fixed" { - size, _ := primaryIt.Size() - if size == 1 { - val, ok := primaryIt.Next() - if !ok { - panic("Sizes lie") - } - newIt := ts.GetTripleIterator(it.Direction(), val) - newIt.CopyTagsFrom(it) - for _, tag := range primaryIt.Tags() { - newIt.AddFixedTag(tag, val) - } - return newIt, true - } - } - it.Close() - return it, false -} diff --git a/graph/memstore/memtriplestore_test.go b/graph/memstore/memtriplestore_test.go deleted file mode 100644 index 71d7016..0000000 --- a/graph/memstore/memtriplestore_test.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package memstore - -import ( - "sort" - "testing" - - . "github.com/smartystreets/goconvey/convey" - - "github.com/google/cayley/graph" -) - -func TestMemstore(t *testing.T) { - Convey("With a simple memstore", t, func() { - ts := MakeTestingMemstore() - Convey("It should have a reasonable size", func() { - So(ts.Size(), ShouldEqual, 11) - }) - Convey("It should have an Id Space that makes sense", func() { - v := ts.GetIdFor("C") - So(v.(int64), ShouldEqual, 4) - }) - }) -} - -func TestIteratorsAndNextResultOrderA(t *testing.T) { - ts := MakeTestingMemstore() - fixed := ts.MakeFixed() - fixed.AddValue(ts.GetIdFor("C")) - all := ts.GetNodesAllIterator() - lto := graph.NewLinksToIterator(ts, all, "o") - innerAnd := graph.NewAndIterator() - - fixed2 := ts.MakeFixed() - fixed2.AddValue(ts.GetIdFor("follows")) - lto2 := graph.NewLinksToIterator(ts, fixed2, "p") - innerAnd.AddSubIterator(lto2) - innerAnd.AddSubIterator(lto) - hasa := graph.NewHasaIterator(ts, innerAnd, "s") - outerAnd := graph.NewAndIterator() - outerAnd.AddSubIterator(fixed) - outerAnd.AddSubIterator(hasa) - val, ok := outerAnd.Next() - if !ok { - t.Error("Expected one matching subtree") - } - if ts.GetNameFor(val) != "C" { - t.Errorf("Matching subtree should be %s, got %s", "barak", ts.GetNameFor(val)) - } - expected := make([]string, 2) - expected[0] = "B" - expected[1] = "D" - actualOut := make([]string, 2) - actualOut[0] = ts.GetNameFor(all.LastResult()) - nresultOk := outerAnd.NextResult() - if !nresultOk { - t.Error("Expected two results got one") - } - actualOut[1] = ts.GetNameFor(all.LastResult()) - nresultOk = outerAnd.NextResult() - if nresultOk { - t.Error("Expected two results got three") - } - CompareStringSlices(t, expected, actualOut) - val, ok = outerAnd.Next() - if ok { - t.Error("More than one possible top level output?") - } -} - -func CompareStringSlices(t *testing.T, expected []string, actual []string) { - if len(expected) != len(actual) { - t.Error("String slices are not the same length") - } - sort.Strings(expected) - sort.Strings(actual) - for i := 0; i < len(expected); i++ { - if expected[i] != actual[i] { - t.Errorf("At index %d, expected \"%s\" and got \"%s\"", i, expected[i], actual[i]) - } - } -} - -func TestLinksToOptimization(t *testing.T) { - ts := MakeTestingMemstore() - fixed := ts.MakeFixed() - fixed.AddValue(ts.GetIdFor("cool")) - lto := graph.NewLinksToIterator(ts, fixed, "o") - lto.AddTag("foo") - newIt, changed := lto.Optimize() - if !changed { - t.Error("Iterator didn't change") - } - if newIt.Type() != "llrb" { - t.Fatal("Didn't swap out to LLRB") - } - v := newIt.(*LlrbIterator) - v_clone := v.Clone() - if v_clone.DebugString(0) != v.DebugString(0) { - t.Fatal("Wrong iterator. Got ", v_clone.DebugString(0)) - } - if len(v_clone.Tags()) < 1 || v_clone.Tags()[0] != "foo" { - t.Fatal("Tag on LinksTo did not persist") - } -} - -func TestRemoveTriple(t *testing.T) { - ts := MakeTestingMemstore() - ts.RemoveTriple(graph.MakeTriple("E", "follows", "F", "")) - fixed := ts.MakeFixed() - fixed.AddValue(ts.GetIdFor("E")) - lto := graph.NewLinksToIterator(ts, fixed, "s") - fixed2 := ts.MakeFixed() - fixed2.AddValue(ts.GetIdFor("follows")) - lto2 := graph.NewLinksToIterator(ts, fixed2, "p") - innerAnd := graph.NewAndIterator() - innerAnd.AddSubIterator(lto2) - innerAnd.AddSubIterator(lto) - hasa := graph.NewHasaIterator(ts, innerAnd, "o") - newIt, _ := hasa.Optimize() - _, ok := newIt.Next() - if ok { - t.Error("E should not have any followers.") - } -} diff --git a/graph/memstore/triplestore.go b/graph/memstore/triplestore.go new file mode 100644 index 0000000..7c7882e --- /dev/null +++ b/graph/memstore/triplestore.go @@ -0,0 +1,268 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "fmt" + + "github.com/barakmich/glog" + "github.com/google/cayley/graph" + + "github.com/petar/GoLLRB/llrb" +) + +type TripleDirectionIndex struct { + subject map[int64]*llrb.LLRB + predicate map[int64]*llrb.LLRB + object map[int64]*llrb.LLRB + provenance map[int64]*llrb.LLRB +} + +func NewTripleDirectionIndex() *TripleDirectionIndex { + var tdi TripleDirectionIndex + tdi.subject = make(map[int64]*llrb.LLRB) + tdi.predicate = make(map[int64]*llrb.LLRB) + tdi.object = make(map[int64]*llrb.LLRB) + tdi.provenance = make(map[int64]*llrb.LLRB) + return &tdi +} + +func (tdi *TripleDirectionIndex) GetForDir(s string) map[int64]*llrb.LLRB { + if s == "s" { + return tdi.subject + } else if s == "o" { + return tdi.object + } else if s == "p" { + return tdi.predicate + } else if s == "c" { + return tdi.provenance + } + panic("Bad direction") +} + +func (tdi *TripleDirectionIndex) GetOrCreate(dir string, id int64) *llrb.LLRB { + directionIndex := tdi.GetForDir(dir) + if _, ok := directionIndex[id]; !ok { + directionIndex[id] = llrb.New() + } + return directionIndex[id] +} + +func (tdi *TripleDirectionIndex) Get(dir string, id int64) (*llrb.LLRB, bool) { + directionIndex := tdi.GetForDir(dir) + tree, exists := directionIndex[id] + return tree, exists +} + +type MemTripleStore struct { + idCounter int64 + tripleIdCounter int64 + idMap map[string]int64 + revIdMap map[int64]string + triples []graph.Triple + size int64 + index TripleDirectionIndex + // vip_index map[string]map[int64]map[string]map[int64]*llrb.Tree +} + +func NewMemTripleStore() *MemTripleStore { + var ts MemTripleStore + ts.idMap = make(map[string]int64) + ts.revIdMap = make(map[int64]string) + ts.triples = make([]graph.Triple, 1, 200) + + // Sentinel null triple so triple indices start at 1 + ts.triples[0] = graph.Triple{} + ts.size = 1 + ts.index = *NewTripleDirectionIndex() + ts.idCounter = 1 + ts.tripleIdCounter = 1 + return &ts +} + +func (ts *MemTripleStore) AddTripleSet(triples []*graph.Triple) { + for _, t := range triples { + ts.AddTriple(t) + } +} + +func (ts *MemTripleStore) tripleExists(t *graph.Triple) (bool, int64) { + smallest := -1 + var smallest_tree *llrb.LLRB + for _, dir := range graph.TripleDirections { + sid := t.Get(dir) + if dir == "c" && sid == "" { + continue + } + id, ok := ts.idMap[sid] + // If we've never heard about a node, it most not exist + if !ok { + return false, 0 + } + index, exists := ts.index.Get(dir, id) + if !exists { + // If it's never been indexed in this direction, it can't exist. + return false, 0 + } + if smallest == -1 || index.Len() < smallest { + smallest = index.Len() + smallest_tree = index + } + } + it := NewLlrbIterator(smallest_tree, "") + + for { + val, ok := it.Next() + if !ok { + break + } + if t.Equals(&ts.triples[val.(int64)]) { + return true, val.(int64) + } + } + return false, 0 +} + +func (ts *MemTripleStore) AddTriple(t *graph.Triple) { + if exists, _ := ts.tripleExists(t); exists { + return + } + var tripleID int64 + ts.triples = append(ts.triples, *t) + tripleID = ts.tripleIdCounter + ts.size++ + ts.tripleIdCounter++ + + for _, dir := range graph.TripleDirections { + sid := t.Get(dir) + if dir == "c" && sid == "" { + continue + } + if _, ok := ts.idMap[sid]; !ok { + ts.idMap[sid] = ts.idCounter + ts.revIdMap[ts.idCounter] = sid + ts.idCounter++ + } + } + + for _, dir := range graph.TripleDirections { + if dir == "c" && t.Get(dir) == "" { + continue + } + id := ts.idMap[t.Get(dir)] + tree := ts.index.GetOrCreate(dir, id) + tree.ReplaceOrInsert(Int64(tripleID)) + } + + // TODO(barakmich): Add VIP indexing +} + +func (ts *MemTripleStore) RemoveTriple(t *graph.Triple) { + var tripleID int64 + var exists bool + tripleID = 0 + if exists, tripleID = ts.tripleExists(t); !exists { + return + } + + ts.triples[tripleID] = graph.Triple{} + ts.size-- + + for _, dir := range graph.TripleDirections { + if dir == "c" && t.Get(dir) == "" { + continue + } + id := ts.idMap[t.Get(dir)] + tree := ts.index.GetOrCreate(dir, id) + tree.Delete(Int64(tripleID)) + } + + for _, dir := range graph.TripleDirections { + if dir == "c" && t.Get(dir) == "" { + continue + } + id, ok := ts.idMap[t.Get(dir)] + if !ok { + continue + } + stillExists := false + for _, dir := range graph.TripleDirections { + if dir == "c" && t.Get(dir) == "" { + continue + } + nodeTree := ts.index.GetOrCreate(dir, id) + if nodeTree.Len() != 0 { + stillExists = true + break + } + } + if !stillExists { + delete(ts.idMap, t.Get(dir)) + delete(ts.revIdMap, id) + } + } +} + +func (ts *MemTripleStore) GetTriple(index graph.TSVal) *graph.Triple { + return &ts.triples[index.(int64)] +} + +func (ts *MemTripleStore) GetTripleIterator(direction string, value graph.TSVal) graph.Iterator { + index, ok := ts.index.Get(direction, value.(int64)) + data := fmt.Sprintf("dir:%s val:%d", direction, value.(int64)) + if ok { + return NewLlrbIterator(index, data) + } + return &graph.NullIterator{} +} + +func (ts *MemTripleStore) Size() int64 { + return ts.size - 1 // Don't count the sentinel +} + +func (ts *MemTripleStore) DebugPrint() { + for i, t := range ts.triples { + if i == 0 { + continue + } + glog.V(2).Infoln("%d: %s", i, t.ToString()) + } +} + +func (ts *MemTripleStore) GetIdFor(name string) graph.TSVal { + return ts.idMap[name] +} + +func (ts *MemTripleStore) GetNameFor(id graph.TSVal) string { + return ts.revIdMap[id.(int64)] +} + +func (ts *MemTripleStore) GetTriplesAllIterator() graph.Iterator { + return graph.NewInt64AllIterator(0, ts.Size()) +} + +func (ts *MemTripleStore) MakeFixed() *graph.FixedIterator { + return graph.NewFixedIteratorWithCompare(graph.BasicEquality) +} + +func (ts *MemTripleStore) GetTripleDirection(val graph.TSVal, direction string) graph.TSVal { + name := ts.GetTriple(val).Get(direction) + return ts.GetIdFor(name) +} + +func (ts *MemTripleStore) GetNodesAllIterator() graph.Iterator { + return NewMemstoreAllIterator(ts) +} +func (ts *MemTripleStore) Close() {} diff --git a/graph/memstore/triplestore_iterator_optimize.go b/graph/memstore/triplestore_iterator_optimize.go new file mode 100644 index 0000000..7c895fc --- /dev/null +++ b/graph/memstore/triplestore_iterator_optimize.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "github.com/google/cayley/graph" +) + +func (ts *MemTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { + switch it.Type() { + case "linksto": + return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) + + } + return it, false +} + +func (ts *MemTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { + l := it.GetSubIterators() + if l.Len() != 1 { + return it, false + } + primaryIt := l.Front().Value.(graph.Iterator) + if primaryIt.Type() == "fixed" { + size, _ := primaryIt.Size() + if size == 1 { + val, ok := primaryIt.Next() + if !ok { + panic("Sizes lie") + } + newIt := ts.GetTripleIterator(it.Direction(), val) + newIt.CopyTagsFrom(it) + for _, tag := range primaryIt.Tags() { + newIt.AddFixedTag(tag, val) + } + return newIt, true + } + } + it.Close() + return it, false +} diff --git a/graph/memstore/triplestore_test.go b/graph/memstore/triplestore_test.go new file mode 100644 index 0000000..71d7016 --- /dev/null +++ b/graph/memstore/triplestore_test.go @@ -0,0 +1,138 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package memstore + +import ( + "sort" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph" +) + +func TestMemstore(t *testing.T) { + Convey("With a simple memstore", t, func() { + ts := MakeTestingMemstore() + Convey("It should have a reasonable size", func() { + So(ts.Size(), ShouldEqual, 11) + }) + Convey("It should have an Id Space that makes sense", func() { + v := ts.GetIdFor("C") + So(v.(int64), ShouldEqual, 4) + }) + }) +} + +func TestIteratorsAndNextResultOrderA(t *testing.T) { + ts := MakeTestingMemstore() + fixed := ts.MakeFixed() + fixed.AddValue(ts.GetIdFor("C")) + all := ts.GetNodesAllIterator() + lto := graph.NewLinksToIterator(ts, all, "o") + innerAnd := graph.NewAndIterator() + + fixed2 := ts.MakeFixed() + fixed2.AddValue(ts.GetIdFor("follows")) + lto2 := graph.NewLinksToIterator(ts, fixed2, "p") + innerAnd.AddSubIterator(lto2) + innerAnd.AddSubIterator(lto) + hasa := graph.NewHasaIterator(ts, innerAnd, "s") + outerAnd := graph.NewAndIterator() + outerAnd.AddSubIterator(fixed) + outerAnd.AddSubIterator(hasa) + val, ok := outerAnd.Next() + if !ok { + t.Error("Expected one matching subtree") + } + if ts.GetNameFor(val) != "C" { + t.Errorf("Matching subtree should be %s, got %s", "barak", ts.GetNameFor(val)) + } + expected := make([]string, 2) + expected[0] = "B" + expected[1] = "D" + actualOut := make([]string, 2) + actualOut[0] = ts.GetNameFor(all.LastResult()) + nresultOk := outerAnd.NextResult() + if !nresultOk { + t.Error("Expected two results got one") + } + actualOut[1] = ts.GetNameFor(all.LastResult()) + nresultOk = outerAnd.NextResult() + if nresultOk { + t.Error("Expected two results got three") + } + CompareStringSlices(t, expected, actualOut) + val, ok = outerAnd.Next() + if ok { + t.Error("More than one possible top level output?") + } +} + +func CompareStringSlices(t *testing.T, expected []string, actual []string) { + if len(expected) != len(actual) { + t.Error("String slices are not the same length") + } + sort.Strings(expected) + sort.Strings(actual) + for i := 0; i < len(expected); i++ { + if expected[i] != actual[i] { + t.Errorf("At index %d, expected \"%s\" and got \"%s\"", i, expected[i], actual[i]) + } + } +} + +func TestLinksToOptimization(t *testing.T) { + ts := MakeTestingMemstore() + fixed := ts.MakeFixed() + fixed.AddValue(ts.GetIdFor("cool")) + lto := graph.NewLinksToIterator(ts, fixed, "o") + lto.AddTag("foo") + newIt, changed := lto.Optimize() + if !changed { + t.Error("Iterator didn't change") + } + if newIt.Type() != "llrb" { + t.Fatal("Didn't swap out to LLRB") + } + v := newIt.(*LlrbIterator) + v_clone := v.Clone() + if v_clone.DebugString(0) != v.DebugString(0) { + t.Fatal("Wrong iterator. Got ", v_clone.DebugString(0)) + } + if len(v_clone.Tags()) < 1 || v_clone.Tags()[0] != "foo" { + t.Fatal("Tag on LinksTo did not persist") + } +} + +func TestRemoveTriple(t *testing.T) { + ts := MakeTestingMemstore() + ts.RemoveTriple(graph.MakeTriple("E", "follows", "F", "")) + fixed := ts.MakeFixed() + fixed.AddValue(ts.GetIdFor("E")) + lto := graph.NewLinksToIterator(ts, fixed, "s") + fixed2 := ts.MakeFixed() + fixed2.AddValue(ts.GetIdFor("follows")) + lto2 := graph.NewLinksToIterator(ts, fixed2, "p") + innerAnd := graph.NewAndIterator() + innerAnd.AddSubIterator(lto2) + innerAnd.AddSubIterator(lto) + hasa := graph.NewHasaIterator(ts, innerAnd, "o") + newIt, _ := hasa.Optimize() + _, ok := newIt.Next() + if ok { + t.Error("E should not have any followers.") + } +} diff --git a/graph/mongo/iterator.go b/graph/mongo/iterator.go new file mode 100644 index 0000000..addcfb0 --- /dev/null +++ b/graph/mongo/iterator.go @@ -0,0 +1,181 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongo + +import ( + "fmt" + "strings" + + "github.com/barakmich/glog" + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + + "github.com/google/cayley/graph" +) + +type MongoIterator struct { + graph.BaseIterator + ts *MongoTripleStore + dir string + iter *mgo.Iter + hash string + name string + size int64 + isAll bool + constraint bson.M + collection string +} + +func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator { + var m MongoIterator + graph.BaseIteratorInit(&m.BaseIterator) + + m.name = ts.GetNameFor(val) + m.collection = collection + switch dir { + + case "s": + m.constraint = bson.M{"Sub": m.name} + case "p": + m.constraint = bson.M{"Pred": m.name} + case "o": + m.constraint = bson.M{"Obj": m.name} + case "c": + m.constraint = bson.M{"Provenance": m.name} + } + + m.ts = ts + m.dir = dir + m.iter = ts.db.C(collection).Find(m.constraint).Iter() + size, err := ts.db.C(collection).Find(m.constraint).Count() + if err != nil { + glog.Errorln("Trouble getting size for iterator! ", err) + return nil + } + m.size = int64(size) + m.hash = val.(string) + m.isAll = false + return &m +} + +func NewMongoAllIterator(ts *MongoTripleStore, collection string) *MongoIterator { + var m MongoIterator + m.ts = ts + m.dir = "all" + m.constraint = nil + m.collection = collection + m.iter = ts.db.C(collection).Find(nil).Iter() + size, err := ts.db.C(collection).Count() + if err != nil { + glog.Errorln("Trouble getting size for iterator! ", err) + return nil + } + m.size = int64(size) + m.hash = "" + m.isAll = true + return &m +} + +func (m *MongoIterator) Reset() { + m.iter.Close() + m.iter = m.ts.db.C(m.collection).Find(m.constraint).Iter() + +} + +func (m *MongoIterator) Close() { + m.iter.Close() +} + +func (m *MongoIterator) Clone() graph.Iterator { + var newM graph.Iterator + if m.isAll { + newM = NewMongoAllIterator(m.ts, m.collection) + } else { + newM = NewMongoIterator(m.ts, m.collection, m.dir, m.hash) + } + newM.CopyTagsFrom(m) + return newM +} + +func (m *MongoIterator) Next() (graph.TSVal, bool) { + var result struct { + Id string "_id" + //Sub string "Sub" + //Pred string "Pred" + //Obj string "Obj" + } + found := m.iter.Next(&result) + if !found { + err := m.iter.Err() + if err != nil { + glog.Errorln("Error Nexting MongoIterator: ", err) + } + return nil, false + } + m.Last = result.Id + return result.Id, true +} + +func (m *MongoIterator) Check(v graph.TSVal) bool { + graph.CheckLogIn(m, v) + if m.isAll { + m.Last = v + return graph.CheckLogOut(m, v, true) + } + var offset int + switch m.dir { + case "s": + offset = 0 + case "p": + offset = (m.ts.hasher.Size() * 2) + case "o": + offset = (m.ts.hasher.Size() * 2) * 2 + case "c": + offset = (m.ts.hasher.Size() * 2) * 3 + } + val := v.(string)[offset : m.ts.hasher.Size()*2+offset] + if val == m.hash { + m.Last = v + return graph.CheckLogOut(m, v, true) + } + return graph.CheckLogOut(m, v, false) +} + +func (m *MongoIterator) Size() (int64, bool) { + return m.size, true +} + +func (m *MongoIterator) Type() string { + if m.isAll { + return "all" + } + return "mongo" +} +func (m *MongoIterator) Sorted() bool { return true } +func (m *MongoIterator) Optimize() (graph.Iterator, bool) { return m, false } + +func (m *MongoIterator) DebugString(indent int) string { + size, _ := m.Size() + return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), m.Type(), size, m.hash, m.name) +} + +func (m *MongoIterator) GetStats() *graph.IteratorStats { + size, _ := m.Size() + return &graph.IteratorStats{ + CheckCost: 1, + NextCost: 5, + Size: size, + } +} diff --git a/graph/mongo/mongo_iterator.go b/graph/mongo/mongo_iterator.go deleted file mode 100644 index addcfb0..0000000 --- a/graph/mongo/mongo_iterator.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mongo - -import ( - "fmt" - "strings" - - "github.com/barakmich/glog" - "labix.org/v2/mgo" - "labix.org/v2/mgo/bson" - - "github.com/google/cayley/graph" -) - -type MongoIterator struct { - graph.BaseIterator - ts *MongoTripleStore - dir string - iter *mgo.Iter - hash string - name string - size int64 - isAll bool - constraint bson.M - collection string -} - -func NewMongoIterator(ts *MongoTripleStore, collection string, dir string, val graph.TSVal) *MongoIterator { - var m MongoIterator - graph.BaseIteratorInit(&m.BaseIterator) - - m.name = ts.GetNameFor(val) - m.collection = collection - switch dir { - - case "s": - m.constraint = bson.M{"Sub": m.name} - case "p": - m.constraint = bson.M{"Pred": m.name} - case "o": - m.constraint = bson.M{"Obj": m.name} - case "c": - m.constraint = bson.M{"Provenance": m.name} - } - - m.ts = ts - m.dir = dir - m.iter = ts.db.C(collection).Find(m.constraint).Iter() - size, err := ts.db.C(collection).Find(m.constraint).Count() - if err != nil { - glog.Errorln("Trouble getting size for iterator! ", err) - return nil - } - m.size = int64(size) - m.hash = val.(string) - m.isAll = false - return &m -} - -func NewMongoAllIterator(ts *MongoTripleStore, collection string) *MongoIterator { - var m MongoIterator - m.ts = ts - m.dir = "all" - m.constraint = nil - m.collection = collection - m.iter = ts.db.C(collection).Find(nil).Iter() - size, err := ts.db.C(collection).Count() - if err != nil { - glog.Errorln("Trouble getting size for iterator! ", err) - return nil - } - m.size = int64(size) - m.hash = "" - m.isAll = true - return &m -} - -func (m *MongoIterator) Reset() { - m.iter.Close() - m.iter = m.ts.db.C(m.collection).Find(m.constraint).Iter() - -} - -func (m *MongoIterator) Close() { - m.iter.Close() -} - -func (m *MongoIterator) Clone() graph.Iterator { - var newM graph.Iterator - if m.isAll { - newM = NewMongoAllIterator(m.ts, m.collection) - } else { - newM = NewMongoIterator(m.ts, m.collection, m.dir, m.hash) - } - newM.CopyTagsFrom(m) - return newM -} - -func (m *MongoIterator) Next() (graph.TSVal, bool) { - var result struct { - Id string "_id" - //Sub string "Sub" - //Pred string "Pred" - //Obj string "Obj" - } - found := m.iter.Next(&result) - if !found { - err := m.iter.Err() - if err != nil { - glog.Errorln("Error Nexting MongoIterator: ", err) - } - return nil, false - } - m.Last = result.Id - return result.Id, true -} - -func (m *MongoIterator) Check(v graph.TSVal) bool { - graph.CheckLogIn(m, v) - if m.isAll { - m.Last = v - return graph.CheckLogOut(m, v, true) - } - var offset int - switch m.dir { - case "s": - offset = 0 - case "p": - offset = (m.ts.hasher.Size() * 2) - case "o": - offset = (m.ts.hasher.Size() * 2) * 2 - case "c": - offset = (m.ts.hasher.Size() * 2) * 3 - } - val := v.(string)[offset : m.ts.hasher.Size()*2+offset] - if val == m.hash { - m.Last = v - return graph.CheckLogOut(m, v, true) - } - return graph.CheckLogOut(m, v, false) -} - -func (m *MongoIterator) Size() (int64, bool) { - return m.size, true -} - -func (m *MongoIterator) Type() string { - if m.isAll { - return "all" - } - return "mongo" -} -func (m *MongoIterator) Sorted() bool { return true } -func (m *MongoIterator) Optimize() (graph.Iterator, bool) { return m, false } - -func (m *MongoIterator) DebugString(indent int) string { - size, _ := m.Size() - return fmt.Sprintf("%s(%s size:%d %s %s)", strings.Repeat(" ", indent), m.Type(), size, m.hash, m.name) -} - -func (m *MongoIterator) GetStats() *graph.IteratorStats { - size, _ := m.Size() - return &graph.IteratorStats{ - CheckCost: 1, - NextCost: 5, - Size: size, - } -} diff --git a/graph/mongo/mongo_triplestore.go b/graph/mongo/mongo_triplestore.go deleted file mode 100644 index 917ab4d..0000000 --- a/graph/mongo/mongo_triplestore.go +++ /dev/null @@ -1,329 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mongo - -import ( - "crypto/sha1" - "encoding/hex" - "hash" - "log" - - "labix.org/v2/mgo" - "labix.org/v2/mgo/bson" - - "github.com/barakmich/glog" - "github.com/google/cayley/graph" -) - -const DefaultDBName = "cayley" - -type MongoTripleStore struct { - session *mgo.Session - db *mgo.Database - hasher hash.Hash - idCache *IDLru -} - -func CreateNewMongoGraph(addr string, options graph.OptionsDict) bool { - conn, err := mgo.Dial(addr) - if err != nil { - glog.Fatal("Error connecting: ", err) - return false - } - conn.SetSafe(&mgo.Safe{}) - dbName := DefaultDBName - if val, ok := options.GetStringKey("database_name"); ok { - dbName = val - } - db := conn.DB(dbName) - indexOpts := mgo.Index{ - Key: []string{"Sub"}, - Unique: false, - DropDups: false, - Background: true, - Sparse: true, - } - db.C("triples").EnsureIndex(indexOpts) - indexOpts.Key = []string{"Pred"} - db.C("triples").EnsureIndex(indexOpts) - indexOpts.Key = []string{"Obj"} - db.C("triples").EnsureIndex(indexOpts) - indexOpts.Key = []string{"Provenance"} - db.C("triples").EnsureIndex(indexOpts) - return true -} - -func NewMongoTripleStore(addr string, options graph.OptionsDict) *MongoTripleStore { - var ts MongoTripleStore - conn, err := mgo.Dial(addr) - if err != nil { - glog.Fatal("Error connecting: ", err) - } - conn.SetSafe(&mgo.Safe{}) - dbName := DefaultDBName - if val, ok := options.GetStringKey("database_name"); ok { - dbName = val - } - ts.db = conn.DB(dbName) - ts.session = conn - ts.hasher = sha1.New() - ts.idCache = NewIDLru(1 << 16) - return &ts -} - -func (ts *MongoTripleStore) getIdForTriple(t *graph.Triple) string { - id := ts.ConvertStringToByteHash(t.Sub) - id += ts.ConvertStringToByteHash(t.Pred) - id += ts.ConvertStringToByteHash(t.Obj) - id += ts.ConvertStringToByteHash(t.Provenance) - return id -} - -func (ts *MongoTripleStore) ConvertStringToByteHash(s string) string { - ts.hasher.Reset() - key := make([]byte, 0, ts.hasher.Size()) - ts.hasher.Write([]byte(s)) - key = ts.hasher.Sum(key) - return hex.EncodeToString(key) -} - -type MongoNode struct { - Id string "_id" - Name string "Name" - Size int "Size" -} - -func (ts *MongoTripleStore) updateNodeBy(node_name string, inc int) { - var size MongoNode - node := ts.GetIdFor(node_name) - err := ts.db.C("nodes").FindId(node).One(&size) - if err != nil { - if err.Error() == "not found" { - // Not found. Okay. - size.Id = node.(string) - size.Name = node_name - size.Size = inc - } else { - glog.Error("Error:", err) - return - } - } else { - size.Id = node.(string) - size.Name = node_name - size.Size += inc - } - - // Removing something... - if inc < 0 { - if size.Size <= 0 { - err := ts.db.C("nodes").RemoveId(node) - if err != nil { - glog.Error("Error: ", err, " while removing node ", node_name) - return - } - } - } - - _, err2 := ts.db.C("nodes").UpsertId(node, size) - if err2 != nil { - glog.Error("Error: ", err) - } -} - -func (ts *MongoTripleStore) writeTriple(t *graph.Triple) bool { - tripledoc := bson.M{"_id": ts.getIdForTriple(t), "Sub": t.Sub, "Pred": t.Pred, "Obj": t.Obj, "Provenance": t.Provenance} - err := ts.db.C("triples").Insert(tripledoc) - if err != nil { - // Among the reasons I hate MongoDB. "Errors don't happen! Right guys?" - if err.(*mgo.LastError).Code == 11000 { - return false - } - glog.Error("Error: ", err) - return false - } - return true -} - -func (ts *MongoTripleStore) AddTriple(t *graph.Triple) { - _ = ts.writeTriple(t) - ts.updateNodeBy(t.Sub, 1) - ts.updateNodeBy(t.Pred, 1) - ts.updateNodeBy(t.Obj, 1) - if t.Provenance != "" { - ts.updateNodeBy(t.Provenance, 1) - } -} - -func (ts *MongoTripleStore) AddTripleSet(in []*graph.Triple) { - ts.session.SetSafe(nil) - idMap := make(map[string]int) - for _, t := range in { - wrote := ts.writeTriple(t) - if wrote { - idMap[t.Sub]++ - idMap[t.Obj]++ - idMap[t.Pred]++ - if t.Provenance != "" { - idMap[t.Provenance]++ - } - } - } - for k, v := range idMap { - ts.updateNodeBy(k, v) - } - ts.session.SetSafe(&mgo.Safe{}) -} - -func (ts *MongoTripleStore) RemoveTriple(t *graph.Triple) { - err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t)) - if err == mgo.ErrNotFound { - return - } else if err != nil { - log.Println("Error: ", err, " while removing triple ", t) - return - } - ts.updateNodeBy(t.Sub, -1) - ts.updateNodeBy(t.Pred, -1) - ts.updateNodeBy(t.Obj, -1) - if t.Provenance != "" { - ts.updateNodeBy(t.Provenance, -1) - } -} - -func (ts *MongoTripleStore) GetTriple(val graph.TSVal) *graph.Triple { - var bsonDoc bson.M - err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc) - if err != nil { - log.Println("Error: Couldn't retrieve triple", val.(string), err) - } - return graph.MakeTriple( - bsonDoc["Sub"].(string), - bsonDoc["Pred"].(string), - bsonDoc["Obj"].(string), - bsonDoc["Provenance"].(string)) -} - -func (ts *MongoTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { - return NewMongoIterator(ts, "triples", dir, val) -} - -func (ts *MongoTripleStore) GetNodesAllIterator() graph.Iterator { - return NewMongoAllIterator(ts, "nodes") -} - -func (ts *MongoTripleStore) GetTriplesAllIterator() graph.Iterator { - return NewMongoAllIterator(ts, "triples") -} - -func (ts *MongoTripleStore) GetIdFor(s string) graph.TSVal { - return ts.ConvertStringToByteHash(s) -} - -func (ts *MongoTripleStore) GetNameFor(v graph.TSVal) string { - val, ok := ts.idCache.Get(v.(string)) - if ok { - return val - } - var node MongoNode - err := ts.db.C("nodes").FindId(v.(string)).One(&node) - if err != nil { - log.Println("Error: Couldn't retrieve node", v.(string), err) - } - ts.idCache.Put(v.(string), node.Name) - return node.Name -} - -func (ts *MongoTripleStore) Size() int64 { - count, err := ts.db.C("triples").Count() - if err != nil { - glog.Error("Error: ", err) - return 0 - } - return int64(count) -} - -func compareStrings(a, b graph.TSVal) bool { - return a.(string) == b.(string) -} - -func (ts *MongoTripleStore) MakeFixed() *graph.FixedIterator { - return graph.NewFixedIteratorWithCompare(compareStrings) -} - -func (ts *MongoTripleStore) Close() { - ts.db.Session.Close() -} - -func (ts *MongoTripleStore) GetTripleDirection(in graph.TSVal, dir string) graph.TSVal { - // Maybe do the trick here - var offset int - switch dir { - case "s": - offset = 0 - case "p": - offset = (ts.hasher.Size() * 2) - case "o": - offset = (ts.hasher.Size() * 2) * 2 - case "c": - offset = (ts.hasher.Size() * 2) * 3 - } - val := in.(string)[offset : ts.hasher.Size()*2+offset] - return val -} - -func (ts *MongoTripleStore) BulkLoad(t_chan chan *graph.Triple) { - ts.session.SetSafe(nil) - for triple := range t_chan { - ts.writeTriple(triple) - } - outputTo := bson.M{"replace": "nodes", "sharded": true} - glog.Infoln("Mapreducing") - job := mgo.MapReduce{ - Map: `function() { - var len = this["_id"].length - var s_key = this["_id"].slice(0, len / 4) - var p_key = this["_id"].slice(len / 4, 2 * len / 4) - var o_key = this["_id"].slice(2 * len / 4, 3 * len / 4) - var c_key = this["_id"].slice(3 * len / 4) - emit(s_key, {"_id": s_key, "Name" : this.Sub, "Size" : 1}) - emit(p_key, {"_id": p_key, "Name" : this.Pred, "Size" : 1}) - emit(o_key, {"_id": o_key, "Name" : this.Obj, "Size" : 1}) - if (this.Provenance != "") { - emit(c_key, {"_id": c_key, "Name" : this.Provenance, "Size" : 1}) - } - } - `, - Reduce: ` - function(key, value_list) { - out = {"_id": key, "Name": value_list[0].Name} - count = 0 - for (var i = 0; i < value_list.length; i++) { - count = count + value_list[i].Size - - } - out["Size"] = count - return out - } - `, - Out: outputTo, - } - ts.db.C("triples").Find(nil).MapReduce(&job, nil) - glog.Infoln("Fixing") - ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) { - db.nodes.update({"_id": result._id}, result.value) - }) }`}, {"args", bson.D{}}}, nil) - - ts.session.SetSafe(&mgo.Safe{}) -} diff --git a/graph/mongo/mongo_triplestore_iterator_optimize.go b/graph/mongo/mongo_triplestore_iterator_optimize.go deleted file mode 100644 index d10bc22..0000000 --- a/graph/mongo/mongo_triplestore_iterator_optimize.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mongo - -import ( - "github.com/google/cayley/graph" -) - -func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { - switch it.Type() { - case "linksto": - return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) - - } - return it, false -} - -func (ts *MongoTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { - l := it.GetSubIterators() - if l.Len() != 1 { - return it, false - } - primaryIt := l.Front().Value.(graph.Iterator) - if primaryIt.Type() == "fixed" { - size, _ := primaryIt.Size() - if size == 1 { - val, ok := primaryIt.Next() - if !ok { - panic("Sizes lie") - } - newIt := ts.GetTripleIterator(it.Direction(), val) - newIt.CopyTagsFrom(it) - for _, tag := range primaryIt.Tags() { - newIt.AddFixedTag(tag, val) - } - it.Close() - return newIt, true - } - } - return it, false -} diff --git a/graph/mongo/triplestore.go b/graph/mongo/triplestore.go new file mode 100644 index 0000000..917ab4d --- /dev/null +++ b/graph/mongo/triplestore.go @@ -0,0 +1,329 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongo + +import ( + "crypto/sha1" + "encoding/hex" + "hash" + "log" + + "labix.org/v2/mgo" + "labix.org/v2/mgo/bson" + + "github.com/barakmich/glog" + "github.com/google/cayley/graph" +) + +const DefaultDBName = "cayley" + +type MongoTripleStore struct { + session *mgo.Session + db *mgo.Database + hasher hash.Hash + idCache *IDLru +} + +func CreateNewMongoGraph(addr string, options graph.OptionsDict) bool { + conn, err := mgo.Dial(addr) + if err != nil { + glog.Fatal("Error connecting: ", err) + return false + } + conn.SetSafe(&mgo.Safe{}) + dbName := DefaultDBName + if val, ok := options.GetStringKey("database_name"); ok { + dbName = val + } + db := conn.DB(dbName) + indexOpts := mgo.Index{ + Key: []string{"Sub"}, + Unique: false, + DropDups: false, + Background: true, + Sparse: true, + } + db.C("triples").EnsureIndex(indexOpts) + indexOpts.Key = []string{"Pred"} + db.C("triples").EnsureIndex(indexOpts) + indexOpts.Key = []string{"Obj"} + db.C("triples").EnsureIndex(indexOpts) + indexOpts.Key = []string{"Provenance"} + db.C("triples").EnsureIndex(indexOpts) + return true +} + +func NewMongoTripleStore(addr string, options graph.OptionsDict) *MongoTripleStore { + var ts MongoTripleStore + conn, err := mgo.Dial(addr) + if err != nil { + glog.Fatal("Error connecting: ", err) + } + conn.SetSafe(&mgo.Safe{}) + dbName := DefaultDBName + if val, ok := options.GetStringKey("database_name"); ok { + dbName = val + } + ts.db = conn.DB(dbName) + ts.session = conn + ts.hasher = sha1.New() + ts.idCache = NewIDLru(1 << 16) + return &ts +} + +func (ts *MongoTripleStore) getIdForTriple(t *graph.Triple) string { + id := ts.ConvertStringToByteHash(t.Sub) + id += ts.ConvertStringToByteHash(t.Pred) + id += ts.ConvertStringToByteHash(t.Obj) + id += ts.ConvertStringToByteHash(t.Provenance) + return id +} + +func (ts *MongoTripleStore) ConvertStringToByteHash(s string) string { + ts.hasher.Reset() + key := make([]byte, 0, ts.hasher.Size()) + ts.hasher.Write([]byte(s)) + key = ts.hasher.Sum(key) + return hex.EncodeToString(key) +} + +type MongoNode struct { + Id string "_id" + Name string "Name" + Size int "Size" +} + +func (ts *MongoTripleStore) updateNodeBy(node_name string, inc int) { + var size MongoNode + node := ts.GetIdFor(node_name) + err := ts.db.C("nodes").FindId(node).One(&size) + if err != nil { + if err.Error() == "not found" { + // Not found. Okay. + size.Id = node.(string) + size.Name = node_name + size.Size = inc + } else { + glog.Error("Error:", err) + return + } + } else { + size.Id = node.(string) + size.Name = node_name + size.Size += inc + } + + // Removing something... + if inc < 0 { + if size.Size <= 0 { + err := ts.db.C("nodes").RemoveId(node) + if err != nil { + glog.Error("Error: ", err, " while removing node ", node_name) + return + } + } + } + + _, err2 := ts.db.C("nodes").UpsertId(node, size) + if err2 != nil { + glog.Error("Error: ", err) + } +} + +func (ts *MongoTripleStore) writeTriple(t *graph.Triple) bool { + tripledoc := bson.M{"_id": ts.getIdForTriple(t), "Sub": t.Sub, "Pred": t.Pred, "Obj": t.Obj, "Provenance": t.Provenance} + err := ts.db.C("triples").Insert(tripledoc) + if err != nil { + // Among the reasons I hate MongoDB. "Errors don't happen! Right guys?" + if err.(*mgo.LastError).Code == 11000 { + return false + } + glog.Error("Error: ", err) + return false + } + return true +} + +func (ts *MongoTripleStore) AddTriple(t *graph.Triple) { + _ = ts.writeTriple(t) + ts.updateNodeBy(t.Sub, 1) + ts.updateNodeBy(t.Pred, 1) + ts.updateNodeBy(t.Obj, 1) + if t.Provenance != "" { + ts.updateNodeBy(t.Provenance, 1) + } +} + +func (ts *MongoTripleStore) AddTripleSet(in []*graph.Triple) { + ts.session.SetSafe(nil) + idMap := make(map[string]int) + for _, t := range in { + wrote := ts.writeTriple(t) + if wrote { + idMap[t.Sub]++ + idMap[t.Obj]++ + idMap[t.Pred]++ + if t.Provenance != "" { + idMap[t.Provenance]++ + } + } + } + for k, v := range idMap { + ts.updateNodeBy(k, v) + } + ts.session.SetSafe(&mgo.Safe{}) +} + +func (ts *MongoTripleStore) RemoveTriple(t *graph.Triple) { + err := ts.db.C("triples").RemoveId(ts.getIdForTriple(t)) + if err == mgo.ErrNotFound { + return + } else if err != nil { + log.Println("Error: ", err, " while removing triple ", t) + return + } + ts.updateNodeBy(t.Sub, -1) + ts.updateNodeBy(t.Pred, -1) + ts.updateNodeBy(t.Obj, -1) + if t.Provenance != "" { + ts.updateNodeBy(t.Provenance, -1) + } +} + +func (ts *MongoTripleStore) GetTriple(val graph.TSVal) *graph.Triple { + var bsonDoc bson.M + err := ts.db.C("triples").FindId(val.(string)).One(&bsonDoc) + if err != nil { + log.Println("Error: Couldn't retrieve triple", val.(string), err) + } + return graph.MakeTriple( + bsonDoc["Sub"].(string), + bsonDoc["Pred"].(string), + bsonDoc["Obj"].(string), + bsonDoc["Provenance"].(string)) +} + +func (ts *MongoTripleStore) GetTripleIterator(dir string, val graph.TSVal) graph.Iterator { + return NewMongoIterator(ts, "triples", dir, val) +} + +func (ts *MongoTripleStore) GetNodesAllIterator() graph.Iterator { + return NewMongoAllIterator(ts, "nodes") +} + +func (ts *MongoTripleStore) GetTriplesAllIterator() graph.Iterator { + return NewMongoAllIterator(ts, "triples") +} + +func (ts *MongoTripleStore) GetIdFor(s string) graph.TSVal { + return ts.ConvertStringToByteHash(s) +} + +func (ts *MongoTripleStore) GetNameFor(v graph.TSVal) string { + val, ok := ts.idCache.Get(v.(string)) + if ok { + return val + } + var node MongoNode + err := ts.db.C("nodes").FindId(v.(string)).One(&node) + if err != nil { + log.Println("Error: Couldn't retrieve node", v.(string), err) + } + ts.idCache.Put(v.(string), node.Name) + return node.Name +} + +func (ts *MongoTripleStore) Size() int64 { + count, err := ts.db.C("triples").Count() + if err != nil { + glog.Error("Error: ", err) + return 0 + } + return int64(count) +} + +func compareStrings(a, b graph.TSVal) bool { + return a.(string) == b.(string) +} + +func (ts *MongoTripleStore) MakeFixed() *graph.FixedIterator { + return graph.NewFixedIteratorWithCompare(compareStrings) +} + +func (ts *MongoTripleStore) Close() { + ts.db.Session.Close() +} + +func (ts *MongoTripleStore) GetTripleDirection(in graph.TSVal, dir string) graph.TSVal { + // Maybe do the trick here + var offset int + switch dir { + case "s": + offset = 0 + case "p": + offset = (ts.hasher.Size() * 2) + case "o": + offset = (ts.hasher.Size() * 2) * 2 + case "c": + offset = (ts.hasher.Size() * 2) * 3 + } + val := in.(string)[offset : ts.hasher.Size()*2+offset] + return val +} + +func (ts *MongoTripleStore) BulkLoad(t_chan chan *graph.Triple) { + ts.session.SetSafe(nil) + for triple := range t_chan { + ts.writeTriple(triple) + } + outputTo := bson.M{"replace": "nodes", "sharded": true} + glog.Infoln("Mapreducing") + job := mgo.MapReduce{ + Map: `function() { + var len = this["_id"].length + var s_key = this["_id"].slice(0, len / 4) + var p_key = this["_id"].slice(len / 4, 2 * len / 4) + var o_key = this["_id"].slice(2 * len / 4, 3 * len / 4) + var c_key = this["_id"].slice(3 * len / 4) + emit(s_key, {"_id": s_key, "Name" : this.Sub, "Size" : 1}) + emit(p_key, {"_id": p_key, "Name" : this.Pred, "Size" : 1}) + emit(o_key, {"_id": o_key, "Name" : this.Obj, "Size" : 1}) + if (this.Provenance != "") { + emit(c_key, {"_id": c_key, "Name" : this.Provenance, "Size" : 1}) + } + } + `, + Reduce: ` + function(key, value_list) { + out = {"_id": key, "Name": value_list[0].Name} + count = 0 + for (var i = 0; i < value_list.length; i++) { + count = count + value_list[i].Size + + } + out["Size"] = count + return out + } + `, + Out: outputTo, + } + ts.db.C("triples").Find(nil).MapReduce(&job, nil) + glog.Infoln("Fixing") + ts.db.Run(bson.D{{"eval", `function() { db.nodes.find().forEach(function (result) { + db.nodes.update({"_id": result._id}, result.value) + }) }`}, {"args", bson.D{}}}, nil) + + ts.session.SetSafe(&mgo.Safe{}) +} diff --git a/graph/mongo/triplestore_iterator_optimize.go b/graph/mongo/triplestore_iterator_optimize.go new file mode 100644 index 0000000..d10bc22 --- /dev/null +++ b/graph/mongo/triplestore_iterator_optimize.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mongo + +import ( + "github.com/google/cayley/graph" +) + +func (ts *MongoTripleStore) OptimizeIterator(it graph.Iterator) (graph.Iterator, bool) { + switch it.Type() { + case "linksto": + return ts.optimizeLinksTo(it.(*graph.LinksToIterator)) + + } + return it, false +} + +func (ts *MongoTripleStore) optimizeLinksTo(it *graph.LinksToIterator) (graph.Iterator, bool) { + l := it.GetSubIterators() + if l.Len() != 1 { + return it, false + } + primaryIt := l.Front().Value.(graph.Iterator) + if primaryIt.Type() == "fixed" { + size, _ := primaryIt.Size() + if size == 1 { + val, ok := primaryIt.Next() + if !ok { + panic("Sizes lie") + } + newIt := ts.GetTripleIterator(it.Direction(), val) + newIt.CopyTagsFrom(it) + for _, tag := range primaryIt.Tags() { + newIt.AddFixedTag(tag, val) + } + it.Close() + return newIt, true + } + } + return it, false +} diff --git a/graph/sexp/session.go b/graph/sexp/session.go new file mode 100644 index 0000000..7065e02 --- /dev/null +++ b/graph/sexp/session.go @@ -0,0 +1,121 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sexp + +// Defines a running session of the sexp query language. + +import ( + "errors" + "fmt" + "sort" + + "github.com/google/cayley/graph" +) + +type SexpSession struct { + ts graph.TripleStore + debug bool +} + +func NewSexpSession(inputTripleStore graph.TripleStore) *SexpSession { + var s SexpSession + s.ts = inputTripleStore + return &s +} + +func (s *SexpSession) ToggleDebug() { + s.debug = !s.debug +} + +func (s *SexpSession) InputParses(input string) (graph.ParseResult, error) { + var parenDepth int + for i, x := range input { + if x == '(' { + parenDepth++ + } + if x == ')' { + parenDepth-- + if parenDepth < 0 { + min := 0 + if (i - 10) > min { + min = i - 10 + } + return graph.ParseFail, errors.New(fmt.Sprintf("Too many close parens at char %d: %s", i, input[min:i])) + } + } + } + if parenDepth > 0 { + return graph.ParseMore, nil + } + if len(ParseString(input)) > 0 { + return graph.Parsed, nil + } + return graph.ParseFail, errors.New("Invalid Syntax") +} + +func (s *SexpSession) ExecInput(input string, out chan interface{}, limit int) { + it := BuildIteratorTreeForQuery(s.ts, input) + newIt, changed := it.Optimize() + if changed { + it = newIt + } + + if s.debug { + fmt.Println(it.DebugString(0)) + } + nResults := 0 + for { + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + out <- &tags + nResults++ + if nResults > limit && limit != -1 { + break + } + for it.NextResult() == true { + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + out <- &tags + nResults++ + if nResults > limit && limit != -1 { + break + } + } + } + close(out) +} + +func (s *SexpSession) ToText(result interface{}) string { + out := fmt.Sprintln("****") + tags := result.(*map[string]graph.TSVal) + tagKeys := make([]string, len(*tags)) + i := 0 + for k, _ := range *tags { + tagKeys[i] = k + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + if k == "$_" { + continue + } + out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) + } + return out +} diff --git a/graph/sexp/sexp_session.go b/graph/sexp/sexp_session.go deleted file mode 100644 index 7065e02..0000000 --- a/graph/sexp/sexp_session.go +++ /dev/null @@ -1,121 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sexp - -// Defines a running session of the sexp query language. - -import ( - "errors" - "fmt" - "sort" - - "github.com/google/cayley/graph" -) - -type SexpSession struct { - ts graph.TripleStore - debug bool -} - -func NewSexpSession(inputTripleStore graph.TripleStore) *SexpSession { - var s SexpSession - s.ts = inputTripleStore - return &s -} - -func (s *SexpSession) ToggleDebug() { - s.debug = !s.debug -} - -func (s *SexpSession) InputParses(input string) (graph.ParseResult, error) { - var parenDepth int - for i, x := range input { - if x == '(' { - parenDepth++ - } - if x == ')' { - parenDepth-- - if parenDepth < 0 { - min := 0 - if (i - 10) > min { - min = i - 10 - } - return graph.ParseFail, errors.New(fmt.Sprintf("Too many close parens at char %d: %s", i, input[min:i])) - } - } - } - if parenDepth > 0 { - return graph.ParseMore, nil - } - if len(ParseString(input)) > 0 { - return graph.Parsed, nil - } - return graph.ParseFail, errors.New("Invalid Syntax") -} - -func (s *SexpSession) ExecInput(input string, out chan interface{}, limit int) { - it := BuildIteratorTreeForQuery(s.ts, input) - newIt, changed := it.Optimize() - if changed { - it = newIt - } - - if s.debug { - fmt.Println(it.DebugString(0)) - } - nResults := 0 - for { - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - out <- &tags - nResults++ - if nResults > limit && limit != -1 { - break - } - for it.NextResult() == true { - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - out <- &tags - nResults++ - if nResults > limit && limit != -1 { - break - } - } - } - close(out) -} - -func (s *SexpSession) ToText(result interface{}) string { - out := fmt.Sprintln("****") - tags := result.(*map[string]graph.TSVal) - tagKeys := make([]string, len(*tags)) - i := 0 - for k, _ := range *tags { - tagKeys[i] = k - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - if k == "$_" { - continue - } - out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) - } - return out -} diff --git a/http/cayley_http.go b/http/cayley_http.go deleted file mode 100644 index f850715..0000000 --- a/http/cayley_http.go +++ /dev/null @@ -1,152 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "flag" - "fmt" - "html/template" - "net/http" - "os" - "time" - - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/config" - "github.com/google/cayley/graph" -) - -type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int - -var assetsPath = flag.String("assets", "", "Explicit path to the HTTP assets.") -var assetsDirs = []string{"templates", "static", "docs"} - -func hasAssets(path string) bool { - for _, dir := range assetsDirs { - if _, err := os.Stat(fmt.Sprint(path, "/", dir)); os.IsNotExist(err) { - return false - } - } - return true -} - -func findAssetsPath() string { - if *assetsPath != "" { - if hasAssets(*assetsPath) { - return *assetsPath - } else { - glog.Fatalln("Cannot find assets at", *assetsPath, ".") - } - } - - if hasAssets(".") { - return "." - } - - gopathPath := os.ExpandEnv("$GOPATH/src/github.com/google/cayley") - if hasAssets(gopathPath) { - return gopathPath - } - glog.Fatalln("Cannot find assets in any of the default search paths. Please run in the same directory, in a Go workspace, or set --assets .") - return "" -} - -func LogRequest(handler ResponseHandler) httprouter.Handle { - return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { - start := time.Now() - addr := req.Header.Get("X-Real-IP") - if addr == "" { - addr = req.Header.Get("X-Forwarded-For") - if addr == "" { - addr = req.RemoteAddr - } - } - glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr) - code := handler(w, req, params) - glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start)) - - } -} - -func FormatJson400(w http.ResponseWriter, err interface{}) int { - return FormatJsonError(w, 400, err) -} - -func FormatJsonError(w http.ResponseWriter, code int, err interface{}) int { - http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code) - return code -} - -type TemplateRequestHandler struct { - templates *template.Template -} - -func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { - uiType := params.ByName("ui_type") - if r.URL.Path == "/" { - uiType = "query" - } - err := h.templates.ExecuteTemplate(w, uiType+".html", h) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - -type Api struct { - config *config.CayleyConfig - ts graph.TripleStore -} - -func (api *Api) ApiV1(r *httprouter.Router) { - r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) - r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) - r.POST("/api/v1/write", LogRequest(api.ServeV1Write)) - r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad)) - //TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form? - r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete)) -} - -func SetupRoutes(ts graph.TripleStore, cfg *config.CayleyConfig) { - r := httprouter.New() - assets := findAssetsPath() - if glog.V(2) { - glog.V(2).Infoln("Found assets at", assets) - } - var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl"))) - templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html")) - root := &TemplateRequestHandler{templates: templates} - docs := &DocRequestHandler{assets: assets} - api := &Api{config: cfg, ts: ts} - api.ApiV1(r) - - //m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true})) - //r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) - r.GET("/docs/:docpage", docs.ServeHTTP) - r.GET("/ui/:ui_type", root.ServeHTTP) - r.GET("/", root.ServeHTTP) - http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/"))))) - http.Handle("/", r) -} - -func CayleyHTTP(ts graph.TripleStore, cfg *config.CayleyConfig) { - SetupRoutes(ts, cfg) - glog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) - fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) - err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil) - if err != nil { - glog.Fatal("ListenAndServe: ", err) - } -} diff --git a/http/cayley_http_docs.go b/http/cayley_http_docs.go deleted file mode 100644 index 0d1d234..0000000 --- a/http/cayley_http_docs.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "fmt" - "io/ioutil" - "net/http" - "os" - - "github.com/julienschmidt/httprouter" - "github.com/russross/blackfriday" -) - -type DocRequestHandler struct { - assets string -} - -func MarkdownWithCSS(input []byte, title string) []byte { - // set up the HTML renderer - htmlFlags := 0 - htmlFlags |= blackfriday.HTML_USE_XHTML - htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS - htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS - htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES - htmlFlags |= blackfriday.HTML_COMPLETE_PAGE - renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS) - - // set up the parser - extensions := 0 - //extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS - extensions |= blackfriday.EXTENSION_TABLES - extensions |= blackfriday.EXTENSION_FENCED_CODE - extensions |= blackfriday.EXTENSION_AUTOLINK - extensions |= blackfriday.EXTENSION_STRIKETHROUGH - //extensions |= blackfriday.EXTENSION_SPACE_HEADERS - extensions |= blackfriday.EXTENSION_HEADER_IDS - extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS - - return blackfriday.Markdown(input, renderer, extensions) -} - -func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { - docpage := params.ByName("docpage") - if docpage == "" { - docpage = "Index" - } - file, err := os.Open(fmt.Sprintf("%s/docs/%s.md", h.assets, docpage)) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - data, err := ioutil.ReadAll(file) - if err != nil { - http.Error(w, err.Error(), http.StatusNoContent) - return - } - output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage)) - fmt.Fprint(w, string(output)) -} - -var markdownCSS = "/static/css/docs.css" diff --git a/http/cayley_http_query.go b/http/cayley_http_query.go deleted file mode 100644 index 8170e13..0000000 --- a/http/cayley_http_query.go +++ /dev/null @@ -1,153 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/graph" - "github.com/google/cayley/query/gremlin" - "github.com/google/cayley/query/mql" -) - -type SuccessQueryWrapper struct { - Result interface{} `json:"result"` -} - -type ErrorQueryWrapper struct { - Error string `json:"error"` -} - -func WrapErrResult(err error) ([]byte, error) { - var wrap ErrorQueryWrapper - wrap.Error = err.Error() - return json.MarshalIndent(wrap, "", " ") -} - -func WrapResult(result interface{}) ([]byte, error) { - var wrap SuccessQueryWrapper - wrap.Result = result - return json.MarshalIndent(wrap, "", " ") -} - -func RunJsonQuery(query string, ses graph.HttpSession) (interface{}, error) { - c := make(chan interface{}, 5) - go ses.ExecInput(query, c, 100) - for res := range c { - ses.BuildJson(res) - } - return ses.GetJson() -} - -func GetQueryShape(query string, ses graph.HttpSession) ([]byte, error) { - c := make(chan map[string]interface{}, 5) - go ses.GetQuery(query, c) - var data map[string]interface{} - for res := range c { - data = res - } - return json.Marshal(data) -} - -// TODO(barakmich): Turn this into proper middleware. -func (api *Api) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - var ses graph.HttpSession - switch params.ByName("query_lang") { - case "gremlin": - ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) - case "mql": - ses = mql.NewMqlSession(api.ts) - default: - return FormatJson400(w, "Need a query language.") - } - var err error - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - code := string(bodyBytes) - result, err := ses.InputParses(code) - switch result { - case graph.Parsed: - var output interface{} - var bytes []byte - var err error - output, err = RunJsonQuery(code, ses) - if err != nil { - bytes, err = WrapErrResult(err) - http.Error(w, string(bytes), 400) - ses = nil - return 400 - } - bytes, err = WrapResult(output) - if err != nil { - ses = nil - return FormatJson400(w, err) - } - fmt.Fprint(w, string(bytes)) - ses = nil - return 200 - case graph.ParseFail: - ses = nil - return FormatJson400(w, err) - default: - ses = nil - return FormatJsonError(w, 500, "Incomplete data?") - } - http.Error(w, "", http.StatusNotFound) - ses = nil - return http.StatusNotFound -} - -func (api *Api) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - var ses graph.HttpSession - switch params.ByName("query_lang") { - case "gremlin": - ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) - case "mql": - ses = mql.NewMqlSession(api.ts) - default: - return FormatJson400(w, "Need a query language.") - } - var err error - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - code := string(bodyBytes) - result, err := ses.InputParses(code) - switch result { - case graph.Parsed: - var output []byte - var err error - output, err = GetQueryShape(code, ses) - if err != nil { - return FormatJson400(w, err) - } - fmt.Fprint(w, string(output)) - return 200 - case graph.ParseFail: - return FormatJson400(w, err) - default: - return FormatJsonError(w, 500, "Incomplete data?") - } - http.Error(w, "", http.StatusNotFound) - return http.StatusNotFound -} diff --git a/http/cayley_http_test.go b/http/cayley_http_test.go deleted file mode 100644 index d59749f..0000000 --- a/http/cayley_http_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "testing" - . "github.com/smartystreets/goconvey/convey" -) - -func TestParseJSONOkay(t *testing.T) { - Convey("Parse JSON", t, func() { - bytelist := []byte(`[ - {"subject": "foo", "predicate": "bar", "object": "baz"}, - {"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"} - ]`) - x, err := ParseJsonToTripleList(bytelist) - So(err, ShouldBeNil) - So(len(x), ShouldEqual, 2) - So(x[0].Sub, ShouldEqual, "foo") - So(x[0].Provenance, ShouldEqual, "") - So(x[1].Provenance, ShouldEqual, "graph") - }) - - Convey("Parse JSON extra field", t, func() { - bytelist := []byte(`[ - {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} - ]`) - _, err := ParseJsonToTripleList(bytelist) - So(err, ShouldBeNil) - }) -} - -func TestParseJSONFail(t *testing.T) { - Convey("Parse JSON Fail", t, func() { - bytelist := []byte(`[ - {"subject": "foo", "predicate": "bar"} - ]`) - _, err := ParseJsonToTripleList(bytelist) - So(err, ShouldNotBeNil) - }) -} diff --git a/http/cayley_http_write.go b/http/cayley_http_write.go deleted file mode 100644 index 20fb60d..0000000 --- a/http/cayley_http_write.go +++ /dev/null @@ -1,119 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package http - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" - "strconv" - - "github.com/barakmich/glog" - "github.com/julienschmidt/httprouter" - - "github.com/google/cayley/graph" - "github.com/google/cayley/nquads" -) - -func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) { - var tripleList []*graph.Triple - err := json.Unmarshal(jsonBody, &tripleList) - if err != nil { - return nil, err - } - for i, t := range tripleList { - if !t.IsValid() { - return nil, errors.New(fmt.Sprintf("Invalid triple at index %d. %s", i, t.ToString())) - } - } - return tripleList, nil -} - -func (api *Api) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int { - if api.config.ReadOnly { - return FormatJson400(w, "Database is read-only.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - tripleList, terr := ParseJsonToTripleList(bodyBytes) - if terr != nil { - return FormatJson400(w, terr) - } - api.ts.AddTripleSet(tripleList) - fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", len(tripleList)) - return 200 -} - -func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - if api.config.ReadOnly { - return FormatJson400(w, "Database is read-only.") - } - - formFile, _, err := r.FormFile("NQuadFile") - if err != nil { - glog.Errorln(err) - return FormatJsonError(w, 500, "Couldn't read file: "+err.Error()) - } - - defer formFile.Close() - - blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) - if blockErr != nil { - blockSize = int64(api.config.LoadSize) - } - - tChan := make(chan *graph.Triple) - go nquads.ReadNQuadsFromReader(tChan, formFile) - tripleblock := make([]*graph.Triple, blockSize) - nTriples := 0 - i := int64(0) - for t := range tChan { - tripleblock[i] = t - i++ - nTriples++ - if i == blockSize { - api.ts.AddTripleSet(tripleblock) - i = 0 - } - } - api.ts.AddTripleSet(tripleblock[0:i]) - fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", nTriples) - return 200 -} - -func (api *Api) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { - if api.config.ReadOnly { - return FormatJson400(w, "Database is read-only.") - } - bodyBytes, err := ioutil.ReadAll(r.Body) - if err != nil { - return FormatJson400(w, err) - } - tripleList, terr := ParseJsonToTripleList(bodyBytes) - if terr != nil { - return FormatJson400(w, terr) - } - count := 0 - for _, triple := range tripleList { - api.ts.RemoveTriple(triple) - count++ - } - fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d triples.\"}", count) - return 200 -} diff --git a/http/docs.go b/http/docs.go new file mode 100644 index 0000000..0d1d234 --- /dev/null +++ b/http/docs.go @@ -0,0 +1,74 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + + "github.com/julienschmidt/httprouter" + "github.com/russross/blackfriday" +) + +type DocRequestHandler struct { + assets string +} + +func MarkdownWithCSS(input []byte, title string) []byte { + // set up the HTML renderer + htmlFlags := 0 + htmlFlags |= blackfriday.HTML_USE_XHTML + htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS + htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS + htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES + htmlFlags |= blackfriday.HTML_COMPLETE_PAGE + renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS) + + // set up the parser + extensions := 0 + //extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS + extensions |= blackfriday.EXTENSION_TABLES + extensions |= blackfriday.EXTENSION_FENCED_CODE + extensions |= blackfriday.EXTENSION_AUTOLINK + extensions |= blackfriday.EXTENSION_STRIKETHROUGH + //extensions |= blackfriday.EXTENSION_SPACE_HEADERS + extensions |= blackfriday.EXTENSION_HEADER_IDS + extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS + + return blackfriday.Markdown(input, renderer, extensions) +} + +func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + docpage := params.ByName("docpage") + if docpage == "" { + docpage = "Index" + } + file, err := os.Open(fmt.Sprintf("%s/docs/%s.md", h.assets, docpage)) + if err != nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + data, err := ioutil.ReadAll(file) + if err != nil { + http.Error(w, err.Error(), http.StatusNoContent) + return + } + output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage)) + fmt.Fprint(w, string(output)) +} + +var markdownCSS = "/static/css/docs.css" diff --git a/http/http.go b/http/http.go new file mode 100644 index 0000000..f850715 --- /dev/null +++ b/http/http.go @@ -0,0 +1,152 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "flag" + "fmt" + "html/template" + "net/http" + "os" + "time" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/config" + "github.com/google/cayley/graph" +) + +type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int + +var assetsPath = flag.String("assets", "", "Explicit path to the HTTP assets.") +var assetsDirs = []string{"templates", "static", "docs"} + +func hasAssets(path string) bool { + for _, dir := range assetsDirs { + if _, err := os.Stat(fmt.Sprint(path, "/", dir)); os.IsNotExist(err) { + return false + } + } + return true +} + +func findAssetsPath() string { + if *assetsPath != "" { + if hasAssets(*assetsPath) { + return *assetsPath + } else { + glog.Fatalln("Cannot find assets at", *assetsPath, ".") + } + } + + if hasAssets(".") { + return "." + } + + gopathPath := os.ExpandEnv("$GOPATH/src/github.com/google/cayley") + if hasAssets(gopathPath) { + return gopathPath + } + glog.Fatalln("Cannot find assets in any of the default search paths. Please run in the same directory, in a Go workspace, or set --assets .") + return "" +} + +func LogRequest(handler ResponseHandler) httprouter.Handle { + return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) { + start := time.Now() + addr := req.Header.Get("X-Real-IP") + if addr == "" { + addr = req.Header.Get("X-Forwarded-For") + if addr == "" { + addr = req.RemoteAddr + } + } + glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr) + code := handler(w, req, params) + glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start)) + + } +} + +func FormatJson400(w http.ResponseWriter, err interface{}) int { + return FormatJsonError(w, 400, err) +} + +func FormatJsonError(w http.ResponseWriter, code int, err interface{}) int { + http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code) + return code +} + +type TemplateRequestHandler struct { + templates *template.Template +} + +func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) { + uiType := params.ByName("ui_type") + if r.URL.Path == "/" { + uiType = "query" + } + err := h.templates.ExecuteTemplate(w, uiType+".html", h) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + } +} + +type Api struct { + config *config.CayleyConfig + ts graph.TripleStore +} + +func (api *Api) ApiV1(r *httprouter.Router) { + r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query)) + r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape)) + r.POST("/api/v1/write", LogRequest(api.ServeV1Write)) + r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad)) + //TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form? + r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete)) +} + +func SetupRoutes(ts graph.TripleStore, cfg *config.CayleyConfig) { + r := httprouter.New() + assets := findAssetsPath() + if glog.V(2) { + glog.V(2).Infoln("Found assets at", assets) + } + var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl"))) + templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html")) + root := &TemplateRequestHandler{templates: templates} + docs := &DocRequestHandler{assets: assets} + api := &Api{config: cfg, ts: ts} + api.ApiV1(r) + + //m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true})) + //r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/")))) + r.GET("/docs/:docpage", docs.ServeHTTP) + r.GET("/ui/:ui_type", root.ServeHTTP) + r.GET("/", root.ServeHTTP) + http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/"))))) + http.Handle("/", r) +} + +func CayleyHTTP(ts graph.TripleStore, cfg *config.CayleyConfig) { + SetupRoutes(ts, cfg) + glog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) + fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort) + err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil) + if err != nil { + glog.Fatal("ListenAndServe: ", err) + } +} diff --git a/http/http_test.go b/http/http_test.go new file mode 100644 index 0000000..d59749f --- /dev/null +++ b/http/http_test.go @@ -0,0 +1,53 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "testing" + . "github.com/smartystreets/goconvey/convey" +) + +func TestParseJSONOkay(t *testing.T) { + Convey("Parse JSON", t, func() { + bytelist := []byte(`[ + {"subject": "foo", "predicate": "bar", "object": "baz"}, + {"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"} + ]`) + x, err := ParseJsonToTripleList(bytelist) + So(err, ShouldBeNil) + So(len(x), ShouldEqual, 2) + So(x[0].Sub, ShouldEqual, "foo") + So(x[0].Provenance, ShouldEqual, "") + So(x[1].Provenance, ShouldEqual, "graph") + }) + + Convey("Parse JSON extra field", t, func() { + bytelist := []byte(`[ + {"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"} + ]`) + _, err := ParseJsonToTripleList(bytelist) + So(err, ShouldBeNil) + }) +} + +func TestParseJSONFail(t *testing.T) { + Convey("Parse JSON Fail", t, func() { + bytelist := []byte(`[ + {"subject": "foo", "predicate": "bar"} + ]`) + _, err := ParseJsonToTripleList(bytelist) + So(err, ShouldNotBeNil) + }) +} diff --git a/http/query.go b/http/query.go new file mode 100644 index 0000000..8170e13 --- /dev/null +++ b/http/query.go @@ -0,0 +1,153 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/query/gremlin" + "github.com/google/cayley/query/mql" +) + +type SuccessQueryWrapper struct { + Result interface{} `json:"result"` +} + +type ErrorQueryWrapper struct { + Error string `json:"error"` +} + +func WrapErrResult(err error) ([]byte, error) { + var wrap ErrorQueryWrapper + wrap.Error = err.Error() + return json.MarshalIndent(wrap, "", " ") +} + +func WrapResult(result interface{}) ([]byte, error) { + var wrap SuccessQueryWrapper + wrap.Result = result + return json.MarshalIndent(wrap, "", " ") +} + +func RunJsonQuery(query string, ses graph.HttpSession) (interface{}, error) { + c := make(chan interface{}, 5) + go ses.ExecInput(query, c, 100) + for res := range c { + ses.BuildJson(res) + } + return ses.GetJson() +} + +func GetQueryShape(query string, ses graph.HttpSession) ([]byte, error) { + c := make(chan map[string]interface{}, 5) + go ses.GetQuery(query, c) + var data map[string]interface{} + for res := range c { + data = res + } + return json.Marshal(data) +} + +// TODO(barakmich): Turn this into proper middleware. +func (api *Api) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + var ses graph.HttpSession + switch params.ByName("query_lang") { + case "gremlin": + ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) + case "mql": + ses = mql.NewMqlSession(api.ts) + default: + return FormatJson400(w, "Need a query language.") + } + var err error + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + code := string(bodyBytes) + result, err := ses.InputParses(code) + switch result { + case graph.Parsed: + var output interface{} + var bytes []byte + var err error + output, err = RunJsonQuery(code, ses) + if err != nil { + bytes, err = WrapErrResult(err) + http.Error(w, string(bytes), 400) + ses = nil + return 400 + } + bytes, err = WrapResult(output) + if err != nil { + ses = nil + return FormatJson400(w, err) + } + fmt.Fprint(w, string(bytes)) + ses = nil + return 200 + case graph.ParseFail: + ses = nil + return FormatJson400(w, err) + default: + ses = nil + return FormatJsonError(w, 500, "Incomplete data?") + } + http.Error(w, "", http.StatusNotFound) + ses = nil + return http.StatusNotFound +} + +func (api *Api) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + var ses graph.HttpSession + switch params.ByName("query_lang") { + case "gremlin": + ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false) + case "mql": + ses = mql.NewMqlSession(api.ts) + default: + return FormatJson400(w, "Need a query language.") + } + var err error + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + code := string(bodyBytes) + result, err := ses.InputParses(code) + switch result { + case graph.Parsed: + var output []byte + var err error + output, err = GetQueryShape(code, ses) + if err != nil { + return FormatJson400(w, err) + } + fmt.Fprint(w, string(output)) + return 200 + case graph.ParseFail: + return FormatJson400(w, err) + default: + return FormatJsonError(w, 500, "Incomplete data?") + } + http.Error(w, "", http.StatusNotFound) + return http.StatusNotFound +} diff --git a/http/write.go b/http/write.go new file mode 100644 index 0000000..20fb60d --- /dev/null +++ b/http/write.go @@ -0,0 +1,119 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package http + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "strconv" + + "github.com/barakmich/glog" + "github.com/julienschmidt/httprouter" + + "github.com/google/cayley/graph" + "github.com/google/cayley/nquads" +) + +func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) { + var tripleList []*graph.Triple + err := json.Unmarshal(jsonBody, &tripleList) + if err != nil { + return nil, err + } + for i, t := range tripleList { + if !t.IsValid() { + return nil, errors.New(fmt.Sprintf("Invalid triple at index %d. %s", i, t.ToString())) + } + } + return tripleList, nil +} + +func (api *Api) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int { + if api.config.ReadOnly { + return FormatJson400(w, "Database is read-only.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + tripleList, terr := ParseJsonToTripleList(bodyBytes) + if terr != nil { + return FormatJson400(w, terr) + } + api.ts.AddTripleSet(tripleList) + fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", len(tripleList)) + return 200 +} + +func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + if api.config.ReadOnly { + return FormatJson400(w, "Database is read-only.") + } + + formFile, _, err := r.FormFile("NQuadFile") + if err != nil { + glog.Errorln(err) + return FormatJsonError(w, 500, "Couldn't read file: "+err.Error()) + } + + defer formFile.Close() + + blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) + if blockErr != nil { + blockSize = int64(api.config.LoadSize) + } + + tChan := make(chan *graph.Triple) + go nquads.ReadNQuadsFromReader(tChan, formFile) + tripleblock := make([]*graph.Triple, blockSize) + nTriples := 0 + i := int64(0) + for t := range tChan { + tripleblock[i] = t + i++ + nTriples++ + if i == blockSize { + api.ts.AddTripleSet(tripleblock) + i = 0 + } + } + api.ts.AddTripleSet(tripleblock[0:i]) + fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", nTriples) + return 200 +} + +func (api *Api) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int { + if api.config.ReadOnly { + return FormatJson400(w, "Database is read-only.") + } + bodyBytes, err := ioutil.ReadAll(r.Body) + if err != nil { + return FormatJson400(w, err) + } + tripleList, terr := ParseJsonToTripleList(bodyBytes) + if terr != nil { + return FormatJson400(w, terr) + } + count := 0 + for _, triple := range tripleList { + api.ts.RemoveTriple(triple) + count++ + } + fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d triples.\"}", count) + return 200 +} diff --git a/query/gremlin/build_iterator.go b/query/gremlin/build_iterator.go new file mode 100644 index 0000000..b6e9a45 --- /dev/null +++ b/query/gremlin/build_iterator.go @@ -0,0 +1,315 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "strconv" + + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" +) + +func getStrings(obj *otto.Object, field string) []string { + strings := make([]string, 0) + val, _ := obj.Get(field) + if !val.IsUndefined() { + export, _ := val.Export() + array := export.([]interface{}) + for _, arg := range array { + strings = append(strings, arg.(string)) + } + } + return strings +} + +func getStringArgs(obj *otto.Object) []string { return getStrings(obj, "string_args") } + +func buildIteratorTree(obj *otto.Object, ts graph.TripleStore) graph.Iterator { + if !isVertexChain(obj) { + return graph.NewNullIterator() + } + return buildIteratorTreeHelper(obj, ts, graph.NewNullIterator()) +} + +func makeListOfStringsFromArrayValue(obj *otto.Object) []string { + var output []string + lengthValue, _ := obj.Get("length") + length, _ := lengthValue.ToInteger() + ulength := uint32(length) + for index := uint32(0); index < ulength; index += 1 { + name := strconv.FormatInt(int64(index), 10) + value, err := obj.Get(name) + if err != nil { + continue + } + if !value.IsString() { + continue + } + s, _ := value.ToString() + output = append(output, s) + } + return output +} + +func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator { + if val.IsNull() || val.IsUndefined() { + return ts.GetNodesAllIterator() + } + if val.IsPrimitive() { + thing, _ := val.Export() + switch v := thing.(type) { + case string: + it := ts.MakeFixed() + it.AddValue(ts.GetIdFor(v)) + return it + default: + glog.Errorln("Trying to build unknown primitive value.") + } + } + switch val.Class() { + case "Object": + return buildIteratorTree(val.Object(), ts) + case "Array": + // Had better be an array of strings + strings := makeListOfStringsFromArrayValue(val.Object()) + it := ts.MakeFixed() + for _, x := range strings { + it.AddValue(ts.GetIdFor(x)) + } + return it + case "Number": + fallthrough + case "Boolean": + fallthrough + case "Date": + fallthrough + case "String": + it := ts.MakeFixed() + str, _ := val.ToString() + it.AddValue(ts.GetIdFor(str)) + return it + default: + glog.Errorln("Trying to handle unsupported Javascript value.") + return graph.NewNullIterator() + } +} + +func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Iterator, isReverse bool) graph.Iterator { + argList, _ := obj.Get("_gremlin_values") + if argList.Class() != "GoArray" { + glog.Errorln("How is arglist not an array? Return nothing.", argList.Class()) + return graph.NewNullIterator() + } + argArray := argList.Object() + lengthVal, _ := argArray.Get("length") + length, _ := lengthVal.ToInteger() + var predicateNodeIterator graph.Iterator + if length == 0 { + predicateNodeIterator = ts.GetNodesAllIterator() + } else { + zero, _ := argArray.Get("0") + predicateNodeIterator = buildIteratorFromValue(zero, ts) + } + if length >= 2 { + var tags []string + one, _ := argArray.Get("1") + if one.IsString() { + s, _ := one.ToString() + tags = append(tags, s) + } else if one.Class() == "Array" { + tags = makeListOfStringsFromArrayValue(one.Object()) + } + for _, tag := range tags { + predicateNodeIterator.AddTag(tag) + } + } + + in, out := "s", "o" + if isReverse { + in, out = out, in + } + lto := graph.NewLinksToIterator(ts, base, in) + and := graph.NewAndIterator() + and.AddSubIterator(graph.NewLinksToIterator(ts, predicateNodeIterator, "p")) + and.AddSubIterator(lto) + return graph.NewHasaIterator(ts, and, out) +} + +func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.Iterator) graph.Iterator { + var it graph.Iterator + it = base + // TODO: Better error handling + kindVal, _ := obj.Get("_gremlin_type") + stringArgs := getStringArgs(obj) + var subIt graph.Iterator + prevVal, _ := obj.Get("_gremlin_prev") + if !prevVal.IsObject() { + subIt = base + } else { + subIt = buildIteratorTreeHelper(prevVal.Object(), ts, base) + } + + kind, _ := kindVal.ToString() + switch kind { + case "vertex": + if len(stringArgs) == 0 { + it = ts.GetNodesAllIterator() + } else { + fixed := ts.MakeFixed() + for _, name := range stringArgs { + fixed.AddValue(ts.GetIdFor(name)) + } + it = fixed + } + case "tag": + it = subIt + for _, tag := range stringArgs { + it.AddTag(tag) + } + case "save": + all := ts.GetNodesAllIterator() + if len(stringArgs) > 2 || len(stringArgs) == 0 { + return graph.NewNullIterator() + } + if len(stringArgs) == 2 { + all.AddTag(stringArgs[1]) + } else { + all.AddTag(stringArgs[0]) + } + predFixed := ts.MakeFixed() + predFixed.AddValue(ts.GetIdFor(stringArgs[0])) + subAnd := graph.NewAndIterator() + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "o")) + hasa := graph.NewHasaIterator(ts, subAnd, "s") + and := graph.NewAndIterator() + and.AddSubIterator(hasa) + and.AddSubIterator(subIt) + it = and + case "saver": + all := ts.GetNodesAllIterator() + if len(stringArgs) > 2 || len(stringArgs) == 0 { + return graph.NewNullIterator() + } + if len(stringArgs) == 2 { + all.AddTag(stringArgs[1]) + } else { + all.AddTag(stringArgs[0]) + } + predFixed := ts.MakeFixed() + predFixed.AddValue(ts.GetIdFor(stringArgs[0])) + subAnd := graph.NewAndIterator() + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "s")) + hasa := graph.NewHasaIterator(ts, subAnd, "o") + and := graph.NewAndIterator() + and.AddSubIterator(hasa) + and.AddSubIterator(subIt) + it = and + case "has": + fixed := ts.MakeFixed() + if len(stringArgs) < 2 { + return graph.NewNullIterator() + } + for _, name := range stringArgs[1:] { + fixed.AddValue(ts.GetIdFor(name)) + } + predFixed := ts.MakeFixed() + predFixed.AddValue(ts.GetIdFor(stringArgs[0])) + subAnd := graph.NewAndIterator() + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) + subAnd.AddSubIterator(graph.NewLinksToIterator(ts, fixed, "o")) + hasa := graph.NewHasaIterator(ts, subAnd, "s") + and := graph.NewAndIterator() + and.AddSubIterator(hasa) + and.AddSubIterator(subIt) + it = and + case "morphism": + it = base + case "and": + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if !isVertexChain(firstArg.Object()) { + return graph.NewNullIterator() + } + argIt := buildIteratorTree(firstArg.Object(), ts) + + and := graph.NewAndIterator() + and.AddSubIterator(subIt) + and.AddSubIterator(argIt) + it = and + case "back": + arg, _ := obj.Get("_gremlin_back_chain") + argIt := buildIteratorTree(arg.Object(), ts) + and := graph.NewAndIterator() + and.AddSubIterator(subIt) + and.AddSubIterator(argIt) + it = and + case "is": + fixed := ts.MakeFixed() + for _, name := range stringArgs { + fixed.AddValue(ts.GetIdFor(name)) + } + and := graph.NewAndIterator() + and.AddSubIterator(fixed) + and.AddSubIterator(subIt) + it = and + case "or": + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if !isVertexChain(firstArg.Object()) { + return graph.NewNullIterator() + } + argIt := buildIteratorTree(firstArg.Object(), ts) + + or := graph.NewOrIterator() + or.AddSubIterator(subIt) + or.AddSubIterator(argIt) + it = or + case "both": + // Hardly the most efficient pattern, but the most general. + // Worth looking into an Optimize() optimization here. + clone := subIt.Clone() + it1 := buildInOutIterator(obj, ts, subIt, false) + it2 := buildInOutIterator(obj, ts, clone, true) + + or := graph.NewOrIterator() + or.AddSubIterator(it1) + or.AddSubIterator(it2) + it = or + case "out": + it = buildInOutIterator(obj, ts, subIt, false) + case "follow": + // Follow a morphism + arg, _ := obj.Get("_gremlin_values") + firstArg, _ := arg.Object().Get("0") + if isVertexChain(firstArg.Object()) { + return graph.NewNullIterator() + } + it = buildIteratorTreeHelper(firstArg.Object(), ts, subIt) + case "followr": + // Follow a morphism + arg, _ := obj.Get("_gremlin_followr") + if isVertexChain(arg.Object()) { + return graph.NewNullIterator() + } + it = buildIteratorTreeHelper(arg.Object(), ts, subIt) + case "in": + it = buildInOutIterator(obj, ts, subIt, true) + } + return it +} diff --git a/query/gremlin/environ.go b/query/gremlin/environ.go new file mode 100644 index 0000000..4e7f332 --- /dev/null +++ b/query/gremlin/environ.go @@ -0,0 +1,95 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +// Builds a new Gremlin environment pointing at a session. + +import ( + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" +) + +func BuildGremlinEnv(ses *GremlinSession) *otto.Otto { + env := otto.New() + setupGremlin(env, ses) + return env +} + +func concatStringArgs(call otto.FunctionCall) *[]interface{} { + outStrings := make([]interface{}, 0) + for _, arg := range call.ArgumentList { + if arg.IsString() { + outStrings = append(outStrings, arg.String()) + } + if arg.IsObject() && arg.Class() == "Array" { + obj, _ := arg.Export() + for _, x := range obj.([]interface{}) { + outStrings = append(outStrings, x.(string)) + } + } + } + return &outStrings +} + +func isVertexChain(obj *otto.Object) bool { + val, _ := obj.Get("_gremlin_type") + if x, _ := val.ToString(); x == "vertex" { + return true + } + val, _ = obj.Get("_gremlin_prev") + if val.IsObject() { + return isVertexChain(val.Object()) + } + return false +} + +func setupGremlin(env *otto.Otto, ses *GremlinSession) { + graph, _ := env.Object("graph = {}") + graph.Set("Vertex", func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, err := call.Otto.Object("out") + if err != nil { + glog.Error(err.Error()) + return otto.TrueValue() + } + out.Set("_gremlin_type", "vertex") + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + embedTraversals(env, ses, out) + embedFinals(env, ses, out) + return out.Value() + }) + + graph.Set("Morphism", func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", "morphism") + embedTraversals(env, ses, out) + return out.Value() + }) + graph.Set("Emit", func(call otto.FunctionCall) otto.Value { + value := call.Argument(0) + if value.IsDefined() { + ses.SendResult(&GremlinResult{metaresult: false, err: "", val: &value, actualResults: nil}) + } + return otto.NullValue() + }) + env.Run("graph.V = graph.Vertex") + env.Run("graph.M = graph.Morphism") + env.Run("g = graph") + +} diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go new file mode 100644 index 0000000..f8c978d --- /dev/null +++ b/query/gremlin/finals.go @@ -0,0 +1,274 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" +) + +const GremlinTopResultTag = "id" + +func embedFinals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { + obj.Set("All", allFunc(env, ses, obj)) + obj.Set("GetLimit", limitFunc(env, ses, obj)) + obj.Set("ToArray", toArrayFunc(env, ses, obj, false)) + obj.Set("ToValue", toValueFunc(env, ses, obj, false)) + obj.Set("TagArray", toArrayFunc(env, ses, obj, true)) + obj.Set("TagValue", toValueFunc(env, ses, obj, true)) + obj.Set("Map", mapFunc(env, ses, obj)) + obj.Set("ForEach", mapFunc(env, ses, obj)) +} + +func allFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + ses.limit = -1 + ses.count = 0 + runIteratorOnSession(it, ses) + return otto.NullValue() + } +} + +func limitFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + if len(call.ArgumentList) > 0 { + limitVal, _ := call.Argument(0).ToInteger() + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + ses.limit = int(limitVal) + ses.count = 0 + runIteratorOnSession(it, ses) + } + return otto.NullValue() + } +} + +func toArrayFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + limit := -1 + if len(call.ArgumentList) > 0 { + limitParsed, _ := call.Argument(0).ToInteger() + limit = int(limitParsed) + } + var val otto.Value + var err error + if !withTags { + array := runIteratorToArrayNoTags(it, ses, limit) + val, err = call.Otto.ToValue(array) + } else { + array := runIteratorToArray(it, ses, limit) + val, err = call.Otto.ToValue(array) + } + + if err != nil { + glog.Error(err) + return otto.NullValue() + } + return val + } +} + +func toValueFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + limit := 1 + var val otto.Value + var err error + if !withTags { + array := runIteratorToArrayNoTags(it, ses, limit) + if len(array) < 1 { + return otto.NullValue() + } + val, err = call.Otto.ToValue(array[0]) + } else { + array := runIteratorToArray(it, ses, limit) + if len(array) < 1 { + return otto.NullValue() + } + val, err = call.Otto.ToValue(array[0]) + } + if err != nil { + glog.Error(err) + return otto.NullValue() + } else { + return val + } + + } +} + +func mapFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + it := buildIteratorTree(obj, ses.ts) + it.AddTag(GremlinTopResultTag) + limit := -1 + if len(call.ArgumentList) == 0 { + return otto.NullValue() + } + callback := call.Argument(len(call.ArgumentList) - 1) + if len(call.ArgumentList) > 1 { + limitParsed, _ := call.Argument(0).ToInteger() + limit = int(limitParsed) + } + runIteratorWithCallback(it, ses, callback, call, limit) + return otto.NullValue() + } +} + +func tagsToValueMap(m map[string]graph.TSVal, ses *GremlinSession) map[string]string { + outputMap := make(map[string]string) + for k, v := range m { + outputMap[k] = ses.ts.GetNameFor(v) + } + return outputMap +} + +func runIteratorToArray(it graph.Iterator, ses *GremlinSession, limit int) []map[string]string { + output := make([]map[string]string, 0) + count := 0 + it, _ = it.Optimize() + for { + if ses.doHalt { + return nil + } + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + output = append(output, tagsToValueMap(tags, ses)) + count++ + if limit >= 0 && count >= limit { + break + } + for it.NextResult() == true { + if ses.doHalt { + return nil + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + output = append(output, tagsToValueMap(tags, ses)) + count++ + if limit >= 0 && count >= limit { + break + } + } + } + it.Close() + return output +} + +func runIteratorToArrayNoTags(it graph.Iterator, ses *GremlinSession, limit int) []string { + output := make([]string, 0) + count := 0 + it, _ = it.Optimize() + for { + if ses.doHalt { + return nil + } + val, ok := it.Next() + if !ok { + break + } + output = append(output, ses.ts.GetNameFor(val)) + count++ + if limit >= 0 && count >= limit { + break + } + } + it.Close() + return output +} + +func runIteratorWithCallback(it graph.Iterator, ses *GremlinSession, callback otto.Value, this otto.FunctionCall, limit int) { + count := 0 + it, _ = it.Optimize() + for { + if ses.doHalt { + return + } + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) + val, _ = callback.Call(this.This, val) + count++ + if limit >= 0 && count >= limit { + break + } + for it.NextResult() == true { + if ses.doHalt { + return + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) + val, _ = callback.Call(this.This, val) + count++ + if limit >= 0 && count >= limit { + break + } + } + } + it.Close() +} + +func runIteratorOnSession(it graph.Iterator, ses *GremlinSession) { + if ses.lookingForQueryShape { + graph.OutputQueryShapeForIterator(it, ses.ts, &(ses.queryShape)) + return + } + it, _ = it.Optimize() + glog.V(2).Infoln(it.DebugString(0)) + for { + // TODO(barakmich): Better halting. + if ses.doHalt { + return + } + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) + if !cont { + break + } + for it.NextResult() == true { + if ses.doHalt { + return + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) + if !cont { + break + } + } + } + it.Close() +} diff --git a/query/gremlin/functional_test.go b/query/gremlin/functional_test.go new file mode 100644 index 0000000..f6c65fb --- /dev/null +++ b/query/gremlin/functional_test.go @@ -0,0 +1,230 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "sort" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph/memstore" +) + +// +---+ +---+ +// | A |------- ->| F |<-- +// +---+ \------>+---+-/ +---+ \--+---+ +// ------>|#B#| | | E | +// +---+-------/ >+---+ | +---+ +// | C | / v +// +---+ -/ +---+ +// ---- +---+/ |#G#| +// \-->|#D#|------------->+---+ +// +---+ +// + +func buildTripleStore() *GremlinSession { + ts := memstore.MakeTestingMemstore() + return NewGremlinSession(ts, -1, false) +} + +func shouldBeUnordered(actual interface{}, expected ...interface{}) string { + if len(expected) != 1 { + return "Only one list supported" + } + actualStr := actual.([]string) + expectedStr := expected[0].([]string) + sort.Strings(actualStr) + sort.Strings(expectedStr) + return ShouldResemble(actualStr, expectedStr) +} + +func runQueryGetTag(query string, tag string) ([]string, int) { + js := buildTripleStore() + output := make([]string, 0) + c := make(chan interface{}, 5) + js.ExecInput(query, c, -1) + count := 0 + for result := range c { + count++ + data := result.(*GremlinResult) + if data.val == nil { + val := (*data.actualResults)[tag] + if val != nil { + output = append(output, js.ts.GetNameFor(val)) + } + } + } + return output, count +} + +func ConveyQuery(doc string, query string, expected []string) { + ConveyQueryTag(doc, query, GremlinTopResultTag, expected) +} + +func ConveyQueryTag(doc string, query string, tag string, expected []string) { + Convey(doc, func() { + actual, _ := runQueryGetTag(query, tag) + So(actual, shouldBeUnordered, expected) + }) +} + +func TestGremlin(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + + ConveyQuery("Can get a single vertex", + `g.V("A").All()`, + []string{"A"}) + + ConveyQuery("Can use .Out()", + `g.V("A").Out("follows").All()`, + []string{"B"}) + + ConveyQuery("Can use .In()", + `g.V("B").In("follows").All()`, + []string{"A", "C", "D"}) + + ConveyQuery("Can use .Both()", + `g.V("F").Both("follows").All()`, + []string{"B", "G", "E"}) + + ConveyQuery("Can use .Tag()-.Is()-.Back()", + `g.V("B").In("follows").Tag("foo").Out("status").Is("cool").Back("foo").All()`, + []string{"D"}) + + ConveyQuery("Can separate .Tag()-.Is()-.Back()", + ` + x = g.V("C").Out("follows").Tag("foo").Out("status").Is("cool").Back("foo") + x.In("follows").Is("D").Back("foo").All() + `, + []string{"B"}) + + Convey("Can do multiple .Back()s", func() { + query := ` + g.V("E").Out("follows").As("f").Out("follows").Out("status").Is("cool").Back("f").In("follows").In("follows").As("acd").Out("status").Is("cool").Back("f").All() + ` + expected := []string{"D"} + actual, _ := runQueryGetTag(query, "acd") + So(actual, shouldBeUnordered, expected) + }) + + }) +} + +func TestGremlinMorphism(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + + ConveyQuery("Simple morphism works", + ` + grandfollows = g.M().Out("follows").Out("follows") + g.V("C").Follow(grandfollows).All() + `, + []string{"G", "F", "B"}) + + ConveyQuery("Reverse morphism works", + ` + grandfollows = g.M().Out("follows").Out("follows") + g.V("F").FollowR(grandfollows).All() + `, []string{"A", "C", "D"}) + + }) +} + +func TestGremlinIntersection(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + ConveyQuery("Simple intersection", + ` + function follows(x) { return g.V(x).Out("follows") } + + follows("D").And(follows("C")).All() + `, []string{"B"}) + + ConveyQuery("Simple Morphism Intersection", + ` + grandfollows = g.M().Out("follows").Out("follows") + function gfollows(x) { return g.V(x).Follow(grandfollows) } + + gfollows("A").And(gfollows("C")).All() + `, []string{"F"}) + + ConveyQuery("Double Morphism Intersection", + ` + grandfollows = g.M().Out("follows").Out("follows") + function gfollows(x) { return g.V(x).Follow(grandfollows) } + + gfollows("E").And(gfollows("C")).And(gfollows("B")).All() + `, []string{"G"}) + + ConveyQuery("Reverse Intersection", + ` + grandfollows = g.M().Out("follows").Out("follows") + + g.V("G").FollowR(grandfollows).Intersect(g.V("F").FollowR(grandfollows)).All() + `, []string{"C"}) + + ConveyQuery("Standard sort of morphism intersection, continue follow", + ` + gfollowers = g.M().In("follows").In("follows") + function cool(x) { return g.V(x).As("a").Out("status").Is("cool").Back("a") } + cool("G").Follow(gfollowers).Intersect(cool("B").Follow(gfollowers)).All() + `, []string{"C"}) + + }) +} + +func TestGremlinHas(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + ConveyQuery("Test a simple Has", + `g.V().Has("status", "cool").All()`, + []string{"G", "D", "B"}) + + ConveyQuery("Test a double Has", + `g.V().Has("status", "cool").Has("follows", "F").All()`, + []string{"B"}) + + }) +} + +func TestGremlinTag(t *testing.T) { + Convey("With a default memtriplestore", t, func() { + ConveyQueryTag("Test a simple save", + `g.V().Save("status", "somecool").All()`, + "somecool", + []string{"cool", "cool", "cool"}) + + ConveyQueryTag("Test a simple saveR", + `g.V("cool").SaveR("status", "who").All()`, + "who", + []string{"G", "D", "B"}) + + ConveyQueryTag("Test an out save", + `g.V("D").Out(null, "pred").All()`, + "pred", + []string{"follows", "follows", "status"}) + + ConveyQueryTag("Test a tag list", + `g.V("D").Out(null, ["pred", "foo", "bar"]).All()`, + "foo", + []string{"follows", "follows", "status"}) + + ConveyQuery("Test a pred list", + `g.V("D").Out(["follows", "status"]).All()`, + []string{"B", "G", "cool"}) + + ConveyQuery("Test a predicate path", + `g.V("D").Out(g.V("follows"), "pred").All()`, + []string{"B", "G"}) + }) +} diff --git a/query/gremlin/gremlin_build_iterator.go b/query/gremlin/gremlin_build_iterator.go deleted file mode 100644 index b6e9a45..0000000 --- a/query/gremlin/gremlin_build_iterator.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "strconv" - - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" - - "github.com/google/cayley/graph" -) - -func getStrings(obj *otto.Object, field string) []string { - strings := make([]string, 0) - val, _ := obj.Get(field) - if !val.IsUndefined() { - export, _ := val.Export() - array := export.([]interface{}) - for _, arg := range array { - strings = append(strings, arg.(string)) - } - } - return strings -} - -func getStringArgs(obj *otto.Object) []string { return getStrings(obj, "string_args") } - -func buildIteratorTree(obj *otto.Object, ts graph.TripleStore) graph.Iterator { - if !isVertexChain(obj) { - return graph.NewNullIterator() - } - return buildIteratorTreeHelper(obj, ts, graph.NewNullIterator()) -} - -func makeListOfStringsFromArrayValue(obj *otto.Object) []string { - var output []string - lengthValue, _ := obj.Get("length") - length, _ := lengthValue.ToInteger() - ulength := uint32(length) - for index := uint32(0); index < ulength; index += 1 { - name := strconv.FormatInt(int64(index), 10) - value, err := obj.Get(name) - if err != nil { - continue - } - if !value.IsString() { - continue - } - s, _ := value.ToString() - output = append(output, s) - } - return output -} - -func buildIteratorFromValue(val otto.Value, ts graph.TripleStore) graph.Iterator { - if val.IsNull() || val.IsUndefined() { - return ts.GetNodesAllIterator() - } - if val.IsPrimitive() { - thing, _ := val.Export() - switch v := thing.(type) { - case string: - it := ts.MakeFixed() - it.AddValue(ts.GetIdFor(v)) - return it - default: - glog.Errorln("Trying to build unknown primitive value.") - } - } - switch val.Class() { - case "Object": - return buildIteratorTree(val.Object(), ts) - case "Array": - // Had better be an array of strings - strings := makeListOfStringsFromArrayValue(val.Object()) - it := ts.MakeFixed() - for _, x := range strings { - it.AddValue(ts.GetIdFor(x)) - } - return it - case "Number": - fallthrough - case "Boolean": - fallthrough - case "Date": - fallthrough - case "String": - it := ts.MakeFixed() - str, _ := val.ToString() - it.AddValue(ts.GetIdFor(str)) - return it - default: - glog.Errorln("Trying to handle unsupported Javascript value.") - return graph.NewNullIterator() - } -} - -func buildInOutIterator(obj *otto.Object, ts graph.TripleStore, base graph.Iterator, isReverse bool) graph.Iterator { - argList, _ := obj.Get("_gremlin_values") - if argList.Class() != "GoArray" { - glog.Errorln("How is arglist not an array? Return nothing.", argList.Class()) - return graph.NewNullIterator() - } - argArray := argList.Object() - lengthVal, _ := argArray.Get("length") - length, _ := lengthVal.ToInteger() - var predicateNodeIterator graph.Iterator - if length == 0 { - predicateNodeIterator = ts.GetNodesAllIterator() - } else { - zero, _ := argArray.Get("0") - predicateNodeIterator = buildIteratorFromValue(zero, ts) - } - if length >= 2 { - var tags []string - one, _ := argArray.Get("1") - if one.IsString() { - s, _ := one.ToString() - tags = append(tags, s) - } else if one.Class() == "Array" { - tags = makeListOfStringsFromArrayValue(one.Object()) - } - for _, tag := range tags { - predicateNodeIterator.AddTag(tag) - } - } - - in, out := "s", "o" - if isReverse { - in, out = out, in - } - lto := graph.NewLinksToIterator(ts, base, in) - and := graph.NewAndIterator() - and.AddSubIterator(graph.NewLinksToIterator(ts, predicateNodeIterator, "p")) - and.AddSubIterator(lto) - return graph.NewHasaIterator(ts, and, out) -} - -func buildIteratorTreeHelper(obj *otto.Object, ts graph.TripleStore, base graph.Iterator) graph.Iterator { - var it graph.Iterator - it = base - // TODO: Better error handling - kindVal, _ := obj.Get("_gremlin_type") - stringArgs := getStringArgs(obj) - var subIt graph.Iterator - prevVal, _ := obj.Get("_gremlin_prev") - if !prevVal.IsObject() { - subIt = base - } else { - subIt = buildIteratorTreeHelper(prevVal.Object(), ts, base) - } - - kind, _ := kindVal.ToString() - switch kind { - case "vertex": - if len(stringArgs) == 0 { - it = ts.GetNodesAllIterator() - } else { - fixed := ts.MakeFixed() - for _, name := range stringArgs { - fixed.AddValue(ts.GetIdFor(name)) - } - it = fixed - } - case "tag": - it = subIt - for _, tag := range stringArgs { - it.AddTag(tag) - } - case "save": - all := ts.GetNodesAllIterator() - if len(stringArgs) > 2 || len(stringArgs) == 0 { - return graph.NewNullIterator() - } - if len(stringArgs) == 2 { - all.AddTag(stringArgs[1]) - } else { - all.AddTag(stringArgs[0]) - } - predFixed := ts.MakeFixed() - predFixed.AddValue(ts.GetIdFor(stringArgs[0])) - subAnd := graph.NewAndIterator() - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "o")) - hasa := graph.NewHasaIterator(ts, subAnd, "s") - and := graph.NewAndIterator() - and.AddSubIterator(hasa) - and.AddSubIterator(subIt) - it = and - case "saver": - all := ts.GetNodesAllIterator() - if len(stringArgs) > 2 || len(stringArgs) == 0 { - return graph.NewNullIterator() - } - if len(stringArgs) == 2 { - all.AddTag(stringArgs[1]) - } else { - all.AddTag(stringArgs[0]) - } - predFixed := ts.MakeFixed() - predFixed.AddValue(ts.GetIdFor(stringArgs[0])) - subAnd := graph.NewAndIterator() - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, all, "s")) - hasa := graph.NewHasaIterator(ts, subAnd, "o") - and := graph.NewAndIterator() - and.AddSubIterator(hasa) - and.AddSubIterator(subIt) - it = and - case "has": - fixed := ts.MakeFixed() - if len(stringArgs) < 2 { - return graph.NewNullIterator() - } - for _, name := range stringArgs[1:] { - fixed.AddValue(ts.GetIdFor(name)) - } - predFixed := ts.MakeFixed() - predFixed.AddValue(ts.GetIdFor(stringArgs[0])) - subAnd := graph.NewAndIterator() - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, predFixed, "p")) - subAnd.AddSubIterator(graph.NewLinksToIterator(ts, fixed, "o")) - hasa := graph.NewHasaIterator(ts, subAnd, "s") - and := graph.NewAndIterator() - and.AddSubIterator(hasa) - and.AddSubIterator(subIt) - it = and - case "morphism": - it = base - case "and": - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if !isVertexChain(firstArg.Object()) { - return graph.NewNullIterator() - } - argIt := buildIteratorTree(firstArg.Object(), ts) - - and := graph.NewAndIterator() - and.AddSubIterator(subIt) - and.AddSubIterator(argIt) - it = and - case "back": - arg, _ := obj.Get("_gremlin_back_chain") - argIt := buildIteratorTree(arg.Object(), ts) - and := graph.NewAndIterator() - and.AddSubIterator(subIt) - and.AddSubIterator(argIt) - it = and - case "is": - fixed := ts.MakeFixed() - for _, name := range stringArgs { - fixed.AddValue(ts.GetIdFor(name)) - } - and := graph.NewAndIterator() - and.AddSubIterator(fixed) - and.AddSubIterator(subIt) - it = and - case "or": - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if !isVertexChain(firstArg.Object()) { - return graph.NewNullIterator() - } - argIt := buildIteratorTree(firstArg.Object(), ts) - - or := graph.NewOrIterator() - or.AddSubIterator(subIt) - or.AddSubIterator(argIt) - it = or - case "both": - // Hardly the most efficient pattern, but the most general. - // Worth looking into an Optimize() optimization here. - clone := subIt.Clone() - it1 := buildInOutIterator(obj, ts, subIt, false) - it2 := buildInOutIterator(obj, ts, clone, true) - - or := graph.NewOrIterator() - or.AddSubIterator(it1) - or.AddSubIterator(it2) - it = or - case "out": - it = buildInOutIterator(obj, ts, subIt, false) - case "follow": - // Follow a morphism - arg, _ := obj.Get("_gremlin_values") - firstArg, _ := arg.Object().Get("0") - if isVertexChain(firstArg.Object()) { - return graph.NewNullIterator() - } - it = buildIteratorTreeHelper(firstArg.Object(), ts, subIt) - case "followr": - // Follow a morphism - arg, _ := obj.Get("_gremlin_followr") - if isVertexChain(arg.Object()) { - return graph.NewNullIterator() - } - it = buildIteratorTreeHelper(arg.Object(), ts, subIt) - case "in": - it = buildInOutIterator(obj, ts, subIt, true) - } - return it -} diff --git a/query/gremlin/gremlin_env.go b/query/gremlin/gremlin_env.go deleted file mode 100644 index 4e7f332..0000000 --- a/query/gremlin/gremlin_env.go +++ /dev/null @@ -1,95 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -// Builds a new Gremlin environment pointing at a session. - -import ( - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" -) - -func BuildGremlinEnv(ses *GremlinSession) *otto.Otto { - env := otto.New() - setupGremlin(env, ses) - return env -} - -func concatStringArgs(call otto.FunctionCall) *[]interface{} { - outStrings := make([]interface{}, 0) - for _, arg := range call.ArgumentList { - if arg.IsString() { - outStrings = append(outStrings, arg.String()) - } - if arg.IsObject() && arg.Class() == "Array" { - obj, _ := arg.Export() - for _, x := range obj.([]interface{}) { - outStrings = append(outStrings, x.(string)) - } - } - } - return &outStrings -} - -func isVertexChain(obj *otto.Object) bool { - val, _ := obj.Get("_gremlin_type") - if x, _ := val.ToString(); x == "vertex" { - return true - } - val, _ = obj.Get("_gremlin_prev") - if val.IsObject() { - return isVertexChain(val.Object()) - } - return false -} - -func setupGremlin(env *otto.Otto, ses *GremlinSession) { - graph, _ := env.Object("graph = {}") - graph.Set("Vertex", func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, err := call.Otto.Object("out") - if err != nil { - glog.Error(err.Error()) - return otto.TrueValue() - } - out.Set("_gremlin_type", "vertex") - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - embedTraversals(env, ses, out) - embedFinals(env, ses, out) - return out.Value() - }) - - graph.Set("Morphism", func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", "morphism") - embedTraversals(env, ses, out) - return out.Value() - }) - graph.Set("Emit", func(call otto.FunctionCall) otto.Value { - value := call.Argument(0) - if value.IsDefined() { - ses.SendResult(&GremlinResult{metaresult: false, err: "", val: &value, actualResults: nil}) - } - return otto.NullValue() - }) - env.Run("graph.V = graph.Vertex") - env.Run("graph.M = graph.Morphism") - env.Run("g = graph") - -} diff --git a/query/gremlin/gremlin_finals.go b/query/gremlin/gremlin_finals.go deleted file mode 100644 index f8c978d..0000000 --- a/query/gremlin/gremlin_finals.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" - - "github.com/google/cayley/graph" -) - -const GremlinTopResultTag = "id" - -func embedFinals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { - obj.Set("All", allFunc(env, ses, obj)) - obj.Set("GetLimit", limitFunc(env, ses, obj)) - obj.Set("ToArray", toArrayFunc(env, ses, obj, false)) - obj.Set("ToValue", toValueFunc(env, ses, obj, false)) - obj.Set("TagArray", toArrayFunc(env, ses, obj, true)) - obj.Set("TagValue", toValueFunc(env, ses, obj, true)) - obj.Set("Map", mapFunc(env, ses, obj)) - obj.Set("ForEach", mapFunc(env, ses, obj)) -} - -func allFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - ses.limit = -1 - ses.count = 0 - runIteratorOnSession(it, ses) - return otto.NullValue() - } -} - -func limitFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - if len(call.ArgumentList) > 0 { - limitVal, _ := call.Argument(0).ToInteger() - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - ses.limit = int(limitVal) - ses.count = 0 - runIteratorOnSession(it, ses) - } - return otto.NullValue() - } -} - -func toArrayFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - limit := -1 - if len(call.ArgumentList) > 0 { - limitParsed, _ := call.Argument(0).ToInteger() - limit = int(limitParsed) - } - var val otto.Value - var err error - if !withTags { - array := runIteratorToArrayNoTags(it, ses, limit) - val, err = call.Otto.ToValue(array) - } else { - array := runIteratorToArray(it, ses, limit) - val, err = call.Otto.ToValue(array) - } - - if err != nil { - glog.Error(err) - return otto.NullValue() - } - return val - } -} - -func toValueFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object, withTags bool) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - limit := 1 - var val otto.Value - var err error - if !withTags { - array := runIteratorToArrayNoTags(it, ses, limit) - if len(array) < 1 { - return otto.NullValue() - } - val, err = call.Otto.ToValue(array[0]) - } else { - array := runIteratorToArray(it, ses, limit) - if len(array) < 1 { - return otto.NullValue() - } - val, err = call.Otto.ToValue(array[0]) - } - if err != nil { - glog.Error(err) - return otto.NullValue() - } else { - return val - } - - } -} - -func mapFunc(env *otto.Otto, ses *GremlinSession, obj *otto.Object) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - it := buildIteratorTree(obj, ses.ts) - it.AddTag(GremlinTopResultTag) - limit := -1 - if len(call.ArgumentList) == 0 { - return otto.NullValue() - } - callback := call.Argument(len(call.ArgumentList) - 1) - if len(call.ArgumentList) > 1 { - limitParsed, _ := call.Argument(0).ToInteger() - limit = int(limitParsed) - } - runIteratorWithCallback(it, ses, callback, call, limit) - return otto.NullValue() - } -} - -func tagsToValueMap(m map[string]graph.TSVal, ses *GremlinSession) map[string]string { - outputMap := make(map[string]string) - for k, v := range m { - outputMap[k] = ses.ts.GetNameFor(v) - } - return outputMap -} - -func runIteratorToArray(it graph.Iterator, ses *GremlinSession, limit int) []map[string]string { - output := make([]map[string]string, 0) - count := 0 - it, _ = it.Optimize() - for { - if ses.doHalt { - return nil - } - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - output = append(output, tagsToValueMap(tags, ses)) - count++ - if limit >= 0 && count >= limit { - break - } - for it.NextResult() == true { - if ses.doHalt { - return nil - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - output = append(output, tagsToValueMap(tags, ses)) - count++ - if limit >= 0 && count >= limit { - break - } - } - } - it.Close() - return output -} - -func runIteratorToArrayNoTags(it graph.Iterator, ses *GremlinSession, limit int) []string { - output := make([]string, 0) - count := 0 - it, _ = it.Optimize() - for { - if ses.doHalt { - return nil - } - val, ok := it.Next() - if !ok { - break - } - output = append(output, ses.ts.GetNameFor(val)) - count++ - if limit >= 0 && count >= limit { - break - } - } - it.Close() - return output -} - -func runIteratorWithCallback(it graph.Iterator, ses *GremlinSession, callback otto.Value, this otto.FunctionCall, limit int) { - count := 0 - it, _ = it.Optimize() - for { - if ses.doHalt { - return - } - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) - val, _ = callback.Call(this.This, val) - count++ - if limit >= 0 && count >= limit { - break - } - for it.NextResult() == true { - if ses.doHalt { - return - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - val, _ := this.Otto.ToValue(tagsToValueMap(tags, ses)) - val, _ = callback.Call(this.This, val) - count++ - if limit >= 0 && count >= limit { - break - } - } - } - it.Close() -} - -func runIteratorOnSession(it graph.Iterator, ses *GremlinSession) { - if ses.lookingForQueryShape { - graph.OutputQueryShapeForIterator(it, ses.ts, &(ses.queryShape)) - return - } - it, _ = it.Optimize() - glog.V(2).Infoln(it.DebugString(0)) - for { - // TODO(barakmich): Better halting. - if ses.doHalt { - return - } - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) - if !cont { - break - } - for it.NextResult() == true { - if ses.doHalt { - return - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - cont := ses.SendResult(&GremlinResult{metaresult: false, err: "", val: nil, actualResults: &tags}) - if !cont { - break - } - } - } - it.Close() -} diff --git a/query/gremlin/gremlin_functional_test.go b/query/gremlin/gremlin_functional_test.go deleted file mode 100644 index f6c65fb..0000000 --- a/query/gremlin/gremlin_functional_test.go +++ /dev/null @@ -1,230 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "sort" - "testing" - - . "github.com/smartystreets/goconvey/convey" - - "github.com/google/cayley/graph/memstore" -) - -// +---+ +---+ -// | A |------- ->| F |<-- -// +---+ \------>+---+-/ +---+ \--+---+ -// ------>|#B#| | | E | -// +---+-------/ >+---+ | +---+ -// | C | / v -// +---+ -/ +---+ -// ---- +---+/ |#G#| -// \-->|#D#|------------->+---+ -// +---+ -// - -func buildTripleStore() *GremlinSession { - ts := memstore.MakeTestingMemstore() - return NewGremlinSession(ts, -1, false) -} - -func shouldBeUnordered(actual interface{}, expected ...interface{}) string { - if len(expected) != 1 { - return "Only one list supported" - } - actualStr := actual.([]string) - expectedStr := expected[0].([]string) - sort.Strings(actualStr) - sort.Strings(expectedStr) - return ShouldResemble(actualStr, expectedStr) -} - -func runQueryGetTag(query string, tag string) ([]string, int) { - js := buildTripleStore() - output := make([]string, 0) - c := make(chan interface{}, 5) - js.ExecInput(query, c, -1) - count := 0 - for result := range c { - count++ - data := result.(*GremlinResult) - if data.val == nil { - val := (*data.actualResults)[tag] - if val != nil { - output = append(output, js.ts.GetNameFor(val)) - } - } - } - return output, count -} - -func ConveyQuery(doc string, query string, expected []string) { - ConveyQueryTag(doc, query, GremlinTopResultTag, expected) -} - -func ConveyQueryTag(doc string, query string, tag string, expected []string) { - Convey(doc, func() { - actual, _ := runQueryGetTag(query, tag) - So(actual, shouldBeUnordered, expected) - }) -} - -func TestGremlin(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - - ConveyQuery("Can get a single vertex", - `g.V("A").All()`, - []string{"A"}) - - ConveyQuery("Can use .Out()", - `g.V("A").Out("follows").All()`, - []string{"B"}) - - ConveyQuery("Can use .In()", - `g.V("B").In("follows").All()`, - []string{"A", "C", "D"}) - - ConveyQuery("Can use .Both()", - `g.V("F").Both("follows").All()`, - []string{"B", "G", "E"}) - - ConveyQuery("Can use .Tag()-.Is()-.Back()", - `g.V("B").In("follows").Tag("foo").Out("status").Is("cool").Back("foo").All()`, - []string{"D"}) - - ConveyQuery("Can separate .Tag()-.Is()-.Back()", - ` - x = g.V("C").Out("follows").Tag("foo").Out("status").Is("cool").Back("foo") - x.In("follows").Is("D").Back("foo").All() - `, - []string{"B"}) - - Convey("Can do multiple .Back()s", func() { - query := ` - g.V("E").Out("follows").As("f").Out("follows").Out("status").Is("cool").Back("f").In("follows").In("follows").As("acd").Out("status").Is("cool").Back("f").All() - ` - expected := []string{"D"} - actual, _ := runQueryGetTag(query, "acd") - So(actual, shouldBeUnordered, expected) - }) - - }) -} - -func TestGremlinMorphism(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - - ConveyQuery("Simple morphism works", - ` - grandfollows = g.M().Out("follows").Out("follows") - g.V("C").Follow(grandfollows).All() - `, - []string{"G", "F", "B"}) - - ConveyQuery("Reverse morphism works", - ` - grandfollows = g.M().Out("follows").Out("follows") - g.V("F").FollowR(grandfollows).All() - `, []string{"A", "C", "D"}) - - }) -} - -func TestGremlinIntersection(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - ConveyQuery("Simple intersection", - ` - function follows(x) { return g.V(x).Out("follows") } - - follows("D").And(follows("C")).All() - `, []string{"B"}) - - ConveyQuery("Simple Morphism Intersection", - ` - grandfollows = g.M().Out("follows").Out("follows") - function gfollows(x) { return g.V(x).Follow(grandfollows) } - - gfollows("A").And(gfollows("C")).All() - `, []string{"F"}) - - ConveyQuery("Double Morphism Intersection", - ` - grandfollows = g.M().Out("follows").Out("follows") - function gfollows(x) { return g.V(x).Follow(grandfollows) } - - gfollows("E").And(gfollows("C")).And(gfollows("B")).All() - `, []string{"G"}) - - ConveyQuery("Reverse Intersection", - ` - grandfollows = g.M().Out("follows").Out("follows") - - g.V("G").FollowR(grandfollows).Intersect(g.V("F").FollowR(grandfollows)).All() - `, []string{"C"}) - - ConveyQuery("Standard sort of morphism intersection, continue follow", - ` - gfollowers = g.M().In("follows").In("follows") - function cool(x) { return g.V(x).As("a").Out("status").Is("cool").Back("a") } - cool("G").Follow(gfollowers).Intersect(cool("B").Follow(gfollowers)).All() - `, []string{"C"}) - - }) -} - -func TestGremlinHas(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - ConveyQuery("Test a simple Has", - `g.V().Has("status", "cool").All()`, - []string{"G", "D", "B"}) - - ConveyQuery("Test a double Has", - `g.V().Has("status", "cool").Has("follows", "F").All()`, - []string{"B"}) - - }) -} - -func TestGremlinTag(t *testing.T) { - Convey("With a default memtriplestore", t, func() { - ConveyQueryTag("Test a simple save", - `g.V().Save("status", "somecool").All()`, - "somecool", - []string{"cool", "cool", "cool"}) - - ConveyQueryTag("Test a simple saveR", - `g.V("cool").SaveR("status", "who").All()`, - "who", - []string{"G", "D", "B"}) - - ConveyQueryTag("Test an out save", - `g.V("D").Out(null, "pred").All()`, - "pred", - []string{"follows", "follows", "status"}) - - ConveyQueryTag("Test a tag list", - `g.V("D").Out(null, ["pred", "foo", "bar"]).All()`, - "foo", - []string{"follows", "follows", "status"}) - - ConveyQuery("Test a pred list", - `g.V("D").Out(["follows", "status"]).All()`, - []string{"B", "G", "cool"}) - - ConveyQuery("Test a predicate path", - `g.V("D").Out(g.V("follows"), "pred").All()`, - []string{"B", "G"}) - }) -} diff --git a/query/gremlin/gremlin_session.go b/query/gremlin/gremlin_session.go deleted file mode 100644 index a0b0483..0000000 --- a/query/gremlin/gremlin_session.go +++ /dev/null @@ -1,266 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -import ( - "errors" - "fmt" - "sort" - "time" - - "github.com/robertkrimen/otto" - - "github.com/google/cayley/graph" -) - -type GremlinSession struct { - ts graph.TripleStore - currentChannel chan interface{} - env *otto.Otto - debug bool - limit int - count int - dataOutput []interface{} - lookingForQueryShape bool - queryShape map[string]interface{} - err error - script *otto.Script - doHalt bool - timeoutSec time.Duration - emptyEnv *otto.Otto -} - -func NewGremlinSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool) *GremlinSession { - var g GremlinSession - g.ts = inputTripleStore - g.env = BuildGremlinEnv(&g) - g.limit = -1 - g.count = 0 - g.lookingForQueryShape = false - if persist { - g.emptyEnv = g.env - } - if timeoutSec < 0 { - g.timeoutSec = time.Duration(-1) - } else { - g.timeoutSec = time.Duration(timeoutSec) - } - g.ClearJson() - return &g -} - -type GremlinResult struct { - metaresult bool - err string - val *otto.Value - actualResults *map[string]graph.TSVal -} - -func (g *GremlinSession) ToggleDebug() { - g.debug = !g.debug -} - -func (g *GremlinSession) GetQuery(input string, output_struct chan map[string]interface{}) { - defer close(output_struct) - g.queryShape = make(map[string]interface{}) - g.lookingForQueryShape = true - g.env.Run(input) - output_struct <- g.queryShape - g.queryShape = nil -} - -func (g *GremlinSession) InputParses(input string) (graph.ParseResult, error) { - script, err := g.env.Compile("", input) - if err != nil { - return graph.ParseFail, err - } - g.script = script - return graph.Parsed, nil -} - -func (g *GremlinSession) SendResult(result *GremlinResult) bool { - if g.limit >= 0 && g.limit == g.count { - return false - } - if g.doHalt { - return false - } - if g.currentChannel != nil { - g.currentChannel <- result - g.count++ - if g.limit >= 0 && g.limit == g.count { - return false - } else { - return true - } - } - return false -} - -var halt = errors.New("Query Timeout") - -func (g *GremlinSession) runUnsafe(input interface{}) (otto.Value, error) { - g.doHalt = false - defer func() { - if caught := recover(); caught != nil { - if caught == halt { - g.err = halt - return - } - panic(caught) // Something else happened, repanic! - } - }() - - g.env.Interrupt = make(chan func(), 1) // The buffer prevents blocking - - if g.timeoutSec != -1 { - go func() { - time.Sleep(g.timeoutSec * time.Second) // Stop after two seconds - g.doHalt = true - if g.env != nil { - g.env.Interrupt <- func() { - panic(halt) - } - g.env = g.emptyEnv - } - }() - } - - return g.env.Run(input) // Here be dragons (risky code) -} - -func (g *GremlinSession) ExecInput(input string, out chan interface{}, limit int) { - defer close(out) - g.err = nil - g.currentChannel = out - var err error - var value otto.Value - if g.script == nil { - value, err = g.runUnsafe(input) - } else { - value, err = g.runUnsafe(g.script) - } - if err != nil { - out <- &GremlinResult{metaresult: true, - err: err.Error(), - val: &value, - actualResults: nil} - } else { - out <- &GremlinResult{metaresult: true, - err: "", - val: &value, - actualResults: nil} - } - g.currentChannel = nil - g.script = nil - g.env = g.emptyEnv - return -} - -func (s *GremlinSession) ToText(result interface{}) string { - data := result.(*GremlinResult) - if data.metaresult { - if data.err != "" { - return fmt.Sprintln("Error: ", data.err) - } - if data.val != nil { - s, _ := data.val.Export() - if data.val.IsObject() { - typeVal, _ := data.val.Object().Get("_gremlin_type") - if !typeVal.IsUndefined() { - s = "[internal Iterator]" - } - } - return fmt.Sprintln("=>", s) - } - return "" - } - var out string - out = fmt.Sprintln("****") - if data.val == nil { - tags := data.actualResults - tagKeys := make([]string, len(*tags)) - i := 0 - for k, _ := range *tags { - tagKeys[i] = k - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - if k == "$_" { - continue - } - out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) - } - } else { - if data.val.IsObject() { - export, _ := data.val.Export() - mapExport := export.(map[string]string) - for k, v := range mapExport { - out += fmt.Sprintf("%s : %v\n", k, v) - } - } else { - strVersion, _ := data.val.ToString() - out += fmt.Sprintf("%s\n", strVersion) - } - } - return out -} - -// Web stuff -func (ses *GremlinSession) BuildJson(result interface{}) { - data := result.(*GremlinResult) - if !data.metaresult { - if data.val == nil { - obj := make(map[string]string) - tags := data.actualResults - tagKeys := make([]string, len(*tags)) - i := 0 - for k, _ := range *tags { - tagKeys[i] = k - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - obj[k] = ses.ts.GetNameFor((*tags)[k]) - } - ses.dataOutput = append(ses.dataOutput, obj) - } else { - if data.val.IsObject() { - export, _ := data.val.Export() - ses.dataOutput = append(ses.dataOutput, export) - } else { - strVersion, _ := data.val.ToString() - ses.dataOutput = append(ses.dataOutput, strVersion) - } - } - } - -} - -func (ses *GremlinSession) GetJson() (interface{}, error) { - defer ses.ClearJson() - if ses.err != nil { - return nil, ses.err - } - if ses.doHalt { - return nil, halt - } - return ses.dataOutput, nil -} - -func (ses *GremlinSession) ClearJson() { - ses.dataOutput = nil -} diff --git a/query/gremlin/gremlin_traversals.go b/query/gremlin/gremlin_traversals.go deleted file mode 100644 index c0f4704..0000000 --- a/query/gremlin/gremlin_traversals.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gremlin - -// Adds special traversal functions to JS Gremlin objects. Most of these just build the chain of objects, and won't often need the session. - -import ( - "github.com/barakmich/glog" - "github.com/robertkrimen/otto" -) - -func embedTraversals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { - obj.Set("In", gremlinFunc("in", obj, env, ses)) - obj.Set("Out", gremlinFunc("out", obj, env, ses)) - obj.Set("Is", gremlinFunc("is", obj, env, ses)) - obj.Set("Both", gremlinFunc("both", obj, env, ses)) - obj.Set("Follow", gremlinFunc("follow", obj, env, ses)) - obj.Set("FollowR", gremlinFollowR("followr", obj, env, ses)) - obj.Set("And", gremlinFunc("and", obj, env, ses)) - obj.Set("Intersect", gremlinFunc("and", obj, env, ses)) - obj.Set("Union", gremlinFunc("or", obj, env, ses)) - obj.Set("Or", gremlinFunc("or", obj, env, ses)) - obj.Set("Back", gremlinBack("back", obj, env, ses)) - obj.Set("Tag", gremlinFunc("tag", obj, env, ses)) - obj.Set("As", gremlinFunc("tag", obj, env, ses)) - obj.Set("Has", gremlinFunc("has", obj, env, ses)) - obj.Set("Save", gremlinFunc("save", obj, env, ses)) - obj.Set("SaveR", gremlinFunc("saver", obj, env, ses)) -} - -func gremlinFunc(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", kind) - out.Set("_gremlin_values", call.ArgumentList) - out.Set("_gremlin_prev", prevObj) - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - embedTraversals(env, ses, out) - if isVertexChain(call.This.Object()) { - embedFinals(env, ses, out) - } - return out.Value() - } -} - -func gremlinBack(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", kind) - out.Set("_gremlin_values", call.ArgumentList) - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - var otherChain *otto.Object - var thisObj *otto.Object - if len(*outStrings) != 0 { - otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, (*outStrings)[0].(string)) - } else { - otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, "") - } - out.Set("_gremlin_prev", thisObj) - out.Set("_gremlin_back_chain", otherChain) - embedTraversals(env, ses, out) - if isVertexChain(call.This.Object()) { - embedFinals(env, ses, out) - } - return out.Value() - - } -} - -func gremlinFollowR(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { - return func(call otto.FunctionCall) otto.Value { - call.Otto.Run("var out = {}") - out, _ := call.Otto.Object("out") - out.Set("_gremlin_type", kind) - out.Set("_gremlin_values", call.ArgumentList) - outStrings := concatStringArgs(call) - if len(*outStrings) > 0 { - out.Set("string_args", *outStrings) - } - if len(call.ArgumentList) == 0 { - return prevObj.Value() - } - arg := call.Argument(0) - if isVertexChain(arg.Object()) { - return prevObj.Value() - } - newChain, _ := reverseGremlinChainTo(call.Otto, arg.Object(), "") - out.Set("_gremlin_prev", prevObj) - out.Set("_gremlin_followr", newChain) - embedTraversals(env, ses, out) - if isVertexChain(call.This.Object()) { - embedFinals(env, ses, out) - } - return out.Value() - - } -} - -func reverseGremlinChainTo(env *otto.Otto, prevObj *otto.Object, tag string) (*otto.Object, *otto.Object) { - env.Run("var _base_object = {}") - base, err := env.Object("_base_object") - if err != nil { - glog.Error(err) - return otto.NullValue().Object(), otto.NullValue().Object() - } - if isVertexChain(prevObj) { - base.Set("_gremlin_type", "vertex") - } else { - base.Set("_gremlin_type", "morphism") - } - return reverseGremlinChainHelper(env, prevObj, base, tag) -} - -func reverseGremlinChainHelper(env *otto.Otto, chain *otto.Object, newBase *otto.Object, tag string) (*otto.Object, *otto.Object) { - kindVal, _ := chain.Get("_gremlin_type") - kind, _ := kindVal.ToString() - - if tag != "" { - if kind == "tag" { - tags := getStringArgs(chain) - for _, t := range tags { - if t == tag { - return newBase, chain - } - } - } - } - - if kind == "morphism" || kind == "vertex" { - return newBase, chain - } - var newKind string - switch kind { - case "in": - newKind = "out" - case "out": - newKind = "in" - default: - newKind = kind - } - prev, _ := chain.Get("_gremlin_prev") - env.Run("var out = {}") - out, _ := env.Object("out") - out.Set("_gremlin_type", newKind) - values, _ := chain.Get("_gremlin_values") - out.Set("_gremlin_values", values) - back, _ := chain.Get("_gremlin_back_chain") - out.Set("_gremlin_back_chain", back) - out.Set("_gremlin_prev", newBase) - strings, _ := chain.Get("string_args") - out.Set("string_args", strings) - return reverseGremlinChainHelper(env, prev.Object(), out, tag) -} - -func debugChain(obj *otto.Object) bool { - val, _ := obj.Get("_gremlin_type") - x, _ := val.ToString() - glog.V(2).Infoln(x) - val, _ = obj.Get("_gremlin_prev") - if val.IsObject() { - return debugChain(val.Object()) - } - return false -} diff --git a/query/gremlin/session.go b/query/gremlin/session.go new file mode 100644 index 0000000..a0b0483 --- /dev/null +++ b/query/gremlin/session.go @@ -0,0 +1,266 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +import ( + "errors" + "fmt" + "sort" + "time" + + "github.com/robertkrimen/otto" + + "github.com/google/cayley/graph" +) + +type GremlinSession struct { + ts graph.TripleStore + currentChannel chan interface{} + env *otto.Otto + debug bool + limit int + count int + dataOutput []interface{} + lookingForQueryShape bool + queryShape map[string]interface{} + err error + script *otto.Script + doHalt bool + timeoutSec time.Duration + emptyEnv *otto.Otto +} + +func NewGremlinSession(inputTripleStore graph.TripleStore, timeoutSec int, persist bool) *GremlinSession { + var g GremlinSession + g.ts = inputTripleStore + g.env = BuildGremlinEnv(&g) + g.limit = -1 + g.count = 0 + g.lookingForQueryShape = false + if persist { + g.emptyEnv = g.env + } + if timeoutSec < 0 { + g.timeoutSec = time.Duration(-1) + } else { + g.timeoutSec = time.Duration(timeoutSec) + } + g.ClearJson() + return &g +} + +type GremlinResult struct { + metaresult bool + err string + val *otto.Value + actualResults *map[string]graph.TSVal +} + +func (g *GremlinSession) ToggleDebug() { + g.debug = !g.debug +} + +func (g *GremlinSession) GetQuery(input string, output_struct chan map[string]interface{}) { + defer close(output_struct) + g.queryShape = make(map[string]interface{}) + g.lookingForQueryShape = true + g.env.Run(input) + output_struct <- g.queryShape + g.queryShape = nil +} + +func (g *GremlinSession) InputParses(input string) (graph.ParseResult, error) { + script, err := g.env.Compile("", input) + if err != nil { + return graph.ParseFail, err + } + g.script = script + return graph.Parsed, nil +} + +func (g *GremlinSession) SendResult(result *GremlinResult) bool { + if g.limit >= 0 && g.limit == g.count { + return false + } + if g.doHalt { + return false + } + if g.currentChannel != nil { + g.currentChannel <- result + g.count++ + if g.limit >= 0 && g.limit == g.count { + return false + } else { + return true + } + } + return false +} + +var halt = errors.New("Query Timeout") + +func (g *GremlinSession) runUnsafe(input interface{}) (otto.Value, error) { + g.doHalt = false + defer func() { + if caught := recover(); caught != nil { + if caught == halt { + g.err = halt + return + } + panic(caught) // Something else happened, repanic! + } + }() + + g.env.Interrupt = make(chan func(), 1) // The buffer prevents blocking + + if g.timeoutSec != -1 { + go func() { + time.Sleep(g.timeoutSec * time.Second) // Stop after two seconds + g.doHalt = true + if g.env != nil { + g.env.Interrupt <- func() { + panic(halt) + } + g.env = g.emptyEnv + } + }() + } + + return g.env.Run(input) // Here be dragons (risky code) +} + +func (g *GremlinSession) ExecInput(input string, out chan interface{}, limit int) { + defer close(out) + g.err = nil + g.currentChannel = out + var err error + var value otto.Value + if g.script == nil { + value, err = g.runUnsafe(input) + } else { + value, err = g.runUnsafe(g.script) + } + if err != nil { + out <- &GremlinResult{metaresult: true, + err: err.Error(), + val: &value, + actualResults: nil} + } else { + out <- &GremlinResult{metaresult: true, + err: "", + val: &value, + actualResults: nil} + } + g.currentChannel = nil + g.script = nil + g.env = g.emptyEnv + return +} + +func (s *GremlinSession) ToText(result interface{}) string { + data := result.(*GremlinResult) + if data.metaresult { + if data.err != "" { + return fmt.Sprintln("Error: ", data.err) + } + if data.val != nil { + s, _ := data.val.Export() + if data.val.IsObject() { + typeVal, _ := data.val.Object().Get("_gremlin_type") + if !typeVal.IsUndefined() { + s = "[internal Iterator]" + } + } + return fmt.Sprintln("=>", s) + } + return "" + } + var out string + out = fmt.Sprintln("****") + if data.val == nil { + tags := data.actualResults + tagKeys := make([]string, len(*tags)) + i := 0 + for k, _ := range *tags { + tagKeys[i] = k + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + if k == "$_" { + continue + } + out += fmt.Sprintf("%s : %s\n", k, s.ts.GetNameFor((*tags)[k])) + } + } else { + if data.val.IsObject() { + export, _ := data.val.Export() + mapExport := export.(map[string]string) + for k, v := range mapExport { + out += fmt.Sprintf("%s : %v\n", k, v) + } + } else { + strVersion, _ := data.val.ToString() + out += fmt.Sprintf("%s\n", strVersion) + } + } + return out +} + +// Web stuff +func (ses *GremlinSession) BuildJson(result interface{}) { + data := result.(*GremlinResult) + if !data.metaresult { + if data.val == nil { + obj := make(map[string]string) + tags := data.actualResults + tagKeys := make([]string, len(*tags)) + i := 0 + for k, _ := range *tags { + tagKeys[i] = k + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + obj[k] = ses.ts.GetNameFor((*tags)[k]) + } + ses.dataOutput = append(ses.dataOutput, obj) + } else { + if data.val.IsObject() { + export, _ := data.val.Export() + ses.dataOutput = append(ses.dataOutput, export) + } else { + strVersion, _ := data.val.ToString() + ses.dataOutput = append(ses.dataOutput, strVersion) + } + } + } + +} + +func (ses *GremlinSession) GetJson() (interface{}, error) { + defer ses.ClearJson() + if ses.err != nil { + return nil, ses.err + } + if ses.doHalt { + return nil, halt + } + return ses.dataOutput, nil +} + +func (ses *GremlinSession) ClearJson() { + ses.dataOutput = nil +} diff --git a/query/gremlin/traversals.go b/query/gremlin/traversals.go new file mode 100644 index 0000000..c0f4704 --- /dev/null +++ b/query/gremlin/traversals.go @@ -0,0 +1,184 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gremlin + +// Adds special traversal functions to JS Gremlin objects. Most of these just build the chain of objects, and won't often need the session. + +import ( + "github.com/barakmich/glog" + "github.com/robertkrimen/otto" +) + +func embedTraversals(env *otto.Otto, ses *GremlinSession, obj *otto.Object) { + obj.Set("In", gremlinFunc("in", obj, env, ses)) + obj.Set("Out", gremlinFunc("out", obj, env, ses)) + obj.Set("Is", gremlinFunc("is", obj, env, ses)) + obj.Set("Both", gremlinFunc("both", obj, env, ses)) + obj.Set("Follow", gremlinFunc("follow", obj, env, ses)) + obj.Set("FollowR", gremlinFollowR("followr", obj, env, ses)) + obj.Set("And", gremlinFunc("and", obj, env, ses)) + obj.Set("Intersect", gremlinFunc("and", obj, env, ses)) + obj.Set("Union", gremlinFunc("or", obj, env, ses)) + obj.Set("Or", gremlinFunc("or", obj, env, ses)) + obj.Set("Back", gremlinBack("back", obj, env, ses)) + obj.Set("Tag", gremlinFunc("tag", obj, env, ses)) + obj.Set("As", gremlinFunc("tag", obj, env, ses)) + obj.Set("Has", gremlinFunc("has", obj, env, ses)) + obj.Set("Save", gremlinFunc("save", obj, env, ses)) + obj.Set("SaveR", gremlinFunc("saver", obj, env, ses)) +} + +func gremlinFunc(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", kind) + out.Set("_gremlin_values", call.ArgumentList) + out.Set("_gremlin_prev", prevObj) + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + embedTraversals(env, ses, out) + if isVertexChain(call.This.Object()) { + embedFinals(env, ses, out) + } + return out.Value() + } +} + +func gremlinBack(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", kind) + out.Set("_gremlin_values", call.ArgumentList) + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + var otherChain *otto.Object + var thisObj *otto.Object + if len(*outStrings) != 0 { + otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, (*outStrings)[0].(string)) + } else { + otherChain, thisObj = reverseGremlinChainTo(call.Otto, prevObj, "") + } + out.Set("_gremlin_prev", thisObj) + out.Set("_gremlin_back_chain", otherChain) + embedTraversals(env, ses, out) + if isVertexChain(call.This.Object()) { + embedFinals(env, ses, out) + } + return out.Value() + + } +} + +func gremlinFollowR(kind string, prevObj *otto.Object, env *otto.Otto, ses *GremlinSession) func(otto.FunctionCall) otto.Value { + return func(call otto.FunctionCall) otto.Value { + call.Otto.Run("var out = {}") + out, _ := call.Otto.Object("out") + out.Set("_gremlin_type", kind) + out.Set("_gremlin_values", call.ArgumentList) + outStrings := concatStringArgs(call) + if len(*outStrings) > 0 { + out.Set("string_args", *outStrings) + } + if len(call.ArgumentList) == 0 { + return prevObj.Value() + } + arg := call.Argument(0) + if isVertexChain(arg.Object()) { + return prevObj.Value() + } + newChain, _ := reverseGremlinChainTo(call.Otto, arg.Object(), "") + out.Set("_gremlin_prev", prevObj) + out.Set("_gremlin_followr", newChain) + embedTraversals(env, ses, out) + if isVertexChain(call.This.Object()) { + embedFinals(env, ses, out) + } + return out.Value() + + } +} + +func reverseGremlinChainTo(env *otto.Otto, prevObj *otto.Object, tag string) (*otto.Object, *otto.Object) { + env.Run("var _base_object = {}") + base, err := env.Object("_base_object") + if err != nil { + glog.Error(err) + return otto.NullValue().Object(), otto.NullValue().Object() + } + if isVertexChain(prevObj) { + base.Set("_gremlin_type", "vertex") + } else { + base.Set("_gremlin_type", "morphism") + } + return reverseGremlinChainHelper(env, prevObj, base, tag) +} + +func reverseGremlinChainHelper(env *otto.Otto, chain *otto.Object, newBase *otto.Object, tag string) (*otto.Object, *otto.Object) { + kindVal, _ := chain.Get("_gremlin_type") + kind, _ := kindVal.ToString() + + if tag != "" { + if kind == "tag" { + tags := getStringArgs(chain) + for _, t := range tags { + if t == tag { + return newBase, chain + } + } + } + } + + if kind == "morphism" || kind == "vertex" { + return newBase, chain + } + var newKind string + switch kind { + case "in": + newKind = "out" + case "out": + newKind = "in" + default: + newKind = kind + } + prev, _ := chain.Get("_gremlin_prev") + env.Run("var out = {}") + out, _ := env.Object("out") + out.Set("_gremlin_type", newKind) + values, _ := chain.Get("_gremlin_values") + out.Set("_gremlin_values", values) + back, _ := chain.Get("_gremlin_back_chain") + out.Set("_gremlin_back_chain", back) + out.Set("_gremlin_prev", newBase) + strings, _ := chain.Get("string_args") + out.Set("string_args", strings) + return reverseGremlinChainHelper(env, prev.Object(), out, tag) +} + +func debugChain(obj *otto.Object) bool { + val, _ := obj.Get("_gremlin_type") + x, _ := val.ToString() + glog.V(2).Infoln(x) + val, _ = obj.Get("_gremlin_prev") + if val.IsObject() { + return debugChain(val.Object()) + } + return false +} diff --git a/query/mql/build_iterator.go b/query/mql/build_iterator.go new file mode 100644 index 0000000..6273696 --- /dev/null +++ b/query/mql/build_iterator.go @@ -0,0 +1,181 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "errors" + "fmt" + "log" + "math" + "strings" + + "github.com/google/cayley/graph" +) + +func (m *MqlQuery) buildFixed(s string) graph.Iterator { + f := m.ses.ts.MakeFixed() + f.AddValue(m.ses.ts.GetIdFor(s)) + return f +} + +func (m *MqlQuery) buildResultIterator(path MqlPath) graph.Iterator { + all := m.ses.ts.GetNodesAllIterator() + all.AddTag(string(path)) + return graph.NewOptionalIterator(all) +} + +func (m *MqlQuery) BuildIteratorTree(query interface{}) { + m.isRepeated = make(map[MqlPath]bool) + m.queryStructure = make(map[MqlPath]map[string]interface{}) + m.queryResult = make(map[MqlResultPath]map[string]interface{}) + m.queryResult[""] = make(map[string]interface{}) + + m.it, m.err = m.buildIteratorTreeInternal(query, NewMqlPath()) + if m.err != nil { + m.isError = true + } +} + +func (m *MqlQuery) buildIteratorTreeInternal(query interface{}, path MqlPath) (graph.Iterator, error) { + var it graph.Iterator + var err error + err = nil + switch t := query.(type) { + case bool: + // for JSON booleans + // Treat the bool as a string and call it a day. + // Things which are really bool-like are special cases and will be dealt with separately. + if t { + it = m.buildFixed("true") + } + it = m.buildFixed("false") + case float64: + // for JSON numbers + // Damn you, Javascript, and your lack of integer values. + if math.Floor(t) == t { + // Treat it like an integer. + it = m.buildFixed(fmt.Sprintf("%d", t)) + } else { + it = m.buildFixed(fmt.Sprintf("%f", t)) + } + case string: + // for JSON strings + it = m.buildFixed(t) + case []interface{}: + // for JSON arrays + m.isRepeated[path] = true + if len(t) == 0 { + it = m.buildResultIterator(path) + } else if len(t) == 1 { + it, err = m.buildIteratorTreeInternal(t[0], path) + } else { + err = errors.New(fmt.Sprintf("Multiple fields at location root%s", path.DisplayString())) + } + case map[string]interface{}: + // for JSON objects + it, err = m.buildIteratorTreeMapInternal(t, path) + case nil: + it = m.buildResultIterator(path) + default: + log.Fatal("Unknown JSON type?", query) + } + if err != nil { + return nil, err + } + it.AddTag(string(path)) + return it, nil +} + +func (m *MqlQuery) buildIteratorTreeMapInternal(query map[string]interface{}, path MqlPath) (graph.Iterator, error) { + it := graph.NewAndIterator() + it.AddSubIterator(m.ses.ts.GetNodesAllIterator()) + var err error + err = nil + outputStructure := make(map[string]interface{}) + for key, subquery := range query { + outputStructure[key] = nil + reverse := false + pred := key + if strings.HasPrefix(pred, "@") { + i := strings.Index(pred, ":") + if i != -1 { + pred = pred[(i + 1):] + } + } + if strings.HasPrefix(pred, "!") { + reverse = true + pred = strings.TrimPrefix(pred, "!") + } + + // Other special constructs here + var subit graph.Iterator + if key == "id" { + subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) + if err != nil { + return nil, err + } + it.AddSubIterator(subit) + } else { + subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) + if err != nil { + return nil, err + } + subAnd := graph.NewAndIterator() + predFixed := m.ses.ts.MakeFixed() + predFixed.AddValue(m.ses.ts.GetIdFor(pred)) + subAnd.AddSubIterator(graph.NewLinksToIterator(m.ses.ts, predFixed, "p")) + if reverse { + lto := graph.NewLinksToIterator(m.ses.ts, subit, "s") + subAnd.AddSubIterator(lto) + hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "o") + it.AddSubIterator(hasa) + } else { + lto := graph.NewLinksToIterator(m.ses.ts, subit, "o") + subAnd.AddSubIterator(lto) + hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "s") + it.AddSubIterator(hasa) + } + } + } + if err != nil { + return nil, err + } + m.queryStructure[path] = outputStructure + return it, nil +} + +type MqlResultPathSlice []MqlResultPath + +func (sl MqlResultPathSlice) Len() int { + return len(sl) +} + +func (sl MqlResultPathSlice) Less(i, j int) bool { + iLen := len(strings.Split(string(sl[i]), "\x30")) + jLen := len(strings.Split(string(sl[j]), "\x30")) + if iLen < jLen { + return true + } + if iLen == jLen { + if len(string(sl[i])) < len(string(sl[j])) { + return true + } + } + return false +} + +func (sl MqlResultPathSlice) Swap(i, j int) { + sl[i], sl[j] = sl[j], sl[i] +} diff --git a/query/mql/fill.go b/query/mql/fill.go new file mode 100644 index 0000000..26de32a --- /dev/null +++ b/query/mql/fill.go @@ -0,0 +1,114 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "sort" + + "github.com/google/cayley/graph" +) + +func (m *MqlQuery) treeifyResult(tags map[string]graph.TSVal) map[MqlResultPath]string { + // Transform the map into something a little more interesting. + results := make(map[MqlPath]string) + for k, v := range tags { + results[MqlPath(k)] = m.ses.ts.GetNameFor(v) + } + resultPaths := make(map[MqlResultPath]string) + for k, v := range results { + resultPaths[k.ToResultPathFromMap(results)] = v + } + + var paths MqlResultPathSlice + + for path, _ := range resultPaths { + paths = append(paths, path) + } + + sort.Sort(paths) + + // Build Structure + for _, path := range paths { + currentPath := path.getPath() + value := resultPaths[path] + namePath := path.AppendValue(value) + if _, ok := m.queryResult[namePath]; !ok { + targetPath, key := path.splitLastPath() + if path == "" { + targetPath, key = "", value + if _, ok := m.queryResult[""][value]; !ok { + m.resultOrder = append(m.resultOrder, value) + } + } + if _, ok := m.queryStructure[currentPath]; ok { + // If there's substructure, then copy that in. + newStruct := m.copyPathStructure(currentPath) + if m.isRepeated[currentPath] && currentPath != "" { + switch t := m.queryResult[targetPath][key].(type) { + case nil: + x := make([]interface{}, 0) + x = append(x, newStruct) + m.queryResult[targetPath][key] = x + m.queryResult[namePath] = newStruct + case []interface{}: + m.queryResult[targetPath][key] = append(t, newStruct) + m.queryResult[namePath] = newStruct + } + + } else { + m.queryResult[namePath] = newStruct + m.queryResult[targetPath][key] = newStruct + } + } + } + } + + // Fill values + for _, path := range paths { + currentPath := path.getPath() + value := resultPaths[path] + namePath := path.AppendValue(value) + if _, ok := m.queryStructure[currentPath]; ok { + // We're dealing with ids. + if _, ok := m.queryResult[namePath]["id"]; ok { + m.queryResult[namePath]["id"] = value + } + } else { + // Just a value. + targetPath, key := path.splitLastPath() + if m.isRepeated[currentPath] { + switch t := m.queryResult[targetPath][key].(type) { + case nil: + x := make([]interface{}, 0) + x = append(x, value) + m.queryResult[targetPath][key] = x + case []interface{}: + m.queryResult[targetPath][key] = append(t, value) + } + + } else { + m.queryResult[targetPath][key] = value + } + } + } + + return resultPaths +} + +func (m *MqlQuery) buildResults() { + for _, v := range m.resultOrder { + m.results = append(m.results, m.queryResult[""][v]) + } +} diff --git a/query/mql/functional_test.go b/query/mql/functional_test.go new file mode 100644 index 0000000..97c2eac --- /dev/null +++ b/query/mql/functional_test.go @@ -0,0 +1,264 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "encoding/json" + "testing" + + . "github.com/smartystreets/goconvey/convey" + + "github.com/google/cayley/graph/memstore" +) + +// +---+ +---+ +// | A |------- ->| F |<-- +// +---+ \------>+---+-/ +---+ \--+---+ +// ------>|#B#| | | E | +// +---+-------/ >+---+ | +---+ +// | C | / v +// +---+ -/ +---+ +// ---- +---+/ |#G#| +// \-->|#D#|------------->+---+ +// +---+ +// + +func buildTripleStore() *MqlSession { + ts := memstore.MakeTestingMemstore() + return NewMqlSession(ts) +} + +func compareJsonInterfaces(actual interface{}, expected interface{}, path MqlPath, t *testing.T) { + isError := false + switch ex := expected.(type) { + case bool: + switch ac := actual.(type) { + case bool: + if ac != ex { + isError = true + } + default: + t.Log("Mismatched type") + isError = true + } + case float64: + switch ac := actual.(type) { + case float64: + if ac != ex { + isError = true + } + default: + t.Log("Mismatched type") + isError = true + } + case string: + switch ac := actual.(type) { + case string: + if ac != ex { + isError = true + } + default: + isError = true + } + case []interface{}: + switch ac := actual.(type) { + case []interface{}: + if len(ac) != len(ex) { + t.Log("Different lengths") + isError = true + } else { + for i, elem := range ex { + compareJsonInterfaces(ac[i], elem, path.Follow(string(i)), t) + } + } + default: + t.Log("Mismatched type") + isError = true + } + case map[string]interface{}: + switch ac := actual.(type) { + case map[string]interface{}: + for k, v := range ex { + actual_value, ok := ac[k] + if !ok { + t.Log("Key", k, "not in actual output.") + isError = true + } else { + compareJsonInterfaces(actual_value, v, path.Follow(string(k)), t) + } + } + default: + t.Log("Mismatched type") + isError = true + } + case nil: + switch ac := actual.(type) { + case nil: + if ac != ex { + isError = true + } + default: + t.Log("Mismatched type") + isError = true + } + default: + t.Error("Unknown JSON type?", expected) + } + + if isError { + actual_bytes, _ := json.MarshalIndent(actual, "", " ") + expected_bytes, _ := json.MarshalIndent(expected, "", " ") + t.Error(path.DisplayString(), ":\n", string(actual_bytes), "\nexpected", string(expected_bytes)) + } +} + +func runAndTestQuery(query string, expected string, t *testing.T) { + ses := buildTripleStore() + c := make(chan interface{}, 5) + go ses.ExecInput(query, c, -1) + for result := range c { + ses.BuildJson(result) + } + actual_struct, _ := ses.GetJson() + var expected_struct interface{} + json.Unmarshal([]byte(expected), &expected_struct) + compareJsonInterfaces(actual_struct, expected_struct, NewMqlPath(), t) + ses.ClearJson() +} + +func TestGetAllIds(t *testing.T) { + Convey("Should get all IDs in the database", t, func() { + query := ` + [{"id": null}] + ` + expected := ` + [ + {"id": "A"}, + {"id": "follows"}, + {"id": "B"}, + {"id": "C"}, + {"id": "D"}, + {"id": "F"}, + {"id": "G"}, + {"id": "E"}, + {"id": "status"}, + {"id": "cool"}, + {"id": "status_graph"} + ] + ` + runAndTestQuery(query, expected, t) + }) +} + +func TestGetCool(t *testing.T) { + query := ` + [{"id": null, "status": "cool"}] + ` + expected := ` + [ + {"id": "B", "status": "cool"}, + {"id": "D", "status": "cool"}, + {"id": "G", "status": "cool"} + ] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetFollowsList(t *testing.T) { + query := ` + [{"id": "C", "follows": []}] + ` + expected := ` + [{ + "id": "C", + "follows": [ + "B", "D" + ] + }] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetFollowsStruct(t *testing.T) { + query := ` + [{"id": null, "follows": {"id": null, "status": "cool"}}] + ` + expected := ` + [ + {"id": "A", "follows": {"id": "B", "status": "cool"}}, + {"id": "C", "follows": {"id": "D", "status": "cool"}}, + {"id": "D", "follows": {"id": "G", "status": "cool"}}, + {"id": "F", "follows": {"id": "G", "status": "cool"}} + ] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetFollowsReverseStructList(t *testing.T) { + query := ` + [{"id": null, "!follows": [{"id": null, "status" : "cool"}]}] + ` + expected := ` + [ + {"id": "F", "!follows": [{"id": "B", "status": "cool"}]}, + {"id": "B", "!follows": [{"id": "D", "status": "cool"}]}, + {"id": "G", "!follows": [{"id": "D", "status": "cool"}]} + ] + ` + runAndTestQuery(query, expected, t) +} + +func TestGetRevFollowsList(t *testing.T) { + query := ` + [{"id": "F", "!follows": []}] + ` + expected := ` + [{ + "id": "F", + "!follows": [ + "B", "E" + ] + }] + ` + runAndTestQuery(query, expected, t) +} + +func TestCoFollows(t *testing.T) { + query := ` + [{"id": null, "@A:follows": "B", "@B:follows": "D"}] + ` + expected := ` + [{ + "id": "C", + "@A:follows": "B", + "@B:follows": "D" + }] + ` + runAndTestQuery(query, expected, t) +} + +func TestRevCoFollows(t *testing.T) { + query := ` + [{"id": null, "!follows": {"id": "C"}, "@a:!follows": "D"}] + ` + expected := ` + [{ + "id": "B", + "!follows": {"id": "C"}, + "@a:!follows": "D" + }] + ` + runAndTestQuery(query, expected, t) +} diff --git a/query/mql/mql_build_iterator.go b/query/mql/mql_build_iterator.go deleted file mode 100644 index 6273696..0000000 --- a/query/mql/mql_build_iterator.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "errors" - "fmt" - "log" - "math" - "strings" - - "github.com/google/cayley/graph" -) - -func (m *MqlQuery) buildFixed(s string) graph.Iterator { - f := m.ses.ts.MakeFixed() - f.AddValue(m.ses.ts.GetIdFor(s)) - return f -} - -func (m *MqlQuery) buildResultIterator(path MqlPath) graph.Iterator { - all := m.ses.ts.GetNodesAllIterator() - all.AddTag(string(path)) - return graph.NewOptionalIterator(all) -} - -func (m *MqlQuery) BuildIteratorTree(query interface{}) { - m.isRepeated = make(map[MqlPath]bool) - m.queryStructure = make(map[MqlPath]map[string]interface{}) - m.queryResult = make(map[MqlResultPath]map[string]interface{}) - m.queryResult[""] = make(map[string]interface{}) - - m.it, m.err = m.buildIteratorTreeInternal(query, NewMqlPath()) - if m.err != nil { - m.isError = true - } -} - -func (m *MqlQuery) buildIteratorTreeInternal(query interface{}, path MqlPath) (graph.Iterator, error) { - var it graph.Iterator - var err error - err = nil - switch t := query.(type) { - case bool: - // for JSON booleans - // Treat the bool as a string and call it a day. - // Things which are really bool-like are special cases and will be dealt with separately. - if t { - it = m.buildFixed("true") - } - it = m.buildFixed("false") - case float64: - // for JSON numbers - // Damn you, Javascript, and your lack of integer values. - if math.Floor(t) == t { - // Treat it like an integer. - it = m.buildFixed(fmt.Sprintf("%d", t)) - } else { - it = m.buildFixed(fmt.Sprintf("%f", t)) - } - case string: - // for JSON strings - it = m.buildFixed(t) - case []interface{}: - // for JSON arrays - m.isRepeated[path] = true - if len(t) == 0 { - it = m.buildResultIterator(path) - } else if len(t) == 1 { - it, err = m.buildIteratorTreeInternal(t[0], path) - } else { - err = errors.New(fmt.Sprintf("Multiple fields at location root%s", path.DisplayString())) - } - case map[string]interface{}: - // for JSON objects - it, err = m.buildIteratorTreeMapInternal(t, path) - case nil: - it = m.buildResultIterator(path) - default: - log.Fatal("Unknown JSON type?", query) - } - if err != nil { - return nil, err - } - it.AddTag(string(path)) - return it, nil -} - -func (m *MqlQuery) buildIteratorTreeMapInternal(query map[string]interface{}, path MqlPath) (graph.Iterator, error) { - it := graph.NewAndIterator() - it.AddSubIterator(m.ses.ts.GetNodesAllIterator()) - var err error - err = nil - outputStructure := make(map[string]interface{}) - for key, subquery := range query { - outputStructure[key] = nil - reverse := false - pred := key - if strings.HasPrefix(pred, "@") { - i := strings.Index(pred, ":") - if i != -1 { - pred = pred[(i + 1):] - } - } - if strings.HasPrefix(pred, "!") { - reverse = true - pred = strings.TrimPrefix(pred, "!") - } - - // Other special constructs here - var subit graph.Iterator - if key == "id" { - subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) - if err != nil { - return nil, err - } - it.AddSubIterator(subit) - } else { - subit, err = m.buildIteratorTreeInternal(subquery, path.Follow(key)) - if err != nil { - return nil, err - } - subAnd := graph.NewAndIterator() - predFixed := m.ses.ts.MakeFixed() - predFixed.AddValue(m.ses.ts.GetIdFor(pred)) - subAnd.AddSubIterator(graph.NewLinksToIterator(m.ses.ts, predFixed, "p")) - if reverse { - lto := graph.NewLinksToIterator(m.ses.ts, subit, "s") - subAnd.AddSubIterator(lto) - hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "o") - it.AddSubIterator(hasa) - } else { - lto := graph.NewLinksToIterator(m.ses.ts, subit, "o") - subAnd.AddSubIterator(lto) - hasa := graph.NewHasaIterator(m.ses.ts, subAnd, "s") - it.AddSubIterator(hasa) - } - } - } - if err != nil { - return nil, err - } - m.queryStructure[path] = outputStructure - return it, nil -} - -type MqlResultPathSlice []MqlResultPath - -func (sl MqlResultPathSlice) Len() int { - return len(sl) -} - -func (sl MqlResultPathSlice) Less(i, j int) bool { - iLen := len(strings.Split(string(sl[i]), "\x30")) - jLen := len(strings.Split(string(sl[j]), "\x30")) - if iLen < jLen { - return true - } - if iLen == jLen { - if len(string(sl[i])) < len(string(sl[j])) { - return true - } - } - return false -} - -func (sl MqlResultPathSlice) Swap(i, j int) { - sl[i], sl[j] = sl[j], sl[i] -} diff --git a/query/mql/mql_fill.go b/query/mql/mql_fill.go deleted file mode 100644 index 26de32a..0000000 --- a/query/mql/mql_fill.go +++ /dev/null @@ -1,114 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "sort" - - "github.com/google/cayley/graph" -) - -func (m *MqlQuery) treeifyResult(tags map[string]graph.TSVal) map[MqlResultPath]string { - // Transform the map into something a little more interesting. - results := make(map[MqlPath]string) - for k, v := range tags { - results[MqlPath(k)] = m.ses.ts.GetNameFor(v) - } - resultPaths := make(map[MqlResultPath]string) - for k, v := range results { - resultPaths[k.ToResultPathFromMap(results)] = v - } - - var paths MqlResultPathSlice - - for path, _ := range resultPaths { - paths = append(paths, path) - } - - sort.Sort(paths) - - // Build Structure - for _, path := range paths { - currentPath := path.getPath() - value := resultPaths[path] - namePath := path.AppendValue(value) - if _, ok := m.queryResult[namePath]; !ok { - targetPath, key := path.splitLastPath() - if path == "" { - targetPath, key = "", value - if _, ok := m.queryResult[""][value]; !ok { - m.resultOrder = append(m.resultOrder, value) - } - } - if _, ok := m.queryStructure[currentPath]; ok { - // If there's substructure, then copy that in. - newStruct := m.copyPathStructure(currentPath) - if m.isRepeated[currentPath] && currentPath != "" { - switch t := m.queryResult[targetPath][key].(type) { - case nil: - x := make([]interface{}, 0) - x = append(x, newStruct) - m.queryResult[targetPath][key] = x - m.queryResult[namePath] = newStruct - case []interface{}: - m.queryResult[targetPath][key] = append(t, newStruct) - m.queryResult[namePath] = newStruct - } - - } else { - m.queryResult[namePath] = newStruct - m.queryResult[targetPath][key] = newStruct - } - } - } - } - - // Fill values - for _, path := range paths { - currentPath := path.getPath() - value := resultPaths[path] - namePath := path.AppendValue(value) - if _, ok := m.queryStructure[currentPath]; ok { - // We're dealing with ids. - if _, ok := m.queryResult[namePath]["id"]; ok { - m.queryResult[namePath]["id"] = value - } - } else { - // Just a value. - targetPath, key := path.splitLastPath() - if m.isRepeated[currentPath] { - switch t := m.queryResult[targetPath][key].(type) { - case nil: - x := make([]interface{}, 0) - x = append(x, value) - m.queryResult[targetPath][key] = x - case []interface{}: - m.queryResult[targetPath][key] = append(t, value) - } - - } else { - m.queryResult[targetPath][key] = value - } - } - } - - return resultPaths -} - -func (m *MqlQuery) buildResults() { - for _, v := range m.resultOrder { - m.results = append(m.results, m.queryResult[""][v]) - } -} diff --git a/query/mql/mql_functional_test.go b/query/mql/mql_functional_test.go deleted file mode 100644 index 97c2eac..0000000 --- a/query/mql/mql_functional_test.go +++ /dev/null @@ -1,264 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "encoding/json" - "testing" - - . "github.com/smartystreets/goconvey/convey" - - "github.com/google/cayley/graph/memstore" -) - -// +---+ +---+ -// | A |------- ->| F |<-- -// +---+ \------>+---+-/ +---+ \--+---+ -// ------>|#B#| | | E | -// +---+-------/ >+---+ | +---+ -// | C | / v -// +---+ -/ +---+ -// ---- +---+/ |#G#| -// \-->|#D#|------------->+---+ -// +---+ -// - -func buildTripleStore() *MqlSession { - ts := memstore.MakeTestingMemstore() - return NewMqlSession(ts) -} - -func compareJsonInterfaces(actual interface{}, expected interface{}, path MqlPath, t *testing.T) { - isError := false - switch ex := expected.(type) { - case bool: - switch ac := actual.(type) { - case bool: - if ac != ex { - isError = true - } - default: - t.Log("Mismatched type") - isError = true - } - case float64: - switch ac := actual.(type) { - case float64: - if ac != ex { - isError = true - } - default: - t.Log("Mismatched type") - isError = true - } - case string: - switch ac := actual.(type) { - case string: - if ac != ex { - isError = true - } - default: - isError = true - } - case []interface{}: - switch ac := actual.(type) { - case []interface{}: - if len(ac) != len(ex) { - t.Log("Different lengths") - isError = true - } else { - for i, elem := range ex { - compareJsonInterfaces(ac[i], elem, path.Follow(string(i)), t) - } - } - default: - t.Log("Mismatched type") - isError = true - } - case map[string]interface{}: - switch ac := actual.(type) { - case map[string]interface{}: - for k, v := range ex { - actual_value, ok := ac[k] - if !ok { - t.Log("Key", k, "not in actual output.") - isError = true - } else { - compareJsonInterfaces(actual_value, v, path.Follow(string(k)), t) - } - } - default: - t.Log("Mismatched type") - isError = true - } - case nil: - switch ac := actual.(type) { - case nil: - if ac != ex { - isError = true - } - default: - t.Log("Mismatched type") - isError = true - } - default: - t.Error("Unknown JSON type?", expected) - } - - if isError { - actual_bytes, _ := json.MarshalIndent(actual, "", " ") - expected_bytes, _ := json.MarshalIndent(expected, "", " ") - t.Error(path.DisplayString(), ":\n", string(actual_bytes), "\nexpected", string(expected_bytes)) - } -} - -func runAndTestQuery(query string, expected string, t *testing.T) { - ses := buildTripleStore() - c := make(chan interface{}, 5) - go ses.ExecInput(query, c, -1) - for result := range c { - ses.BuildJson(result) - } - actual_struct, _ := ses.GetJson() - var expected_struct interface{} - json.Unmarshal([]byte(expected), &expected_struct) - compareJsonInterfaces(actual_struct, expected_struct, NewMqlPath(), t) - ses.ClearJson() -} - -func TestGetAllIds(t *testing.T) { - Convey("Should get all IDs in the database", t, func() { - query := ` - [{"id": null}] - ` - expected := ` - [ - {"id": "A"}, - {"id": "follows"}, - {"id": "B"}, - {"id": "C"}, - {"id": "D"}, - {"id": "F"}, - {"id": "G"}, - {"id": "E"}, - {"id": "status"}, - {"id": "cool"}, - {"id": "status_graph"} - ] - ` - runAndTestQuery(query, expected, t) - }) -} - -func TestGetCool(t *testing.T) { - query := ` - [{"id": null, "status": "cool"}] - ` - expected := ` - [ - {"id": "B", "status": "cool"}, - {"id": "D", "status": "cool"}, - {"id": "G", "status": "cool"} - ] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetFollowsList(t *testing.T) { - query := ` - [{"id": "C", "follows": []}] - ` - expected := ` - [{ - "id": "C", - "follows": [ - "B", "D" - ] - }] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetFollowsStruct(t *testing.T) { - query := ` - [{"id": null, "follows": {"id": null, "status": "cool"}}] - ` - expected := ` - [ - {"id": "A", "follows": {"id": "B", "status": "cool"}}, - {"id": "C", "follows": {"id": "D", "status": "cool"}}, - {"id": "D", "follows": {"id": "G", "status": "cool"}}, - {"id": "F", "follows": {"id": "G", "status": "cool"}} - ] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetFollowsReverseStructList(t *testing.T) { - query := ` - [{"id": null, "!follows": [{"id": null, "status" : "cool"}]}] - ` - expected := ` - [ - {"id": "F", "!follows": [{"id": "B", "status": "cool"}]}, - {"id": "B", "!follows": [{"id": "D", "status": "cool"}]}, - {"id": "G", "!follows": [{"id": "D", "status": "cool"}]} - ] - ` - runAndTestQuery(query, expected, t) -} - -func TestGetRevFollowsList(t *testing.T) { - query := ` - [{"id": "F", "!follows": []}] - ` - expected := ` - [{ - "id": "F", - "!follows": [ - "B", "E" - ] - }] - ` - runAndTestQuery(query, expected, t) -} - -func TestCoFollows(t *testing.T) { - query := ` - [{"id": null, "@A:follows": "B", "@B:follows": "D"}] - ` - expected := ` - [{ - "id": "C", - "@A:follows": "B", - "@B:follows": "D" - }] - ` - runAndTestQuery(query, expected, t) -} - -func TestRevCoFollows(t *testing.T) { - query := ` - [{"id": null, "!follows": {"id": "C"}, "@a:!follows": "D"}] - ` - expected := ` - [{ - "id": "B", - "!follows": {"id": "C"}, - "@a:!follows": "D" - }] - ` - runAndTestQuery(query, expected, t) -} diff --git a/query/mql/mql_query.go b/query/mql/mql_query.go deleted file mode 100644 index 66d8179..0000000 --- a/query/mql/mql_query.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "fmt" - "strings" - - "github.com/google/cayley/graph" -) - -type MqlPath string -type MqlResultPath string - -type MqlQuery struct { - ses *MqlSession - it graph.Iterator - isRepeated map[MqlPath]bool - queryStructure map[MqlPath]map[string]interface{} - queryResult map[MqlResultPath]map[string]interface{} - results []interface{} - resultOrder []string - isError bool - err error -} - -func (mqlQuery *MqlQuery) copyPathStructure(path MqlPath) map[string]interface{} { - output := make(map[string]interface{}) - for k, v := range mqlQuery.queryStructure[path] { - output[k] = v - } - return output -} - -func NewMqlPath() MqlPath { - return "" -} -func (p MqlPath) Follow(s string) MqlPath { - return MqlPath(fmt.Sprintf("%s\x1E%s", p, s)) -} - -func (p MqlPath) DisplayString() string { - return strings.Replace(string(p), "\x1E", ".", -1) -} - -func NewMqlResultPath() MqlResultPath { - return "" -} - -func (p MqlResultPath) FollowPath(followPiece string, value string) MqlResultPath { - if string(p) == "" { - return MqlResultPath(fmt.Sprintf("%s\x1E%s", value, followPiece)) - } - return MqlResultPath(fmt.Sprintf("%s\x1E%s\x1E%s", p, value, followPiece)) -} - -func (p MqlResultPath) getPath() MqlPath { - out := NewMqlPath() - pathPieces := strings.Split(string(p), "\x1E") - for len(pathPieces) > 1 { - a := pathPieces[1] - pathPieces = pathPieces[2:] - out = out.Follow(a) - } - return out -} - -func (p MqlResultPath) splitLastPath() (MqlResultPath, string) { - pathPieces := strings.Split(string(p), "\x1E") - return MqlResultPath(strings.Join(pathPieces[:len(pathPieces)-1], "\x1E")), pathPieces[len(pathPieces)-1] -} - -func (p MqlResultPath) AppendValue(value string) MqlResultPath { - if string(p) == "" { - return MqlResultPath(value) - } - return MqlResultPath(fmt.Sprintf("%s\x1E%s", p, value)) -} - -func (p MqlPath) ToResultPathFromMap(resultMap map[MqlPath]string) MqlResultPath { - output := NewMqlResultPath() - pathPieces := strings.Split(string(p), "\x1E")[1:] - pathSoFar := NewMqlPath() - for _, piece := range pathPieces { - output = output.FollowPath(piece, resultMap[pathSoFar]) - pathSoFar = pathSoFar.Follow(piece) - } - return output -} - -func NewMqlQuery(ses *MqlSession) *MqlQuery { - var q MqlQuery - q.ses = ses - q.results = make([]interface{}, 0) - q.resultOrder = make([]string, 0) - q.err = nil - q.isError = false - return &q -} diff --git a/query/mql/mql_session.go b/query/mql/mql_session.go deleted file mode 100644 index 96d88bf..0000000 --- a/query/mql/mql_session.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package mql - -import ( - "encoding/json" - "fmt" - "sort" - - "github.com/barakmich/glog" - - "github.com/google/cayley/graph" -) - -type MqlSession struct { - ts graph.TripleStore - currentQuery *MqlQuery - debug bool -} - -func NewMqlSession(ts graph.TripleStore) *MqlSession { - var m MqlSession - m.ts = ts - return &m -} - -func (m *MqlSession) ToggleDebug() { - m.debug = !m.debug -} - -func (m *MqlSession) GetQuery(input string, output_struct chan map[string]interface{}) { - defer close(output_struct) - var mqlQuery interface{} - err := json.Unmarshal([]byte(input), &mqlQuery) - if err != nil { - return - } - m.currentQuery = NewMqlQuery(m) - m.currentQuery.BuildIteratorTree(mqlQuery) - output := make(map[string]interface{}) - graph.OutputQueryShapeForIterator(m.currentQuery.it, m.ts, &output) - nodes := output["nodes"].([]graph.Node) - new_nodes := make([]graph.Node, 0) - for _, n := range nodes { - n.Tags = nil - new_nodes = append(new_nodes, n) - } - output["nodes"] = new_nodes - output_struct <- output -} - -func (m *MqlSession) InputParses(input string) (graph.ParseResult, error) { - var x interface{} - err := json.Unmarshal([]byte(input), &x) - if err != nil { - return graph.ParseFail, err - } - return graph.Parsed, nil -} - -func (m *MqlSession) ExecInput(input string, c chan interface{}, limit int) { - defer close(c) - var mqlQuery interface{} - err := json.Unmarshal([]byte(input), &mqlQuery) - if err != nil { - return - } - m.currentQuery = NewMqlQuery(m) - m.currentQuery.BuildIteratorTree(mqlQuery) - if m.currentQuery.isError { - return - } - it, _ := m.currentQuery.it.Optimize() - if glog.V(2) { - glog.V(2).Infoln(it.DebugString(0)) - } - for { - _, ok := it.Next() - if !ok { - break - } - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - c <- &tags - for it.NextResult() == true { - tags := make(map[string]graph.TSVal) - it.TagResults(&tags) - c <- &tags - } - } -} - -func (m *MqlSession) ToText(result interface{}) string { - tags := *(result.(*map[string]graph.TSVal)) - out := fmt.Sprintln("****") - tagKeys := make([]string, len(tags)) - m.currentQuery.treeifyResult(tags) - m.currentQuery.buildResults() - r, _ := json.MarshalIndent(m.currentQuery.results, "", " ") - fmt.Println(string(r)) - i := 0 - for k, _ := range tags { - tagKeys[i] = string(k) - i++ - } - sort.Strings(tagKeys) - for _, k := range tagKeys { - if k == "$_" { - continue - } - out += fmt.Sprintf("%s : %s\n", k, m.ts.GetNameFor(tags[k])) - } - return out -} - -func (m *MqlSession) BuildJson(result interface{}) { - m.currentQuery.treeifyResult(*(result.(*map[string]graph.TSVal))) -} - -func (m *MqlSession) GetJson() (interface{}, error) { - m.currentQuery.buildResults() - if m.currentQuery.isError { - return nil, m.currentQuery.err - } else { - return m.currentQuery.results, nil - } -} - -func (m *MqlSession) ClearJson() { - // Since we create a new MqlQuery underneath every query, clearing isn't necessary. - return -} diff --git a/query/mql/query.go b/query/mql/query.go new file mode 100644 index 0000000..66d8179 --- /dev/null +++ b/query/mql/query.go @@ -0,0 +1,111 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "fmt" + "strings" + + "github.com/google/cayley/graph" +) + +type MqlPath string +type MqlResultPath string + +type MqlQuery struct { + ses *MqlSession + it graph.Iterator + isRepeated map[MqlPath]bool + queryStructure map[MqlPath]map[string]interface{} + queryResult map[MqlResultPath]map[string]interface{} + results []interface{} + resultOrder []string + isError bool + err error +} + +func (mqlQuery *MqlQuery) copyPathStructure(path MqlPath) map[string]interface{} { + output := make(map[string]interface{}) + for k, v := range mqlQuery.queryStructure[path] { + output[k] = v + } + return output +} + +func NewMqlPath() MqlPath { + return "" +} +func (p MqlPath) Follow(s string) MqlPath { + return MqlPath(fmt.Sprintf("%s\x1E%s", p, s)) +} + +func (p MqlPath) DisplayString() string { + return strings.Replace(string(p), "\x1E", ".", -1) +} + +func NewMqlResultPath() MqlResultPath { + return "" +} + +func (p MqlResultPath) FollowPath(followPiece string, value string) MqlResultPath { + if string(p) == "" { + return MqlResultPath(fmt.Sprintf("%s\x1E%s", value, followPiece)) + } + return MqlResultPath(fmt.Sprintf("%s\x1E%s\x1E%s", p, value, followPiece)) +} + +func (p MqlResultPath) getPath() MqlPath { + out := NewMqlPath() + pathPieces := strings.Split(string(p), "\x1E") + for len(pathPieces) > 1 { + a := pathPieces[1] + pathPieces = pathPieces[2:] + out = out.Follow(a) + } + return out +} + +func (p MqlResultPath) splitLastPath() (MqlResultPath, string) { + pathPieces := strings.Split(string(p), "\x1E") + return MqlResultPath(strings.Join(pathPieces[:len(pathPieces)-1], "\x1E")), pathPieces[len(pathPieces)-1] +} + +func (p MqlResultPath) AppendValue(value string) MqlResultPath { + if string(p) == "" { + return MqlResultPath(value) + } + return MqlResultPath(fmt.Sprintf("%s\x1E%s", p, value)) +} + +func (p MqlPath) ToResultPathFromMap(resultMap map[MqlPath]string) MqlResultPath { + output := NewMqlResultPath() + pathPieces := strings.Split(string(p), "\x1E")[1:] + pathSoFar := NewMqlPath() + for _, piece := range pathPieces { + output = output.FollowPath(piece, resultMap[pathSoFar]) + pathSoFar = pathSoFar.Follow(piece) + } + return output +} + +func NewMqlQuery(ses *MqlSession) *MqlQuery { + var q MqlQuery + q.ses = ses + q.results = make([]interface{}, 0) + q.resultOrder = make([]string, 0) + q.err = nil + q.isError = false + return &q +} diff --git a/query/mql/session.go b/query/mql/session.go new file mode 100644 index 0000000..96d88bf --- /dev/null +++ b/query/mql/session.go @@ -0,0 +1,144 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mql + +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/barakmich/glog" + + "github.com/google/cayley/graph" +) + +type MqlSession struct { + ts graph.TripleStore + currentQuery *MqlQuery + debug bool +} + +func NewMqlSession(ts graph.TripleStore) *MqlSession { + var m MqlSession + m.ts = ts + return &m +} + +func (m *MqlSession) ToggleDebug() { + m.debug = !m.debug +} + +func (m *MqlSession) GetQuery(input string, output_struct chan map[string]interface{}) { + defer close(output_struct) + var mqlQuery interface{} + err := json.Unmarshal([]byte(input), &mqlQuery) + if err != nil { + return + } + m.currentQuery = NewMqlQuery(m) + m.currentQuery.BuildIteratorTree(mqlQuery) + output := make(map[string]interface{}) + graph.OutputQueryShapeForIterator(m.currentQuery.it, m.ts, &output) + nodes := output["nodes"].([]graph.Node) + new_nodes := make([]graph.Node, 0) + for _, n := range nodes { + n.Tags = nil + new_nodes = append(new_nodes, n) + } + output["nodes"] = new_nodes + output_struct <- output +} + +func (m *MqlSession) InputParses(input string) (graph.ParseResult, error) { + var x interface{} + err := json.Unmarshal([]byte(input), &x) + if err != nil { + return graph.ParseFail, err + } + return graph.Parsed, nil +} + +func (m *MqlSession) ExecInput(input string, c chan interface{}, limit int) { + defer close(c) + var mqlQuery interface{} + err := json.Unmarshal([]byte(input), &mqlQuery) + if err != nil { + return + } + m.currentQuery = NewMqlQuery(m) + m.currentQuery.BuildIteratorTree(mqlQuery) + if m.currentQuery.isError { + return + } + it, _ := m.currentQuery.it.Optimize() + if glog.V(2) { + glog.V(2).Infoln(it.DebugString(0)) + } + for { + _, ok := it.Next() + if !ok { + break + } + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + c <- &tags + for it.NextResult() == true { + tags := make(map[string]graph.TSVal) + it.TagResults(&tags) + c <- &tags + } + } +} + +func (m *MqlSession) ToText(result interface{}) string { + tags := *(result.(*map[string]graph.TSVal)) + out := fmt.Sprintln("****") + tagKeys := make([]string, len(tags)) + m.currentQuery.treeifyResult(tags) + m.currentQuery.buildResults() + r, _ := json.MarshalIndent(m.currentQuery.results, "", " ") + fmt.Println(string(r)) + i := 0 + for k, _ := range tags { + tagKeys[i] = string(k) + i++ + } + sort.Strings(tagKeys) + for _, k := range tagKeys { + if k == "$_" { + continue + } + out += fmt.Sprintf("%s : %s\n", k, m.ts.GetNameFor(tags[k])) + } + return out +} + +func (m *MqlSession) BuildJson(result interface{}) { + m.currentQuery.treeifyResult(*(result.(*map[string]graph.TSVal))) +} + +func (m *MqlSession) GetJson() (interface{}, error) { + m.currentQuery.buildResults() + if m.currentQuery.isError { + return nil, m.currentQuery.err + } else { + return m.currentQuery.results, nil + } +} + +func (m *MqlSession) ClearJson() { + // Since we create a new MqlQuery underneath every query, clearing isn't necessary. + return +}