From b754810c6e3e349f51edcf602822210d26676fe2 Mon Sep 17 00:00:00 2001 From: Barak Michener Date: Tue, 28 Jul 2015 16:31:11 -0400 Subject: [PATCH] Remove old iterator code, leaving a simple all-iterator for the sql backend --- graph/iterator/and_iterator_optimize.go | 2 +- graph/sql/all_iterator.go | 212 ++++++++++++++++++++++++ graph/sql/iterator.go | 274 -------------------------------- integration/integration_test.go | 5 +- 4 files changed, 215 insertions(+), 278 deletions(-) create mode 100644 graph/sql/all_iterator.go delete mode 100644 graph/sql/iterator.go diff --git a/graph/iterator/and_iterator_optimize.go b/graph/iterator/and_iterator_optimize.go index cec5960..10aa803 100644 --- a/graph/iterator/and_iterator_optimize.go +++ b/graph/iterator/and_iterator_optimize.go @@ -331,7 +331,7 @@ func materializeIts(its []graph.Iterator) []graph.Iterator { out = append(out, its[0]) for _, it := range its[1:] { stats := it.Stats() - if false && stats.Size*stats.NextCost < (stats.ContainsCost*(1+(stats.Size/(allStats.Size+1)))) { + if stats.Size*stats.NextCost < (stats.ContainsCost * (1 + (stats.Size / (allStats.Size + 1)))) { if graph.Height(it, graph.Materialize) > 10 { out = append(out, NewMaterialize(it)) continue diff --git a/graph/sql/all_iterator.go b/graph/sql/all_iterator.go new file mode 100644 index 0000000..05f94b3 --- /dev/null +++ b/graph/sql/all_iterator.go @@ -0,0 +1,212 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sql + +import ( + "database/sql" + + "github.com/barakmich/glog" + + "github.com/google/cayley/graph" + "github.com/google/cayley/graph/iterator" + "github.com/google/cayley/quad" +) + +type AllIterator struct { + uid uint64 + tags graph.Tagger + qs *QuadStore + dir quad.Direction + val graph.Value + table string + cursor *sql.Rows + result graph.Value + err error +} + +func (it *AllIterator) makeCursor() { + var cursor *sql.Rows + var err error + if it.cursor != nil { + it.cursor.Close() + } + if it.table == "quads" { + cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`) + if err != nil { + glog.Errorln("Couldn't get cursor from SQL database: %v", err) + cursor = nil + } + } else { + glog.V(4).Infoln("sql: getting node query") + cursor, err = it.qs.db.Query(`SELECT node FROM + ( + SELECT subject FROM quads + UNION + SELECT predicate FROM quads + UNION + SELECT object FROM quads + UNION + SELECT label FROM quads + ) AS DistinctNodes (node) WHERE node IS NOT NULL;`) + if err != nil { + glog.Errorln("Couldn't get cursor from SQL database: %v", err) + cursor = nil + } + glog.V(4).Infoln("sql: got node query") + } + it.cursor = cursor +} + +func NewAllIterator(qs *QuadStore, table string) *AllIterator { + it := &AllIterator{ + uid: iterator.NextUID(), + qs: qs, + table: table, + } + return it +} + +func (it *AllIterator) UID() uint64 { + return it.uid +} + +func (it *AllIterator) Reset() { + it.err = nil + it.Close() +} + +func (it *AllIterator) Err() error { + return it.err +} + +func (it *AllIterator) Close() error { + if it.cursor != nil { + err := it.cursor.Close() + if err != nil { + return err + } + it.cursor = nil + } + return nil +} + +func (it *AllIterator) Tagger() *graph.Tagger { + return &it.tags +} + +func (it *AllIterator) TagResults(dst map[string]graph.Value) { + for _, tag := range it.tags.Tags() { + dst[tag] = it.Result() + } + + for tag, value := range it.tags.Fixed() { + dst[tag] = value + } +} + +func (it *AllIterator) Clone() graph.Iterator { + var m *AllIterator + m = NewAllIterator(it.qs, it.table) + m.tags.CopyFrom(it) + return m +} + +func (it *AllIterator) SubIterators() []graph.Iterator { + return nil +} + +func (it *AllIterator) Next() bool { + graph.NextLogIn(it) + if it.cursor == nil { + it.makeCursor() + } + if !it.cursor.Next() { + glog.V(4).Infoln("sql: No next") + err := it.cursor.Err() + if err != nil { + glog.Errorf("Cursor error in SQL: %v", err) + it.err = err + } + it.cursor.Close() + return false + } + if it.table == "nodes" { + var node string + err := it.cursor.Scan(&node) + if err != nil { + glog.Errorf("Error nexting node iterator: %v", err) + it.err = err + return false + } + it.result = node + return true + } + var q quad.Quad + err := it.cursor.Scan(&q.Subject, &q.Predicate, &q.Object, &q.Label) + if err != nil { + glog.Errorf("Error scanning sql iterator: %v", err) + it.err = err + return false + } + it.result = q + return graph.NextLogOut(it, it.result, true) +} + +func (it *AllIterator) Contains(v graph.Value) bool { + graph.ContainsLogIn(it, v) + it.result = v + return graph.ContainsLogOut(it, v, true) +} + +func (it *AllIterator) Size() (int64, bool) { + return it.qs.Size(), true +} + +func (it *AllIterator) Result() graph.Value { + if it.result == nil { + glog.Fatalln("result was nil", it) + } + return it.result +} + +func (it *AllIterator) NextPath() bool { + return false +} + +func (it *AllIterator) Type() graph.Type { + return graph.All +} + +func (it *AllIterator) Sorted() bool { return false } +func (it *AllIterator) Optimize() (graph.Iterator, bool) { return it, false } + +func (it *AllIterator) Describe() graph.Description { + size, _ := it.Size() + return graph.Description{ + UID: it.UID(), + Name: "sql/all", + Type: it.Type(), + Size: size, + } +} + +func (it *AllIterator) Stats() graph.IteratorStats { + size, _ := it.Size() + return graph.IteratorStats{ + ContainsCost: 1, + NextCost: 9999, + Size: size, + } +} diff --git a/graph/sql/iterator.go b/graph/sql/iterator.go deleted file mode 100644 index 1482eaa..0000000 --- a/graph/sql/iterator.go +++ /dev/null @@ -1,274 +0,0 @@ -// Copyright 2014 The Cayley Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sql - -import ( - "database/sql" - "fmt" - - "github.com/barakmich/glog" - - "github.com/google/cayley/graph" - "github.com/google/cayley/graph/iterator" - "github.com/google/cayley/quad" -) - -type Iterator struct { - uid uint64 - tags graph.Tagger - qs *QuadStore - dir quad.Direction - val graph.Value - size int64 - isAll bool - table string - cursor *sql.Rows - result graph.Value - err error -} - -func (it *Iterator) makeCursor() { - var cursor *sql.Rows - var err error - if it.cursor != nil { - it.cursor.Close() - } - if it.isAll { - if it.table == "quads" { - cursor, err = it.qs.db.Query(`SELECT subject, predicate, object, label FROM quads;`) - if err != nil { - glog.Errorln("Couldn't get cursor from SQL database: %v", err) - cursor = nil - } - } else { - glog.V(4).Infoln("sql: getting node query") - cursor, err = it.qs.db.Query(`SELECT node FROM - ( - SELECT subject FROM quads - UNION - SELECT predicate FROM quads - UNION - SELECT object FROM quads - UNION - SELECT label FROM quads - ) AS DistinctNodes (node) WHERE node IS NOT NULL;`) - if err != nil { - glog.Errorln("Couldn't get cursor from SQL database: %v", err) - cursor = nil - } - glog.V(4).Infoln("sql: got node query") - } - } else { - cursor, err = it.qs.db.Query( - fmt.Sprintf("SELECT subject, predicate, object, label FROM quads WHERE %s = $1;", it.dir.String()), it.val.(string)) - if err != nil { - glog.Errorln("Couldn't get cursor from SQL database: %v", err) - cursor = nil - } - } - it.cursor = cursor -} - -func NewIterator(qs *QuadStore, d quad.Direction, val graph.Value) *Iterator { - it := &Iterator{ - uid: iterator.NextUID(), - qs: qs, - dir: d, - size: -1, - val: val, - table: "quads", - isAll: false, - } - return it -} - -func NewAllIterator(qs *QuadStore, table string) *Iterator { - it := &Iterator{ - uid: iterator.NextUID(), - qs: qs, - dir: quad.Any, - size: qs.Size(), - table: table, - isAll: true, - } - return it -} - -func (it *Iterator) UID() uint64 { - return it.uid -} - -func (it *Iterator) Reset() { - it.err = nil - it.Close() -} - -func (it *Iterator) Err() error { - return it.err -} - -func (it *Iterator) Close() error { - if it.cursor != nil { - err := it.cursor.Close() - if err != nil { - return err - } - it.cursor = nil - } - return nil -} - -func (it *Iterator) Tagger() *graph.Tagger { - return &it.tags -} - -func (it *Iterator) TagResults(dst map[string]graph.Value) { - for _, tag := range it.tags.Tags() { - dst[tag] = it.Result() - } - - for tag, value := range it.tags.Fixed() { - dst[tag] = value - } -} - -func (it *Iterator) Clone() graph.Iterator { - var m *Iterator - if it.isAll { - m = NewAllIterator(it.qs, it.table) - } else { - m = NewIterator(it.qs, it.dir, it.val) - } - m.tags.CopyFrom(it) - return m -} - -func (it *Iterator) SubIterators() []graph.Iterator { - return nil -} - -func (it *Iterator) Next() bool { - graph.NextLogIn(it) - if it.cursor == nil { - it.makeCursor() - } - if !it.cursor.Next() { - glog.V(4).Infoln("sql: No next") - err := it.cursor.Err() - if err != nil { - glog.Errorf("Cursor error in SQL: %v", err) - it.err = err - } - it.cursor.Close() - return false - } - if it.table == "nodes" { - var node string - err := it.cursor.Scan(&node) - if err != nil { - glog.Errorf("Error nexting node iterator: %v", err) - it.err = err - return false - } - it.result = node - return true - } - var q quad.Quad - err := it.cursor.Scan(&q.Subject, &q.Predicate, &q.Object, &q.Label) - if err != nil { - glog.Errorf("Error scanning sql iterator: %v", err) - it.err = err - return false - } - it.result = q - return graph.NextLogOut(it, it.result, true) -} - -func (it *Iterator) Contains(v graph.Value) bool { - graph.ContainsLogIn(it, v) - if it.isAll { - it.result = v - return graph.ContainsLogOut(it, v, true) - } - q := v.(quad.Quad) - if q.Get(it.dir) == it.val.(string) { - it.result = v - return graph.ContainsLogOut(it, v, true) - } - return graph.ContainsLogOut(it, v, false) -} - -func (it *Iterator) Size() (int64, bool) { - if it.size != -1 { - return it.size, true - } - it.size = it.qs.sizeForIterator(it.isAll, it.dir, it.val.(string)) - return it.size, true -} - -func (it *Iterator) Result() graph.Value { - if it.result == nil { - glog.Fatalln("result was nil", it) - } - return it.result -} - -func (it *Iterator) NextPath() bool { - return false -} - -var sqlType graph.Type - -func init() { - sqlType = graph.RegisterIterator("sql") -} - -func Type() graph.Type { return sqlType } - -func (it *Iterator) Type() graph.Type { - if it.isAll { - return graph.All - } - return sqlType -} - -func (it *Iterator) Sorted() bool { return false } -func (it *Iterator) Optimize() (graph.Iterator, bool) { return it, false } - -func (it *Iterator) Describe() graph.Description { - size, _ := it.Size() - return graph.Description{ - UID: it.UID(), - Name: fmt.Sprintf("%s/%s", it.val, it.dir), - Type: it.Type(), - Size: size, - } -} - -func (it *Iterator) Stats() graph.IteratorStats { - size, _ := it.Size() - if it.table == "nodes" || it.isAll { - return graph.IteratorStats{ - ContainsCost: 1, - NextCost: 9999, - Size: size, - } - } - return graph.IteratorStats{ - ContainsCost: 1, - NextCost: 5, - Size: size, - } -} diff --git a/integration/integration_test.go b/integration/integration_test.go index a57a469..51ab30a 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -524,7 +524,6 @@ func TestQueries(t *testing.T) { } func TestDeletedAndRecreatedQueries(t *testing.T) { - t.Skip() if testing.Short() { t.Skip() } @@ -541,7 +540,7 @@ func checkQueries(t *testing.T) { continue } tInit := time.Now() - fmt.Printf("Now testing %s ", test.message) + t.Logf("Now testing %s ", test.message) ses := gremlin.NewSession(handle.QuadStore, cfg.Timeout, true) _, err := ses.Parse(test.query) if err != nil { @@ -570,7 +569,7 @@ func checkQueries(t *testing.T) { t.Error("Query timed out: skipping validation.") continue } - fmt.Printf("(%v)\n", time.Since(tInit)) + t.Logf("(%v)\n", time.Since(tInit)) if len(got) != len(test.expect) { t.Errorf("Unexpected number of results, got:%d expect:%d on %s.", len(got), len(test.expect), test.message)