diff --git a/cayley_test.go b/cayley_test.go index a92d947..02d4f4a 100644 --- a/cayley_test.go +++ b/cayley_test.go @@ -18,8 +18,10 @@ import ( "bytes" "compress/bzip2" "compress/gzip" + "fmt" "io" "reflect" + "sort" "strings" "sync" "testing" @@ -81,13 +83,13 @@ var benchmarkQueries = []struct { a.Intersect(b).Intersect(c).ForEach(function (d) { if (!(d.id in seen)) { seen[d.id] = true; - g.Emit(d.id) + g.Emit(d) } }) `, expect: [][]interface{}{ - {"/en/billy_gilbert"}, - {"/en/sterling_holloway"}, + {map[string]string{"id": "/en/sterling_holloway"}}, + {map[string]string{"id": "/en/billy_gilbert"}}, }, }, @@ -102,21 +104,21 @@ var benchmarkQueries = []struct { `, tag: "person", expect: [][]interface{}{ - {map[string]string{"id": "Casablanca", "person": "Claude Rains"}}, - {map[string]string{"id": "Casablanca", "person": "Conrad Veidt"}}, - {map[string]string{"id": "Casablanca", "person": "Dooley Wilson"}}, - {map[string]string{"id": "Casablanca", "person": "Helmut Dantine"}}, - {map[string]string{"id": "Casablanca", "person": "Humphrey Bogart"}}, {map[string]string{"id": "Casablanca", "person": "Ingrid Bergman"}}, - {map[string]string{"id": "Casablanca", "person": "John Qualen"}}, - {map[string]string{"id": "Casablanca", "person": "Joy Page"}}, - {map[string]string{"id": "Casablanca", "person": "Leonid Kinskey"}}, - {map[string]string{"id": "Casablanca", "person": "Lou Marcelle"}}, {map[string]string{"id": "Casablanca", "person": "Madeleine LeBeau"}}, + {map[string]string{"id": "Casablanca", "person": "Joy Page"}}, + {map[string]string{"id": "Casablanca", "person": "Claude Rains"}}, + {map[string]string{"id": "Casablanca", "person": "S.Z. Sakall"}}, + {map[string]string{"id": "Casablanca", "person": "Helmut Dantine"}}, + {map[string]string{"id": "Casablanca", "person": "Conrad Veidt"}}, {map[string]string{"id": "Casablanca", "person": "Paul Henreid"}}, {map[string]string{"id": "Casablanca", "person": "Peter Lorre"}}, {map[string]string{"id": "Casablanca", "person": "Sydney Greenstreet"}}, - {map[string]string{"id": "Casablanca", "person": "S.Z. Sakall"}}, + {map[string]string{"id": "Casablanca", "person": "Leonid Kinskey"}}, + {map[string]string{"id": "Casablanca", "person": "Lou Marcelle"}}, + {map[string]string{"id": "Casablanca", "person": "Dooley Wilson"}}, + {map[string]string{"id": "Casablanca", "person": "John Qualen"}}, + {map[string]string{"id": "Casablanca", "person": "Humphrey Bogart"}}, }, }, @@ -175,42 +177,98 @@ var benchmarkQueries = []struct { `, expect: [][]interface{}{ {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Proposal", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Crash", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Gun Shy", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Demolition Man", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Divine Secrets of the Ya-Ya Sisterhood", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "A Time to Kill", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Forces of Nature", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Hope Floats", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Infamous", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Love Potion No. 9", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality 2: Armed and Fabulous", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Murder by Numbers", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Practical Magic", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed 2: Cruise Control", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Net", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Prince of Egypt", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Two Weeks Notice", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "While You Were Sleeping", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "28 Days", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Premonition", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Wrestling Ernest Hemingway", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Fire on the Amazon", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Thing Called Love", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "In Love and War", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "In Love and War", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Proposal", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Proposal", "costar2_actor": "Keanu Reeves", "costar2_movie": "Parenthood", "id": "Mary Steenburgen"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Proposal", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Devil's Advocate", "id": "Craig T. Nelson"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Crash", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Crash", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Gun Shy", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Gun Shy", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Demolition Man", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Demolition Man", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Demolition Man", "costar2_actor": "Keanu Reeves", "costar2_movie": "Thumbsucker", "id": "Benjamin Bratt"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality", "costar2_actor": "Keanu Reeves", "costar2_movie": "Thumbsucker", "id": "Benjamin Bratt"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Divine Secrets of the Ya-Ya Sisterhood", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Divine Secrets of the Ya-Ya Sisterhood", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Divine Secrets of the Ya-Ya Sisterhood", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Private Lives of Pippa Lee", "id": "Shirley Knight"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "A Time to Kill", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "A Time to Kill", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Forces of Nature", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Forces of Nature", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Hope Floats", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Hope Floats", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Infamous", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Infamous", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Infamous", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Jeff Daniels"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Jeff Daniels"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Love Potion No. 9", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Love Potion No. 9", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality", "costar2_actor": "Keanu Reeves", "costar2_movie": "Thumbsucker", "id": "Benjamin Bratt"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality 2: Armed and Fabulous", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Miss Congeniality 2: Armed and Fabulous", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Murder by Numbers", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Murder by Numbers", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Practical Magic", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Practical Magic", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Practical Magic", "costar2_actor": "Keanu Reeves", "costar2_movie": "Parenthood", "id": "Dianne Wiest"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Flying", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Animatrix", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Tune in Tomorrow", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Last Time I Committed Suicide", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Constantine", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Permanent Record", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Dangerous Liaisons", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Private Lives of Pippa Lee", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "A Scanner Darkly", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "A Walk in the Clouds", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Hardball", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Life Under Water", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Much Ado About Nothing", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "My Own Private Idaho", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Parenthood", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Point Break", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Providence", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "River's Edge", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Something's Gotta Give", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Sweet November", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Matrix Reloaded", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Matrix Revisited", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Prince of Pennsylvania", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Replacements", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Even Cowgirls Get the Blues", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Youngblood", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Bill \u0026 Ted's Bogus Journey", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Bill \u0026 Ted's Excellent Adventure", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Johnny Mnemonic", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Devil's Advocate", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Thumbsucker", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "I Love You to Death", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Bram Stoker's Dracula", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Gift", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Little Buddha", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Night Watchman", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Chain Reaction", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Babes in Toyland", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Day the Earth Stood Still", "id": "Keanu Reeves"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "River's Edge", "id": "Dennis Hopper"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Dennis Hopper"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Jeff Daniels"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Joe Morton"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Alan Ruck"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Glenn Plummer"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Carlos Carrasco"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Beth Grant"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Richard Lineback"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Hawthorne James"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Jordan Lund"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Thomas Rosales, Jr."}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed 2: Cruise Control", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed 2: Cruise Control", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed 2: Cruise Control", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Glenn Plummer"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "Flying", "id": "Keanu Reeves"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Animatrix", "id": "Keanu Reeves"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "Tune in Tomorrow", "id": "Keanu Reeves"}}, @@ -252,28 +310,38 @@ var benchmarkQueries = []struct { {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "Chain Reaction", "id": "Keanu Reeves"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "Babes in Toyland", "id": "Keanu Reeves"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Day the Earth Stood Still", "id": "Keanu Reeves"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "River's Edge", "id": "Dennis Hopper"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Dennis Hopper"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Joe Morton"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Alan Ruck"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Glenn Plummer"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed 2: Cruise Control", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Glenn Plummer"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Carlos Carrasco"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Beth Grant"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Richard Lineback"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Hawthorne James"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Jordan Lund"}}, - {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Speed", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Thomas Rosales, Jr."}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Christopher Plummer"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Dylan Walsh"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Shohreh Aghdashloo"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Lake House", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Lynn Collins"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Net", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Net", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Prince of Egypt", "costar2_actor": "Keanu Reeves", "costar2_movie": "Dangerous Liaisons", "id": "Michelle Pfeiffer"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Prince of Egypt", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Prince of Egypt", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Prince of Egypt", "costar2_actor": "Keanu Reeves", "costar2_movie": "Parenthood", "id": "Steve Martin"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Two Weeks Notice", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Two Weeks Notice", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "While You Were Sleeping", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "While You Were Sleeping", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "While You Were Sleeping", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Replacements", "id": "Jack Warden"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "28 Days", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "28 Days", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Premonition", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Premonition", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Premonition", "costar2_actor": "Keanu Reeves", "costar2_movie": "Constantine", "id": "Peter Stormare"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Wrestling Ernest Hemingway", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Wrestling Ernest Hemingway", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Fire on the Amazon", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "Fire on the Amazon", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Thing Called Love", "costar2_actor": "Keanu Reeves", "costar2_movie": "My Own Private Idaho", "id": "River Phoenix"}}, {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Thing Called Love", "costar2_actor": "Keanu Reeves", "costar2_movie": "I Love You to Death", "id": "River Phoenix"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Thing Called Love", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "The Thing Called Love", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "In Love and War", "costar2_actor": "Keanu Reeves", "costar2_movie": "Speed", "id": "Sandra Bullock"}}, + {map[string]string{"costar1_actor": "Sandra Bullock", "costar1_movie": "In Love and War", "costar2_actor": "Keanu Reeves", "costar2_movie": "The Lake House", "id": "Sandra Bullock"}}, }, }, } @@ -394,7 +462,7 @@ func checkQueries(t *testing.T) { t.Errorf("Unexpected number of results, got:%d expect:%d on %s.", len(got), len(test.expect), test.message) continue } - if reflect.DeepEqual(got, test.expect) { + if unsortedEqual(got, test.expect) { continue } t.Errorf("Unexpected results for %s:\n", test.message) @@ -404,6 +472,29 @@ func checkQueries(t *testing.T) { } } +func unsortedEqual(got, expect [][]interface{}) bool { + gotList := convertToStringList(got) + expectList := convertToStringList(expect) + return reflect.DeepEqual(gotList, expectList) +} + +func convertToStringList(in [][]interface{}) []string { + var out []string + for _, x := range in { + var subEntry []string + for _, y := range x { + m := y.(map[string]string) + for k, v := range m { + subEntry = append(subEntry, fmt.Sprint(k, ":", v)) + } + } + sort.Strings(subEntry) + out = append(out, strings.Join(subEntry, ",")) + } + sort.Strings(out) + return out +} + func runBench(n int, b *testing.B) { if testing.Short() && benchmarkQueries[n].long { b.Skip() diff --git a/graph/iterator.go b/graph/iterator.go index 96f1587..ca17cb5 100644 --- a/graph/iterator.go +++ b/graph/iterator.go @@ -183,6 +183,9 @@ type IteratorStats struct { ContainsCost int64 NextCost int64 Size int64 + Next int64 + Contains int64 + ContainsNext int64 } // Type enumerates the set of Iterator types. @@ -250,6 +253,24 @@ func (t Type) String() string { return types[t] } +type StatsContainer struct { + IteratorStats + Kind string + Uid uint64 + SubIts []StatsContainer +} + +func DumpStats(it Iterator) StatsContainer { + var out StatsContainer + out.IteratorStats = it.Stats() + out.Kind = it.Type().String() + out.Uid = it.UID() + for _, sub := range it.SubIterators() { + out.SubIts = append(out.SubIts, DumpStats(sub)) + } + return out +} + // Utility logging functions for when an iterator gets called Next upon, or Contains upon, as // well as what they return. Highly useful for tracing the execution path of a query. func ContainsLogIn(it Iterator, val Value) { diff --git a/graph/iterator/all_iterator.go b/graph/iterator/all_iterator.go index 80e471c..0a890cf 100644 --- a/graph/iterator/all_iterator.go +++ b/graph/iterator/all_iterator.go @@ -36,6 +36,7 @@ type Int64 struct { max, min int64 at int64 result graph.Value + runstats graph.IteratorStats } // Creates a new Int64 with the given range. @@ -89,6 +90,7 @@ func (it *Int64) DebugString(indent int) string { // Return the next integer, and mark it as the result. func (it *Int64) Next() bool { graph.NextLogIn(it) + it.runstats.Next += 1 if it.at == -1 { return graph.NextLogOut(it, nil, false) } @@ -130,6 +132,7 @@ func (it *Int64) Size() (int64, bool) { // withing the range, assuming the value is an int64. func (it *Int64) Contains(tsv graph.Value) bool { graph.ContainsLogIn(it, tsv) + it.runstats.Contains += 1 v := tsv.(int64) if it.min <= v && v <= it.max { it.result = v @@ -153,5 +156,7 @@ func (it *Int64) Stats() graph.IteratorStats { ContainsCost: 1, NextCost: 1, Size: s, + Next: it.runstats.Next, + Contains: it.runstats.Contains, } } diff --git a/graph/iterator/and_iterator.go b/graph/iterator/and_iterator.go index c18cd70..a585cd9 100644 --- a/graph/iterator/and_iterator.go +++ b/graph/iterator/and_iterator.go @@ -32,6 +32,7 @@ type And struct { primaryIt graph.Iterator checkList []graph.Iterator result graph.Value + runstats graph.IteratorStats } // Creates a new And iterator. @@ -158,6 +159,7 @@ func (it *And) AddSubIterator(sub graph.Iterator) { // is therefore very important. func (it *And) Next() bool { graph.NextLogIn(it) + it.runstats.Next += 1 for graph.Next(it.primaryIt) { curr := it.primaryIt.Result() if it.subItsContain(curr, nil) { @@ -211,6 +213,7 @@ func (it *And) checkContainsList(val graph.Value, lastResult graph.Value) bool { // Check a value against the entire iterator, in order. func (it *And) Contains(val graph.Value) bool { graph.ContainsLogIn(it, val) + it.runstats.Contains += 1 lastResult := it.result if it.checkList != nil { return it.checkContainsList(val, lastResult) diff --git a/graph/iterator/and_iterator_optimize.go b/graph/iterator/and_iterator_optimize.go index cb8030a7..c3af04a 100644 --- a/graph/iterator/and_iterator_optimize.go +++ b/graph/iterator/and_iterator_optimize.go @@ -17,6 +17,8 @@ package iterator import ( "sort" + "github.com/barakmich/glog" + "github.com/google/cayley/graph" ) @@ -68,7 +70,7 @@ func (it *And) Optimize() (graph.Iterator, bool) { // And now, without changing any of the iterators, we reorder them. it_list is // now a permutation of itself, but the contents are unchanged. - its = optimizeOrder(its) + its = it.optimizeOrder(its) its = materializeIts(its) @@ -87,6 +89,7 @@ func (it *And) Optimize() (graph.Iterator, bool) { newAnd.tags.CopyFrom(it) newAnd.optimizeContains() + glog.V(3).Infoln(it.UID(), "became", newAnd.UID()) // And close ourselves but not our subiterators -- some may still be alive in // the new And (they were unchanged upon calling Optimize() on them, at the @@ -133,7 +136,7 @@ func (_ *And) optimizeReplacement(its []graph.Iterator) graph.Iterator { // optimizeOrder(l) takes a list and returns a list, containing the same contents // but with a new ordering, however it wishes. -func optimizeOrder(its []graph.Iterator) []graph.Iterator { +func (it *And) optimizeOrder(its []graph.Iterator) []graph.Iterator { var ( // bad contains iterators that can't be (efficiently) nexted, such as // graph.Optional or graph.Not. Separate them out and tack them on at the end. @@ -146,29 +149,35 @@ func optimizeOrder(its []graph.Iterator) []graph.Iterator { // Total cost is defined as The Next()ed iterator's cost to Next() out // all of it's contents, and to Contains() each of those against everyone // else. - for _, it := range its { - if _, canNext := it.(graph.Nexter); !canNext { - bad = append(bad, it) + for _, root := range its { + if _, canNext := root.(graph.Nexter); !canNext { + bad = append(bad, root) continue } - rootStats := it.Stats() + rootStats := root.Stats() cost := rootStats.NextCost for _, f := range its { - if _, canNext := it.(graph.Nexter); !canNext { + if _, canNext := f.(graph.Nexter); !canNext { continue } - if f == it { + if f == root { continue } stats := f.Stats() - cost += stats.ContainsCost + cost += stats.ContainsCost * (1 + (rootStats.Size / (stats.Size + 1))) } cost *= rootStats.Size + if glog.V(3) { + glog.V(3).Infoln("And:", it.UID(), "Root:", root.UID(), "Total Cost:", cost, "Best:", bestCost) + } if cost < bestCost { - best = it + best = root bestCost = cost } } + if glog.V(3) { + glog.V(3).Infoln("And:", it.UID(), "Choosing:", best.UID(), "Best:", bestCost) + } // TODO(barakmich): Optimization of order need not stop here. Picking a smart // Contains() order based on probability of getting a false Contains() first is @@ -297,9 +306,12 @@ func hasOneUsefulIterator(its []graph.Iterator) graph.Iterator { func materializeIts(its []graph.Iterator) []graph.Iterator { var out []graph.Iterator - for _, it := range its { + + allStats := getStatsForSlice(its) + out = append(out, its[0]) + for _, it := range its[1:] { stats := it.Stats() - if stats.Size*stats.NextCost < stats.ContainsCost { + if stats.Size*stats.NextCost < (stats.ContainsCost * (1 + (stats.Size / (allStats.Size + 1)))) { if graph.Height(it, graph.Materialize) > 10 { out = append(out, NewMaterialize(it)) continue @@ -310,26 +322,34 @@ func materializeIts(its []graph.Iterator) []graph.Iterator { return out } -// and.Stats() lives here in and-iterator-optimize.go because it may -// in the future return different statistics based on how it is optimized. -// For now, however, it's pretty static. -func (it *And) Stats() graph.IteratorStats { - primaryStats := it.primaryIt.Stats() +func getStatsForSlice(its []graph.Iterator) graph.IteratorStats { + primary := its[0] + primaryStats := primary.Stats() ContainsCost := primaryStats.ContainsCost NextCost := primaryStats.NextCost Size := primaryStats.Size - for _, sub := range it.internalIterators { + for _, sub := range its[1:] { stats := sub.Stats() - NextCost += stats.ContainsCost + NextCost += stats.ContainsCost * (1 + (primaryStats.Size / (stats.Size + 1))) ContainsCost += stats.ContainsCost if Size > stats.Size { Size = stats.Size } } return graph.IteratorStats{ - ContainsCost: ContainsCost * 2, + ContainsCost: ContainsCost, NextCost: NextCost, Size: Size, } } + +// and.Stats() lives here in and-iterator-optimize.go because it may +// in the future return different statistics based on how it is optimized. +// For now, however, it's pretty static. +func (it *And) Stats() graph.IteratorStats { + stats := getStatsForSlice(it.SubIterators()) + stats.Next = it.runstats.Next + stats.Contains = it.runstats.Contains + return stats +} diff --git a/graph/iterator/fixed_iterator.go b/graph/iterator/fixed_iterator.go index a2d57b1..8ecda34 100644 --- a/graph/iterator/fixed_iterator.go +++ b/graph/iterator/fixed_iterator.go @@ -108,9 +108,10 @@ func (it *Fixed) DebugString(indent int) string { if len(it.values) > 0 { value = fmt.Sprint(it.values[0]) } - return fmt.Sprintf("%s(%s tags: %s Size: %d id0: %d)", + return fmt.Sprintf("%s(%s %d tags: %s Size: %d id0: %d)", strings.Repeat(" ", indent), it.Type(), + it.UID(), it.tags.Fixed(), len(it.values), value, diff --git a/graph/iterator/hasa_iterator.go b/graph/iterator/hasa_iterator.go index b88f58b..b8aae90 100644 --- a/graph/iterator/hasa_iterator.go +++ b/graph/iterator/hasa_iterator.go @@ -54,6 +54,7 @@ type HasA struct { dir quad.Direction resultIt graph.Iterator result graph.Value + runstats graph.IteratorStats } // Construct a new HasA iterator, given the triple subiterator, and the triple @@ -143,6 +144,7 @@ func (it *HasA) DebugString(indent int) string { // and then Next() values out of that iterator and Contains() them against our subiterator. func (it *HasA) Contains(val graph.Value) bool { graph.ContainsLogIn(it, val) + it.runstats.Contains += 1 if glog.V(4) { glog.V(4).Infoln("Id is", it.ts.NameOf(val)) } @@ -159,6 +161,7 @@ func (it *HasA) Contains(val graph.Value) bool { // another match is made. func (it *HasA) NextContains() bool { for graph.Next(it.resultIt) { + it.runstats.ContainsNext += 1 link := it.resultIt.Result() if glog.V(4) { glog.V(4).Infoln("Quad is", it.ts.Quad(link)) @@ -193,6 +196,7 @@ func (it *HasA) NextPath() bool { // pull our direction out of it, and return that. func (it *HasA) Next() bool { graph.NextLogIn(it) + it.runstats.Next += 1 if it.resultIt != nil { it.resultIt.Close() } @@ -229,6 +233,9 @@ func (it *HasA) Stats() graph.IteratorStats { NextCost: tripleConstant + subitStats.NextCost, ContainsCost: (fanoutFactor * nextConstant) * subitStats.ContainsCost, Size: faninFactor * subitStats.Size, + Next: it.runstats.Next, + Contains: it.runstats.Contains, + ContainsNext: it.runstats.ContainsNext, } } @@ -244,5 +251,5 @@ func (it *HasA) Close() { func (it *HasA) Type() graph.Type { return graph.HasA } func (it *HasA) Size() (int64, bool) { - return 0, true + return it.Stats().Size, false } diff --git a/graph/iterator/iterator.go b/graph/iterator/iterator.go index 67d8a80..ab36b7b 100644 --- a/graph/iterator/iterator.go +++ b/graph/iterator/iterator.go @@ -25,6 +25,10 @@ import ( var nextIteratorID uint64 +func init() { + atomic.StoreUint64(&nextIteratorID, 1) +} + func NextUID() uint64 { return atomic.AddUint64(&nextIteratorID, 1) - 1 } diff --git a/graph/iterator/linksto_iterator.go b/graph/iterator/linksto_iterator.go index 517fc11..d5bd3d8 100644 --- a/graph/iterator/linksto_iterator.go +++ b/graph/iterator/linksto_iterator.go @@ -48,6 +48,7 @@ type LinksTo struct { dir quad.Direction nextIt graph.Iterator result graph.Value + runstats graph.IteratorStats } // Construct a new LinksTo iterator around a direction and a subiterator of @@ -118,6 +119,7 @@ func (it *LinksTo) DebugString(indent int) string { // for the LinksTo. func (it *LinksTo) Contains(val graph.Value) bool { graph.ContainsLogIn(it, val) + it.runstats.Contains += 1 node := it.ts.TripleDirection(val, it.dir) if it.primaryIt.Contains(node) { it.result = val @@ -155,7 +157,9 @@ func (it *LinksTo) Optimize() (graph.Iterator, bool) { // Next()ing a LinksTo operates as described above. func (it *LinksTo) Next() bool { graph.NextLogIn(it) + it.runstats.Next += 1 if graph.Next(it.nextIt) { + it.runstats.ContainsNext += 1 it.result = it.nextIt.Result() return graph.NextLogOut(it, it.nextIt, true) } @@ -201,9 +205,12 @@ func (it *LinksTo) Stats() graph.IteratorStats { NextCost: nextConstant + subitStats.NextCost, ContainsCost: checkConstant + subitStats.ContainsCost, Size: fanoutFactor * subitStats.Size, + Next: it.runstats.Next, + Contains: it.runstats.Contains, + ContainsNext: it.runstats.ContainsNext, } } func (it *LinksTo) Size() (int64, bool) { - return 0, true + return it.Stats().Size, false } diff --git a/graph/iterator/materialize_iterator.go b/graph/iterator/materialize_iterator.go index 8528f4f..46f2d57 100644 --- a/graph/iterator/materialize_iterator.go +++ b/graph/iterator/materialize_iterator.go @@ -45,11 +45,13 @@ type Materialize struct { tags graph.Tagger containsMap map[graph.Value]int values [][]result + actualSize int64 index int subindex int subIt graph.Iterator hasRun bool aborted bool + runstats graph.IteratorStats } func NewMaterialize(sub graph.Iterator) *Materialize { @@ -111,6 +113,7 @@ func (it *Materialize) Clone() graph.Iterator { out.aborted = it.aborted out.values = it.values out.containsMap = it.containsMap + out.actualSize = it.actualSize } return out } @@ -171,8 +174,10 @@ func (it *Materialize) Optimize() (graph.Iterator, bool) { // Otherwise, guess based on the size of the subiterator. func (it *Materialize) Size() (int64, bool) { if it.hasRun && !it.aborted { - return int64(len(it.values)), true + glog.V(2).Infoln("returning size", it.actualSize) + return it.actualSize, true } + glog.V(2).Infoln("bailing size", it.actualSize) return it.subIt.Size() } @@ -186,11 +191,14 @@ func (it *Materialize) Stats() graph.IteratorStats { ContainsCost: overhead * subitStats.NextCost, NextCost: overhead * subitStats.NextCost, Size: size, + Next: it.runstats.Next, + Contains: it.runstats.Contains, } } func (it *Materialize) Next() bool { graph.NextLogIn(it) + it.runstats.Next += 1 if !it.hasRun { it.materializeSet() } @@ -208,6 +216,7 @@ func (it *Materialize) Next() bool { func (it *Materialize) Contains(v graph.Value) bool { graph.ContainsLogIn(it, v) + it.runstats.Contains += 1 if !it.hasRun { it.materializeSet() } @@ -264,10 +273,17 @@ func (it *Materialize) materializeSet() { tags := make(map[string]graph.Value) it.subIt.TagResults(tags) it.values[index] = append(it.values[index], result{id: id, tags: tags}) + it.actualSize += 1 for it.subIt.NextPath() { + i++ + if i > abortMaterializeAt { + it.aborted = true + break + } tags := make(map[string]graph.Value) it.subIt.TagResults(tags) it.values[index] = append(it.values[index], result{id: id, tags: tags}) + it.actualSize += 1 } } if it.aborted { diff --git a/query/gremlin/finals.go b/query/gremlin/finals.go index bdd7de2..a82da13 100644 --- a/query/gremlin/finals.go +++ b/query/gremlin/finals.go @@ -15,6 +15,8 @@ package gremlin import ( + "encoding/json" + "github.com/barakmich/glog" "github.com/robertkrimen/otto" @@ -278,5 +280,9 @@ func runIteratorOnSession(it graph.Iterator, ses *Session) { } } } + if glog.V(2) { + bytes, _ := json.MarshalIndent(graph.DumpStats(it), "", " ") + glog.V(2).Infoln(string(bytes)) + } it.Close() }