diff --git a/appengine/app.yaml b/appengine/app.yaml index 41b96a2..947054d 100644 --- a/appengine/app.yaml +++ b/appengine/app.yaml @@ -1,7 +1,7 @@ application: cayley-test version: 1 runtime: go -api_version: go1 +api_version: go1.4beta handlers: - url: /.* diff --git a/graph/gaedatastore/iterator.go b/graph/gaedatastore/iterator.go index 29900fd..e60d442 100644 --- a/graph/gaedatastore/iterator.go +++ b/graph/gaedatastore/iterator.go @@ -228,7 +228,7 @@ func (it *Iterator) Next() bool { it.offset = 0 it.buffer = make([]string, 0, bufferSize) // Create query - // TODO (stefankoshiw) Keys only query? + // TODO (panamafrancis) Keys only query? q := datastore.NewQuery(it.kind).Limit(bufferSize) if !it.isAll { // Filter on the direction {subject,objekt...} diff --git a/graph/gaedatastore/quadstore.go b/graph/gaedatastore/quadstore.go index 69952e8..34e7c82 100644 --- a/graph/gaedatastore/quadstore.go +++ b/graph/gaedatastore/quadstore.go @@ -62,12 +62,12 @@ type Token struct { type QuadEntry struct { Hash string - Added []int64 `datastore:",noindex"` - Deleted []int64 `datastore:",noindex"` - Subject string `datastore:"subject"` - Predicate string `datastore:"predicate"` - Object string `datastore:"object"` - Label string `datastore:"label"` + Added []string `datastore:",noindex"` + Deleted []string `datastore:",noindex"` + Subject string `datastore:"subject"` + Predicate string `datastore:"predicate"` + Object string `datastore:"object"` + Label string `datastore:"label"` } type NodeEntry struct { @@ -76,7 +76,7 @@ type NodeEntry struct { } type LogEntry struct { - LogID int64 + LogID string Action string Key string Timestamp int64 @@ -127,8 +127,8 @@ func (qs *QuadStore) createKeyForMetadata() *datastore.Key { return qs.createKeyFromToken(&Token{"metadata", "metadataentry"}) } -func (qs *QuadStore) createKeyForLog(deltaID int64) *datastore.Key { - return datastore.NewKey(qs.context, "logentry", "", deltaID, nil) +func (qs *QuadStore) createKeyForLog(deltaID graph.PrimaryKey) *datastore.Key { + return datastore.NewKey(qs.context, "logentry", deltaID.String(), 0, nil) } func (qs *QuadStore) createKeyFromToken(t *Token) *datastore.Key { @@ -161,35 +161,42 @@ func getContext(opts graph.Options) (appengine.Context, error) { return appengine.NewContext(req), nil } -func (qs *QuadStore) ApplyDeltas(in []graph.Delta) error { +func (qs *QuadStore) ApplyDeltas(in []graph.Delta, ignoreOpts graph.IgnoreOpts) error { if qs.context == nil { return errors.New("No context, graph not correctly initialised") } toKeep := make([]graph.Delta, 0) for _, d := range in { + if d.Action != graph.Add && d.Action != graph.Delete { + //Defensive shortcut + return errors.New("Datastore: invalid action") + } key := qs.createKeyForQuad(d.Quad) keep := false switch d.Action { case graph.Add: - if found, err := qs.checkValid(key); !found && err == nil { - keep = true - } else if err != nil { + found, err := qs.checkValid(key) + if err != nil { return err + } + if !found || ignoreOpts.IgnoreDup { + keep = true } else { glog.Warningf("Quad exists already: %v", d) } case graph.Delete: - if found, err := qs.checkValid(key); found && err == nil { - keep = true - } else if err != nil { + found, err := qs.checkValid(key) + if err != nil { return err + } + if found || ignoreOpts.IgnoreMissing { + keep = true } else { glog.Warningf("Quad does not exist and so cannot be deleted: %v", d) } default: - keep = true + keep = false } - if keep { toKeep = append(toKeep, d) } @@ -309,10 +316,10 @@ func (qs *QuadStore) updateQuads(in []graph.Delta) (int64, error) { // If the quad exists the Added[] will be non-empty if in[x].Action == graph.Add { - foundQuads[k].Added = append(foundQuads[k].Added, in[x].ID) + foundQuads[k].Added = append(foundQuads[k].Added, in[x].ID.String()) quadCount += 1 } else { - foundQuads[k].Deleted = append(foundQuads[k].Deleted, in[x].ID) + foundQuads[k].Deleted = append(foundQuads[k].Deleted, in[x].ID.String()) quadCount -= 1 } } @@ -365,7 +372,7 @@ func (qs *QuadStore) updateLog(in []graph.Delta) error { } entry := LogEntry{ - LogID: d.ID, + LogID: d.ID.String(), Action: action, Key: qs.createKeyForQuad(d.Quad).String(), Timestamp: d.Timestamp.UnixNano(), @@ -481,10 +488,10 @@ func (qs *QuadStore) NodeSize() int64 { return foundMetadata.NodeCount } -func (qs *QuadStore) Horizon() int64 { +func (qs *QuadStore) Horizon() graph.PrimaryKey { if qs.context == nil { glog.Warning("Warning: HTTP Request context is nil, cannot get horizon from datastore.") - return 0 + return graph.NewUniqueKey("") } // Query log for last entry... q := datastore.NewQuery("logentry").Order("-Timestamp").Limit(1) @@ -492,9 +499,9 @@ func (qs *QuadStore) Horizon() int64 { _, err := q.GetAll(qs.context, &logEntries) if err != nil || len(logEntries) == 0 { // Error fetching horizon, probably graph is empty - return 0 + return graph.NewUniqueKey("") } - return logEntries[0].LogID + return graph.NewUniqueKey(logEntries[0].LogID) } func compareTokens(a, b graph.Value) bool { diff --git a/graph/gaedatastore/quadstore_test.go b/graph/gaedatastore/quadstore_test.go index ae3d6be..ea9d2b0 100644 --- a/graph/gaedatastore/quadstore_test.go +++ b/graph/gaedatastore/quadstore_test.go @@ -134,11 +134,6 @@ func createInstance() (aetest.Instance, graph.Options, error) { } opts := make(graph.Options) opts["HTTPRequest"] = req1 - - if inst == nil { - glog.Info("help") - } - return inst, opts, nil } diff --git a/graph/primarykey.go b/graph/primarykey.go index 3ab0260..d2d49e2 100644 --- a/graph/primarykey.go +++ b/graph/primarykey.go @@ -15,46 +15,64 @@ package graph import ( + "code.google.com/p/go-uuid/uuid" "encoding/json" + "errors" + "github.com/barakmich/glog" "strconv" "sync" ) -// Defines the PrimaryKey interface, this abstracts the generation of IDs - type primaryKeyType uint8 const ( none primaryKeyType = iota sequential + unique ) type PrimaryKey struct { - keyType primaryKeyType + KeyType primaryKeyType mut sync.Mutex - sequentialID int64 + SequentialID int64 + UniqueID string } func NewSequentialKey(horizon int64) PrimaryKey { return PrimaryKey{ - keyType: sequential, - sequentialID: horizon, + KeyType: sequential, + SequentialID: horizon, + } +} + +func NewUniqueKey(horizon string) PrimaryKey { + id := uuid.Parse(horizon) + if id == nil { + id = uuid.NewUUID() + } + return PrimaryKey{ + KeyType: unique, + UniqueID: id.String(), } } func (p *PrimaryKey) Next() PrimaryKey { - switch p.keyType { + switch p.KeyType { case sequential: p.mut.Lock() defer p.mut.Unlock() - p.sequentialID++ - if p.sequentialID <= 0 { - p.sequentialID = 1 + p.SequentialID++ + if p.SequentialID <= 0 { + p.SequentialID = 1 } return PrimaryKey{ - keyType: sequential, - sequentialID: p.sequentialID, + KeyType: sequential, + SequentialID: p.SequentialID, } + case unique: + id := uuid.NewUUID() + p.UniqueID = id.String() + return *p case none: panic("Calling next() on a none PrimaryKey") } @@ -62,27 +80,50 @@ func (p *PrimaryKey) Next() PrimaryKey { } func (p *PrimaryKey) Int() int64 { - switch p.keyType { + switch p.KeyType { case sequential: - return p.sequentialID + return p.SequentialID + case unique: + glog.Fatal("UUID cannot be cast to an int64") + return -1 } return -1 } func (p *PrimaryKey) String() string { - // More options for more keyTypes - return strconv.FormatInt(p.sequentialID, 10) + switch p.KeyType { + case sequential: + return strconv.FormatInt(p.SequentialID, 10) + case unique: + return p.UniqueID + case none: + panic("Calling String() on a none PrimaryKey") + } + return "" } func (p *PrimaryKey) MarshalJSON() ([]byte, error) { - return json.Marshal(p.sequentialID) + if p.KeyType == none { + return nil, errors.New("Cannot marshal PrimaryKey with KeyType of 'none'") + } + return json.Marshal(*p) } +//To avoid recursion in the implmentation of the UnmarshalJSON interface below +type primaryKey PrimaryKey + func (p *PrimaryKey) UnmarshalJSON(bytes []byte) error { /* Careful special casing here. For example, string-related things might begin if bytes[0] == '"' { } */ - p.keyType = sequential - return json.Unmarshal(bytes, &p.sequentialID) + temp := primaryKey{} + if err := json.Unmarshal(bytes, &temp); err != nil { + return err + } + *p = (PrimaryKey)(temp) + if p.KeyType == none { + return errors.New("Could not properly unmarshal primary key, 'none' keytype detected") + } + return nil } diff --git a/graph/primarykey_test.go b/graph/primarykey_test.go new file mode 100644 index 0000000..aa142ef --- /dev/null +++ b/graph/primarykey_test.go @@ -0,0 +1,92 @@ +// Copyright 2015 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph_test + +import ( + "code.google.com/p/go-uuid/uuid" + . "github.com/google/cayley/graph" + "testing" +) + +func TestSequentialKeyCreation(t *testing.T) { + { + seqKey := NewSequentialKey(666) + seqKey.Next() + + var expected int64 = 667 + result := seqKey.Int() + if expected != result { + t.Errorf("Expected %q got %q", expected, result) + } + } + { + seqKey := NewSequentialKey(0) + seqKey.Next() + + var expected int64 = 1 + result := seqKey.Int() + if expected != result { + t.Errorf("Expected %q got %q", expected, result) + } + } +} + +func TestUniqueKeyCreation(t *testing.T) { + uniqueKey := NewUniqueKey("") + if uuid.Parse(uniqueKey.String()) == nil { + t.Error("Invalid uuid generated") + } + uniqueKey.Next() + if uuid.Parse(uniqueKey.String()) == nil { + t.Error("Invalid uuid generated") + } +} + +func TestSequentialKeyMarshaling(t *testing.T) { + seqKey := NewSequentialKey(666) + seqKeyBytes, err := seqKey.MarshalJSON() + if err != nil { + t.Errorf("Marshaling of sequential key failed with : %v", err) + } + + seqKey2 := PrimaryKey{} + err = seqKey2.UnmarshalJSON(seqKeyBytes) + if err != nil { + t.Errorf("Unmarshaling of sequential key failed with : %v", err) + } + if seqKey.Int() != seqKey2.Int() { + t.Errorf("Unmarshaling failed: Expected: %d, got: %d", seqKey.Int(), seqKey2.Int()) + } +} + +func TestUniqueKeyMarshaling(t *testing.T) { + uniqueKey := NewUniqueKey("") + uniqueKeyBytes, err := uniqueKey.MarshalJSON() + if err != nil { + t.Errorf("Marshaling of unique key failed with : %v", err) + } + + uniqueKey2 := PrimaryKey{} + err = uniqueKey2.UnmarshalJSON(uniqueKeyBytes) + if err != nil { + t.Errorf("Unmarshaling of unique key failed with : %v", err) + } + if uuid.Parse(uniqueKey2.String()) == nil { + t.Error("Unique Key incorrectly unmarshaled") + } + if uniqueKey.String() != uniqueKey2.String() { + t.Error("Unique Key incorrectly unmarshaled") + } +}