Primary key refactoring re. Un/Marshaling of JSON and the unique keys, plus the ApplyDelta() and 'defensive' changes introduced by Kortshack, also did some tidying up.

appengine/app.yaml

@@ -1,7 +1,7 @@
 application: cayley-test
 version: 1
 runtime: go
-api_version: go1
+api_version: go1.4beta
 
 handlers:
 - url: /.*

graph/gaedatastore/iterator.go

@@ -228,7 +228,7 @@ func (it *Iterator) Next() bool {
 	it.offset = 0
 	it.buffer = make([]string, 0, bufferSize)
 	// Create query
-	// TODO (stefankoshiw) Keys only query?
+	// TODO (panamafrancis) Keys only query?
 	q := datastore.NewQuery(it.kind).Limit(bufferSize)
 	if !it.isAll {
 		// Filter on the direction {subject,objekt...}

graph/gaedatastore/quadstore.go

@@ -62,12 +62,12 @@ type Token struct {
 
 type QuadEntry struct {
 	Hash      string
-	Added     []int64 `datastore:",noindex"`
+	Added     []string `datastore:",noindex"`
-	Deleted   []int64 `datastore:",noindex"`
+	Deleted   []string `datastore:",noindex"`
-	Subject   string  `datastore:"subject"`
+	Subject   string   `datastore:"subject"`
-	Predicate string  `datastore:"predicate"`
+	Predicate string   `datastore:"predicate"`
-	Object    string  `datastore:"object"`
+	Object    string   `datastore:"object"`
-	Label     string  `datastore:"label"`
+	Label     string   `datastore:"label"`
 }
 
 type NodeEntry struct {
@@ -76,7 +76,7 @@ type NodeEntry struct {
 }
 
 type LogEntry struct {
-	LogID     int64
+	LogID     string
 	Action    string
 	Key       string
 	Timestamp int64
@@ -127,8 +127,8 @@ func (qs *QuadStore) createKeyForMetadata() *datastore.Key {
 	return qs.createKeyFromToken(&Token{"metadata", "metadataentry"})
 }
 
-func (qs *QuadStore) createKeyForLog(deltaID int64) *datastore.Key {
+func (qs *QuadStore) createKeyForLog(deltaID graph.PrimaryKey) *datastore.Key {
-	return datastore.NewKey(qs.context, "logentry", "", deltaID, nil)
+	return datastore.NewKey(qs.context, "logentry", deltaID.String(), 0, nil)
 }
 
 func (qs *QuadStore) createKeyFromToken(t *Token) *datastore.Key {
@@ -161,35 +161,42 @@ func getContext(opts graph.Options) (appengine.Context, error) {
 	return appengine.NewContext(req), nil
 }
 
-func (qs *QuadStore) ApplyDeltas(in []graph.Delta) error {
+func (qs *QuadStore) ApplyDeltas(in []graph.Delta, ignoreOpts graph.IgnoreOpts) error {
 	if qs.context == nil {
 		return errors.New("No context, graph not correctly initialised")
 	}
 	toKeep := make([]graph.Delta, 0)
 	for _, d := range in {
+		if d.Action != graph.Add && d.Action != graph.Delete {
+			//Defensive shortcut
+			return errors.New("Datastore: invalid action")
+		}
 		key := qs.createKeyForQuad(d.Quad)
 		keep := false
 		switch d.Action {
 		case graph.Add:
-			if found, err := qs.checkValid(key); !found && err == nil {
+			found, err := qs.checkValid(key)
-				keep = true
+			if err != nil {
-			} else if err != nil {
 				return err
+			}
+			if !found || ignoreOpts.IgnoreDup {
+				keep = true
 			} else {
 				glog.Warningf("Quad exists already: %v", d)
 			}
 		case graph.Delete:
-			if found, err := qs.checkValid(key); found && err == nil {
+			found, err := qs.checkValid(key)
-				keep = true
+			if err != nil {
-			} else if err != nil {
 				return err
+			}
+			if found || ignoreOpts.IgnoreMissing {
+				keep = true
 			} else {
 				glog.Warningf("Quad does not exist and so cannot be deleted: %v", d)
 			}
 		default:
-			keep = true
+			keep = false
 		}
-
 		if keep {
 			toKeep = append(toKeep, d)
 		}
@@ -309,10 +316,10 @@ func (qs *QuadStore) updateQuads(in []graph.Delta) (int64, error) {
 
 				// If the quad exists the Added[] will be non-empty
 				if in[x].Action == graph.Add {
-					foundQuads[k].Added = append(foundQuads[k].Added, in[x].ID)
+					foundQuads[k].Added = append(foundQuads[k].Added, in[x].ID.String())
 					quadCount += 1
 				} else {
-					foundQuads[k].Deleted = append(foundQuads[k].Deleted, in[x].ID)
+					foundQuads[k].Deleted = append(foundQuads[k].Deleted, in[x].ID.String())
 					quadCount -= 1
 				}
 			}
@@ -365,7 +372,7 @@ func (qs *QuadStore) updateLog(in []graph.Delta) error {
 		}
 
 		entry := LogEntry{
-			LogID:     d.ID,
+			LogID:     d.ID.String(),
 			Action:    action,
 			Key:       qs.createKeyForQuad(d.Quad).String(),
 			Timestamp: d.Timestamp.UnixNano(),
@@ -481,10 +488,10 @@ func (qs *QuadStore) NodeSize() int64 {
 	return foundMetadata.NodeCount
 }
 
-func (qs *QuadStore) Horizon() int64 {
+func (qs *QuadStore) Horizon() graph.PrimaryKey {
 	if qs.context == nil {
 		glog.Warning("Warning: HTTP Request context is nil, cannot get horizon from datastore.")
-		return 0
+		return graph.NewUniqueKey("")
 	}
 	// Query log for last entry...
 	q := datastore.NewQuery("logentry").Order("-Timestamp").Limit(1)
@@ -492,9 +499,9 @@ func (qs *QuadStore) Horizon() int64 {
 	_, err := q.GetAll(qs.context, &logEntries)
 	if err != nil || len(logEntries) == 0 {
 		// Error fetching horizon, probably graph is empty
-		return 0
+		return graph.NewUniqueKey("")
 	}
-	return logEntries[0].LogID
+	return graph.NewUniqueKey(logEntries[0].LogID)
 }
 
 func compareTokens(a, b graph.Value) bool {

graph/gaedatastore/quadstore_test.go

@@ -134,11 +134,6 @@ func createInstance() (aetest.Instance, graph.Options, error) {
 	}
 	opts := make(graph.Options)
 	opts["HTTPRequest"] = req1
-
-	if inst == nil {
-		glog.Info("help")
-	}
-
 	return inst, opts, nil
 }
 

graph/primarykey.go

@@ -15,46 +15,64 @@
 package graph
 
 import (
+	"code.google.com/p/go-uuid/uuid"
 	"encoding/json"
+	"errors"
+	"github.com/barakmich/glog"
 	"strconv"
 	"sync"
 )
 
-// Defines the PrimaryKey interface, this abstracts the generation of IDs
-
 type primaryKeyType uint8
 
 const (
 	none primaryKeyType = iota
 	sequential
+	unique
 )
 
 type PrimaryKey struct {
-	keyType      primaryKeyType
+	KeyType      primaryKeyType
 	mut          sync.Mutex
-	sequentialID int64
+	SequentialID int64
+	UniqueID     string
 }
 
 func NewSequentialKey(horizon int64) PrimaryKey {
 	return PrimaryKey{
-		keyType:      sequential,
+		KeyType:      sequential,
-		sequentialID: horizon,
+		SequentialID: horizon,
+	}
+}
+
+func NewUniqueKey(horizon string) PrimaryKey {
+	id := uuid.Parse(horizon)
+	if id == nil {
+		id = uuid.NewUUID()
+	}
+	return PrimaryKey{
+		KeyType:  unique,
+		UniqueID: id.String(),
 	}
 }
 
 func (p *PrimaryKey) Next() PrimaryKey {
-	switch p.keyType {
+	switch p.KeyType {
 	case sequential:
 		p.mut.Lock()
 		defer p.mut.Unlock()
-		p.sequentialID++
+		p.SequentialID++
-		if p.sequentialID <= 0 {
+		if p.SequentialID <= 0 {
-			p.sequentialID = 1
+			p.SequentialID = 1
 		}
 		return PrimaryKey{
-			keyType:      sequential,
+			KeyType:      sequential,
-			sequentialID: p.sequentialID,
+			SequentialID: p.SequentialID,
 		}
+	case unique:
+		id := uuid.NewUUID()
+		p.UniqueID = id.String()
+		return *p
 	case none:
 		panic("Calling next() on a none PrimaryKey")
 	}
@@ -62,27 +80,50 @@ func (p *PrimaryKey) Next() PrimaryKey {
 }
 
 func (p *PrimaryKey) Int() int64 {
-	switch p.keyType {
+	switch p.KeyType {
 	case sequential:
-		return p.sequentialID
+		return p.SequentialID
+	case unique:
+		glog.Fatal("UUID cannot be cast to an int64")
+		return -1
 	}
 	return -1
 }
 
 func (p *PrimaryKey) String() string {
-	// More options for more keyTypes
+	switch p.KeyType {
-	return strconv.FormatInt(p.sequentialID, 10)
+	case sequential:
+		return strconv.FormatInt(p.SequentialID, 10)
+	case unique:
+		return p.UniqueID
+	case none:
+		panic("Calling String() on a none PrimaryKey")
+	}
+	return ""
 }
 
 func (p *PrimaryKey) MarshalJSON() ([]byte, error) {
-	return json.Marshal(p.sequentialID)
+	if p.KeyType == none {
+		return nil, errors.New("Cannot marshal PrimaryKey with KeyType of 'none'")
+	}
+	return json.Marshal(*p)
 }
 
+//To avoid recursion in the implmentation of the UnmarshalJSON interface below
+type primaryKey PrimaryKey
+
 func (p *PrimaryKey) UnmarshalJSON(bytes []byte) error {
 	/* Careful special casing here. For example, string-related things might begin
 	if bytes[0] == '"' {
 	}
 	*/
-	p.keyType = sequential
+	temp := primaryKey{}
-	return json.Unmarshal(bytes, &p.sequentialID)
+	if err := json.Unmarshal(bytes, &temp); err != nil {
+		return err
+	}
+	*p = (PrimaryKey)(temp)
+	if p.KeyType == none {
+		return errors.New("Could not properly unmarshal primary key, 'none' keytype detected")
+	}
+	return nil
 }

graph/primarykey_test.go

@@ -0,0 +1,92 @@
+// Copyright 2015 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package graph_test
+
+import (
+	"code.google.com/p/go-uuid/uuid"
+	. "github.com/google/cayley/graph"
+	"testing"
+)
+
+func TestSequentialKeyCreation(t *testing.T) {
+	{
+		seqKey := NewSequentialKey(666)
+		seqKey.Next()
+
+		var expected int64 = 667
+		result := seqKey.Int()
+		if expected != result {
+			t.Errorf("Expected %q got %q", expected, result)
+		}
+	}
+	{
+		seqKey := NewSequentialKey(0)
+		seqKey.Next()
+
+		var expected int64 = 1
+		result := seqKey.Int()
+		if expected != result {
+			t.Errorf("Expected %q got %q", expected, result)
+		}
+	}
+}
+
+func TestUniqueKeyCreation(t *testing.T) {
+	uniqueKey := NewUniqueKey("")
+	if uuid.Parse(uniqueKey.String()) == nil {
+		t.Error("Invalid uuid generated")
+	}
+	uniqueKey.Next()
+	if uuid.Parse(uniqueKey.String()) == nil {
+		t.Error("Invalid uuid generated")
+	}
+}
+
+func TestSequentialKeyMarshaling(t *testing.T) {
+	seqKey := NewSequentialKey(666)
+	seqKeyBytes, err := seqKey.MarshalJSON()
+	if err != nil {
+		t.Errorf("Marshaling of sequential key failed with : %v", err)
+	}
+
+	seqKey2 := PrimaryKey{}
+	err = seqKey2.UnmarshalJSON(seqKeyBytes)
+	if err != nil {
+		t.Errorf("Unmarshaling of sequential key failed with : %v", err)
+	}
+	if seqKey.Int() != seqKey2.Int() {
+		t.Errorf("Unmarshaling failed: Expected: %d, got: %d", seqKey.Int(), seqKey2.Int())
+	}
+}
+
+func TestUniqueKeyMarshaling(t *testing.T) {
+	uniqueKey := NewUniqueKey("")
+	uniqueKeyBytes, err := uniqueKey.MarshalJSON()
+	if err != nil {
+		t.Errorf("Marshaling of unique key failed with : %v", err)
+	}
+
+	uniqueKey2 := PrimaryKey{}
+	err = uniqueKey2.UnmarshalJSON(uniqueKeyBytes)
+	if err != nil {
+		t.Errorf("Unmarshaling of unique key failed with : %v", err)
+	}
+	if uuid.Parse(uniqueKey2.String()) == nil {
+		t.Error("Unique Key incorrectly unmarshaled")
+	}
+	if uniqueKey.String() != uniqueKey2.String() {
+		t.Error("Unique Key incorrectly unmarshaled")
+	}
+}