Reorganize to go get will work
This makes almost no changes to source, but touches every almost file. Also fixes error in gremlin test code.
This commit is contained in:
kortschak
parent
e46a5bbe4a
commit
e0df752618
130 changed files with 8766 additions and 10167 deletions
73
http/cayley-http-docs.go
Normal file
73
http/cayley-http-docs.go
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
// Copyright 2014 The Cayley Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
"github.com/russross/blackfriday"
|
||||
)
|
||||
|
||||
type DocRequestHandler struct {
|
||||
}
|
||||
|
||||
func MarkdownWithCSS(input []byte, title string) []byte {
|
||||
// set up the HTML renderer
|
||||
htmlFlags := 0
|
||||
htmlFlags |= blackfriday.HTML_USE_XHTML
|
||||
htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
|
||||
htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
|
||||
htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
|
||||
htmlFlags |= blackfriday.HTML_COMPLETE_PAGE
|
||||
renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS)
|
||||
|
||||
// set up the parser
|
||||
extensions := 0
|
||||
//extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
|
||||
extensions |= blackfriday.EXTENSION_TABLES
|
||||
extensions |= blackfriday.EXTENSION_FENCED_CODE
|
||||
extensions |= blackfriday.EXTENSION_AUTOLINK
|
||||
extensions |= blackfriday.EXTENSION_STRIKETHROUGH
|
||||
//extensions |= blackfriday.EXTENSION_SPACE_HEADERS
|
||||
extensions |= blackfriday.EXTENSION_HEADER_IDS
|
||||
extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS
|
||||
|
||||
return blackfriday.Markdown(input, renderer, extensions)
|
||||
}
|
||||
|
||||
func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
|
||||
docpage := params.ByName("docpage")
|
||||
if docpage == "" {
|
||||
docpage = "Index"
|
||||
}
|
||||
file, err := os.Open(fmt.Sprintf("docs/%s.md", docpage))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
data, err := ioutil.ReadAll(file)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNoContent)
|
||||
return
|
||||
}
|
||||
output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage))
|
||||
fmt.Fprint(w, string(output))
|
||||
}
|
||||
|
||||
var markdownCSS = "/static/css/docs.css"
|
||||
153
http/cayley-http-query.go
Normal file
153
http/cayley-http-query.go
Normal file
|
|
@ -0,0 +1,153 @@
|
|||
// Copyright 2014 The Cayley Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
|
||||
"github.com/google/cayley/graph"
|
||||
"github.com/google/cayley/gremlin"
|
||||
"github.com/google/cayley/mql"
|
||||
)
|
||||
|
||||
type SuccessQueryWrapper struct {
|
||||
Result interface{} `json:"result"`
|
||||
}
|
||||
|
||||
type ErrorQueryWrapper struct {
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
func WrapErrResult(err error) ([]byte, error) {
|
||||
var wrap ErrorQueryWrapper
|
||||
wrap.Error = err.Error()
|
||||
return json.MarshalIndent(wrap, "", " ")
|
||||
}
|
||||
|
||||
func WrapResult(result interface{}) ([]byte, error) {
|
||||
var wrap SuccessQueryWrapper
|
||||
wrap.Result = result
|
||||
return json.MarshalIndent(wrap, "", " ")
|
||||
}
|
||||
|
||||
func RunJsonQuery(query string, ses graph.HttpSession) (interface{}, error) {
|
||||
c := make(chan interface{}, 5)
|
||||
go ses.ExecInput(query, c, 100)
|
||||
for res := range c {
|
||||
ses.BuildJson(res)
|
||||
}
|
||||
return ses.GetJson()
|
||||
}
|
||||
|
||||
func GetQueryShape(query string, ses graph.HttpSession) ([]byte, error) {
|
||||
c := make(chan map[string]interface{}, 5)
|
||||
go ses.GetQuery(query, c)
|
||||
var data map[string]interface{}
|
||||
for res := range c {
|
||||
data = res
|
||||
}
|
||||
return json.Marshal(data)
|
||||
}
|
||||
|
||||
// TODO(barakmich): Turn this into proper middleware.
|
||||
func (api *Api) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
|
||||
var ses graph.HttpSession
|
||||
switch params.ByName("query_lang") {
|
||||
case "gremlin":
|
||||
ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false)
|
||||
case "mql":
|
||||
ses = mql.NewMqlSession(api.ts)
|
||||
default:
|
||||
return FormatJson400(w, "Need a query language.")
|
||||
}
|
||||
var err error
|
||||
bodyBytes, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return FormatJson400(w, err)
|
||||
}
|
||||
code := string(bodyBytes)
|
||||
result, err := ses.InputParses(code)
|
||||
switch result {
|
||||
case graph.Parsed:
|
||||
var output interface{}
|
||||
var bytes []byte
|
||||
var err error
|
||||
output, err = RunJsonQuery(code, ses)
|
||||
if err != nil {
|
||||
bytes, err = WrapErrResult(err)
|
||||
http.Error(w, string(bytes), 400)
|
||||
ses = nil
|
||||
return 400
|
||||
}
|
||||
bytes, err = WrapResult(output)
|
||||
if err != nil {
|
||||
ses = nil
|
||||
return FormatJson400(w, err)
|
||||
}
|
||||
fmt.Fprint(w, string(bytes))
|
||||
ses = nil
|
||||
return 200
|
||||
case graph.ParseFail:
|
||||
ses = nil
|
||||
return FormatJson400(w, err)
|
||||
default:
|
||||
ses = nil
|
||||
return FormatJsonError(w, 500, "Incomplete data?")
|
||||
}
|
||||
http.Error(w, "", http.StatusNotFound)
|
||||
ses = nil
|
||||
return http.StatusNotFound
|
||||
}
|
||||
|
||||
func (api *Api) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
|
||||
var ses graph.HttpSession
|
||||
switch params.ByName("query_lang") {
|
||||
case "gremlin":
|
||||
ses = gremlin.NewGremlinSession(api.ts, api.config.GremlinTimeout, false)
|
||||
case "mql":
|
||||
ses = mql.NewMqlSession(api.ts)
|
||||
default:
|
||||
return FormatJson400(w, "Need a query language.")
|
||||
}
|
||||
var err error
|
||||
bodyBytes, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return FormatJson400(w, err)
|
||||
}
|
||||
code := string(bodyBytes)
|
||||
result, err := ses.InputParses(code)
|
||||
switch result {
|
||||
case graph.Parsed:
|
||||
var output []byte
|
||||
var err error
|
||||
output, err = GetQueryShape(code, ses)
|
||||
if err != nil {
|
||||
return FormatJson400(w, err)
|
||||
}
|
||||
fmt.Fprint(w, string(output))
|
||||
return 200
|
||||
case graph.ParseFail:
|
||||
return FormatJson400(w, err)
|
||||
default:
|
||||
return FormatJsonError(w, 500, "Incomplete data?")
|
||||
}
|
||||
http.Error(w, "", http.StatusNotFound)
|
||||
return http.StatusNotFound
|
||||
}
|
||||
119
http/cayley-http-write.go
Normal file
119
http/cayley-http-write.go
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
// Copyright 2014 The Cayley Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/barakmich/glog"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
|
||||
"github.com/google/cayley/graph"
|
||||
"github.com/google/cayley/nquads"
|
||||
)
|
||||
|
||||
func ParseJsonToTripleList(jsonBody []byte) ([]*graph.Triple, error) {
|
||||
var tripleList []*graph.Triple
|
||||
err := json.Unmarshal(jsonBody, &tripleList)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, t := range tripleList {
|
||||
if !t.IsValid() {
|
||||
return nil, errors.New(fmt.Sprintf("Invalid triple at index %d. %s", i, t.ToString()))
|
||||
}
|
||||
}
|
||||
return tripleList, nil
|
||||
}
|
||||
|
||||
func (api *Api) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int {
|
||||
if api.config.ReadOnly {
|
||||
return FormatJson400(w, "Database is read-only.")
|
||||
}
|
||||
bodyBytes, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return FormatJson400(w, err)
|
||||
}
|
||||
tripleList, terr := ParseJsonToTripleList(bodyBytes)
|
||||
if terr != nil {
|
||||
return FormatJson400(w, terr)
|
||||
}
|
||||
api.ts.AddTripleSet(tripleList)
|
||||
fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", len(tripleList))
|
||||
return 200
|
||||
}
|
||||
|
||||
func (api *Api) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
|
||||
if api.config.ReadOnly {
|
||||
return FormatJson400(w, "Database is read-only.")
|
||||
}
|
||||
|
||||
formFile, _, err := r.FormFile("NQuadFile")
|
||||
if err != nil {
|
||||
glog.Errorln(err)
|
||||
return FormatJsonError(w, 500, "Couldn't read file: "+err.Error())
|
||||
}
|
||||
|
||||
defer formFile.Close()
|
||||
|
||||
blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
|
||||
if blockErr != nil {
|
||||
blockSize = int64(api.config.LoadSize)
|
||||
}
|
||||
|
||||
tChan := make(chan *graph.Triple)
|
||||
go nquads.ReadNQuadsFromReader(tChan, formFile)
|
||||
tripleblock := make([]*graph.Triple, blockSize)
|
||||
nTriples := 0
|
||||
i := int64(0)
|
||||
for t := range tChan {
|
||||
tripleblock[i] = t
|
||||
i++
|
||||
nTriples++
|
||||
if i == blockSize {
|
||||
api.ts.AddTripleSet(tripleblock)
|
||||
i = 0
|
||||
}
|
||||
}
|
||||
api.ts.AddTripleSet(tripleblock[0:i])
|
||||
fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d triples.\"}", nTriples)
|
||||
return 200
|
||||
}
|
||||
|
||||
func (api *Api) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
|
||||
if api.config.ReadOnly {
|
||||
return FormatJson400(w, "Database is read-only.")
|
||||
}
|
||||
bodyBytes, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
return FormatJson400(w, err)
|
||||
}
|
||||
tripleList, terr := ParseJsonToTripleList(bodyBytes)
|
||||
if terr != nil {
|
||||
return FormatJson400(w, terr)
|
||||
}
|
||||
count := 0
|
||||
for _, triple := range tripleList {
|
||||
api.ts.RemoveTriple(triple)
|
||||
count++
|
||||
}
|
||||
fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d triples.\"}", count)
|
||||
return 200
|
||||
}
|
||||
113
http/cayley-http.go
Normal file
113
http/cayley-http.go
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
// Copyright 2014 The Cayley Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/barakmich/glog"
|
||||
"github.com/julienschmidt/httprouter"
|
||||
|
||||
cfg "github.com/google/cayley/config"
|
||||
"github.com/google/cayley/graph"
|
||||
)
|
||||
|
||||
type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int
|
||||
|
||||
func LogRequest(handler ResponseHandler) httprouter.Handle {
|
||||
return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
|
||||
start := time.Now()
|
||||
addr := req.Header.Get("X-Real-IP")
|
||||
if addr == "" {
|
||||
addr = req.Header.Get("X-Forwarded-For")
|
||||
if addr == "" {
|
||||
addr = req.RemoteAddr
|
||||
}
|
||||
}
|
||||
glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr)
|
||||
code := handler(w, req, params)
|
||||
glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start))
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
func FormatJson400(w http.ResponseWriter, err interface{}) int {
|
||||
return FormatJsonError(w, 400, err)
|
||||
}
|
||||
|
||||
func FormatJsonError(w http.ResponseWriter, code int, err interface{}) int {
|
||||
http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code)
|
||||
return code
|
||||
}
|
||||
|
||||
type TemplateRequestHandler struct {
|
||||
templates *template.Template
|
||||
}
|
||||
|
||||
func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
|
||||
uiType := params.ByName("ui_type")
|
||||
if r.URL.Path == "/" {
|
||||
uiType = "query"
|
||||
}
|
||||
err := h.templates.ExecuteTemplate(w, uiType+".html", h)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
type Api struct {
|
||||
config *cfg.CayleyConfig
|
||||
ts graph.TripleStore
|
||||
}
|
||||
|
||||
func (api *Api) ApiV1(r *httprouter.Router) {
|
||||
r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query))
|
||||
r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape))
|
||||
r.POST("/api/v1/write", LogRequest(api.ServeV1Write))
|
||||
r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad))
|
||||
//TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form?
|
||||
r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete))
|
||||
}
|
||||
|
||||
func SetupRoutes(ts graph.TripleStore, config *cfg.CayleyConfig) {
|
||||
r := httprouter.New()
|
||||
var templates = template.Must(template.ParseGlob("templates/*.tmpl"))
|
||||
templates.ParseGlob("templates/*.html")
|
||||
root := &TemplateRequestHandler{templates: templates}
|
||||
docs := &DocRequestHandler{}
|
||||
api := &Api{config: config, ts: ts}
|
||||
api.ApiV1(r)
|
||||
|
||||
//m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true}))
|
||||
//r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/"))))
|
||||
r.GET("/docs/:docpage", docs.ServeHTTP)
|
||||
r.GET("/ui/:ui_type", root.ServeHTTP)
|
||||
r.GET("/", root.ServeHTTP)
|
||||
http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir("static/"))))
|
||||
http.Handle("/", r)
|
||||
}
|
||||
|
||||
func CayleyHTTP(ts graph.TripleStore, config *cfg.CayleyConfig) {
|
||||
SetupRoutes(ts, config)
|
||||
glog.Infof("Cayley now listening on %s:%s\n", config.ListenHost, config.ListenPort)
|
||||
fmt.Printf("Cayley now listening on %s:%s\n", config.ListenHost, config.ListenPort)
|
||||
err := http.ListenAndServe(fmt.Sprintf("%s:%s", config.ListenHost, config.ListenPort), nil)
|
||||
if err != nil {
|
||||
glog.Fatal("ListenAndServe: ", err)
|
||||
}
|
||||
}
|
||||
53
http/cayley-http_test.go
Normal file
53
http/cayley-http_test.go
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
// Copyright 2014 The Cayley Authors. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package http
|
||||
|
||||
import (
|
||||
"testing"
|
||||
. "github.com/smartystreets/goconvey/convey"
|
||||
)
|
||||
|
||||
func TestParseJSONOkay(t *testing.T) {
|
||||
Convey("Parse JSON", t, func() {
|
||||
bytelist := []byte(`[
|
||||
{"subject": "foo", "predicate": "bar", "object": "baz"},
|
||||
{"subject": "foo", "predicate": "bar", "object": "baz", "provenance": "graph"}
|
||||
]`)
|
||||
x, err := ParseJsonToTripleList(bytelist)
|
||||
So(err, ShouldBeNil)
|
||||
So(len(x), ShouldEqual, 2)
|
||||
So(x[0].Sub, ShouldEqual, "foo")
|
||||
So(x[0].Provenance, ShouldEqual, "")
|
||||
So(x[1].Provenance, ShouldEqual, "graph")
|
||||
})
|
||||
|
||||
Convey("Parse JSON extra field", t, func() {
|
||||
bytelist := []byte(`[
|
||||
{"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"}
|
||||
]`)
|
||||
_, err := ParseJsonToTripleList(bytelist)
|
||||
So(err, ShouldBeNil)
|
||||
})
|
||||
}
|
||||
|
||||
func TestParseJSONFail(t *testing.T) {
|
||||
Convey("Parse JSON Fail", t, func() {
|
||||
bytelist := []byte(`[
|
||||
{"subject": "foo", "predicate": "bar"}
|
||||
]`)
|
||||
_, err := ParseJsonToTripleList(bytelist)
|
||||
So(err, ShouldNotBeNil)
|
||||
})
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue