internal: move {config,db,http} into internal

internal/http/docs.go

@@ -0,0 +1,74 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+
+	"github.com/julienschmidt/httprouter"
+	"github.com/russross/blackfriday"
+)
+
+type DocRequestHandler struct {
+	assets string
+}
+
+func MarkdownWithCSS(input []byte, title string) []byte {
+	// set up the HTML renderer
+	htmlFlags := 0
+	htmlFlags |= blackfriday.HTML_USE_XHTML
+	htmlFlags |= blackfriday.HTML_USE_SMARTYPANTS
+	htmlFlags |= blackfriday.HTML_SMARTYPANTS_FRACTIONS
+	htmlFlags |= blackfriday.HTML_SMARTYPANTS_LATEX_DASHES
+	htmlFlags |= blackfriday.HTML_COMPLETE_PAGE
+	renderer := blackfriday.HtmlRenderer(htmlFlags, title, markdownCSS)
+
+	// set up the parser
+	extensions := 0
+	//extensions |= blackfriday.EXTENSION_NO_INTRA_EMPHASIS
+	extensions |= blackfriday.EXTENSION_TABLES
+	extensions |= blackfriday.EXTENSION_FENCED_CODE
+	extensions |= blackfriday.EXTENSION_AUTOLINK
+	extensions |= blackfriday.EXTENSION_STRIKETHROUGH
+	//extensions |= blackfriday.EXTENSION_SPACE_HEADERS
+	extensions |= blackfriday.EXTENSION_HEADER_IDS
+	extensions |= blackfriday.EXTENSION_LAX_HTML_BLOCKS
+
+	return blackfriday.Markdown(input, renderer, extensions)
+}
+
+func (h *DocRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	docpage := params.ByName("docpage")
+	if docpage == "" {
+		docpage = "Index"
+	}
+	file, err := os.Open(fmt.Sprintf("%s/docs/%s.md", h.assets, docpage))
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNoContent)
+		return
+	}
+	output := MarkdownWithCSS(data, fmt.Sprintf("Cayley Docs - %s", docpage))
+	fmt.Fprint(w, string(output))
+}
+
+var markdownCSS = "/static/css/docs.css"

internal/http/http.go

@@ -0,0 +1,171 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"flag"
+	"fmt"
+	"html/template"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/barakmich/glog"
+	"github.com/julienschmidt/httprouter"
+
+	"github.com/google/cayley/graph"
+	"github.com/google/cayley/internal/config"
+	"github.com/google/cayley/internal/db"
+)
+
+type ResponseHandler func(http.ResponseWriter, *http.Request, httprouter.Params) int
+
+var assetsPath = flag.String("assets", "", "Explicit path to the HTTP assets.")
+var assetsDirs = []string{"templates", "static", "docs"}
+
+func hasAssets(path string) bool {
+	for _, dir := range assetsDirs {
+		if _, err := os.Stat(fmt.Sprint(path, "/", dir)); os.IsNotExist(err) {
+			return false
+		}
+	}
+	return true
+}
+
+func findAssetsPath() string {
+	if *assetsPath != "" {
+		if hasAssets(*assetsPath) {
+			return *assetsPath
+		}
+		glog.Fatalln("Cannot find assets at", *assetsPath, ".")
+	}
+
+	if hasAssets(".") {
+		return "."
+	}
+
+	if hasAssets("..") {
+		return ".."
+	}
+
+	gopathPath := os.ExpandEnv("$GOPATH/src/github.com/google/cayley")
+	if hasAssets(gopathPath) {
+		return gopathPath
+	}
+	glog.Fatalln("Cannot find assets in any of the default search paths. Please run in the same directory, in a Go workspace, or set --assets .")
+	panic("cannot reach")
+}
+
+func LogRequest(handler ResponseHandler) httprouter.Handle {
+	return func(w http.ResponseWriter, req *http.Request, params httprouter.Params) {
+		start := time.Now()
+		addr := req.Header.Get("X-Real-IP")
+		if addr == "" {
+			addr = req.Header.Get("X-Forwarded-For")
+			if addr == "" {
+				addr = req.RemoteAddr
+			}
+		}
+		glog.Infof("Started %s %s for %s", req.Method, req.URL.Path, addr)
+		code := handler(w, req, params)
+		glog.Infof("Completed %v %s %s in %v", code, http.StatusText(code), req.URL.Path, time.Since(start))
+
+	}
+}
+
+func jsonResponse(w http.ResponseWriter, code int, err interface{}) int {
+	http.Error(w, fmt.Sprintf("{\"error\" : \"%s\"}", err), code)
+	return code
+}
+
+type TemplateRequestHandler struct {
+	templates *template.Template
+}
+
+func (h *TemplateRequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
+	uiType := params.ByName("ui_type")
+	if r.URL.Path == "/" {
+		uiType = "query"
+	}
+	err := h.templates.ExecuteTemplate(w, uiType+".html", h)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}
+
+type API struct {
+	config *config.Config
+	handle *graph.Handle
+}
+
+func (api *API) GetHandleForRequest(r *http.Request) (*graph.Handle, error) {
+	if !api.config.RequiresHTTPRequestContext {
+		return api.handle, nil
+	}
+
+	opts := make(graph.Options)
+	opts["HTTPRequest"] = r
+
+	qs, err := graph.NewQuadStoreForRequest(api.handle.QuadStore, opts)
+	if err != nil {
+		return nil, err
+	}
+	qw, err := db.OpenQuadWriter(qs, api.config)
+	if err != nil {
+		return nil, err
+	}
+	return &graph.Handle{QuadStore: qs, QuadWriter: qw}, nil
+}
+
+func (api *API) APIv1(r *httprouter.Router) {
+	r.POST("/api/v1/query/:query_lang", LogRequest(api.ServeV1Query))
+	r.POST("/api/v1/shape/:query_lang", LogRequest(api.ServeV1Shape))
+	r.POST("/api/v1/write", LogRequest(api.ServeV1Write))
+	r.POST("/api/v1/write/file/nquad", LogRequest(api.ServeV1WriteNQuad))
+	//TODO(barakmich): /write/text/nquad, which reads from request.body instead of HTML5 file form?
+	r.POST("/api/v1/delete", LogRequest(api.ServeV1Delete))
+}
+
+func SetupRoutes(handle *graph.Handle, cfg *config.Config) {
+	r := httprouter.New()
+	assets := findAssetsPath()
+	if glog.V(2) {
+		glog.V(2).Infoln("Found assets at", assets)
+	}
+	var templates = template.Must(template.ParseGlob(fmt.Sprint(assets, "/templates/*.tmpl")))
+	templates.ParseGlob(fmt.Sprint(assets, "/templates/*.html"))
+	root := &TemplateRequestHandler{templates: templates}
+	docs := &DocRequestHandler{assets: assets}
+	api := &API{config: cfg, handle: handle}
+	api.APIv1(r)
+
+	//m.Use(martini.Static("static", martini.StaticOptions{Prefix: "/static", SkipLogging: true}))
+	//r.Handler("GET", "/static", http.StripPrefix("/static", http.FileServer(http.Dir("static/"))))
+	r.GET("/docs/:docpage", docs.ServeHTTP)
+	r.GET("/ui/:ui_type", root.ServeHTTP)
+	r.GET("/", root.ServeHTTP)
+	http.Handle("/static/", http.StripPrefix("/static", http.FileServer(http.Dir(fmt.Sprint(assets, "/static/")))))
+	http.Handle("/", r)
+}
+
+func Serve(handle *graph.Handle, cfg *config.Config) {
+	SetupRoutes(handle, cfg)
+	glog.Infof("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort)
+	fmt.Printf("Cayley now listening on %s:%s\n", cfg.ListenHost, cfg.ListenPort)
+	err := http.ListenAndServe(fmt.Sprintf("%s:%s", cfg.ListenHost, cfg.ListenPort), nil)
+	if err != nil {
+		glog.Fatal("ListenAndServe: ", err)
+	}
+}

internal/http/http_test.go

@@ -0,0 +1,73 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/google/cayley/quad"
+)
+
+var parseTests = []struct {
+	message string
+	input   string
+	expect  []quad.Quad
+	err     error
+}{
+	{
+		message: "parse correct JSON",
+		input: `[
+			{"subject": "foo", "predicate": "bar", "object": "baz"},
+			{"subject": "foo", "predicate": "bar", "object": "baz", "label": "graph"}
+		]`,
+		expect: []quad.Quad{
+			{"foo", "bar", "baz", ""},
+			{"foo", "bar", "baz", "graph"},
+		},
+		err: nil,
+	},
+	{
+		message: "parse correct JSON with extra field",
+		input: `[
+			{"subject": "foo", "predicate": "bar", "object": "foo", "something_else": "extra data"}
+		]`,
+		expect: []quad.Quad{
+			{"foo", "bar", "foo", ""},
+		},
+		err: nil,
+	},
+	{
+		message: "reject incorrect JSON",
+		input: `[
+			{"subject": "foo", "predicate": "bar"}
+		]`,
+		expect: nil,
+		err:    fmt.Errorf("invalid quad at index %d. %v", 0, quad.Quad{"foo", "bar", "", ""}),
+	},
+}
+
+func TestParseJSON(t *testing.T) {
+	for _, test := range parseTests {
+		got, err := ParseJSONToQuadList([]byte(test.input))
+		if fmt.Sprint(err) != fmt.Sprint(test.err) {
+			t.Errorf("Failed to %v with unexpected error, got:%v expected %v", test.message, err, test.err)
+		}
+		if !reflect.DeepEqual(got, test.expect) {
+			t.Errorf("Failed to %v, got:%v expect:%v", test.message, got, test.expect)
+		}
+	}
+}

internal/http/query.go

@@ -0,0 +1,146 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/julienschmidt/httprouter"
+
+	"github.com/google/cayley/query"
+	"github.com/google/cayley/query/gremlin"
+	"github.com/google/cayley/query/mql"
+)
+
+type SuccessQueryWrapper struct {
+	Result interface{} `json:"result"`
+}
+
+type ErrorQueryWrapper struct {
+	Error string `json:"error"`
+}
+
+func WrapErrResult(err error) ([]byte, error) {
+	var wrap ErrorQueryWrapper
+	wrap.Error = err.Error()
+	return json.MarshalIndent(wrap, "", " ")
+}
+
+func WrapResult(result interface{}) ([]byte, error) {
+	var wrap SuccessQueryWrapper
+	wrap.Result = result
+	return json.MarshalIndent(wrap, "", " ")
+}
+
+func Run(q string, ses query.HTTP) (interface{}, error) {
+	c := make(chan interface{}, 5)
+	go ses.Execute(q, c, 100)
+	for res := range c {
+		ses.Collate(res)
+	}
+	return ses.Results()
+}
+
+func GetQueryShape(q string, ses query.HTTP) ([]byte, error) {
+	s, err := ses.ShapeOf(q)
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(s)
+}
+
+// TODO(barakmich): Turn this into proper middleware.
+func (api *API) ServeV1Query(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
+	h, err := api.GetHandleForRequest(r)
+	var ses query.HTTP
+	switch params.ByName("query_lang") {
+	case "gremlin":
+		ses = gremlin.NewSession(h.QuadStore, api.config.Timeout, false)
+	case "mql":
+		ses = mql.NewSession(h.QuadStore)
+	default:
+		return jsonResponse(w, 400, "Need a query language.")
+	}
+	bodyBytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	code := string(bodyBytes)
+	result, err := ses.Parse(code)
+	switch result {
+	case query.Parsed:
+		var output interface{}
+		var bytes []byte
+		var err error
+		output, err = Run(code, ses)
+		if err != nil {
+			bytes, err = WrapErrResult(err)
+			http.Error(w, string(bytes), 400)
+			ses = nil
+			return 400
+		}
+		bytes, err = WrapResult(output)
+		if err != nil {
+			ses = nil
+			return jsonResponse(w, 400, err)
+		}
+		fmt.Fprint(w, string(bytes))
+		ses = nil
+		return 200
+	case query.ParseFail:
+		ses = nil
+		return jsonResponse(w, 400, err)
+	default:
+		ses = nil
+		return jsonResponse(w, 500, "Incomplete data?")
+	}
+}
+
+func (api *API) ServeV1Shape(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
+	h, err := api.GetHandleForRequest(r)
+	var ses query.HTTP
+	switch params.ByName("query_lang") {
+	case "gremlin":
+		ses = gremlin.NewSession(h.QuadStore, api.config.Timeout, false)
+	case "mql":
+		ses = mql.NewSession(h.QuadStore)
+	default:
+		return jsonResponse(w, 400, "Need a query language.")
+	}
+	bodyBytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	code := string(bodyBytes)
+	result, err := ses.Parse(code)
+	switch result {
+	case query.Parsed:
+		var output []byte
+		var err error
+		output, err = GetQueryShape(code, ses)
+		if err != nil {
+			return jsonResponse(w, 400, err)
+		}
+		fmt.Fprint(w, string(output))
+		return 200
+	case query.ParseFail:
+		return jsonResponse(w, 400, err)
+	default:
+		return jsonResponse(w, 500, "Incomplete data?")
+	}
+}

internal/http/write.go

@@ -0,0 +1,144 @@
+// Copyright 2014 The Cayley Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strconv"
+
+	"github.com/barakmich/glog"
+	"github.com/julienschmidt/httprouter"
+
+	"github.com/google/cayley/internal"
+	"github.com/google/cayley/quad"
+	"github.com/google/cayley/quad/cquads"
+)
+
+func ParseJSONToQuadList(jsonBody []byte) ([]quad.Quad, error) {
+	var quads []quad.Quad
+	err := json.Unmarshal(jsonBody, &quads)
+	if err != nil {
+		return nil, err
+	}
+	for i, q := range quads {
+		if !q.IsValid() {
+			return nil, fmt.Errorf("invalid quad at index %d. %s", i, q)
+		}
+	}
+	return quads, nil
+}
+
+func (api *API) ServeV1Write(w http.ResponseWriter, r *http.Request, _ httprouter.Params) int {
+	if api.config.ReadOnly {
+		return jsonResponse(w, 400, "Database is read-only.")
+	}
+	bodyBytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	quads, err := ParseJSONToQuadList(bodyBytes)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	h, err := api.GetHandleForRequest(r)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+
+	h.QuadWriter.AddQuadSet(quads)
+	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", len(quads))
+	return 200
+}
+
+func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
+	if api.config.ReadOnly {
+		return jsonResponse(w, 400, "Database is read-only.")
+	}
+
+	formFile, _, err := r.FormFile("NQuadFile")
+	if err != nil {
+		glog.Errorln(err)
+		return jsonResponse(w, 500, "Couldn't read file: "+err.Error())
+	}
+	defer formFile.Close()
+
+	blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64)
+	if blockErr != nil {
+		blockSize = int64(api.config.LoadSize)
+	}
+
+	quadReader, err := internal.Decompressor(formFile)
+	// TODO(kortschak) Make this configurable from the web UI.
+	dec := cquads.NewDecoder(quadReader)
+
+	h, err := api.GetHandleForRequest(r)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+
+	var (
+		n     int
+		block = make([]quad.Quad, 0, blockSize)
+	)
+	for {
+		t, err := dec.Unmarshal()
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			glog.Fatalln("what can do this here?", err) // FIXME(kortschak)
+		}
+		block = append(block, t)
+		n++
+		if len(block) == cap(block) {
+			h.QuadWriter.AddQuadSet(block)
+			block = block[:0]
+		}
+	}
+	h.QuadWriter.AddQuadSet(block)
+
+	fmt.Fprintf(w, "{\"result\": \"Successfully wrote %d quads.\"}", n)
+
+	return 200
+}
+
+func (api *API) ServeV1Delete(w http.ResponseWriter, r *http.Request, params httprouter.Params) int {
+	if api.config.ReadOnly {
+		return jsonResponse(w, 400, "Database is read-only.")
+	}
+	bodyBytes, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	quads, err := ParseJSONToQuadList(bodyBytes)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	h, err := api.GetHandleForRequest(r)
+	if err != nil {
+		return jsonResponse(w, 400, err)
+	}
+	count := 0
+	for _, q := range quads {
+		h.QuadWriter.RemoveQuad(q)
+		count++
+	}
+	fmt.Fprintf(w, "{\"result\": \"Successfully deleted %d quads.\"}", count)
+	return 200
+}