diff --git a/cayley.go b/cayley.go index 4d2bfdc..eae0ff9 100644 --- a/cayley.go +++ b/cayley.go @@ -17,10 +17,6 @@ package main import ( - "bufio" - "bytes" - "compress/bzip2" - "compress/gzip" "flag" "fmt" "io" @@ -290,8 +286,11 @@ func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string r = res.Body } - r, err = decompressor(r) + r, err = quad.Decompressor(r) if err != nil { + if err == io.EOF { + return nil + } return err } @@ -311,24 +310,3 @@ func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string return db.Load(qw, cfg, dec) } - -const ( - gzipMagic = "\x1f\x8b" - b2zipMagic = "BZh" -) - -func decompressor(r io.Reader) (io.Reader, error) { - br := bufio.NewReader(r) - buf, err := br.Peek(3) - if err != nil { - return nil, err - } - switch { - case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0: - return gzip.NewReader(br) - case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0: - return bzip2.NewReader(br), nil - default: - return br, nil - } -} diff --git a/http/write.go b/http/write.go index 78743f4..0c71f13 100644 --- a/http/write.go +++ b/http/write.go @@ -75,7 +75,6 @@ func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params glog.Errorln(err) return jsonResponse(w, 500, "Couldn't read file: "+err.Error()) } - defer formFile.Close() blockSize, blockErr := strconv.ParseInt(r.URL.Query().Get("block_size"), 10, 64) @@ -83,8 +82,9 @@ func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params blockSize = int64(api.config.LoadSize) } + quadReader, err := quad.Decompressor(formFile) // TODO(kortschak) Make this configurable from the web UI. - dec := cquads.NewDecoder(formFile) + dec := cquads.NewDecoder(quadReader) h, err := api.GetHandleForRequest(r) if err != nil { @@ -101,7 +101,7 @@ func (api *API) ServeV1WriteNQuad(w http.ResponseWriter, r *http.Request, params if err == io.EOF { break } - panic("what can do this here?") // FIXME(kortschak) + glog.Fatalln("what can do this here?", err) // FIXME(kortschak) } block = append(block, t) n++ diff --git a/quad/decompressor.go b/quad/decompressor.go new file mode 100644 index 0000000..35ee714 --- /dev/null +++ b/quad/decompressor.go @@ -0,0 +1,46 @@ +// Copyright 2014 The Cayley Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package quad + +import ( + "bufio" + "bytes" + "compress/bzip2" + "compress/gzip" + "io" +) + +const ( + gzipMagic = "\x1f\x8b" + b2zipMagic = "BZh" +) + +// Decompressor detects the file type of an io.Reader between +// bzip, gzip, or raw quad file. +func Decompressor(r io.Reader) (io.Reader, error) { + br := bufio.NewReader(r) + buf, err := br.Peek(3) + if err != nil { + return nil, err + } + switch { + case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0: + return gzip.NewReader(br) + case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0: + return bzip2.NewReader(br), nil + default: + return br, nil + } +}