fix decompressing for http

This commit is contained in:
Barak Michener 2015-04-16 17:43:25 -04:00
parent a8f2db71b4
commit 2b86c283ec
3 changed files with 53 additions and 29 deletions

View file

@ -17,10 +17,6 @@
package main
import (
"bufio"
"bytes"
"compress/bzip2"
"compress/gzip"
"flag"
"fmt"
"io"
@ -290,8 +286,11 @@ func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string
r = res.Body
}
r, err = decompressor(r)
r, err = quad.Decompressor(r)
if err != nil {
if err == io.EOF {
return nil
}
return err
}
@ -311,24 +310,3 @@ func decompressAndLoad(qw graph.QuadWriter, cfg *config.Config, path, typ string
return db.Load(qw, cfg, dec)
}
const (
gzipMagic = "\x1f\x8b"
b2zipMagic = "BZh"
)
func decompressor(r io.Reader) (io.Reader, error) {
br := bufio.NewReader(r)
buf, err := br.Peek(3)
if err != nil {
return nil, err
}
switch {
case bytes.Compare(buf[:2], []byte(gzipMagic)) == 0:
return gzip.NewReader(br)
case bytes.Compare(buf[:3], []byte(b2zipMagic)) == 0:
return bzip2.NewReader(br), nil
default:
return br, nil
}
}