Add glide.yaml and vendor deps

vendor/github.com/docker/distribution/registry/storage/blobcachemetrics.go

@@ -0,0 +1,60 @@
+package storage
+
+import (
+	"expvar"
+	"sync/atomic"
+
+	"github.com/docker/distribution/registry/storage/cache"
+)
+
+type blobStatCollector struct {
+	metrics cache.Metrics
+}
+
+func (bsc *blobStatCollector) Hit() {
+	atomic.AddUint64(&bsc.metrics.Requests, 1)
+	atomic.AddUint64(&bsc.metrics.Hits, 1)
+}
+
+func (bsc *blobStatCollector) Miss() {
+	atomic.AddUint64(&bsc.metrics.Requests, 1)
+	atomic.AddUint64(&bsc.metrics.Misses, 1)
+}
+
+func (bsc *blobStatCollector) Metrics() cache.Metrics {
+	return bsc.metrics
+}
+
+// blobStatterCacheMetrics keeps track of cache metrics for blob descriptor
+// cache requests. Note this is kept globally and made available via expvar.
+// For more detailed metrics, its recommend to instrument a particular cache
+// implementation.
+var blobStatterCacheMetrics cache.MetricsTracker = &blobStatCollector{}
+
+func init() {
+	registry := expvar.Get("registry")
+	if registry == nil {
+		registry = expvar.NewMap("registry")
+	}
+
+	cache := registry.(*expvar.Map).Get("cache")
+	if cache == nil {
+		cache = &expvar.Map{}
+		cache.(*expvar.Map).Init()
+		registry.(*expvar.Map).Set("cache", cache)
+	}
+
+	storage := cache.(*expvar.Map).Get("storage")
+	if storage == nil {
+		storage = &expvar.Map{}
+		storage.(*expvar.Map).Init()
+		cache.(*expvar.Map).Set("storage", storage)
+	}
+
+	storage.(*expvar.Map).Set("blobdescriptor", expvar.Func(func() interface{} {
+		// no need for synchronous access: the increments are atomic and
+		// during reading, we don't care if the data is up to date. The
+		// numbers will always *eventually* be reported correctly.
+		return blobStatterCacheMetrics
+	}))
+}

vendor/github.com/docker/distribution/registry/storage/blobserver.go

@@ -0,0 +1,78 @@
+package storage
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// TODO(stevvooe): This should configurable in the future.
+const blobCacheControlMaxAge = 365 * 24 * time.Hour
+
+// blobServer simply serves blobs from a driver instance using a path function
+// to identify paths and a descriptor service to fill in metadata.
+type blobServer struct {
+	driver   driver.StorageDriver
+	statter  distribution.BlobStatter
+	pathFn   func(dgst digest.Digest) (string, error)
+	redirect bool // allows disabling URLFor redirects
+}
+
+func (bs *blobServer) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
+	desc, err := bs.statter.Stat(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	path, err := bs.pathFn(desc.Digest)
+	if err != nil {
+		return err
+	}
+
+	if bs.redirect {
+		redirectURL, err := bs.driver.URLFor(ctx, path, map[string]interface{}{"method": r.Method})
+		switch err.(type) {
+		case nil:
+			// Redirect to storage URL.
+			http.Redirect(w, r, redirectURL, http.StatusTemporaryRedirect)
+			return err
+
+		case driver.ErrUnsupportedMethod:
+			// Fallback to serving the content directly.
+		default:
+			// Some unexpected error.
+			return err
+		}
+	}
+
+	br, err := newFileReader(ctx, bs.driver, path, desc.Size)
+	if err != nil {
+		return err
+	}
+	defer br.Close()
+
+	w.Header().Set("ETag", fmt.Sprintf(`"%s"`, desc.Digest)) // If-None-Match handled by ServeContent
+	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%.f", blobCacheControlMaxAge.Seconds()))
+
+	if w.Header().Get("Docker-Content-Digest") == "" {
+		w.Header().Set("Docker-Content-Digest", desc.Digest.String())
+	}
+
+	if w.Header().Get("Content-Type") == "" {
+		// Set the content type if not already set.
+		w.Header().Set("Content-Type", desc.MediaType)
+	}
+
+	if w.Header().Get("Content-Length") == "" {
+		// Set the content length if not already set.
+		w.Header().Set("Content-Length", fmt.Sprint(desc.Size))
+	}
+
+	http.ServeContent(w, r, desc.Digest.String(), time.Time{}, br)
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/blobstore.go

@@ -0,0 +1,223 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// blobStore implements the read side of the blob store interface over a
+// driver without enforcing per-repository membership. This object is
+// intentionally a leaky abstraction, providing utility methods that support
+// creating and traversing backend links.
+type blobStore struct {
+	driver  driver.StorageDriver
+	statter distribution.BlobStatter
+}
+
+var _ distribution.BlobProvider = &blobStore{}
+
+// Get implements the BlobReadService.Get call.
+func (bs *blobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	bp, err := bs.path(dgst)
+	if err != nil {
+		return nil, err
+	}
+
+	p, err := bs.driver.GetContent(ctx, bp)
+	if err != nil {
+		switch err.(type) {
+		case driver.PathNotFoundError:
+			return nil, distribution.ErrBlobUnknown
+		}
+
+		return nil, err
+	}
+
+	return p, err
+}
+
+func (bs *blobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
+	desc, err := bs.statter.Stat(ctx, dgst)
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := bs.path(desc.Digest)
+	if err != nil {
+		return nil, err
+	}
+
+	return newFileReader(ctx, bs.driver, path, desc.Size)
+}
+
+// Put stores the content p in the blob store, calculating the digest. If the
+// content is already present, only the digest will be returned. This should
+// only be used for small objects, such as manifests. This implemented as a convenience for other Put implementations
+func (bs *blobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
+	dgst := digest.FromBytes(p)
+	desc, err := bs.statter.Stat(ctx, dgst)
+	if err == nil {
+		// content already present
+		return desc, nil
+	} else if err != distribution.ErrBlobUnknown {
+		context.GetLogger(ctx).Errorf("blobStore: error stating content (%v): %v", dgst, err)
+		// real error, return it
+		return distribution.Descriptor{}, err
+	}
+
+	bp, err := bs.path(dgst)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	// TODO(stevvooe): Write out mediatype here, as well.
+	return distribution.Descriptor{
+		Size: int64(len(p)),
+
+		// NOTE(stevvooe): The central blob store firewalls media types from
+		// other users. The caller should look this up and override the value
+		// for the specific repository.
+		MediaType: "application/octet-stream",
+		Digest:    dgst,
+	}, bs.driver.PutContent(ctx, bp, p)
+}
+
+func (bs *blobStore) Enumerate(ctx context.Context, ingester func(dgst digest.Digest) error) error {
+
+	specPath, err := pathFor(blobsPathSpec{})
+	if err != nil {
+		return err
+	}
+
+	err = Walk(ctx, bs.driver, specPath, func(fileInfo driver.FileInfo) error {
+		// skip directories
+		if fileInfo.IsDir() {
+			return nil
+		}
+
+		currentPath := fileInfo.Path()
+		// we only want to parse paths that end with /data
+		_, fileName := path.Split(currentPath)
+		if fileName != "data" {
+			return nil
+		}
+
+		digest, err := digestFromPath(currentPath)
+		if err != nil {
+			return err
+		}
+
+		return ingester(digest)
+	})
+	return err
+}
+
+// path returns the canonical path for the blob identified by digest. The blob
+// may or may not exist.
+func (bs *blobStore) path(dgst digest.Digest) (string, error) {
+	bp, err := pathFor(blobDataPathSpec{
+		digest: dgst,
+	})
+
+	if err != nil {
+		return "", err
+	}
+
+	return bp, nil
+}
+
+// link links the path to the provided digest by writing the digest into the
+// target file. Caller must ensure that the blob actually exists.
+func (bs *blobStore) link(ctx context.Context, path string, dgst digest.Digest) error {
+	// The contents of the "link" file are the exact string contents of the
+	// digest, which is specified in that package.
+	return bs.driver.PutContent(ctx, path, []byte(dgst))
+}
+
+// readlink returns the linked digest at path.
+func (bs *blobStore) readlink(ctx context.Context, path string) (digest.Digest, error) {
+	content, err := bs.driver.GetContent(ctx, path)
+	if err != nil {
+		return "", err
+	}
+
+	linked, err := digest.ParseDigest(string(content))
+	if err != nil {
+		return "", err
+	}
+
+	return linked, nil
+}
+
+// resolve reads the digest link at path and returns the blob store path.
+func (bs *blobStore) resolve(ctx context.Context, path string) (string, error) {
+	dgst, err := bs.readlink(ctx, path)
+	if err != nil {
+		return "", err
+	}
+
+	return bs.path(dgst)
+}
+
+type blobStatter struct {
+	driver driver.StorageDriver
+}
+
+var _ distribution.BlobDescriptorService = &blobStatter{}
+
+// Stat implements BlobStatter.Stat by returning the descriptor for the blob
+// in the main blob store. If this method returns successfully, there is
+// strong guarantee that the blob exists and is available.
+func (bs *blobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	path, err := pathFor(blobDataPathSpec{
+		digest: dgst,
+	})
+
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	fi, err := bs.driver.Stat(ctx, path)
+	if err != nil {
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			return distribution.Descriptor{}, distribution.ErrBlobUnknown
+		default:
+			return distribution.Descriptor{}, err
+		}
+	}
+
+	if fi.IsDir() {
+		// NOTE(stevvooe): This represents a corruption situation. Somehow, we
+		// calculated a blob path and then detected a directory. We log the
+		// error and then error on the side of not knowing about the blob.
+		context.GetLogger(ctx).Warnf("blob path should not be a directory: %q", path)
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	// TODO(stevvooe): Add method to resolve the mediatype. We can store and
+	// cache a "global" media type for the blob, even if a specific repo has a
+	// mediatype that overrides the main one.
+
+	return distribution.Descriptor{
+		Size: fi.Size(),
+
+		// NOTE(stevvooe): The central blob store firewalls media types from
+		// other users. The caller should look this up and override the value
+		// for the specific repository.
+		MediaType: "application/octet-stream",
+		Digest:    dgst,
+	}, nil
+}
+
+func (bs *blobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
+	return distribution.ErrUnsupported
+}
+
+func (bs *blobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	return distribution.ErrUnsupported
+}

vendor/github.com/docker/distribution/registry/storage/blobwriter.go

@@ -0,0 +1,399 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"path"
+	"time"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+var (
+	errResumableDigestNotAvailable = errors.New("resumable digest not available")
+)
+
+// blobWriter is used to control the various aspects of resumable
+// blob upload.
+type blobWriter struct {
+	ctx       context.Context
+	blobStore *linkedBlobStore
+
+	id        string
+	startedAt time.Time
+	digester  digest.Digester
+	written   int64 // track the contiguous write
+
+	fileWriter storagedriver.FileWriter
+	driver     storagedriver.StorageDriver
+	path       string
+
+	resumableDigestEnabled bool
+	committed              bool
+}
+
+var _ distribution.BlobWriter = &blobWriter{}
+
+// ID returns the identifier for this upload.
+func (bw *blobWriter) ID() string {
+	return bw.id
+}
+
+func (bw *blobWriter) StartedAt() time.Time {
+	return bw.startedAt
+}
+
+// Commit marks the upload as completed, returning a valid descriptor. The
+// final size and digest are checked against the first descriptor provided.
+func (bw *blobWriter) Commit(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) {
+	context.GetLogger(ctx).Debug("(*blobWriter).Commit")
+
+	if err := bw.fileWriter.Commit(); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	bw.Close()
+	desc.Size = bw.Size()
+
+	canonical, err := bw.validateBlob(ctx, desc)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if err := bw.moveBlob(ctx, canonical); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if err := bw.blobStore.linkBlob(ctx, canonical, desc.Digest); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if err := bw.removeResources(ctx); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	err = bw.blobStore.blobAccessController.SetDescriptor(ctx, canonical.Digest, canonical)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	bw.committed = true
+	return canonical, nil
+}
+
+// Cancel the blob upload process, releasing any resources associated with
+// the writer and canceling the operation.
+func (bw *blobWriter) Cancel(ctx context.Context) error {
+	context.GetLogger(ctx).Debug("(*blobWriter).Cancel")
+	if err := bw.fileWriter.Cancel(); err != nil {
+		return err
+	}
+
+	if err := bw.Close(); err != nil {
+		context.GetLogger(ctx).Errorf("error closing blobwriter: %s", err)
+	}
+
+	if err := bw.removeResources(ctx); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (bw *blobWriter) Size() int64 {
+	return bw.fileWriter.Size()
+}
+
+func (bw *blobWriter) Write(p []byte) (int, error) {
+	// Ensure that the current write offset matches how many bytes have been
+	// written to the digester. If not, we need to update the digest state to
+	// match the current write position.
+	if err := bw.resumeDigest(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable {
+		return 0, err
+	}
+
+	n, err := io.MultiWriter(bw.fileWriter, bw.digester.Hash()).Write(p)
+	bw.written += int64(n)
+
+	return n, err
+}
+
+func (bw *blobWriter) ReadFrom(r io.Reader) (n int64, err error) {
+	// Ensure that the current write offset matches how many bytes have been
+	// written to the digester. If not, we need to update the digest state to
+	// match the current write position.
+	if err := bw.resumeDigest(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable {
+		return 0, err
+	}
+
+	nn, err := io.Copy(io.MultiWriter(bw.fileWriter, bw.digester.Hash()), r)
+	bw.written += nn
+
+	return nn, err
+}
+
+func (bw *blobWriter) Close() error {
+	if bw.committed {
+		return errors.New("blobwriter close after commit")
+	}
+
+	if err := bw.storeHashState(bw.blobStore.ctx); err != nil && err != errResumableDigestNotAvailable {
+		return err
+	}
+
+	return bw.fileWriter.Close()
+}
+
+// validateBlob checks the data against the digest, returning an error if it
+// does not match. The canonical descriptor is returned.
+func (bw *blobWriter) validateBlob(ctx context.Context, desc distribution.Descriptor) (distribution.Descriptor, error) {
+	var (
+		verified, fullHash bool
+		canonical          digest.Digest
+	)
+
+	if desc.Digest == "" {
+		// if no descriptors are provided, we have nothing to validate
+		// against. We don't really want to support this for the registry.
+		return distribution.Descriptor{}, distribution.ErrBlobInvalidDigest{
+			Reason: fmt.Errorf("cannot validate against empty digest"),
+		}
+	}
+
+	var size int64
+
+	// Stat the on disk file
+	if fi, err := bw.driver.Stat(ctx, bw.path); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			// NOTE(stevvooe): We really don't care if the file is
+			// not actually present for the reader. We now assume
+			// that the desc length is zero.
+			desc.Size = 0
+		default:
+			// Any other error we want propagated up the stack.
+			return distribution.Descriptor{}, err
+		}
+	} else {
+		if fi.IsDir() {
+			return distribution.Descriptor{}, fmt.Errorf("unexpected directory at upload location %q", bw.path)
+		}
+
+		size = fi.Size()
+	}
+
+	if desc.Size > 0 {
+		if desc.Size != size {
+			return distribution.Descriptor{}, distribution.ErrBlobInvalidLength
+		}
+	} else {
+		// if provided 0 or negative length, we can assume caller doesn't know or
+		// care about length.
+		desc.Size = size
+	}
+
+	// TODO(stevvooe): This section is very meandering. Need to be broken down
+	// to be a lot more clear.
+
+	if err := bw.resumeDigest(ctx); err == nil {
+		canonical = bw.digester.Digest()
+
+		if canonical.Algorithm() == desc.Digest.Algorithm() {
+			// Common case: client and server prefer the same canonical digest
+			// algorithm - currently SHA256.
+			verified = desc.Digest == canonical
+		} else {
+			// The client wants to use a different digest algorithm. They'll just
+			// have to be patient and wait for us to download and re-hash the
+			// uploaded content using that digest algorithm.
+			fullHash = true
+		}
+	} else if err == errResumableDigestNotAvailable {
+		// Not using resumable digests, so we need to hash the entire layer.
+		fullHash = true
+	} else {
+		return distribution.Descriptor{}, err
+	}
+
+	if fullHash {
+		// a fantastic optimization: if the the written data and the size are
+		// the same, we don't need to read the data from the backend. This is
+		// because we've written the entire file in the lifecycle of the
+		// current instance.
+		if bw.written == size && digest.Canonical == desc.Digest.Algorithm() {
+			canonical = bw.digester.Digest()
+			verified = desc.Digest == canonical
+		}
+
+		// If the check based on size fails, we fall back to the slowest of
+		// paths. We may be able to make the size-based check a stronger
+		// guarantee, so this may be defensive.
+		if !verified {
+			digester := digest.Canonical.New()
+
+			digestVerifier, err := digest.NewDigestVerifier(desc.Digest)
+			if err != nil {
+				return distribution.Descriptor{}, err
+			}
+
+			// Read the file from the backend driver and validate it.
+			fr, err := newFileReader(ctx, bw.driver, bw.path, desc.Size)
+			if err != nil {
+				return distribution.Descriptor{}, err
+			}
+			defer fr.Close()
+
+			tr := io.TeeReader(fr, digester.Hash())
+
+			if _, err := io.Copy(digestVerifier, tr); err != nil {
+				return distribution.Descriptor{}, err
+			}
+
+			canonical = digester.Digest()
+			verified = digestVerifier.Verified()
+		}
+	}
+
+	if !verified {
+		context.GetLoggerWithFields(ctx,
+			map[interface{}]interface{}{
+				"canonical": canonical,
+				"provided":  desc.Digest,
+			}, "canonical", "provided").
+			Errorf("canonical digest does match provided digest")
+		return distribution.Descriptor{}, distribution.ErrBlobInvalidDigest{
+			Digest: desc.Digest,
+			Reason: fmt.Errorf("content does not match digest"),
+		}
+	}
+
+	// update desc with canonical hash
+	desc.Digest = canonical
+
+	if desc.MediaType == "" {
+		desc.MediaType = "application/octet-stream"
+	}
+
+	return desc, nil
+}
+
+// moveBlob moves the data into its final, hash-qualified destination,
+// identified by dgst. The layer should be validated before commencing the
+// move.
+func (bw *blobWriter) moveBlob(ctx context.Context, desc distribution.Descriptor) error {
+	blobPath, err := pathFor(blobDataPathSpec{
+		digest: desc.Digest,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	// Check for existence
+	if _, err := bw.blobStore.driver.Stat(ctx, blobPath); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			break // ensure that it doesn't exist.
+		default:
+			return err
+		}
+	} else {
+		// If the path exists, we can assume that the content has already
+		// been uploaded, since the blob storage is content-addressable.
+		// While it may be corrupted, detection of such corruption belongs
+		// elsewhere.
+		return nil
+	}
+
+	// If no data was received, we may not actually have a file on disk. Check
+	// the size here and write a zero-length file to blobPath if this is the
+	// case. For the most part, this should only ever happen with zero-length
+	// tars.
+	if _, err := bw.blobStore.driver.Stat(ctx, bw.path); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			// HACK(stevvooe): This is slightly dangerous: if we verify above,
+			// get a hash, then the underlying file is deleted, we risk moving
+			// a zero-length blob into a nonzero-length blob location. To
+			// prevent this horrid thing, we employ the hack of only allowing
+			// to this happen for the digest of an empty tar.
+			if desc.Digest == digest.DigestSha256EmptyTar {
+				return bw.blobStore.driver.PutContent(ctx, blobPath, []byte{})
+			}
+
+			// We let this fail during the move below.
+			logrus.
+				WithField("upload.id", bw.ID()).
+				WithField("digest", desc.Digest).Warnf("attempted to move zero-length content with non-zero digest")
+		default:
+			return err // unrelated error
+		}
+	}
+
+	// TODO(stevvooe): We should also write the mediatype when executing this move.
+
+	return bw.blobStore.driver.Move(ctx, bw.path, blobPath)
+}
+
+// removeResources should clean up all resources associated with the upload
+// instance. An error will be returned if the clean up cannot proceed. If the
+// resources are already not present, no error will be returned.
+func (bw *blobWriter) removeResources(ctx context.Context) error {
+	dataPath, err := pathFor(uploadDataPathSpec{
+		name: bw.blobStore.repository.Named().Name(),
+		id:   bw.id,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	// Resolve and delete the containing directory, which should include any
+	// upload related files.
+	dirPath := path.Dir(dataPath)
+	if err := bw.blobStore.driver.Delete(ctx, dirPath); err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			break // already gone!
+		default:
+			// This should be uncommon enough such that returning an error
+			// should be okay. At this point, the upload should be mostly
+			// complete, but perhaps the backend became unaccessible.
+			context.GetLogger(ctx).Errorf("unable to delete layer upload resources %q: %v", dirPath, err)
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (bw *blobWriter) Reader() (io.ReadCloser, error) {
+	// todo(richardscothern): Change to exponential backoff, i=0.5, e=2, n=4
+	try := 1
+	for try <= 5 {
+		_, err := bw.driver.Stat(bw.ctx, bw.path)
+		if err == nil {
+			break
+		}
+		switch err.(type) {
+		case storagedriver.PathNotFoundError:
+			context.GetLogger(bw.ctx).Debugf("Nothing found on try %d, sleeping...", try)
+			time.Sleep(1 * time.Second)
+			try++
+		default:
+			return nil, err
+		}
+	}
+
+	readCloser, err := bw.driver.Reader(bw.ctx, bw.path, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	return readCloser, nil
+}

vendor/github.com/docker/distribution/registry/storage/blobwriter_nonresumable.go

@@ -0,0 +1,17 @@
+// +build noresumabledigest
+
+package storage
+
+import (
+	"github.com/docker/distribution/context"
+)
+
+// resumeHashAt is a noop when resumable digest support is disabled.
+func (bw *blobWriter) resumeDigest(ctx context.Context) error {
+	return errResumableDigestNotAvailable
+}
+
+// storeHashState is a noop when resumable digest support is disabled.
+func (bw *blobWriter) storeHashState(ctx context.Context) error {
+	return errResumableDigestNotAvailable
+}

vendor/github.com/docker/distribution/registry/storage/blobwriter_resumable.go

@@ -0,0 +1,145 @@
+// +build !noresumabledigest
+
+package storage
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/stevvooe/resumable"
+
+	// register resumable hashes with import
+	_ "github.com/stevvooe/resumable/sha256"
+	_ "github.com/stevvooe/resumable/sha512"
+)
+
+// resumeDigest attempts to restore the state of the internal hash function
+// by loading the most recent saved hash state equal to the current size of the blob.
+func (bw *blobWriter) resumeDigest(ctx context.Context) error {
+	if !bw.resumableDigestEnabled {
+		return errResumableDigestNotAvailable
+	}
+
+	h, ok := bw.digester.Hash().(resumable.Hash)
+	if !ok {
+		return errResumableDigestNotAvailable
+	}
+	offset := bw.fileWriter.Size()
+	if offset == int64(h.Len()) {
+		// State of digester is already at the requested offset.
+		return nil
+	}
+
+	// List hash states from storage backend.
+	var hashStateMatch hashStateEntry
+	hashStates, err := bw.getStoredHashStates(ctx)
+	if err != nil {
+		return fmt.Errorf("unable to get stored hash states with offset %d: %s", offset, err)
+	}
+
+	// Find the highest stored hashState with offset equal to
+	// the requested offset.
+	for _, hashState := range hashStates {
+		if hashState.offset == offset {
+			hashStateMatch = hashState
+			break // Found an exact offset match.
+		}
+	}
+
+	if hashStateMatch.offset == 0 {
+		// No need to load any state, just reset the hasher.
+		h.Reset()
+	} else {
+		storedState, err := bw.driver.GetContent(ctx, hashStateMatch.path)
+		if err != nil {
+			return err
+		}
+
+		if err = h.Restore(storedState); err != nil {
+			return err
+		}
+	}
+
+	// Mind the gap.
+	if gapLen := offset - int64(h.Len()); gapLen > 0 {
+		return errResumableDigestNotAvailable
+	}
+
+	return nil
+}
+
+type hashStateEntry struct {
+	offset int64
+	path   string
+}
+
+// getStoredHashStates returns a slice of hashStateEntries for this upload.
+func (bw *blobWriter) getStoredHashStates(ctx context.Context) ([]hashStateEntry, error) {
+	uploadHashStatePathPrefix, err := pathFor(uploadHashStatePathSpec{
+		name: bw.blobStore.repository.Named().String(),
+		id:   bw.id,
+		alg:  bw.digester.Digest().Algorithm(),
+		list: true,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	paths, err := bw.blobStore.driver.List(ctx, uploadHashStatePathPrefix)
+	if err != nil {
+		if _, ok := err.(storagedriver.PathNotFoundError); !ok {
+			return nil, err
+		}
+		// Treat PathNotFoundError as no entries.
+		paths = nil
+	}
+
+	hashStateEntries := make([]hashStateEntry, 0, len(paths))
+
+	for _, p := range paths {
+		pathSuffix := path.Base(p)
+		// The suffix should be the offset.
+		offset, err := strconv.ParseInt(pathSuffix, 0, 64)
+		if err != nil {
+			logrus.Errorf("unable to parse offset from upload state path %q: %s", p, err)
+		}
+
+		hashStateEntries = append(hashStateEntries, hashStateEntry{offset: offset, path: p})
+	}
+
+	return hashStateEntries, nil
+}
+
+func (bw *blobWriter) storeHashState(ctx context.Context) error {
+	if !bw.resumableDigestEnabled {
+		return errResumableDigestNotAvailable
+	}
+
+	h, ok := bw.digester.Hash().(resumable.Hash)
+	if !ok {
+		return errResumableDigestNotAvailable
+	}
+
+	uploadHashStatePath, err := pathFor(uploadHashStatePathSpec{
+		name:   bw.blobStore.repository.Named().String(),
+		id:     bw.id,
+		alg:    bw.digester.Digest().Algorithm(),
+		offset: int64(h.Len()),
+	})
+
+	if err != nil {
+		return err
+	}
+
+	hashState, err := h.State()
+	if err != nil {
+		return err
+	}
+
+	return bw.driver.PutContent(ctx, uploadHashStatePath, hashState)
+}

vendor/github.com/docker/distribution/registry/storage/cache/cache.go

@@ -0,0 +1,35 @@
+// Package cache provides facilities to speed up access to the storage
+// backend.
+package cache
+
+import (
+	"fmt"
+
+	"github.com/docker/distribution"
+)
+
+// BlobDescriptorCacheProvider provides repository scoped
+// BlobDescriptorService cache instances and a global descriptor cache.
+type BlobDescriptorCacheProvider interface {
+	distribution.BlobDescriptorService
+
+	RepositoryScoped(repo string) (distribution.BlobDescriptorService, error)
+}
+
+// ValidateDescriptor provides a helper function to ensure that caches have
+// common criteria for admitting descriptors.
+func ValidateDescriptor(desc distribution.Descriptor) error {
+	if err := desc.Digest.Validate(); err != nil {
+		return err
+	}
+
+	if desc.Size < 0 {
+		return fmt.Errorf("cache: invalid length in descriptor: %v < 0", desc.Size)
+	}
+
+	if desc.MediaType == "" {
+		return fmt.Errorf("cache: empty mediatype on descriptor: %v", desc)
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/cache/cachecheck/suite.go

@@ -0,0 +1,180 @@
+package cachecheck
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/registry/storage/cache"
+)
+
+// CheckBlobDescriptorCache takes a cache implementation through a common set
+// of operations. If adding new tests, please add them here so new
+// implementations get the benefit. This should be used for unit tests.
+func CheckBlobDescriptorCache(t *testing.T, provider cache.BlobDescriptorCacheProvider) {
+	ctx := context.Background()
+
+	checkBlobDescriptorCacheEmptyRepository(ctx, t, provider)
+	checkBlobDescriptorCacheSetAndRead(ctx, t, provider)
+	checkBlobDescriptorCacheClear(ctx, t, provider)
+}
+
+func checkBlobDescriptorCacheEmptyRepository(ctx context.Context, t *testing.T, provider cache.BlobDescriptorCacheProvider) {
+	if _, err := provider.Stat(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"); err != distribution.ErrBlobUnknown {
+		t.Fatalf("expected unknown blob error with empty store: %v", err)
+	}
+
+	cache, err := provider.RepositoryScoped("")
+	if err == nil {
+		t.Fatalf("expected an error when asking for invalid repo")
+	}
+
+	cache, err = provider.RepositoryScoped("foo/bar")
+	if err != nil {
+		t.Fatalf("unexpected error getting repository: %v", err)
+	}
+
+	if err := cache.SetDescriptor(ctx, "", distribution.Descriptor{
+		Digest:    "sha384:abc",
+		Size:      10,
+		MediaType: "application/octet-stream"}); err != digest.ErrDigestInvalidFormat {
+		t.Fatalf("expected error with invalid digest: %v", err)
+	}
+
+	if err := cache.SetDescriptor(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111", distribution.Descriptor{
+		Digest:    "",
+		Size:      10,
+		MediaType: "application/octet-stream"}); err == nil {
+		t.Fatalf("expected error setting value on invalid descriptor")
+	}
+
+	if _, err := cache.Stat(ctx, ""); err != digest.ErrDigestInvalidFormat {
+		t.Fatalf("expected error checking for cache item with empty digest: %v", err)
+	}
+
+	if _, err := cache.Stat(ctx, "sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111"); err != distribution.ErrBlobUnknown {
+		t.Fatalf("expected unknown blob error with empty repo: %v", err)
+	}
+}
+
+func checkBlobDescriptorCacheSetAndRead(ctx context.Context, t *testing.T, provider cache.BlobDescriptorCacheProvider) {
+	localDigest := digest.Digest("sha384:abc111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")
+	expected := distribution.Descriptor{
+		Digest:    "sha256:abc1111111111111111111111111111111111111111111111111111111111111",
+		Size:      10,
+		MediaType: "application/octet-stream"}
+
+	cache, err := provider.RepositoryScoped("foo/bar")
+	if err != nil {
+		t.Fatalf("unexpected error getting scoped cache: %v", err)
+	}
+
+	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
+		t.Fatalf("error setting descriptor: %v", err)
+	}
+
+	desc, err := cache.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("unexpected error statting fake2:abc: %v", err)
+	}
+
+	if !reflect.DeepEqual(expected, desc) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
+	}
+
+	// also check that we set the canonical key ("fake:abc")
+	desc, err = cache.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("descriptor not returned for canonical key: %v", err)
+	}
+
+	if !reflect.DeepEqual(expected, desc) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
+	}
+
+	// ensure that global gets extra descriptor mapping
+	desc, err = provider.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("expected blob unknown in global cache: %v, %v", err, desc)
+	}
+
+	if !reflect.DeepEqual(desc, expected) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
+	}
+
+	// get at it through canonical descriptor
+	desc, err = provider.Stat(ctx, expected.Digest)
+	if err != nil {
+		t.Fatalf("unexpected error checking glboal descriptor: %v", err)
+	}
+
+	if !reflect.DeepEqual(desc, expected) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
+	}
+
+	// now, we set the repo local mediatype to something else and ensure it
+	// doesn't get changed in the provider cache.
+	expected.MediaType = "application/json"
+
+	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
+		t.Fatalf("unexpected error setting descriptor: %v", err)
+	}
+
+	desc, err = cache.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("unexpected error getting descriptor: %v", err)
+	}
+
+	if !reflect.DeepEqual(desc, expected) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
+	}
+
+	desc, err = provider.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("unexpected error getting global descriptor: %v", err)
+	}
+
+	expected.MediaType = "application/octet-stream" // expect original mediatype in global
+
+	if !reflect.DeepEqual(desc, expected) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", desc, expected)
+	}
+}
+
+func checkBlobDescriptorCacheClear(ctx context.Context, t *testing.T, provider cache.BlobDescriptorCacheProvider) {
+	localDigest := digest.Digest("sha384:def111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111111")
+	expected := distribution.Descriptor{
+		Digest:    "sha256:def1111111111111111111111111111111111111111111111111111111111111",
+		Size:      10,
+		MediaType: "application/octet-stream"}
+
+	cache, err := provider.RepositoryScoped("foo/bar")
+	if err != nil {
+		t.Fatalf("unexpected error getting scoped cache: %v", err)
+	}
+
+	if err := cache.SetDescriptor(ctx, localDigest, expected); err != nil {
+		t.Fatalf("error setting descriptor: %v", err)
+	}
+
+	desc, err := cache.Stat(ctx, localDigest)
+	if err != nil {
+		t.Fatalf("unexpected error statting fake2:abc: %v", err)
+	}
+
+	if !reflect.DeepEqual(expected, desc) {
+		t.Fatalf("unexpected descriptor: %#v != %#v", expected, desc)
+	}
+
+	err = cache.Clear(ctx, localDigest)
+	if err != nil {
+		t.Error(err)
+	}
+
+	desc, err = cache.Stat(ctx, localDigest)
+	if err == nil {
+		t.Fatalf("expected error statting deleted blob: %v", err)
+	}
+}

vendor/github.com/docker/distribution/registry/storage/cache/cachedblobdescriptorstore.go

@@ -0,0 +1,101 @@
+package cache
+
+import (
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+
+	"github.com/docker/distribution"
+)
+
+// Metrics is used to hold metric counters
+// related to the number of times a cache was
+// hit or missed.
+type Metrics struct {
+	Requests uint64
+	Hits     uint64
+	Misses   uint64
+}
+
+// MetricsTracker represents a metric tracker
+// which simply counts the number of hits and misses.
+type MetricsTracker interface {
+	Hit()
+	Miss()
+	Metrics() Metrics
+}
+
+type cachedBlobStatter struct {
+	cache   distribution.BlobDescriptorService
+	backend distribution.BlobDescriptorService
+	tracker MetricsTracker
+}
+
+// NewCachedBlobStatter creates a new statter which prefers a cache and
+// falls back to a backend.
+func NewCachedBlobStatter(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService) distribution.BlobDescriptorService {
+	return &cachedBlobStatter{
+		cache:   cache,
+		backend: backend,
+	}
+}
+
+// NewCachedBlobStatterWithMetrics creates a new statter which prefers a cache and
+// falls back to a backend. Hits and misses will send to the tracker.
+func NewCachedBlobStatterWithMetrics(cache distribution.BlobDescriptorService, backend distribution.BlobDescriptorService, tracker MetricsTracker) distribution.BlobStatter {
+	return &cachedBlobStatter{
+		cache:   cache,
+		backend: backend,
+		tracker: tracker,
+	}
+}
+
+func (cbds *cachedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	desc, err := cbds.cache.Stat(ctx, dgst)
+	if err != nil {
+		if err != distribution.ErrBlobUnknown {
+			context.GetLogger(ctx).Errorf("error retrieving descriptor from cache: %v", err)
+		}
+
+		goto fallback
+	}
+
+	if cbds.tracker != nil {
+		cbds.tracker.Hit()
+	}
+	return desc, nil
+fallback:
+	if cbds.tracker != nil {
+		cbds.tracker.Miss()
+	}
+	desc, err = cbds.backend.Stat(ctx, dgst)
+	if err != nil {
+		return desc, err
+	}
+
+	if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
+		context.GetLogger(ctx).Errorf("error adding descriptor %v to cache: %v", desc.Digest, err)
+	}
+
+	return desc, err
+
+}
+
+func (cbds *cachedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) error {
+	err := cbds.cache.Clear(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	err = cbds.backend.Clear(ctx, dgst)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (cbds *cachedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	if err := cbds.cache.SetDescriptor(ctx, dgst, desc); err != nil {
+		context.GetLogger(ctx).Errorf("error adding descriptor %v to cache: %v", desc.Digest, err)
+	}
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/cache/memory/memory.go

@@ -0,0 +1,179 @@
+package memory
+
+import (
+	"sync"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/cache"
+)
+
+type inMemoryBlobDescriptorCacheProvider struct {
+	global       *mapBlobDescriptorCache
+	repositories map[string]*mapBlobDescriptorCache
+	mu           sync.RWMutex
+}
+
+// NewInMemoryBlobDescriptorCacheProvider returns a new mapped-based cache for
+// storing blob descriptor data.
+func NewInMemoryBlobDescriptorCacheProvider() cache.BlobDescriptorCacheProvider {
+	return &inMemoryBlobDescriptorCacheProvider{
+		global:       newMapBlobDescriptorCache(),
+		repositories: make(map[string]*mapBlobDescriptorCache),
+	}
+}
+
+func (imbdcp *inMemoryBlobDescriptorCacheProvider) RepositoryScoped(repo string) (distribution.BlobDescriptorService, error) {
+	if _, err := reference.ParseNamed(repo); err != nil {
+		return nil, err
+	}
+
+	imbdcp.mu.RLock()
+	defer imbdcp.mu.RUnlock()
+
+	return &repositoryScopedInMemoryBlobDescriptorCache{
+		repo:       repo,
+		parent:     imbdcp,
+		repository: imbdcp.repositories[repo],
+	}, nil
+}
+
+func (imbdcp *inMemoryBlobDescriptorCacheProvider) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	return imbdcp.global.Stat(ctx, dgst)
+}
+
+func (imbdcp *inMemoryBlobDescriptorCacheProvider) Clear(ctx context.Context, dgst digest.Digest) error {
+	return imbdcp.global.Clear(ctx, dgst)
+}
+
+func (imbdcp *inMemoryBlobDescriptorCacheProvider) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	_, err := imbdcp.Stat(ctx, dgst)
+	if err == distribution.ErrBlobUnknown {
+
+		if dgst.Algorithm() != desc.Digest.Algorithm() && dgst != desc.Digest {
+			// if the digests differ, set the other canonical mapping
+			if err := imbdcp.global.SetDescriptor(ctx, desc.Digest, desc); err != nil {
+				return err
+			}
+		}
+
+		// unknown, just set it
+		return imbdcp.global.SetDescriptor(ctx, dgst, desc)
+	}
+
+	// we already know it, do nothing
+	return err
+}
+
+// repositoryScopedInMemoryBlobDescriptorCache provides the request scoped
+// repository cache. Instances are not thread-safe but the delegated
+// operations are.
+type repositoryScopedInMemoryBlobDescriptorCache struct {
+	repo       string
+	parent     *inMemoryBlobDescriptorCacheProvider // allows lazy allocation of repo's map
+	repository *mapBlobDescriptorCache
+}
+
+func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	rsimbdcp.parent.mu.Lock()
+	repo := rsimbdcp.repository
+	rsimbdcp.parent.mu.Unlock()
+
+	if repo == nil {
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	return repo.Stat(ctx, dgst)
+}
+
+func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) Clear(ctx context.Context, dgst digest.Digest) error {
+	rsimbdcp.parent.mu.Lock()
+	repo := rsimbdcp.repository
+	rsimbdcp.parent.mu.Unlock()
+
+	if repo == nil {
+		return distribution.ErrBlobUnknown
+	}
+
+	return repo.Clear(ctx, dgst)
+}
+
+func (rsimbdcp *repositoryScopedInMemoryBlobDescriptorCache) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	rsimbdcp.parent.mu.Lock()
+	repo := rsimbdcp.repository
+	if repo == nil {
+		// allocate map since we are setting it now.
+		var ok bool
+		// have to read back value since we may have allocated elsewhere.
+		repo, ok = rsimbdcp.parent.repositories[rsimbdcp.repo]
+		if !ok {
+			repo = newMapBlobDescriptorCache()
+			rsimbdcp.parent.repositories[rsimbdcp.repo] = repo
+		}
+		rsimbdcp.repository = repo
+	}
+	rsimbdcp.parent.mu.Unlock()
+
+	if err := repo.SetDescriptor(ctx, dgst, desc); err != nil {
+		return err
+	}
+
+	return rsimbdcp.parent.SetDescriptor(ctx, dgst, desc)
+}
+
+// mapBlobDescriptorCache provides a simple map-based implementation of the
+// descriptor cache.
+type mapBlobDescriptorCache struct {
+	descriptors map[digest.Digest]distribution.Descriptor
+	mu          sync.RWMutex
+}
+
+var _ distribution.BlobDescriptorService = &mapBlobDescriptorCache{}
+
+func newMapBlobDescriptorCache() *mapBlobDescriptorCache {
+	return &mapBlobDescriptorCache{
+		descriptors: make(map[digest.Digest]distribution.Descriptor),
+	}
+}
+
+func (mbdc *mapBlobDescriptorCache) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	if err := dgst.Validate(); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	mbdc.mu.RLock()
+	defer mbdc.mu.RUnlock()
+
+	desc, ok := mbdc.descriptors[dgst]
+	if !ok {
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	return desc, nil
+}
+
+func (mbdc *mapBlobDescriptorCache) Clear(ctx context.Context, dgst digest.Digest) error {
+	mbdc.mu.Lock()
+	defer mbdc.mu.Unlock()
+
+	delete(mbdc.descriptors, dgst)
+	return nil
+}
+
+func (mbdc *mapBlobDescriptorCache) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	if err := cache.ValidateDescriptor(desc); err != nil {
+		return err
+	}
+
+	mbdc.mu.Lock()
+	defer mbdc.mu.Unlock()
+
+	mbdc.descriptors[dgst] = desc
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/cache/memory/memory_test.go

@@ -0,0 +1,13 @@
+package memory
+
+import (
+	"testing"
+
+	"github.com/docker/distribution/registry/storage/cache/cachecheck"
+)
+
+// TestInMemoryBlobInfoCache checks the in memory implementation is working
+// correctly.
+func TestInMemoryBlobInfoCache(t *testing.T) {
+	cachecheck.CheckBlobDescriptorCache(t, NewInMemoryBlobDescriptorCacheProvider())
+}

vendor/github.com/docker/distribution/registry/storage/cache/redis/redis.go

@@ -0,0 +1,268 @@
+package redis
+
+import (
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/cache"
+	"github.com/garyburd/redigo/redis"
+)
+
+// redisBlobStatService provides an implementation of
+// BlobDescriptorCacheProvider based on redis. Blob descriptors are stored in
+// two parts. The first provide fast access to repository membership through a
+// redis set for each repo. The second is a redis hash keyed by the digest of
+// the layer, providing path, length and mediatype information. There is also
+// a per-repository redis hash of the blob descriptor, allowing override of
+// data. This is currently used to override the mediatype on a per-repository
+// basis.
+//
+// Note that there is no implied relationship between these two caches. The
+// layer may exist in one, both or none and the code must be written this way.
+type redisBlobDescriptorService struct {
+	pool *redis.Pool
+
+	// TODO(stevvooe): We use a pool because we don't have great control over
+	// the cache lifecycle to manage connections. A new connection if fetched
+	// for each operation. Once we have better lifecycle management of the
+	// request objects, we can change this to a connection.
+}
+
+// NewRedisBlobDescriptorCacheProvider returns a new redis-based
+// BlobDescriptorCacheProvider using the provided redis connection pool.
+func NewRedisBlobDescriptorCacheProvider(pool *redis.Pool) cache.BlobDescriptorCacheProvider {
+	return &redisBlobDescriptorService{
+		pool: pool,
+	}
+}
+
+// RepositoryScoped returns the scoped cache.
+func (rbds *redisBlobDescriptorService) RepositoryScoped(repo string) (distribution.BlobDescriptorService, error) {
+	if _, err := reference.ParseNamed(repo); err != nil {
+		return nil, err
+	}
+
+	return &repositoryScopedRedisBlobDescriptorService{
+		repo:     repo,
+		upstream: rbds,
+	}, nil
+}
+
+// Stat retrieves the descriptor data from the redis hash entry.
+func (rbds *redisBlobDescriptorService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	if err := dgst.Validate(); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	conn := rbds.pool.Get()
+	defer conn.Close()
+
+	return rbds.stat(ctx, conn, dgst)
+}
+
+func (rbds *redisBlobDescriptorService) Clear(ctx context.Context, dgst digest.Digest) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	conn := rbds.pool.Get()
+	defer conn.Close()
+
+	// Not atomic in redis <= 2.3
+	reply, err := conn.Do("HDEL", rbds.blobDescriptorHashKey(dgst), "digest", "length", "mediatype")
+	if err != nil {
+		return err
+	}
+
+	if reply == 0 {
+		return distribution.ErrBlobUnknown
+	}
+
+	return nil
+}
+
+// stat provides an internal stat call that takes a connection parameter. This
+// allows some internal management of the connection scope.
+func (rbds *redisBlobDescriptorService) stat(ctx context.Context, conn redis.Conn, dgst digest.Digest) (distribution.Descriptor, error) {
+	reply, err := redis.Values(conn.Do("HMGET", rbds.blobDescriptorHashKey(dgst), "digest", "size", "mediatype"))
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	// NOTE(stevvooe): The "size" field used to be "length". We treat a
+	// missing "size" field here as an unknown blob, which causes a cache
+	// miss, effectively migrating the field.
+	if len(reply) < 3 || reply[0] == nil || reply[1] == nil { // don't care if mediatype is nil
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	var desc distribution.Descriptor
+	if _, err := redis.Scan(reply, &desc.Digest, &desc.Size, &desc.MediaType); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	return desc, nil
+}
+
+// SetDescriptor sets the descriptor data for the given digest using a redis
+// hash. A hash is used here since we may store unrelated fields about a layer
+// in the future.
+func (rbds *redisBlobDescriptorService) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	if err := cache.ValidateDescriptor(desc); err != nil {
+		return err
+	}
+
+	conn := rbds.pool.Get()
+	defer conn.Close()
+
+	return rbds.setDescriptor(ctx, conn, dgst, desc)
+}
+
+func (rbds *redisBlobDescriptorService) setDescriptor(ctx context.Context, conn redis.Conn, dgst digest.Digest, desc distribution.Descriptor) error {
+	if _, err := conn.Do("HMSET", rbds.blobDescriptorHashKey(dgst),
+		"digest", desc.Digest,
+		"size", desc.Size); err != nil {
+		return err
+	}
+
+	// Only set mediatype if not already set.
+	if _, err := conn.Do("HSETNX", rbds.blobDescriptorHashKey(dgst),
+		"mediatype", desc.MediaType); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (rbds *redisBlobDescriptorService) blobDescriptorHashKey(dgst digest.Digest) string {
+	return "blobs::" + dgst.String()
+}
+
+type repositoryScopedRedisBlobDescriptorService struct {
+	repo     string
+	upstream *redisBlobDescriptorService
+}
+
+var _ distribution.BlobDescriptorService = &repositoryScopedRedisBlobDescriptorService{}
+
+// Stat ensures that the digest is a member of the specified repository and
+// forwards the descriptor request to the global blob store. If the media type
+// differs for the repository, we override it.
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	if err := dgst.Validate(); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	conn := rsrbds.upstream.pool.Get()
+	defer conn.Close()
+
+	// Check membership to repository first
+	member, err := redis.Bool(conn.Do("SISMEMBER", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst))
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if !member {
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	upstream, err := rsrbds.upstream.stat(ctx, conn, dgst)
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	// We allow a per repository mediatype, let's look it up here.
+	mediatype, err := redis.String(conn.Do("HGET", rsrbds.blobDescriptorHashKey(dgst), "mediatype"))
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	if mediatype != "" {
+		upstream.MediaType = mediatype
+	}
+
+	return upstream, nil
+}
+
+// Clear removes the descriptor from the cache and forwards to the upstream descriptor store
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) Clear(ctx context.Context, dgst digest.Digest) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	conn := rsrbds.upstream.pool.Get()
+	defer conn.Close()
+
+	// Check membership to repository first
+	member, err := redis.Bool(conn.Do("SISMEMBER", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst))
+	if err != nil {
+		return err
+	}
+
+	if !member {
+		return distribution.ErrBlobUnknown
+	}
+
+	return rsrbds.upstream.Clear(ctx, dgst)
+}
+
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	if err := dgst.Validate(); err != nil {
+		return err
+	}
+
+	if err := cache.ValidateDescriptor(desc); err != nil {
+		return err
+	}
+
+	if dgst != desc.Digest {
+		if dgst.Algorithm() == desc.Digest.Algorithm() {
+			return fmt.Errorf("redis cache: digest for descriptors differ but algorthim does not: %q != %q", dgst, desc.Digest)
+		}
+	}
+
+	conn := rsrbds.upstream.pool.Get()
+	defer conn.Close()
+
+	return rsrbds.setDescriptor(ctx, conn, dgst, desc)
+}
+
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) setDescriptor(ctx context.Context, conn redis.Conn, dgst digest.Digest, desc distribution.Descriptor) error {
+	if _, err := conn.Do("SADD", rsrbds.repositoryBlobSetKey(rsrbds.repo), dgst); err != nil {
+		return err
+	}
+
+	if err := rsrbds.upstream.setDescriptor(ctx, conn, dgst, desc); err != nil {
+		return err
+	}
+
+	// Override repository mediatype.
+	if _, err := conn.Do("HSET", rsrbds.blobDescriptorHashKey(dgst), "mediatype", desc.MediaType); err != nil {
+		return err
+	}
+
+	// Also set the values for the primary descriptor, if they differ by
+	// algorithm (ie sha256 vs sha512).
+	if desc.Digest != "" && dgst != desc.Digest && dgst.Algorithm() != desc.Digest.Algorithm() {
+		if err := rsrbds.setDescriptor(ctx, conn, desc.Digest, desc); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) blobDescriptorHashKey(dgst digest.Digest) string {
+	return "repository::" + rsrbds.repo + "::blobs::" + dgst.String()
+}
+
+func (rsrbds *repositoryScopedRedisBlobDescriptorService) repositoryBlobSetKey(repo string) string {
+	return "repository::" + rsrbds.repo + "::blobs"
+}

vendor/github.com/docker/distribution/registry/storage/cache/redis/redis_test.go

@@ -0,0 +1,53 @@
+package redis
+
+import (
+	"flag"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/docker/distribution/registry/storage/cache/cachecheck"
+	"github.com/garyburd/redigo/redis"
+)
+
+var redisAddr string
+
+func init() {
+	flag.StringVar(&redisAddr, "test.registry.storage.cache.redis.addr", "", "configure the address of a test instance of redis")
+}
+
+// TestRedisLayerInfoCache exercises a live redis instance using the cache
+// implementation.
+func TestRedisBlobDescriptorCacheProvider(t *testing.T) {
+	if redisAddr == "" {
+		// fallback to an environement variable
+		redisAddr = os.Getenv("TEST_REGISTRY_STORAGE_CACHE_REDIS_ADDR")
+	}
+
+	if redisAddr == "" {
+		// skip if still not set
+		t.Skip("please set -test.registry.storage.cache.redis.addr to test layer info cache against redis")
+	}
+
+	pool := &redis.Pool{
+		Dial: func() (redis.Conn, error) {
+			return redis.Dial("tcp", redisAddr)
+		},
+		MaxIdle:   1,
+		MaxActive: 2,
+		TestOnBorrow: func(c redis.Conn, t time.Time) error {
+			_, err := c.Do("PING")
+			return err
+		},
+		Wait: false, // if a connection is not avialable, proceed without cache.
+	}
+
+	// Clear the database
+	conn := pool.Get()
+	if _, err := conn.Do("FLUSHDB"); err != nil {
+		t.Fatalf("unexpected error flushing redis db: %v", err)
+	}
+	conn.Close()
+
+	cachecheck.CheckBlobDescriptorCache(t, NewRedisBlobDescriptorCacheProvider(pool))
+}

vendor/github.com/docker/distribution/registry/storage/catalog.go

@@ -0,0 +1,153 @@
+package storage
+
+import (
+	"errors"
+	"io"
+	"path"
+	"strings"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// errFinishedWalk signals an early exit to the walk when the current query
+// is satisfied.
+var errFinishedWalk = errors.New("finished walk")
+
+// Returns a list, or partial list, of repositories in the registry.
+// Because it's a quite expensive operation, it should only be used when building up
+// an initial set of repositories.
+func (reg *registry) Repositories(ctx context.Context, repos []string, last string) (n int, err error) {
+	var foundRepos []string
+
+	if len(repos) == 0 {
+		return 0, errors.New("no space in slice")
+	}
+
+	root, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return 0, err
+	}
+
+	err = Walk(ctx, reg.blobStore.driver, root, func(fileInfo driver.FileInfo) error {
+		err := handleRepository(fileInfo, root, last, func(repoPath string) error {
+			foundRepos = append(foundRepos, repoPath)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		// if we've filled our array, no need to walk any further
+		if len(foundRepos) == len(repos) {
+			return errFinishedWalk
+		}
+
+		return nil
+	})
+
+	n = copy(repos, foundRepos)
+
+	switch err {
+	case nil:
+		// nil means that we completed walk and didn't fill buffer. No more
+		// records are available.
+		err = io.EOF
+	case errFinishedWalk:
+		// more records are available.
+		err = nil
+	}
+
+	return n, err
+}
+
+// Enumerate applies ingester to each repository
+func (reg *registry) Enumerate(ctx context.Context, ingester func(string) error) error {
+	root, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return err
+	}
+
+	err = Walk(ctx, reg.blobStore.driver, root, func(fileInfo driver.FileInfo) error {
+		return handleRepository(fileInfo, root, "", ingester)
+	})
+
+	return err
+}
+
+// lessPath returns true if one path a is less than path b.
+//
+// A component-wise comparison is done, rather than the lexical comparison of
+// strings.
+func lessPath(a, b string) bool {
+	// we provide this behavior by making separator always sort first.
+	return compareReplaceInline(a, b, '/', '\x00') < 0
+}
+
+// compareReplaceInline modifies runtime.cmpstring to replace old with new
+// during a byte-wise comparison.
+func compareReplaceInline(s1, s2 string, old, new byte) int {
+	// TODO(stevvooe): We are missing an optimization when the s1 and s2 have
+	// the exact same slice header. It will make the code unsafe but can
+	// provide some extra performance.
+
+	l := len(s1)
+	if len(s2) < l {
+		l = len(s2)
+	}
+
+	for i := 0; i < l; i++ {
+		c1, c2 := s1[i], s2[i]
+		if c1 == old {
+			c1 = new
+		}
+
+		if c2 == old {
+			c2 = new
+		}
+
+		if c1 < c2 {
+			return -1
+		}
+
+		if c1 > c2 {
+			return +1
+		}
+	}
+
+	if len(s1) < len(s2) {
+		return -1
+	}
+
+	if len(s1) > len(s2) {
+		return +1
+	}
+
+	return 0
+}
+
+// handleRepository calls function fn with a repository path if fileInfo
+// has a path of a repository under root and that it is lexographically
+// after last. Otherwise, it will return ErrSkipDir. This should be used
+// with Walk to do handling with repositories in a storage.
+func handleRepository(fileInfo driver.FileInfo, root, last string, fn func(repoPath string) error) error {
+	filePath := fileInfo.Path()
+
+	// lop the base path off
+	repo := filePath[len(root)+1:]
+
+	_, file := path.Split(repo)
+	if file == "_layers" {
+		repo = strings.TrimSuffix(repo, "/_layers")
+		if lessPath(last, repo) {
+			if err := fn(repo); err != nil {
+				return err
+			}
+		}
+		return ErrSkipDir
+	} else if strings.HasPrefix(file, "_") {
+		return ErrSkipDir
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/catalog_test.go

@@ -0,0 +1,324 @@
+package storage
+
+import (
+	"fmt"
+	"io"
+	"math/rand"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/cache/memory"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/testutil"
+)
+
+type setupEnv struct {
+	ctx      context.Context
+	driver   driver.StorageDriver
+	expected []string
+	registry distribution.Namespace
+}
+
+func setupFS(t *testing.T) *setupEnv {
+	d := inmemory.New()
+	ctx := context.Background()
+	registry, err := NewRegistry(ctx, d, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
+	if err != nil {
+		t.Fatalf("error creating registry: %v", err)
+	}
+
+	repos := []string{
+		"foo/a",
+		"foo/b",
+		"foo-bar/a",
+		"bar/c",
+		"bar/d",
+		"bar/e",
+		"foo/d/in",
+		"foo-bar/b",
+		"test",
+	}
+
+	for _, repo := range repos {
+		makeRepo(ctx, t, repo, registry)
+	}
+
+	expected := []string{
+		"bar/c",
+		"bar/d",
+		"bar/e",
+		"foo/a",
+		"foo/b",
+		"foo/d/in",
+		"foo-bar/a",
+		"foo-bar/b",
+		"test",
+	}
+
+	return &setupEnv{
+		ctx:      ctx,
+		driver:   d,
+		expected: expected,
+		registry: registry,
+	}
+}
+
+func makeRepo(ctx context.Context, t *testing.T, name string, reg distribution.Namespace) {
+	named, err := reference.ParseNamed(name)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	repo, _ := reg.Repository(ctx, named)
+	manifests, _ := repo.Manifests(ctx)
+
+	layers, err := testutil.CreateRandomLayers(1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = testutil.UploadBlobs(repo, layers)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	getKeys := func(digests map[digest.Digest]io.ReadSeeker) (ds []digest.Digest) {
+		for d := range digests {
+			ds = append(ds, d)
+		}
+		return
+	}
+
+	manifest, err := testutil.MakeSchema1Manifest(getKeys(layers))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = manifests.Put(ctx, manifest)
+	if err != nil {
+		t.Fatalf("manifest upload failed: %v", err)
+	}
+
+}
+
+func TestCatalog(t *testing.T) {
+	env := setupFS(t)
+
+	p := make([]string, 50)
+
+	numFilled, err := env.registry.Repositories(env.ctx, p, "")
+	if numFilled != len(env.expected) {
+		t.Errorf("missing items in catalog")
+	}
+
+	if !testEq(p, env.expected, len(env.expected)) {
+		t.Errorf("Expected catalog repos err")
+	}
+
+	if err != io.EOF {
+		t.Errorf("Catalog has more values which we aren't expecting")
+	}
+}
+
+func TestCatalogInParts(t *testing.T) {
+	env := setupFS(t)
+
+	chunkLen := 3
+	p := make([]string, chunkLen)
+
+	numFilled, err := env.registry.Repositories(env.ctx, p, "")
+	if err == io.EOF || numFilled != len(p) {
+		t.Errorf("Expected more values in catalog")
+	}
+
+	if !testEq(p, env.expected[0:chunkLen], numFilled) {
+		t.Errorf("Expected catalog first chunk err")
+	}
+
+	lastRepo := p[len(p)-1]
+	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)
+
+	if err == io.EOF || numFilled != len(p) {
+		t.Errorf("Expected more values in catalog")
+	}
+
+	if !testEq(p, env.expected[chunkLen:chunkLen*2], numFilled) {
+		t.Errorf("Expected catalog second chunk err")
+	}
+
+	lastRepo = p[len(p)-1]
+	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)
+
+	if err != io.EOF || numFilled != len(p) {
+		t.Errorf("Expected end of catalog")
+	}
+
+	if !testEq(p, env.expected[chunkLen*2:chunkLen*3], numFilled) {
+		t.Errorf("Expected catalog third chunk err")
+	}
+
+	lastRepo = p[len(p)-1]
+	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)
+
+	if err != io.EOF {
+		t.Errorf("Catalog has more values which we aren't expecting")
+	}
+
+	if numFilled != 0 {
+		t.Errorf("Expected catalog fourth chunk err")
+	}
+}
+
+func TestCatalogEnumerate(t *testing.T) {
+	env := setupFS(t)
+
+	var repos []string
+	repositoryEnumerator := env.registry.(distribution.RepositoryEnumerator)
+	err := repositoryEnumerator.Enumerate(env.ctx, func(repoName string) error {
+		repos = append(repos, repoName)
+		return nil
+	})
+	if err != nil {
+		t.Errorf("Expected catalog enumerate err")
+	}
+
+	if len(repos) != len(env.expected) {
+		t.Errorf("Expected catalog enumerate doesn't have correct number of values")
+	}
+
+	if !testEq(repos, env.expected, len(env.expected)) {
+		t.Errorf("Expected catalog enumerate not over all values")
+	}
+}
+
+func testEq(a, b []string, size int) bool {
+	for cnt := 0; cnt < size-1; cnt++ {
+		if a[cnt] != b[cnt] {
+			return false
+		}
+	}
+	return true
+}
+
+func setupBadWalkEnv(t *testing.T) *setupEnv {
+	d := newBadListDriver()
+	ctx := context.Background()
+	registry, err := NewRegistry(ctx, d, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
+	if err != nil {
+		t.Fatalf("error creating registry: %v", err)
+	}
+
+	return &setupEnv{
+		ctx:      ctx,
+		driver:   d,
+		registry: registry,
+	}
+}
+
+type badListDriver struct {
+	driver.StorageDriver
+}
+
+var _ driver.StorageDriver = &badListDriver{}
+
+func newBadListDriver() *badListDriver {
+	return &badListDriver{StorageDriver: inmemory.New()}
+}
+
+func (d *badListDriver) List(ctx context.Context, path string) ([]string, error) {
+	return nil, fmt.Errorf("List error")
+}
+
+func TestCatalogWalkError(t *testing.T) {
+	env := setupBadWalkEnv(t)
+	p := make([]string, 1)
+
+	_, err := env.registry.Repositories(env.ctx, p, "")
+	if err == io.EOF {
+		t.Errorf("Expected catalog driver list error")
+	}
+}
+
+func BenchmarkPathCompareEqual(B *testing.B) {
+	B.StopTimer()
+	pp := randomPath(100)
+	// make a real copy
+	ppb := append([]byte{}, []byte(pp)...)
+	a, b := pp, string(ppb)
+
+	B.StartTimer()
+	for i := 0; i < B.N; i++ {
+		lessPath(a, b)
+	}
+}
+
+func BenchmarkPathCompareNotEqual(B *testing.B) {
+	B.StopTimer()
+	a, b := randomPath(100), randomPath(100)
+	B.StartTimer()
+
+	for i := 0; i < B.N; i++ {
+		lessPath(a, b)
+	}
+}
+
+func BenchmarkPathCompareNative(B *testing.B) {
+	B.StopTimer()
+	a, b := randomPath(100), randomPath(100)
+	B.StartTimer()
+
+	for i := 0; i < B.N; i++ {
+		c := a < b
+		c = c && false
+	}
+}
+
+func BenchmarkPathCompareNativeEqual(B *testing.B) {
+	B.StopTimer()
+	pp := randomPath(100)
+	a, b := pp, pp
+	B.StartTimer()
+
+	for i := 0; i < B.N; i++ {
+		c := a < b
+		c = c && false
+	}
+}
+
+var filenameChars = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
+var separatorChars = []byte("._-")
+
+func randomPath(length int64) string {
+	path := "/"
+	for int64(len(path)) < length {
+		chunkLength := rand.Int63n(length-int64(len(path))) + 1
+		chunk := randomFilename(chunkLength)
+		path += chunk
+		remaining := length - int64(len(path))
+		if remaining == 1 {
+			path += randomFilename(1)
+		} else if remaining > 1 {
+			path += "/"
+		}
+	}
+	return path
+}
+
+func randomFilename(length int64) string {
+	b := make([]byte, length)
+	wasSeparator := true
+	for i := range b {
+		if !wasSeparator && i < len(b)-1 && rand.Intn(4) == 0 {
+			b[i] = separatorChars[rand.Intn(len(separatorChars))]
+			wasSeparator = true
+		} else {
+			b[i] = filenameChars[rand.Intn(len(filenameChars))]
+			wasSeparator = false
+		}
+	}
+	return string(b)
+}

vendor/github.com/docker/distribution/registry/storage/doc.go

@@ -0,0 +1,3 @@
+// Package storage contains storage services for use in the registry
+// application. It should be considered an internal package, as of Go 1.4.
+package storage

vendor/github.com/docker/distribution/registry/storage/driver/azure/azure.go

@@ -0,0 +1,483 @@
+// Package azure provides a storagedriver.StorageDriver implementation to
+// store blobs in Microsoft Azure Blob Storage Service.
+package azure
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/base"
+	"github.com/docker/distribution/registry/storage/driver/factory"
+
+	azure "github.com/Azure/azure-sdk-for-go/storage"
+)
+
+const driverName = "azure"
+
+const (
+	paramAccountName = "accountname"
+	paramAccountKey  = "accountkey"
+	paramContainer   = "container"
+	paramRealm       = "realm"
+	maxChunkSize     = 4 * 1024 * 1024
+)
+
+type driver struct {
+	client    azure.BlobStorageClient
+	container string
+}
+
+type baseEmbed struct{ base.Base }
+
+// Driver is a storagedriver.StorageDriver implementation backed by
+// Microsoft Azure Blob Storage Service.
+type Driver struct{ baseEmbed }
+
+func init() {
+	factory.Register(driverName, &azureDriverFactory{})
+}
+
+type azureDriverFactory struct{}
+
+func (factory *azureDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
+	return FromParameters(parameters)
+}
+
+// FromParameters constructs a new Driver with a given parameters map.
+func FromParameters(parameters map[string]interface{}) (*Driver, error) {
+	accountName, ok := parameters[paramAccountName]
+	if !ok || fmt.Sprint(accountName) == "" {
+		return nil, fmt.Errorf("No %s parameter provided", paramAccountName)
+	}
+
+	accountKey, ok := parameters[paramAccountKey]
+	if !ok || fmt.Sprint(accountKey) == "" {
+		return nil, fmt.Errorf("No %s parameter provided", paramAccountKey)
+	}
+
+	container, ok := parameters[paramContainer]
+	if !ok || fmt.Sprint(container) == "" {
+		return nil, fmt.Errorf("No %s parameter provided", paramContainer)
+	}
+
+	realm, ok := parameters[paramRealm]
+	if !ok || fmt.Sprint(realm) == "" {
+		realm = azure.DefaultBaseURL
+	}
+
+	return New(fmt.Sprint(accountName), fmt.Sprint(accountKey), fmt.Sprint(container), fmt.Sprint(realm))
+}
+
+// New constructs a new Driver with the given Azure Storage Account credentials
+func New(accountName, accountKey, container, realm string) (*Driver, error) {
+	api, err := azure.NewClient(accountName, accountKey, realm, azure.DefaultAPIVersion, true)
+	if err != nil {
+		return nil, err
+	}
+
+	blobClient := api.GetBlobService()
+
+	// Create registry container
+	if _, err = blobClient.CreateContainerIfNotExists(container, azure.ContainerAccessTypePrivate); err != nil {
+		return nil, err
+	}
+
+	d := &driver{
+		client:    blobClient,
+		container: container}
+	return &Driver{baseEmbed: baseEmbed{Base: base.Base{StorageDriver: d}}}, nil
+}
+
+// Implement the storagedriver.StorageDriver interface.
+func (d *driver) Name() string {
+	return driverName
+}
+
+// GetContent retrieves the content stored at "path" as a []byte.
+func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
+	blob, err := d.client.GetBlob(d.container, path)
+	if err != nil {
+		if is404(err) {
+			return nil, storagedriver.PathNotFoundError{Path: path}
+		}
+		return nil, err
+	}
+
+	defer blob.Close()
+	return ioutil.ReadAll(blob)
+}
+
+// PutContent stores the []byte content at a location designated by "path".
+func (d *driver) PutContent(ctx context.Context, path string, contents []byte) error {
+	if _, err := d.client.DeleteBlobIfExists(d.container, path, nil); err != nil {
+		return err
+	}
+	writer, err := d.Writer(ctx, path, false)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+	_, err = writer.Write(contents)
+	if err != nil {
+		return err
+	}
+	return writer.Commit()
+}
+
+// Reader retrieves an io.ReadCloser for the content stored at "path" with a
+// given byte offset.
+func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
+	if ok, err := d.client.BlobExists(d.container, path); err != nil {
+		return nil, err
+	} else if !ok {
+		return nil, storagedriver.PathNotFoundError{Path: path}
+	}
+
+	info, err := d.client.GetBlobProperties(d.container, path)
+	if err != nil {
+		return nil, err
+	}
+
+	size := int64(info.ContentLength)
+	if offset >= size {
+		return ioutil.NopCloser(bytes.NewReader(nil)), nil
+	}
+
+	bytesRange := fmt.Sprintf("%v-", offset)
+	resp, err := d.client.GetBlobRange(d.container, path, bytesRange, nil)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+// Writer returns a FileWriter which will store the content written to it
+// at the location designated by "path" after the call to Commit.
+func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
+	blobExists, err := d.client.BlobExists(d.container, path)
+	if err != nil {
+		return nil, err
+	}
+	var size int64
+	if blobExists {
+		if append {
+			blobProperties, err := d.client.GetBlobProperties(d.container, path)
+			if err != nil {
+				return nil, err
+			}
+			size = blobProperties.ContentLength
+		} else {
+			err := d.client.DeleteBlob(d.container, path, nil)
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		if append {
+			return nil, storagedriver.PathNotFoundError{Path: path}
+		}
+		err := d.client.PutAppendBlob(d.container, path, nil)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return d.newWriter(path, size), nil
+}
+
+// Stat retrieves the FileInfo for the given path, including the current size
+// in bytes and the creation time.
+func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
+	// Check if the path is a blob
+	if ok, err := d.client.BlobExists(d.container, path); err != nil {
+		return nil, err
+	} else if ok {
+		blob, err := d.client.GetBlobProperties(d.container, path)
+		if err != nil {
+			return nil, err
+		}
+
+		mtim, err := time.Parse(http.TimeFormat, blob.LastModified)
+		if err != nil {
+			return nil, err
+		}
+
+		return storagedriver.FileInfoInternal{FileInfoFields: storagedriver.FileInfoFields{
+			Path:    path,
+			Size:    int64(blob.ContentLength),
+			ModTime: mtim,
+			IsDir:   false,
+		}}, nil
+	}
+
+	// Check if path is a virtual container
+	virtContainerPath := path
+	if !strings.HasSuffix(virtContainerPath, "/") {
+		virtContainerPath += "/"
+	}
+	blobs, err := d.client.ListBlobs(d.container, azure.ListBlobsParameters{
+		Prefix:     virtContainerPath,
+		MaxResults: 1,
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(blobs.Blobs) > 0 {
+		// path is a virtual container
+		return storagedriver.FileInfoInternal{FileInfoFields: storagedriver.FileInfoFields{
+			Path:  path,
+			IsDir: true,
+		}}, nil
+	}
+
+	// path is not a blob or virtual container
+	return nil, storagedriver.PathNotFoundError{Path: path}
+}
+
+// List returns a list of the objects that are direct descendants of the given
+// path.
+func (d *driver) List(ctx context.Context, path string) ([]string, error) {
+	if path == "/" {
+		path = ""
+	}
+
+	blobs, err := d.listBlobs(d.container, path)
+	if err != nil {
+		return blobs, err
+	}
+
+	list := directDescendants(blobs, path)
+	if path != "" && len(list) == 0 {
+		return nil, storagedriver.PathNotFoundError{Path: path}
+	}
+	return list, nil
+}
+
+// Move moves an object stored at sourcePath to destPath, removing the original
+// object.
+func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
+	sourceBlobURL := d.client.GetBlobURL(d.container, sourcePath)
+	err := d.client.CopyBlob(d.container, destPath, sourceBlobURL)
+	if err != nil {
+		if is404(err) {
+			return storagedriver.PathNotFoundError{Path: sourcePath}
+		}
+		return err
+	}
+
+	return d.client.DeleteBlob(d.container, sourcePath, nil)
+}
+
+// Delete recursively deletes all objects stored at "path" and its subpaths.
+func (d *driver) Delete(ctx context.Context, path string) error {
+	ok, err := d.client.DeleteBlobIfExists(d.container, path, nil)
+	if err != nil {
+		return err
+	}
+	if ok {
+		return nil // was a blob and deleted, return
+	}
+
+	// Not a blob, see if path is a virtual container with blobs
+	blobs, err := d.listBlobs(d.container, path)
+	if err != nil {
+		return err
+	}
+
+	for _, b := range blobs {
+		if err = d.client.DeleteBlob(d.container, b, nil); err != nil {
+			return err
+		}
+	}
+
+	if len(blobs) == 0 {
+		return storagedriver.PathNotFoundError{Path: path}
+	}
+	return nil
+}
+
+// URLFor returns a publicly accessible URL for the blob stored at given path
+// for specified duration by making use of Azure Storage Shared Access Signatures (SAS).
+// See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx for more info.
+func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	expiresTime := time.Now().UTC().Add(20 * time.Minute) // default expiration
+	expires, ok := options["expiry"]
+	if ok {
+		t, ok := expires.(time.Time)
+		if ok {
+			expiresTime = t
+		}
+	}
+	return d.client.GetBlobSASURI(d.container, path, expiresTime, "r")
+}
+
+// directDescendants will find direct descendants (blobs or virtual containers)
+// of from list of blob paths and will return their full paths. Elements in blobs
+// list must be prefixed with a "/" and
+//
+// Example: direct descendants of "/" in {"/foo", "/bar/1", "/bar/2"} is
+// {"/foo", "/bar"} and direct descendants of "bar" is {"/bar/1", "/bar/2"}
+func directDescendants(blobs []string, prefix string) []string {
+	if !strings.HasPrefix(prefix, "/") { // add trailing '/'
+		prefix = "/" + prefix
+	}
+	if !strings.HasSuffix(prefix, "/") { // containerify the path
+		prefix += "/"
+	}
+
+	out := make(map[string]bool)
+	for _, b := range blobs {
+		if strings.HasPrefix(b, prefix) {
+			rel := b[len(prefix):]
+			c := strings.Count(rel, "/")
+			if c == 0 {
+				out[b] = true
+			} else {
+				out[prefix+rel[:strings.Index(rel, "/")]] = true
+			}
+		}
+	}
+
+	var keys []string
+	for k := range out {
+		keys = append(keys, k)
+	}
+	return keys
+}
+
+func (d *driver) listBlobs(container, virtPath string) ([]string, error) {
+	if virtPath != "" && !strings.HasSuffix(virtPath, "/") { // containerify the path
+		virtPath += "/"
+	}
+
+	out := []string{}
+	marker := ""
+	for {
+		resp, err := d.client.ListBlobs(d.container, azure.ListBlobsParameters{
+			Marker: marker,
+			Prefix: virtPath,
+		})
+
+		if err != nil {
+			return out, err
+		}
+
+		for _, b := range resp.Blobs {
+			out = append(out, b.Name)
+		}
+
+		if len(resp.Blobs) == 0 || resp.NextMarker == "" {
+			break
+		}
+		marker = resp.NextMarker
+	}
+	return out, nil
+}
+
+func is404(err error) bool {
+	statusCodeErr, ok := err.(azure.AzureStorageServiceError)
+	return ok && statusCodeErr.StatusCode == http.StatusNotFound
+}
+
+type writer struct {
+	driver    *driver
+	path      string
+	size      int64
+	bw        *bufio.Writer
+	closed    bool
+	committed bool
+	cancelled bool
+}
+
+func (d *driver) newWriter(path string, size int64) storagedriver.FileWriter {
+	return &writer{
+		driver: d,
+		path:   path,
+		size:   size,
+		bw: bufio.NewWriterSize(&blockWriter{
+			client:    d.client,
+			container: d.container,
+			path:      path,
+		}, maxChunkSize),
+	}
+}
+
+func (w *writer) Write(p []byte) (int, error) {
+	if w.closed {
+		return 0, fmt.Errorf("already closed")
+	} else if w.committed {
+		return 0, fmt.Errorf("already committed")
+	} else if w.cancelled {
+		return 0, fmt.Errorf("already cancelled")
+	}
+
+	n, err := w.bw.Write(p)
+	w.size += int64(n)
+	return n, err
+}
+
+func (w *writer) Size() int64 {
+	return w.size
+}
+
+func (w *writer) Close() error {
+	if w.closed {
+		return fmt.Errorf("already closed")
+	}
+	w.closed = true
+	return w.bw.Flush()
+}
+
+func (w *writer) Cancel() error {
+	if w.closed {
+		return fmt.Errorf("already closed")
+	} else if w.committed {
+		return fmt.Errorf("already committed")
+	}
+	w.cancelled = true
+	return w.driver.client.DeleteBlob(w.driver.container, w.path, nil)
+}
+
+func (w *writer) Commit() error {
+	if w.closed {
+		return fmt.Errorf("already closed")
+	} else if w.committed {
+		return fmt.Errorf("already committed")
+	} else if w.cancelled {
+		return fmt.Errorf("already cancelled")
+	}
+	w.committed = true
+	return w.bw.Flush()
+}
+
+type blockWriter struct {
+	client    azure.BlobStorageClient
+	container string
+	path      string
+}
+
+func (bw *blockWriter) Write(p []byte) (int, error) {
+	n := 0
+	for offset := 0; offset < len(p); offset += maxChunkSize {
+		chunkSize := maxChunkSize
+		if offset+chunkSize > len(p) {
+			chunkSize = len(p) - offset
+		}
+		err := bw.client.AppendBlock(bw.container, bw.path, p[offset:offset+chunkSize], nil)
+		if err != nil {
+			return n, err
+		}
+
+		n += chunkSize
+	}
+
+	return n, nil
+}

vendor/github.com/docker/distribution/registry/storage/driver/azure/azure_test.go

@@ -0,0 +1,63 @@
+package azure
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+	. "gopkg.in/check.v1"
+)
+
+const (
+	envAccountName = "AZURE_STORAGE_ACCOUNT_NAME"
+	envAccountKey  = "AZURE_STORAGE_ACCOUNT_KEY"
+	envContainer   = "AZURE_STORAGE_CONTAINER"
+	envRealm       = "AZURE_STORAGE_REALM"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+func init() {
+	var (
+		accountName string
+		accountKey  string
+		container   string
+		realm       string
+	)
+
+	config := []struct {
+		env   string
+		value *string
+	}{
+		{envAccountName, &accountName},
+		{envAccountKey, &accountKey},
+		{envContainer, &container},
+		{envRealm, &realm},
+	}
+
+	missing := []string{}
+	for _, v := range config {
+		*v.value = os.Getenv(v.env)
+		if *v.value == "" {
+			missing = append(missing, v.env)
+		}
+	}
+
+	azureDriverConstructor := func() (storagedriver.StorageDriver, error) {
+		return New(accountName, accountKey, container, realm)
+	}
+
+	// Skip Azure storage driver tests if environment variable parameters are not provided
+	skipCheck := func() string {
+		if len(missing) > 0 {
+			return fmt.Sprintf("Must set %s environment variables to run Azure tests", strings.Join(missing, ", "))
+		}
+		return ""
+	}
+
+	testsuites.RegisterSuite(azureDriverConstructor, skipCheck)
+}

vendor/github.com/docker/distribution/registry/storage/driver/base/base.go

@@ -0,0 +1,198 @@
+// Package base provides a base implementation of the storage driver that can
+// be used to implement common checks. The goal is to increase the amount of
+// code sharing.
+//
+// The canonical approach to use this class is to embed in the exported driver
+// struct such that calls are proxied through this implementation. First,
+// declare the internal driver, as follows:
+//
+// 	type driver struct { ... internal ...}
+//
+// The resulting type should implement StorageDriver such that it can be the
+// target of a Base struct. The exported type can then be declared as follows:
+//
+// 	type Driver struct {
+// 		Base
+// 	}
+//
+// Because Driver embeds Base, it effectively implements Base. If the driver
+// needs to intercept a call, before going to base, Driver should implement
+// that method. Effectively, Driver can intercept calls before coming in and
+// driver implements the actual logic.
+//
+// To further shield the embed from other packages, it is recommended to
+// employ a private embed struct:
+//
+// 	type baseEmbed struct {
+// 		base.Base
+// 	}
+//
+// Then, declare driver to embed baseEmbed, rather than Base directly:
+//
+// 	type Driver struct {
+// 		baseEmbed
+// 	}
+//
+// The type now implements StorageDriver, proxying through Base, without
+// exporting an unnecessary field.
+package base
+
+import (
+	"io"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// Base provides a wrapper around a storagedriver implementation that provides
+// common path and bounds checking.
+type Base struct {
+	storagedriver.StorageDriver
+}
+
+// Format errors received from the storage driver
+func (base *Base) setDriverName(e error) error {
+	switch actual := e.(type) {
+	case nil:
+		return nil
+	case storagedriver.ErrUnsupportedMethod:
+		actual.DriverName = base.StorageDriver.Name()
+		return actual
+	case storagedriver.PathNotFoundError:
+		actual.DriverName = base.StorageDriver.Name()
+		return actual
+	case storagedriver.InvalidPathError:
+		actual.DriverName = base.StorageDriver.Name()
+		return actual
+	case storagedriver.InvalidOffsetError:
+		actual.DriverName = base.StorageDriver.Name()
+		return actual
+	default:
+		storageError := storagedriver.Error{
+			DriverName: base.StorageDriver.Name(),
+			Enclosed:   e,
+		}
+
+		return storageError
+	}
+}
+
+// GetContent wraps GetContent of underlying storage driver.
+func (base *Base) GetContent(ctx context.Context, path string) ([]byte, error) {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.GetContent(%q)", base.Name(), path)
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	b, e := base.StorageDriver.GetContent(ctx, path)
+	return b, base.setDriverName(e)
+}
+
+// PutContent wraps PutContent of underlying storage driver.
+func (base *Base) PutContent(ctx context.Context, path string, content []byte) error {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.PutContent(%q)", base.Name(), path)
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	return base.setDriverName(base.StorageDriver.PutContent(ctx, path, content))
+}
+
+// Reader wraps Reader of underlying storage driver.
+func (base *Base) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.Reader(%q, %d)", base.Name(), path, offset)
+
+	if offset < 0 {
+		return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset, DriverName: base.StorageDriver.Name()}
+	}
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	rc, e := base.StorageDriver.Reader(ctx, path, offset)
+	return rc, base.setDriverName(e)
+}
+
+// Writer wraps Writer of underlying storage driver.
+func (base *Base) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.Writer(%q, %v)", base.Name(), path, append)
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	writer, e := base.StorageDriver.Writer(ctx, path, append)
+	return writer, base.setDriverName(e)
+}
+
+// Stat wraps Stat of underlying storage driver.
+func (base *Base) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.Stat(%q)", base.Name(), path)
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	fi, e := base.StorageDriver.Stat(ctx, path)
+	return fi, base.setDriverName(e)
+}
+
+// List wraps List of underlying storage driver.
+func (base *Base) List(ctx context.Context, path string) ([]string, error) {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.List(%q)", base.Name(), path)
+
+	if !storagedriver.PathRegexp.MatchString(path) && path != "/" {
+		return nil, storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	str, e := base.StorageDriver.List(ctx, path)
+	return str, base.setDriverName(e)
+}
+
+// Move wraps Move of underlying storage driver.
+func (base *Base) Move(ctx context.Context, sourcePath string, destPath string) error {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.Move(%q, %q", base.Name(), sourcePath, destPath)
+
+	if !storagedriver.PathRegexp.MatchString(sourcePath) {
+		return storagedriver.InvalidPathError{Path: sourcePath, DriverName: base.StorageDriver.Name()}
+	} else if !storagedriver.PathRegexp.MatchString(destPath) {
+		return storagedriver.InvalidPathError{Path: destPath, DriverName: base.StorageDriver.Name()}
+	}
+
+	return base.setDriverName(base.StorageDriver.Move(ctx, sourcePath, destPath))
+}
+
+// Delete wraps Delete of underlying storage driver.
+func (base *Base) Delete(ctx context.Context, path string) error {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.Delete(%q)", base.Name(), path)
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	return base.setDriverName(base.StorageDriver.Delete(ctx, path))
+}
+
+// URLFor wraps URLFor of underlying storage driver.
+func (base *Base) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	ctx, done := context.WithTrace(ctx)
+	defer done("%s.URLFor(%q)", base.Name(), path)
+
+	if !storagedriver.PathRegexp.MatchString(path) {
+		return "", storagedriver.InvalidPathError{Path: path, DriverName: base.StorageDriver.Name()}
+	}
+
+	str, e := base.StorageDriver.URLFor(ctx, path, options)
+	return str, base.setDriverName(e)
+}

vendor/github.com/docker/distribution/registry/storage/driver/base/regulator.go

@@ -0,0 +1,145 @@
+package base
+
+import (
+	"io"
+	"sync"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+type regulator struct {
+	storagedriver.StorageDriver
+	*sync.Cond
+
+	available uint64
+}
+
+// NewRegulator wraps the given driver and is used to regulate concurrent calls
+// to the given storage driver to a maximum of the given limit. This is useful
+// for storage drivers that would otherwise create an unbounded number of OS
+// threads if allowed to be called unregulated.
+func NewRegulator(driver storagedriver.StorageDriver, limit uint64) storagedriver.StorageDriver {
+	return &regulator{
+		StorageDriver: driver,
+		Cond:          sync.NewCond(&sync.Mutex{}),
+		available:     limit,
+	}
+}
+
+func (r *regulator) enter() {
+	r.L.Lock()
+	for r.available == 0 {
+		r.Wait()
+	}
+	r.available--
+	r.L.Unlock()
+}
+
+func (r *regulator) exit() {
+	r.L.Lock()
+	// We only need to signal to a waiting FS operation if we're already at the
+	// limit of threads used
+	if r.available == 0 {
+		r.Signal()
+	}
+	r.available++
+	r.L.Unlock()
+}
+
+// Name returns the human-readable "name" of the driver, useful in error
+// messages and logging. By convention, this will just be the registration
+// name, but drivers may provide other information here.
+func (r *regulator) Name() string {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.Name()
+}
+
+// GetContent retrieves the content stored at "path" as a []byte.
+// This should primarily be used for small objects.
+func (r *regulator) GetContent(ctx context.Context, path string) ([]byte, error) {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.GetContent(ctx, path)
+}
+
+// PutContent stores the []byte content at a location designated by "path".
+// This should primarily be used for small objects.
+func (r *regulator) PutContent(ctx context.Context, path string, content []byte) error {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.PutContent(ctx, path, content)
+}
+
+// Reader retrieves an io.ReadCloser for the content stored at "path"
+// with a given byte offset.
+// May be used to resume reading a stream by providing a nonzero offset.
+func (r *regulator) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.Reader(ctx, path, offset)
+}
+
+// Writer stores the contents of the provided io.ReadCloser at a
+// location designated by the given path.
+// May be used to resume writing a stream by providing a nonzero offset.
+// The offset must be no larger than the CurrentSize for this path.
+func (r *regulator) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.Writer(ctx, path, append)
+}
+
+// Stat retrieves the FileInfo for the given path, including the current
+// size in bytes and the creation time.
+func (r *regulator) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.Stat(ctx, path)
+}
+
+// List returns a list of the objects that are direct descendants of the
+//given path.
+func (r *regulator) List(ctx context.Context, path string) ([]string, error) {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.List(ctx, path)
+}
+
+// Move moves an object stored at sourcePath to destPath, removing the
+// original object.
+// Note: This may be no more efficient than a copy followed by a delete for
+// many implementations.
+func (r *regulator) Move(ctx context.Context, sourcePath string, destPath string) error {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.Move(ctx, sourcePath, destPath)
+}
+
+// Delete recursively deletes all objects stored at "path" and its subpaths.
+func (r *regulator) Delete(ctx context.Context, path string) error {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.Delete(ctx, path)
+}
+
+// URLFor returns a URL which may be used to retrieve the content stored at
+// the given path, possibly using the given options.
+// May return an ErrUnsupportedMethod in certain StorageDriver
+// implementations.
+func (r *regulator) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	r.enter()
+	defer r.exit()
+
+	return r.StorageDriver.URLFor(ctx, path, options)
+}

vendor/github.com/docker/distribution/registry/storage/driver/factory/factory.go

@@ -0,0 +1,64 @@
+package factory
+
+import (
+	"fmt"
+
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// driverFactories stores an internal mapping between storage driver names and their respective
+// factories
+var driverFactories = make(map[string]StorageDriverFactory)
+
+// StorageDriverFactory is a factory interface for creating storagedriver.StorageDriver interfaces
+// Storage drivers should call Register() with a factory to make the driver available by name.
+// Individual StorageDriver implementations generally register with the factory via the Register
+// func (below) in their init() funcs, and as such they should be imported anonymously before use.
+// See below for an example of how to register and get a StorageDriver for S3
+//
+//	import _ "github.com/docker/distribution/registry/storage/driver/s3-aws"
+//	s3Driver, err = factory.Create("s3", storageParams)
+//	// assuming no error, s3Driver is the StorageDriver that communicates with S3 according to storageParams
+type StorageDriverFactory interface {
+	// Create returns a new storagedriver.StorageDriver with the given parameters
+	// Parameters will vary by driver and may be ignored
+	// Each parameter key must only consist of lowercase letters and numbers
+	Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error)
+}
+
+// Register makes a storage driver available by the provided name.
+// If Register is called twice with the same name or if driver factory is nil, it panics.
+// Additionally, it is not concurrency safe. Most Storage Drivers call this function
+// in their init() functions. See the documentation for StorageDriverFactory for more.
+func Register(name string, factory StorageDriverFactory) {
+	if factory == nil {
+		panic("Must not provide nil StorageDriverFactory")
+	}
+	_, registered := driverFactories[name]
+	if registered {
+		panic(fmt.Sprintf("StorageDriverFactory named %s already registered", name))
+	}
+
+	driverFactories[name] = factory
+}
+
+// Create a new storagedriver.StorageDriver with the given name and
+// parameters. To use a driver, the StorageDriverFactory must first be
+// registered with the given name. If no drivers are found, an
+// InvalidStorageDriverError is returned
+func Create(name string, parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
+	driverFactory, ok := driverFactories[name]
+	if !ok {
+		return nil, InvalidStorageDriverError{name}
+	}
+	return driverFactory.Create(parameters)
+}
+
+// InvalidStorageDriverError records an attempt to construct an unregistered storage driver
+type InvalidStorageDriverError struct {
+	Name string
+}
+
+func (err InvalidStorageDriverError) Error() string {
+	return fmt.Sprintf("StorageDriver not registered: %s", err.Name)
+}

vendor/github.com/docker/distribution/registry/storage/driver/fileinfo.go

@@ -0,0 +1,79 @@
+package driver
+
+import "time"
+
+// FileInfo returns information about a given path. Inspired by os.FileInfo,
+// it elides the base name method for a full path instead.
+type FileInfo interface {
+	// Path provides the full path of the target of this file info.
+	Path() string
+
+	// Size returns current length in bytes of the file. The return value can
+	// be used to write to the end of the file at path. The value is
+	// meaningless if IsDir returns true.
+	Size() int64
+
+	// ModTime returns the modification time for the file. For backends that
+	// don't have a modification time, the creation time should be returned.
+	ModTime() time.Time
+
+	// IsDir returns true if the path is a directory.
+	IsDir() bool
+}
+
+// NOTE(stevvooe): The next two types, FileInfoFields and FileInfoInternal
+// should only be used by storagedriver implementations. They should moved to
+// a "driver" package, similar to database/sql.
+
+// FileInfoFields provides the exported fields for implementing FileInfo
+// interface in storagedriver implementations. It should be used with
+// InternalFileInfo.
+type FileInfoFields struct {
+	// Path provides the full path of the target of this file info.
+	Path string
+
+	// Size is current length in bytes of the file. The value of this field
+	// can be used to write to the end of the file at path. The value is
+	// meaningless if IsDir is set to true.
+	Size int64
+
+	// ModTime returns the modification time for the file. For backends that
+	// don't have a modification time, the creation time should be returned.
+	ModTime time.Time
+
+	// IsDir returns true if the path is a directory.
+	IsDir bool
+}
+
+// FileInfoInternal implements the FileInfo interface. This should only be
+// used by storagedriver implementations that don't have a specialized
+// FileInfo type.
+type FileInfoInternal struct {
+	FileInfoFields
+}
+
+var _ FileInfo = FileInfoInternal{}
+var _ FileInfo = &FileInfoInternal{}
+
+// Path provides the full path of the target of this file info.
+func (fi FileInfoInternal) Path() string {
+	return fi.FileInfoFields.Path
+}
+
+// Size returns current length in bytes of the file. The return value can
+// be used to write to the end of the file at path. The value is
+// meaningless if IsDir returns true.
+func (fi FileInfoInternal) Size() int64 {
+	return fi.FileInfoFields.Size
+}
+
+// ModTime returns the modification time for the file. For backends that
+// don't have a modification time, the creation time should be returned.
+func (fi FileInfoInternal) ModTime() time.Time {
+	return fi.FileInfoFields.ModTime
+}
+
+// IsDir returns true if the path is a directory.
+func (fi FileInfoInternal) IsDir() bool {
+	return fi.FileInfoFields.IsDir
+}

vendor/github.com/docker/distribution/registry/storage/driver/filesystem/driver.go

@@ -0,0 +1,440 @@
+package filesystem
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"reflect"
+	"strconv"
+	"time"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/base"
+	"github.com/docker/distribution/registry/storage/driver/factory"
+)
+
+const (
+	driverName           = "filesystem"
+	defaultRootDirectory = "/var/lib/registry"
+	defaultMaxThreads    = uint64(100)
+
+	// minThreads is the minimum value for the maxthreads configuration
+	// parameter. If the driver's parameters are less than this we set
+	// the parameters to minThreads
+	minThreads = uint64(25)
+)
+
+// DriverParameters represents all configuration options available for the
+// filesystem driver
+type DriverParameters struct {
+	RootDirectory string
+	MaxThreads    uint64
+}
+
+func init() {
+	factory.Register(driverName, &filesystemDriverFactory{})
+}
+
+// filesystemDriverFactory implements the factory.StorageDriverFactory interface
+type filesystemDriverFactory struct{}
+
+func (factory *filesystemDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
+	return FromParameters(parameters)
+}
+
+type driver struct {
+	rootDirectory string
+}
+
+type baseEmbed struct {
+	base.Base
+}
+
+// Driver is a storagedriver.StorageDriver implementation backed by a local
+// filesystem. All provided paths will be subpaths of the RootDirectory.
+type Driver struct {
+	baseEmbed
+}
+
+// FromParameters constructs a new Driver with a given parameters map
+// Optional Parameters:
+// - rootdirectory
+// - maxthreads
+func FromParameters(parameters map[string]interface{}) (*Driver, error) {
+	params, err := fromParametersImpl(parameters)
+	if err != nil || params == nil {
+		return nil, err
+	}
+	return New(*params), nil
+}
+
+func fromParametersImpl(parameters map[string]interface{}) (*DriverParameters, error) {
+	var (
+		err           error
+		maxThreads    = defaultMaxThreads
+		rootDirectory = defaultRootDirectory
+	)
+
+	if parameters != nil {
+		if rootDir, ok := parameters["rootdirectory"]; ok {
+			rootDirectory = fmt.Sprint(rootDir)
+		}
+
+		// Get maximum number of threads for blocking filesystem operations,
+		// if specified
+		threads := parameters["maxthreads"]
+		switch v := threads.(type) {
+		case string:
+			if maxThreads, err = strconv.ParseUint(v, 0, 64); err != nil {
+				return nil, fmt.Errorf("maxthreads parameter must be an integer, %v invalid", threads)
+			}
+		case uint64:
+			maxThreads = v
+		case int, int32, int64:
+			val := reflect.ValueOf(v).Convert(reflect.TypeOf(threads)).Int()
+			// If threads is negative casting to uint64 will wrap around and
+			// give you the hugest thread limit ever. Let's be sensible, here
+			if val > 0 {
+				maxThreads = uint64(val)
+			}
+		case uint, uint32:
+			maxThreads = reflect.ValueOf(v).Convert(reflect.TypeOf(threads)).Uint()
+		case nil:
+			// do nothing
+		default:
+			return nil, fmt.Errorf("invalid value for maxthreads: %#v", threads)
+		}
+
+		if maxThreads < minThreads {
+			maxThreads = minThreads
+		}
+	}
+
+	params := &DriverParameters{
+		RootDirectory: rootDirectory,
+		MaxThreads:    maxThreads,
+	}
+	return params, nil
+}
+
+// New constructs a new Driver with a given rootDirectory
+func New(params DriverParameters) *Driver {
+	fsDriver := &driver{rootDirectory: params.RootDirectory}
+
+	return &Driver{
+		baseEmbed: baseEmbed{
+			Base: base.Base{
+				StorageDriver: base.NewRegulator(fsDriver, params.MaxThreads),
+			},
+		},
+	}
+}
+
+// Implement the storagedriver.StorageDriver interface
+
+func (d *driver) Name() string {
+	return driverName
+}
+
+// GetContent retrieves the content stored at "path" as a []byte.
+func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
+	rc, err := d.Reader(ctx, path, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+
+	p, err := ioutil.ReadAll(rc)
+	if err != nil {
+		return nil, err
+	}
+
+	return p, nil
+}
+
+// PutContent stores the []byte content at a location designated by "path".
+func (d *driver) PutContent(ctx context.Context, subPath string, contents []byte) error {
+	writer, err := d.Writer(ctx, subPath, false)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+	_, err = io.Copy(writer, bytes.NewReader(contents))
+	if err != nil {
+		writer.Cancel()
+		return err
+	}
+	return writer.Commit()
+}
+
+// Reader retrieves an io.ReadCloser for the content stored at "path" with a
+// given byte offset.
+func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
+	file, err := os.OpenFile(d.fullPath(path), os.O_RDONLY, 0644)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, storagedriver.PathNotFoundError{Path: path}
+		}
+
+		return nil, err
+	}
+
+	seekPos, err := file.Seek(int64(offset), os.SEEK_SET)
+	if err != nil {
+		file.Close()
+		return nil, err
+	} else if seekPos < int64(offset) {
+		file.Close()
+		return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
+	}
+
+	return file, nil
+}
+
+func (d *driver) Writer(ctx context.Context, subPath string, append bool) (storagedriver.FileWriter, error) {
+	fullPath := d.fullPath(subPath)
+	parentDir := path.Dir(fullPath)
+	if err := os.MkdirAll(parentDir, 0777); err != nil {
+		return nil, err
+	}
+
+	fp, err := os.OpenFile(fullPath, os.O_WRONLY|os.O_CREATE, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	var offset int64
+
+	if !append {
+		err := fp.Truncate(0)
+		if err != nil {
+			fp.Close()
+			return nil, err
+		}
+	} else {
+		n, err := fp.Seek(0, os.SEEK_END)
+		if err != nil {
+			fp.Close()
+			return nil, err
+		}
+		offset = int64(n)
+	}
+
+	return newFileWriter(fp, offset), nil
+}
+
+// Stat retrieves the FileInfo for the given path, including the current size
+// in bytes and the creation time.
+func (d *driver) Stat(ctx context.Context, subPath string) (storagedriver.FileInfo, error) {
+	fullPath := d.fullPath(subPath)
+
+	fi, err := os.Stat(fullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, storagedriver.PathNotFoundError{Path: subPath}
+		}
+
+		return nil, err
+	}
+
+	return fileInfo{
+		path:     subPath,
+		FileInfo: fi,
+	}, nil
+}
+
+// List returns a list of the objects that are direct descendants of the given
+// path.
+func (d *driver) List(ctx context.Context, subPath string) ([]string, error) {
+	fullPath := d.fullPath(subPath)
+
+	dir, err := os.Open(fullPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, storagedriver.PathNotFoundError{Path: subPath}
+		}
+		return nil, err
+	}
+
+	defer dir.Close()
+
+	fileNames, err := dir.Readdirnames(0)
+	if err != nil {
+		return nil, err
+	}
+
+	keys := make([]string, 0, len(fileNames))
+	for _, fileName := range fileNames {
+		keys = append(keys, path.Join(subPath, fileName))
+	}
+
+	return keys, nil
+}
+
+// Move moves an object stored at sourcePath to destPath, removing the original
+// object.
+func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
+	source := d.fullPath(sourcePath)
+	dest := d.fullPath(destPath)
+
+	if _, err := os.Stat(source); os.IsNotExist(err) {
+		return storagedriver.PathNotFoundError{Path: sourcePath}
+	}
+
+	if err := os.MkdirAll(path.Dir(dest), 0755); err != nil {
+		return err
+	}
+
+	err := os.Rename(source, dest)
+	return err
+}
+
+// Delete recursively deletes all objects stored at "path" and its subpaths.
+func (d *driver) Delete(ctx context.Context, subPath string) error {
+	fullPath := d.fullPath(subPath)
+
+	_, err := os.Stat(fullPath)
+	if err != nil && !os.IsNotExist(err) {
+		return err
+	} else if err != nil {
+		return storagedriver.PathNotFoundError{Path: subPath}
+	}
+
+	err = os.RemoveAll(fullPath)
+	return err
+}
+
+// URLFor returns a URL which may be used to retrieve the content stored at the given path.
+// May return an UnsupportedMethodErr in certain StorageDriver implementations.
+func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	return "", storagedriver.ErrUnsupportedMethod{}
+}
+
+// fullPath returns the absolute path of a key within the Driver's storage.
+func (d *driver) fullPath(subPath string) string {
+	return path.Join(d.rootDirectory, subPath)
+}
+
+type fileInfo struct {
+	os.FileInfo
+	path string
+}
+
+var _ storagedriver.FileInfo = fileInfo{}
+
+// Path provides the full path of the target of this file info.
+func (fi fileInfo) Path() string {
+	return fi.path
+}
+
+// Size returns current length in bytes of the file. The return value can
+// be used to write to the end of the file at path. The value is
+// meaningless if IsDir returns true.
+func (fi fileInfo) Size() int64 {
+	if fi.IsDir() {
+		return 0
+	}
+
+	return fi.FileInfo.Size()
+}
+
+// ModTime returns the modification time for the file. For backends that
+// don't have a modification time, the creation time should be returned.
+func (fi fileInfo) ModTime() time.Time {
+	return fi.FileInfo.ModTime()
+}
+
+// IsDir returns true if the path is a directory.
+func (fi fileInfo) IsDir() bool {
+	return fi.FileInfo.IsDir()
+}
+
+type fileWriter struct {
+	file      *os.File
+	size      int64
+	bw        *bufio.Writer
+	closed    bool
+	committed bool
+	cancelled bool
+}
+
+func newFileWriter(file *os.File, size int64) *fileWriter {
+	return &fileWriter{
+		file: file,
+		size: size,
+		bw:   bufio.NewWriter(file),
+	}
+}
+
+func (fw *fileWriter) Write(p []byte) (int, error) {
+	if fw.closed {
+		return 0, fmt.Errorf("already closed")
+	} else if fw.committed {
+		return 0, fmt.Errorf("already committed")
+	} else if fw.cancelled {
+		return 0, fmt.Errorf("already cancelled")
+	}
+	n, err := fw.bw.Write(p)
+	fw.size += int64(n)
+	return n, err
+}
+
+func (fw *fileWriter) Size() int64 {
+	return fw.size
+}
+
+func (fw *fileWriter) Close() error {
+	if fw.closed {
+		return fmt.Errorf("already closed")
+	}
+
+	if err := fw.bw.Flush(); err != nil {
+		return err
+	}
+
+	if err := fw.file.Sync(); err != nil {
+		return err
+	}
+
+	if err := fw.file.Close(); err != nil {
+		return err
+	}
+	fw.closed = true
+	return nil
+}
+
+func (fw *fileWriter) Cancel() error {
+	if fw.closed {
+		return fmt.Errorf("already closed")
+	}
+
+	fw.cancelled = true
+	fw.file.Close()
+	return os.Remove(fw.file.Name())
+}
+
+func (fw *fileWriter) Commit() error {
+	if fw.closed {
+		return fmt.Errorf("already closed")
+	} else if fw.committed {
+		return fmt.Errorf("already committed")
+	} else if fw.cancelled {
+		return fmt.Errorf("already cancelled")
+	}
+
+	if err := fw.bw.Flush(); err != nil {
+		return err
+	}
+
+	if err := fw.file.Sync(); err != nil {
+		return err
+	}
+
+	fw.committed = true
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/driver/filesystem/driver_test.go

@@ -0,0 +1,113 @@
+package filesystem
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"testing"
+
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }
+
+func init() {
+	root, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(root)
+
+	driver, err := FromParameters(map[string]interface{}{
+		"rootdirectory": root,
+	})
+	if err != nil {
+		panic(err)
+	}
+
+	testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
+		return driver, nil
+	}, testsuites.NeverSkip)
+}
+
+func TestFromParametersImpl(t *testing.T) {
+
+	tests := []struct {
+		params   map[string]interface{} // techincally the yaml can contain anything
+		expected DriverParameters
+		pass     bool
+	}{
+		// check we use default threads and root dirs
+		{
+			params: map[string]interface{}{},
+			expected: DriverParameters{
+				RootDirectory: defaultRootDirectory,
+				MaxThreads:    defaultMaxThreads,
+			},
+			pass: true,
+		},
+		// Testing initiation with a string maxThreads which can't be parsed
+		{
+			params: map[string]interface{}{
+				"maxthreads": "fail",
+			},
+			expected: DriverParameters{},
+			pass:     false,
+		},
+		{
+			params: map[string]interface{}{
+				"maxthreads": "100",
+			},
+			expected: DriverParameters{
+				RootDirectory: defaultRootDirectory,
+				MaxThreads:    uint64(100),
+			},
+			pass: true,
+		},
+		{
+			params: map[string]interface{}{
+				"maxthreads": 100,
+			},
+			expected: DriverParameters{
+				RootDirectory: defaultRootDirectory,
+				MaxThreads:    uint64(100),
+			},
+			pass: true,
+		},
+		// check that we use minimum thread counts
+		{
+			params: map[string]interface{}{
+				"maxthreads": 1,
+			},
+			expected: DriverParameters{
+				RootDirectory: defaultRootDirectory,
+				MaxThreads:    minThreads,
+			},
+			pass: true,
+		},
+	}
+
+	for _, item := range tests {
+		params, err := fromParametersImpl(item.params)
+
+		if !item.pass {
+			// We only need to assert that expected failures have an error
+			if err == nil {
+				t.Fatalf("expected error configuring filesystem driver with invalid param: %+v", item.params)
+			}
+			continue
+		}
+
+		if err != nil {
+			t.Fatalf("unexpected error creating filesystem driver: %s", err)
+		}
+		// Note that we get a pointer to params back
+		if !reflect.DeepEqual(*params, item.expected) {
+			t.Fatalf("unexpected params from filesystem driver. expected %+v, got %+v", item.expected, params)
+		}
+	}
+
+}

vendor/github.com/docker/distribution/registry/storage/driver/gcs/doc.go

@@ -0,0 +1,3 @@
+// Package gcs implements the Google Cloud Storage driver backend. Support can be
+// enabled by including the "include_gcs" build tag.
+package gcs

vendor/github.com/docker/distribution/registry/storage/driver/gcs/gcs_test.go

@@ -0,0 +1,311 @@
+// +build include_gcs
+
+package gcs
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"fmt"
+	ctx "github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/googleapi"
+	"google.golang.org/cloud/storage"
+	"gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { check.TestingT(t) }
+
+var gcsDriverConstructor func(rootDirectory string) (storagedriver.StorageDriver, error)
+var skipGCS func() string
+
+func init() {
+	bucket := os.Getenv("REGISTRY_STORAGE_GCS_BUCKET")
+	credentials := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+
+	// Skip GCS storage driver tests if environment variable parameters are not provided
+	skipGCS = func() string {
+		if bucket == "" || credentials == "" {
+			return "The following environment variables must be set to enable these tests: REGISTRY_STORAGE_GCS_BUCKET, GOOGLE_APPLICATION_CREDENTIALS"
+		}
+		return ""
+	}
+
+	if skipGCS() != "" {
+		return
+	}
+
+	root, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(root)
+	var ts oauth2.TokenSource
+	var email string
+	var privateKey []byte
+
+	ts, err = google.DefaultTokenSource(ctx.Background(), storage.ScopeFullControl)
+	if err != nil {
+		// Assume that the file contents are within the environment variable since it exists
+		// but does not contain a valid file path
+		jwtConfig, err := google.JWTConfigFromJSON([]byte(credentials), storage.ScopeFullControl)
+		if err != nil {
+			panic(fmt.Sprintf("Error reading JWT config : %s", err))
+		}
+		email = jwtConfig.Email
+		privateKey = []byte(jwtConfig.PrivateKey)
+		if len(privateKey) == 0 {
+			panic("Error reading JWT config : missing private_key property")
+		}
+		if email == "" {
+			panic("Error reading JWT config : missing client_email property")
+		}
+		ts = jwtConfig.TokenSource(ctx.Background())
+	}
+
+	gcsDriverConstructor = func(rootDirectory string) (storagedriver.StorageDriver, error) {
+		parameters := driverParameters{
+			bucket:        bucket,
+			rootDirectory: root,
+			email:         email,
+			privateKey:    privateKey,
+			client:        oauth2.NewClient(ctx.Background(), ts),
+			chunkSize:     defaultChunkSize,
+		}
+
+		return New(parameters)
+	}
+
+	testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
+		return gcsDriverConstructor(root)
+	}, skipGCS)
+}
+
+// Test Committing a FileWriter without having called Write
+func TestCommitEmpty(t *testing.T) {
+	if skipGCS() != "" {
+		t.Skip(skipGCS())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	driver, err := gcsDriverConstructor(validRoot)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	filename := "/test"
+	ctx := ctx.Background()
+
+	writer, err := driver.Writer(ctx, filename, false)
+	defer driver.Delete(ctx, filename)
+	if err != nil {
+		t.Fatalf("driver.Writer: unexpected error: %v", err)
+	}
+	err = writer.Commit()
+	if err != nil {
+		t.Fatalf("writer.Commit: unexpected error: %v", err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatalf("writer.Close: unexpected error: %v", err)
+	}
+	if writer.Size() != 0 {
+		t.Fatalf("writer.Size: %d != 0", writer.Size())
+	}
+	readContents, err := driver.GetContent(ctx, filename)
+	if err != nil {
+		t.Fatalf("driver.GetContent: unexpected error: %v", err)
+	}
+	if len(readContents) != 0 {
+		t.Fatalf("len(driver.GetContent(..)): %d != 0", len(readContents))
+	}
+}
+
+// Test Committing a FileWriter after having written exactly
+// defaultChunksize bytes.
+func TestCommit(t *testing.T) {
+	if skipGCS() != "" {
+		t.Skip(skipGCS())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	driver, err := gcsDriverConstructor(validRoot)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	filename := "/test"
+	ctx := ctx.Background()
+
+	contents := make([]byte, defaultChunkSize)
+	writer, err := driver.Writer(ctx, filename, false)
+	defer driver.Delete(ctx, filename)
+	if err != nil {
+		t.Fatalf("driver.Writer: unexpected error: %v", err)
+	}
+	_, err = writer.Write(contents)
+	if err != nil {
+		t.Fatalf("writer.Write: unexpected error: %v", err)
+	}
+	err = writer.Commit()
+	if err != nil {
+		t.Fatalf("writer.Commit: unexpected error: %v", err)
+	}
+	err = writer.Close()
+	if err != nil {
+		t.Fatalf("writer.Close: unexpected error: %v", err)
+	}
+	if writer.Size() != int64(len(contents)) {
+		t.Fatalf("writer.Size: %d != %d", writer.Size(), len(contents))
+	}
+	readContents, err := driver.GetContent(ctx, filename)
+	if err != nil {
+		t.Fatalf("driver.GetContent: unexpected error: %v", err)
+	}
+	if len(readContents) != len(contents) {
+		t.Fatalf("len(driver.GetContent(..)): %d != %d", len(readContents), len(contents))
+	}
+}
+
+func TestRetry(t *testing.T) {
+	if skipGCS() != "" {
+		t.Skip(skipGCS())
+	}
+
+	assertError := func(expected string, observed error) {
+		observedMsg := "<nil>"
+		if observed != nil {
+			observedMsg = observed.Error()
+		}
+		if observedMsg != expected {
+			t.Fatalf("expected %v, observed %v\n", expected, observedMsg)
+		}
+	}
+
+	err := retry(func() error {
+		return &googleapi.Error{
+			Code:    503,
+			Message: "google api error",
+		}
+	})
+	assertError("googleapi: Error 503: google api error", err)
+
+	err = retry(func() error {
+		return &googleapi.Error{
+			Code:    404,
+			Message: "google api error",
+		}
+	})
+	assertError("googleapi: Error 404: google api error", err)
+
+	err = retry(func() error {
+		return fmt.Errorf("error")
+	})
+	assertError("error", err)
+}
+
+func TestEmptyRootList(t *testing.T) {
+	if skipGCS() != "" {
+		t.Skip(skipGCS())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	rootedDriver, err := gcsDriverConstructor(validRoot)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	emptyRootDriver, err := gcsDriverConstructor("")
+	if err != nil {
+		t.Fatalf("unexpected error creating empty root driver: %v", err)
+	}
+
+	slashRootDriver, err := gcsDriverConstructor("/")
+	if err != nil {
+		t.Fatalf("unexpected error creating slash root driver: %v", err)
+	}
+
+	filename := "/test"
+	contents := []byte("contents")
+	ctx := ctx.Background()
+	err = rootedDriver.PutContent(ctx, filename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer func() {
+		err := rootedDriver.Delete(ctx, filename)
+		if err != nil {
+			t.Fatalf("failed to remove %v due to %v\n", filename, err)
+		}
+	}()
+	keys, err := emptyRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+
+	keys, err = slashRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+}
+
+// TestMoveDirectory checks that moving a directory returns an error.
+func TestMoveDirectory(t *testing.T) {
+	if skipGCS() != "" {
+		t.Skip(skipGCS())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	driver, err := gcsDriverConstructor(validRoot)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	ctx := ctx.Background()
+	contents := []byte("contents")
+	// Create a regular file.
+	err = driver.PutContent(ctx, "/parent/dir/foo", contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer func() {
+		err := driver.Delete(ctx, "/parent")
+		if err != nil {
+			t.Fatalf("failed to remove /parent due to %v\n", err)
+		}
+	}()
+
+	err = driver.Move(ctx, "/parent/dir", "/parent/other")
+	if err == nil {
+		t.Fatalf("Moving directory /parent/dir /parent/other should have return a non-nil error\n")
+	}
+}

vendor/github.com/docker/distribution/registry/storage/driver/inmemory/driver.go

@@ -0,0 +1,312 @@
+package inmemory
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"sync"
+	"time"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/base"
+	"github.com/docker/distribution/registry/storage/driver/factory"
+)
+
+const driverName = "inmemory"
+
+func init() {
+	factory.Register(driverName, &inMemoryDriverFactory{})
+}
+
+// inMemoryDriverFacotry implements the factory.StorageDriverFactory interface.
+type inMemoryDriverFactory struct{}
+
+func (factory *inMemoryDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
+	return New(), nil
+}
+
+type driver struct {
+	root  *dir
+	mutex sync.RWMutex
+}
+
+// baseEmbed allows us to hide the Base embed.
+type baseEmbed struct {
+	base.Base
+}
+
+// Driver is a storagedriver.StorageDriver implementation backed by a local map.
+// Intended solely for example and testing purposes.
+type Driver struct {
+	baseEmbed // embedded, hidden base driver.
+}
+
+var _ storagedriver.StorageDriver = &Driver{}
+
+// New constructs a new Driver.
+func New() *Driver {
+	return &Driver{
+		baseEmbed: baseEmbed{
+			Base: base.Base{
+				StorageDriver: &driver{
+					root: &dir{
+						common: common{
+							p:   "/",
+							mod: time.Now(),
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+// Implement the storagedriver.StorageDriver interface.
+
+func (d *driver) Name() string {
+	return driverName
+}
+
+// GetContent retrieves the content stored at "path" as a []byte.
+func (d *driver) GetContent(ctx context.Context, path string) ([]byte, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
+
+	rc, err := d.Reader(ctx, path, 0)
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+
+	return ioutil.ReadAll(rc)
+}
+
+// PutContent stores the []byte content at a location designated by "path".
+func (d *driver) PutContent(ctx context.Context, p string, contents []byte) error {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	normalized := normalize(p)
+
+	f, err := d.root.mkfile(normalized)
+	if err != nil {
+		// TODO(stevvooe): Again, we need to clarify when this is not a
+		// directory in StorageDriver API.
+		return fmt.Errorf("not a file")
+	}
+
+	f.truncate()
+	f.WriteAt(contents, 0)
+
+	return nil
+}
+
+// Reader retrieves an io.ReadCloser for the content stored at "path" with a
+// given byte offset.
+func (d *driver) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
+
+	if offset < 0 {
+		return nil, storagedriver.InvalidOffsetError{Path: path, Offset: offset}
+	}
+
+	normalized := normalize(path)
+	found := d.root.find(normalized)
+
+	if found.path() != normalized {
+		return nil, storagedriver.PathNotFoundError{Path: path}
+	}
+
+	if found.isdir() {
+		return nil, fmt.Errorf("%q is a directory", path)
+	}
+
+	return ioutil.NopCloser(found.(*file).sectionReader(offset)), nil
+}
+
+// Writer returns a FileWriter which will store the content written to it
+// at the location designated by "path" after the call to Commit.
+func (d *driver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	normalized := normalize(path)
+
+	f, err := d.root.mkfile(normalized)
+	if err != nil {
+		return nil, fmt.Errorf("not a file")
+	}
+
+	if !append {
+		f.truncate()
+	}
+
+	return d.newWriter(f), nil
+}
+
+// Stat returns info about the provided path.
+func (d *driver) Stat(ctx context.Context, path string) (storagedriver.FileInfo, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
+
+	normalized := normalize(path)
+	found := d.root.find(normalized)
+
+	if found.path() != normalized {
+		return nil, storagedriver.PathNotFoundError{Path: path}
+	}
+
+	fi := storagedriver.FileInfoFields{
+		Path:    path,
+		IsDir:   found.isdir(),
+		ModTime: found.modtime(),
+	}
+
+	if !fi.IsDir {
+		fi.Size = int64(len(found.(*file).data))
+	}
+
+	return storagedriver.FileInfoInternal{FileInfoFields: fi}, nil
+}
+
+// List returns a list of the objects that are direct descendants of the given
+// path.
+func (d *driver) List(ctx context.Context, path string) ([]string, error) {
+	d.mutex.RLock()
+	defer d.mutex.RUnlock()
+
+	normalized := normalize(path)
+
+	found := d.root.find(normalized)
+
+	if !found.isdir() {
+		return nil, fmt.Errorf("not a directory") // TODO(stevvooe): Need error type for this...
+	}
+
+	entries, err := found.(*dir).list(normalized)
+
+	if err != nil {
+		switch err {
+		case errNotExists:
+			return nil, storagedriver.PathNotFoundError{Path: path}
+		case errIsNotDir:
+			return nil, fmt.Errorf("not a directory")
+		default:
+			return nil, err
+		}
+	}
+
+	return entries, nil
+}
+
+// Move moves an object stored at sourcePath to destPath, removing the original
+// object.
+func (d *driver) Move(ctx context.Context, sourcePath string, destPath string) error {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	normalizedSrc, normalizedDst := normalize(sourcePath), normalize(destPath)
+
+	err := d.root.move(normalizedSrc, normalizedDst)
+	switch err {
+	case errNotExists:
+		return storagedriver.PathNotFoundError{Path: destPath}
+	default:
+		return err
+	}
+}
+
+// Delete recursively deletes all objects stored at "path" and its subpaths.
+func (d *driver) Delete(ctx context.Context, path string) error {
+	d.mutex.Lock()
+	defer d.mutex.Unlock()
+
+	normalized := normalize(path)
+
+	err := d.root.delete(normalized)
+	switch err {
+	case errNotExists:
+		return storagedriver.PathNotFoundError{Path: path}
+	default:
+		return err
+	}
+}
+
+// URLFor returns a URL which may be used to retrieve the content stored at the given path.
+// May return an UnsupportedMethodErr in certain StorageDriver implementations.
+func (d *driver) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	return "", storagedriver.ErrUnsupportedMethod{}
+}
+
+type writer struct {
+	d         *driver
+	f         *file
+	closed    bool
+	committed bool
+	cancelled bool
+}
+
+func (d *driver) newWriter(f *file) storagedriver.FileWriter {
+	return &writer{
+		d: d,
+		f: f,
+	}
+}
+
+func (w *writer) Write(p []byte) (int, error) {
+	if w.closed {
+		return 0, fmt.Errorf("already closed")
+	} else if w.committed {
+		return 0, fmt.Errorf("already committed")
+	} else if w.cancelled {
+		return 0, fmt.Errorf("already cancelled")
+	}
+
+	w.d.mutex.Lock()
+	defer w.d.mutex.Unlock()
+
+	return w.f.WriteAt(p, int64(len(w.f.data)))
+}
+
+func (w *writer) Size() int64 {
+	w.d.mutex.RLock()
+	defer w.d.mutex.RUnlock()
+
+	return int64(len(w.f.data))
+}
+
+func (w *writer) Close() error {
+	if w.closed {
+		return fmt.Errorf("already closed")
+	}
+	w.closed = true
+	return nil
+}
+
+func (w *writer) Cancel() error {
+	if w.closed {
+		return fmt.Errorf("already closed")
+	} else if w.committed {
+		return fmt.Errorf("already committed")
+	}
+	w.cancelled = true
+
+	w.d.mutex.Lock()
+	defer w.d.mutex.Unlock()
+
+	return w.d.root.delete(w.f.path())
+}
+
+func (w *writer) Commit() error {
+	if w.closed {
+		return fmt.Errorf("already closed")
+	} else if w.committed {
+		return fmt.Errorf("already committed")
+	} else if w.cancelled {
+		return fmt.Errorf("already cancelled")
+	}
+	w.committed = true
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/driver/inmemory/driver_test.go

@@ -0,0 +1,19 @@
+package inmemory
+
+import (
+	"testing"
+
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+	"gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { check.TestingT(t) }
+
+func init() {
+	inmemoryDriverConstructor := func() (storagedriver.StorageDriver, error) {
+		return New(), nil
+	}
+	testsuites.RegisterSuite(inmemoryDriverConstructor, testsuites.NeverSkip)
+}

vendor/github.com/docker/distribution/registry/storage/driver/inmemory/mfs.go

@@ -0,0 +1,338 @@
+package inmemory
+
+import (
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strings"
+	"time"
+)
+
+var (
+	errExists    = fmt.Errorf("exists")
+	errNotExists = fmt.Errorf("notexists")
+	errIsNotDir  = fmt.Errorf("notdir")
+	errIsDir     = fmt.Errorf("isdir")
+)
+
+type node interface {
+	name() string
+	path() string
+	isdir() bool
+	modtime() time.Time
+}
+
+// dir is the central type for the memory-based  storagedriver. All operations
+// are dispatched from a root dir.
+type dir struct {
+	common
+
+	// TODO(stevvooe): Use sorted slice + search.
+	children map[string]node
+}
+
+var _ node = &dir{}
+
+func (d *dir) isdir() bool {
+	return true
+}
+
+// add places the node n into dir d.
+func (d *dir) add(n node) {
+	if d.children == nil {
+		d.children = make(map[string]node)
+	}
+
+	d.children[n.name()] = n
+	d.mod = time.Now()
+}
+
+// find searches for the node, given path q in dir. If the node is found, it
+// will be returned. If the node is not found, the closet existing parent. If
+// the node is found, the returned (node).path() will match q.
+func (d *dir) find(q string) node {
+	q = strings.Trim(q, "/")
+	i := strings.Index(q, "/")
+
+	if q == "" {
+		return d
+	}
+
+	if i == 0 {
+		panic("shouldn't happen, no root paths")
+	}
+
+	var component string
+	if i < 0 {
+		// No more path components
+		component = q
+	} else {
+		component = q[:i]
+	}
+
+	child, ok := d.children[component]
+	if !ok {
+		// Node was not found. Return p and the current node.
+		return d
+	}
+
+	if child.isdir() {
+		// traverse down!
+		q = q[i+1:]
+		return child.(*dir).find(q)
+	}
+
+	return child
+}
+
+func (d *dir) list(p string) ([]string, error) {
+	n := d.find(p)
+
+	if n.path() != p {
+		return nil, errNotExists
+	}
+
+	if !n.isdir() {
+		return nil, errIsNotDir
+	}
+
+	var children []string
+	for _, child := range n.(*dir).children {
+		children = append(children, child.path())
+	}
+
+	sort.Strings(children)
+	return children, nil
+}
+
+// mkfile or return the existing one. returns an error if it exists and is a
+// directory. Essentially, this is open or create.
+func (d *dir) mkfile(p string) (*file, error) {
+	n := d.find(p)
+	if n.path() == p {
+		if n.isdir() {
+			return nil, errIsDir
+		}
+
+		return n.(*file), nil
+	}
+
+	dirpath, filename := path.Split(p)
+	// Make any non-existent directories
+	n, err := d.mkdirs(dirpath)
+	if err != nil {
+		return nil, err
+	}
+
+	dd := n.(*dir)
+	n = &file{
+		common: common{
+			p:   path.Join(dd.path(), filename),
+			mod: time.Now(),
+		},
+	}
+
+	dd.add(n)
+	return n.(*file), nil
+}
+
+// mkdirs creates any missing directory entries in p and returns the result.
+func (d *dir) mkdirs(p string) (*dir, error) {
+	p = normalize(p)
+
+	n := d.find(p)
+
+	if !n.isdir() {
+		// Found something there
+		return nil, errIsNotDir
+	}
+
+	if n.path() == p {
+		return n.(*dir), nil
+	}
+
+	dd := n.(*dir)
+
+	relative := strings.Trim(strings.TrimPrefix(p, n.path()), "/")
+
+	if relative == "" {
+		return dd, nil
+	}
+
+	components := strings.Split(relative, "/")
+	for _, component := range components {
+		d, err := dd.mkdir(component)
+
+		if err != nil {
+			// This should actually never happen, since there are no children.
+			return nil, err
+		}
+		dd = d
+	}
+
+	return dd, nil
+}
+
+// mkdir creates a child directory under d with the given name.
+func (d *dir) mkdir(name string) (*dir, error) {
+	if name == "" {
+		return nil, fmt.Errorf("invalid dirname")
+	}
+
+	_, ok := d.children[name]
+	if ok {
+		return nil, errExists
+	}
+
+	child := &dir{
+		common: common{
+			p:   path.Join(d.path(), name),
+			mod: time.Now(),
+		},
+	}
+	d.add(child)
+	d.mod = time.Now()
+
+	return child, nil
+}
+
+func (d *dir) move(src, dst string) error {
+	dstDirname, _ := path.Split(dst)
+
+	dp, err := d.mkdirs(dstDirname)
+	if err != nil {
+		return err
+	}
+
+	srcDirname, srcFilename := path.Split(src)
+	sp := d.find(srcDirname)
+
+	if normalize(srcDirname) != normalize(sp.path()) {
+		return errNotExists
+	}
+
+	spd, ok := sp.(*dir)
+	if !ok {
+		return errIsNotDir // paranoid.
+	}
+
+	s, ok := spd.children[srcFilename]
+	if !ok {
+		return errNotExists
+	}
+
+	delete(spd.children, srcFilename)
+
+	switch n := s.(type) {
+	case *dir:
+		n.p = dst
+	case *file:
+		n.p = dst
+	}
+
+	dp.add(s)
+
+	return nil
+}
+
+func (d *dir) delete(p string) error {
+	dirname, filename := path.Split(p)
+	parent := d.find(dirname)
+
+	if normalize(dirname) != normalize(parent.path()) {
+		return errNotExists
+	}
+
+	if _, ok := parent.(*dir).children[filename]; !ok {
+		return errNotExists
+	}
+
+	delete(parent.(*dir).children, filename)
+	return nil
+}
+
+// dump outputs a primitive directory structure to stdout.
+func (d *dir) dump(indent string) {
+	fmt.Println(indent, d.name()+"/")
+
+	for _, child := range d.children {
+		if child.isdir() {
+			child.(*dir).dump(indent + "\t")
+		} else {
+			fmt.Println(indent, child.name())
+		}
+
+	}
+}
+
+func (d *dir) String() string {
+	return fmt.Sprintf("&dir{path: %v, children: %v}", d.p, d.children)
+}
+
+// file stores actual data in the fs tree. It acts like an open, seekable file
+// where operations are conducted through ReadAt and WriteAt. Use it with
+// SectionReader for the best effect.
+type file struct {
+	common
+	data []byte
+}
+
+var _ node = &file{}
+
+func (f *file) isdir() bool {
+	return false
+}
+
+func (f *file) truncate() {
+	f.data = f.data[:0]
+}
+
+func (f *file) sectionReader(offset int64) io.Reader {
+	return io.NewSectionReader(f, offset, int64(len(f.data))-offset)
+}
+
+func (f *file) ReadAt(p []byte, offset int64) (n int, err error) {
+	return copy(p, f.data[offset:]), nil
+}
+
+func (f *file) WriteAt(p []byte, offset int64) (n int, err error) {
+	off := int(offset)
+	if cap(f.data) < off+len(p) {
+		data := make([]byte, len(f.data), off+len(p))
+		copy(data, f.data)
+		f.data = data
+	}
+
+	f.mod = time.Now()
+	f.data = f.data[:off+len(p)]
+
+	return copy(f.data[off:off+len(p)], p), nil
+}
+
+func (f *file) String() string {
+	return fmt.Sprintf("&file{path: %q}", f.p)
+}
+
+// common provides shared fields and methods for node implementations.
+type common struct {
+	p   string
+	mod time.Time
+}
+
+func (c *common) name() string {
+	_, name := path.Split(c.p)
+	return name
+}
+
+func (c *common) path() string {
+	return c.p
+}
+
+func (c *common) modtime() time.Time {
+	return c.mod
+}
+
+func normalize(p string) string {
+	return "/" + strings.Trim(p, "/")
+}

vendor/github.com/docker/distribution/registry/storage/driver/middleware/cloudfront/middleware.go

@@ -0,0 +1,136 @@
+// Package middleware - cloudfront wrapper for storage libs
+// N.B. currently only works with S3, not arbitrary sites
+//
+package middleware
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/aws/aws-sdk-go/service/cloudfront/sign"
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
+)
+
+// cloudFrontStorageMiddleware provides a simple implementation of layerHandler that
+// constructs temporary signed CloudFront URLs from the storagedriver layer URL,
+// then issues HTTP Temporary Redirects to this CloudFront content URL.
+type cloudFrontStorageMiddleware struct {
+	storagedriver.StorageDriver
+	urlSigner *sign.URLSigner
+	baseURL   string
+	duration  time.Duration
+}
+
+var _ storagedriver.StorageDriver = &cloudFrontStorageMiddleware{}
+
+// newCloudFrontLayerHandler constructs and returns a new CloudFront
+// LayerHandler implementation.
+// Required options: baseurl, privatekey, keypairid
+func newCloudFrontStorageMiddleware(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) {
+	base, ok := options["baseurl"]
+	if !ok {
+		return nil, fmt.Errorf("no baseurl provided")
+	}
+	baseURL, ok := base.(string)
+	if !ok {
+		return nil, fmt.Errorf("baseurl must be a string")
+	}
+	if !strings.Contains(baseURL, "://") {
+		baseURL = "https://" + baseURL
+	}
+	if !strings.HasSuffix(baseURL, "/") {
+		baseURL += "/"
+	}
+	if _, err := url.Parse(baseURL); err != nil {
+		return nil, fmt.Errorf("invalid baseurl: %v", err)
+	}
+	pk, ok := options["privatekey"]
+	if !ok {
+		return nil, fmt.Errorf("no privatekey provided")
+	}
+	pkPath, ok := pk.(string)
+	if !ok {
+		return nil, fmt.Errorf("privatekey must be a string")
+	}
+	kpid, ok := options["keypairid"]
+	if !ok {
+		return nil, fmt.Errorf("no keypairid provided")
+	}
+	keypairID, ok := kpid.(string)
+	if !ok {
+		return nil, fmt.Errorf("keypairid must be a string")
+	}
+
+	pkBytes, err := ioutil.ReadFile(pkPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read privatekey file: %s", err)
+	}
+
+	block, _ := pem.Decode([]byte(pkBytes))
+	if block == nil {
+		return nil, fmt.Errorf("failed to decode private key as an rsa private key")
+	}
+	privateKey, err := x509.ParsePKCS1PrivateKey(block.Bytes)
+	if err != nil {
+		return nil, err
+	}
+
+	urlSigner := sign.NewURLSigner(keypairID, privateKey)
+
+	duration := 20 * time.Minute
+	d, ok := options["duration"]
+	if ok {
+		switch d := d.(type) {
+		case time.Duration:
+			duration = d
+		case string:
+			dur, err := time.ParseDuration(d)
+			if err != nil {
+				return nil, fmt.Errorf("invalid duration: %s", err)
+			}
+			duration = dur
+		}
+	}
+
+	return &cloudFrontStorageMiddleware{
+		StorageDriver: storageDriver,
+		urlSigner:     urlSigner,
+		baseURL:       baseURL,
+		duration:      duration,
+	}, nil
+}
+
+// S3BucketKeyer is any type that is capable of returning the S3 bucket key
+// which should be cached by AWS CloudFront.
+type S3BucketKeyer interface {
+	S3BucketKey(path string) string
+}
+
+// Resolve returns an http.Handler which can serve the contents of the given
+// Layer, or an error if not supported by the storagedriver.
+func (lh *cloudFrontStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	// TODO(endophage): currently only supports S3
+	keyer, ok := lh.StorageDriver.(S3BucketKeyer)
+	if !ok {
+		context.GetLogger(ctx).Warn("the CloudFront middleware does not support this backend storage driver")
+		return lh.StorageDriver.URLFor(ctx, path, options)
+	}
+
+	cfURL, err := lh.urlSigner.Sign(lh.baseURL+keyer.S3BucketKey(path), time.Now().Add(lh.duration))
+	if err != nil {
+		return "", err
+	}
+	return cfURL, nil
+}
+
+// init registers the cloudfront layerHandler backend.
+func init() {
+	storagemiddleware.Register("cloudfront", storagemiddleware.InitFunc(newCloudFrontStorageMiddleware))
+}

vendor/github.com/docker/distribution/registry/storage/driver/middleware/redirect/middleware.go

@@ -0,0 +1,50 @@
+package middleware
+
+import (
+	"fmt"
+	"net/url"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	storagemiddleware "github.com/docker/distribution/registry/storage/driver/middleware"
+)
+
+type redirectStorageMiddleware struct {
+	storagedriver.StorageDriver
+	scheme string
+	host   string
+}
+
+var _ storagedriver.StorageDriver = &redirectStorageMiddleware{}
+
+func newRedirectStorageMiddleware(sd storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error) {
+	o, ok := options["baseurl"]
+	if !ok {
+		return nil, fmt.Errorf("no baseurl provided")
+	}
+	b, ok := o.(string)
+	if !ok {
+		return nil, fmt.Errorf("baseurl must be a string")
+	}
+	u, err := url.Parse(b)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse redirect baseurl: %s", b)
+	}
+	if u.Scheme == "" {
+		return nil, fmt.Errorf("no scheme specified for redirect baseurl")
+	}
+	if u.Host == "" {
+		return nil, fmt.Errorf("no host specified for redirect baseurl")
+	}
+
+	return &redirectStorageMiddleware{StorageDriver: sd, scheme: u.Scheme, host: u.Host}, nil
+}
+
+func (r *redirectStorageMiddleware) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
+	u := &url.URL{Scheme: r.scheme, Host: r.host, Path: path}
+	return u.String(), nil
+}
+
+func init() {
+	storagemiddleware.Register("redirect", storagemiddleware.InitFunc(newRedirectStorageMiddleware))
+}

vendor/github.com/docker/distribution/registry/storage/driver/middleware/redirect/middleware_test.go

@@ -0,0 +1,58 @@
+package middleware
+
+import (
+	"testing"
+
+	check "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+type MiddlewareSuite struct{}
+
+var _ = check.Suite(&MiddlewareSuite{})
+
+func (s *MiddlewareSuite) TestNoConfig(c *check.C) {
+	options := make(map[string]interface{})
+	_, err := newRedirectStorageMiddleware(nil, options)
+	c.Assert(err, check.ErrorMatches, "no baseurl provided")
+}
+
+func (s *MiddlewareSuite) TestMissingScheme(c *check.C) {
+	options := make(map[string]interface{})
+	options["baseurl"] = "example.com"
+	_, err := newRedirectStorageMiddleware(nil, options)
+	c.Assert(err, check.ErrorMatches, "no scheme specified for redirect baseurl")
+}
+
+func (s *MiddlewareSuite) TestHttpsPort(c *check.C) {
+	options := make(map[string]interface{})
+	options["baseurl"] = "https://example.com:5443"
+	middleware, err := newRedirectStorageMiddleware(nil, options)
+	c.Assert(err, check.Equals, nil)
+
+	m, ok := middleware.(*redirectStorageMiddleware)
+	c.Assert(ok, check.Equals, true)
+	c.Assert(m.scheme, check.Equals, "https")
+	c.Assert(m.host, check.Equals, "example.com:5443")
+
+	url, err := middleware.URLFor(nil, "/rick/data", nil)
+	c.Assert(err, check.Equals, nil)
+	c.Assert(url, check.Equals, "https://example.com:5443/rick/data")
+}
+
+func (s *MiddlewareSuite) TestHTTP(c *check.C) {
+	options := make(map[string]interface{})
+	options["baseurl"] = "http://example.com"
+	middleware, err := newRedirectStorageMiddleware(nil, options)
+	c.Assert(err, check.Equals, nil)
+
+	m, ok := middleware.(*redirectStorageMiddleware)
+	c.Assert(ok, check.Equals, true)
+	c.Assert(m.scheme, check.Equals, "http")
+	c.Assert(m.host, check.Equals, "example.com")
+
+	url, err := middleware.URLFor(nil, "morty/data", nil)
+	c.Assert(err, check.Equals, nil)
+	c.Assert(url, check.Equals, "http://example.com/morty/data")
+}

vendor/github.com/docker/distribution/registry/storage/driver/middleware/storagemiddleware.go

@@ -0,0 +1,39 @@
+package storagemiddleware
+
+import (
+	"fmt"
+
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// InitFunc is the type of a StorageMiddleware factory function and is
+// used to register the constructor for different StorageMiddleware backends.
+type InitFunc func(storageDriver storagedriver.StorageDriver, options map[string]interface{}) (storagedriver.StorageDriver, error)
+
+var storageMiddlewares map[string]InitFunc
+
+// Register is used to register an InitFunc for
+// a StorageMiddleware backend with the given name.
+func Register(name string, initFunc InitFunc) error {
+	if storageMiddlewares == nil {
+		storageMiddlewares = make(map[string]InitFunc)
+	}
+	if _, exists := storageMiddlewares[name]; exists {
+		return fmt.Errorf("name already registered: %s", name)
+	}
+
+	storageMiddlewares[name] = initFunc
+
+	return nil
+}
+
+// Get constructs a StorageMiddleware with the given options using the named backend.
+func Get(name string, options map[string]interface{}, storageDriver storagedriver.StorageDriver) (storagedriver.StorageDriver, error) {
+	if storageMiddlewares != nil {
+		if initFunc, exists := storageMiddlewares[name]; exists {
+			return initFunc(storageDriver, options)
+		}
+	}
+
+	return nil, fmt.Errorf("no storage middleware registered with name: %s", name)
+}

vendor/github.com/docker/distribution/registry/storage/driver/oss/doc.go

@@ -0,0 +1,3 @@
+// Package oss implements the Aliyun OSS Storage driver backend. Support can be
+// enabled by including the "include_oss" build tag.
+package oss

vendor/github.com/docker/distribution/registry/storage/driver/oss/oss_test.go

@@ -0,0 +1,144 @@
+// +build include_oss
+
+package oss
+
+import (
+	"io/ioutil"
+
+	alioss "github.com/denverdino/aliyungo/oss"
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+	//"log"
+	"os"
+	"strconv"
+	"testing"
+
+	"gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { check.TestingT(t) }
+
+var ossDriverConstructor func(rootDirectory string) (*Driver, error)
+
+var skipCheck func() string
+
+func init() {
+	accessKey := os.Getenv("ALIYUN_ACCESS_KEY_ID")
+	secretKey := os.Getenv("ALIYUN_ACCESS_KEY_SECRET")
+	bucket := os.Getenv("OSS_BUCKET")
+	region := os.Getenv("OSS_REGION")
+	internal := os.Getenv("OSS_INTERNAL")
+	encrypt := os.Getenv("OSS_ENCRYPT")
+	secure := os.Getenv("OSS_SECURE")
+	endpoint := os.Getenv("OSS_ENDPOINT")
+	root, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(root)
+
+	ossDriverConstructor = func(rootDirectory string) (*Driver, error) {
+		encryptBool := false
+		if encrypt != "" {
+			encryptBool, err = strconv.ParseBool(encrypt)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		secureBool := false
+		if secure != "" {
+			secureBool, err = strconv.ParseBool(secure)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		internalBool := false
+		if internal != "" {
+			internalBool, err = strconv.ParseBool(internal)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		parameters := DriverParameters{
+			AccessKeyID:     accessKey,
+			AccessKeySecret: secretKey,
+			Bucket:          bucket,
+			Region:          alioss.Region(region),
+			Internal:        internalBool,
+			ChunkSize:       minChunkSize,
+			RootDirectory:   rootDirectory,
+			Encrypt:         encryptBool,
+			Secure:          secureBool,
+			Endpoint:        endpoint,
+		}
+
+		return New(parameters)
+	}
+
+	// Skip OSS storage driver tests if environment variable parameters are not provided
+	skipCheck = func() string {
+		if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
+			return "Must set ALIYUN_ACCESS_KEY_ID, ALIYUN_ACCESS_KEY_SECRET, OSS_REGION, OSS_BUCKET, and OSS_ENCRYPT to run OSS tests"
+		}
+		return ""
+	}
+
+	testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
+		return ossDriverConstructor(root)
+	}, skipCheck)
+}
+
+func TestEmptyRootList(t *testing.T) {
+	if skipCheck() != "" {
+		t.Skip(skipCheck())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	rootedDriver, err := ossDriverConstructor(validRoot)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	emptyRootDriver, err := ossDriverConstructor("")
+	if err != nil {
+		t.Fatalf("unexpected error creating empty root driver: %v", err)
+	}
+
+	slashRootDriver, err := ossDriverConstructor("/")
+	if err != nil {
+		t.Fatalf("unexpected error creating slash root driver: %v", err)
+	}
+
+	filename := "/test"
+	contents := []byte("contents")
+	ctx := context.Background()
+	err = rootedDriver.PutContent(ctx, filename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer rootedDriver.Delete(ctx, filename)
+
+	keys, err := emptyRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+
+	keys, err = slashRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+}

vendor/github.com/docker/distribution/registry/storage/driver/s3-aws/s3_test.go

@@ -0,0 +1,313 @@
+package s3
+
+import (
+	"bytes"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strconv"
+	"testing"
+
+	"gopkg.in/check.v1"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/s3"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { check.TestingT(t) }
+
+var s3DriverConstructor func(rootDirectory, storageClass string) (*Driver, error)
+var skipS3 func() string
+
+func init() {
+	accessKey := os.Getenv("AWS_ACCESS_KEY")
+	secretKey := os.Getenv("AWS_SECRET_KEY")
+	bucket := os.Getenv("S3_BUCKET")
+	encrypt := os.Getenv("S3_ENCRYPT")
+	keyID := os.Getenv("S3_KEY_ID")
+	secure := os.Getenv("S3_SECURE")
+	v4Auth := os.Getenv("S3_V4_AUTH")
+	region := os.Getenv("AWS_REGION")
+	objectACL := os.Getenv("S3_OBJECT_ACL")
+	root, err := ioutil.TempDir("", "driver-")
+	regionEndpoint := os.Getenv("REGION_ENDPOINT")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(root)
+
+	s3DriverConstructor = func(rootDirectory, storageClass string) (*Driver, error) {
+		encryptBool := false
+		if encrypt != "" {
+			encryptBool, err = strconv.ParseBool(encrypt)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		secureBool := true
+		if secure != "" {
+			secureBool, err = strconv.ParseBool(secure)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		v4Bool := true
+		if v4Auth != "" {
+			v4Bool, err = strconv.ParseBool(v4Auth)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		parameters := DriverParameters{
+			accessKey,
+			secretKey,
+			bucket,
+			region,
+			regionEndpoint,
+			encryptBool,
+			keyID,
+			secureBool,
+			v4Bool,
+			minChunkSize,
+			defaultMultipartCopyChunkSize,
+			defaultMultipartCopyMaxConcurrency,
+			defaultMultipartCopyThresholdSize,
+			rootDirectory,
+			storageClass,
+			driverName + "-test",
+			objectACL,
+		}
+
+		return New(parameters)
+	}
+
+	// Skip S3 storage driver tests if environment variable parameters are not provided
+	skipS3 = func() string {
+		if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
+			return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
+		}
+		return ""
+	}
+
+	testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
+		return s3DriverConstructor(root, s3.StorageClassStandard)
+	}, skipS3)
+}
+
+func TestEmptyRootList(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	rootedDriver, err := s3DriverConstructor(validRoot, s3.StorageClassStandard)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	emptyRootDriver, err := s3DriverConstructor("", s3.StorageClassStandard)
+	if err != nil {
+		t.Fatalf("unexpected error creating empty root driver: %v", err)
+	}
+
+	slashRootDriver, err := s3DriverConstructor("/", s3.StorageClassStandard)
+	if err != nil {
+		t.Fatalf("unexpected error creating slash root driver: %v", err)
+	}
+
+	filename := "/test"
+	contents := []byte("contents")
+	ctx := context.Background()
+	err = rootedDriver.PutContent(ctx, filename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer rootedDriver.Delete(ctx, filename)
+
+	keys, err := emptyRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+
+	keys, err = slashRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+}
+
+func TestStorageClass(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	rootDir, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(rootDir)
+
+	standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with standard storage: %v", err)
+	}
+
+	rrDriver, err := s3DriverConstructor(rootDir, s3.StorageClassReducedRedundancy)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
+	}
+
+	if _, err = s3DriverConstructor(rootDir, noStorageClass); err != nil {
+		t.Fatalf("unexpected error creating driver without storage class: %v", err)
+	}
+
+	standardFilename := "/test-standard"
+	rrFilename := "/test-rr"
+	contents := []byte("contents")
+	ctx := context.Background()
+
+	err = standardDriver.PutContent(ctx, standardFilename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer standardDriver.Delete(ctx, standardFilename)
+
+	err = rrDriver.PutContent(ctx, rrFilename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer rrDriver.Delete(ctx, rrFilename)
+
+	standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
+	resp, err := standardDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(standardDriverUnwrapped.Bucket),
+		Key:    aws.String(standardDriverUnwrapped.s3Path(standardFilename)),
+	})
+	if err != nil {
+		t.Fatalf("unexpected error retrieving standard storage file: %v", err)
+	}
+	defer resp.Body.Close()
+	// Amazon only populates this header value for non-standard storage classes
+	if resp.StorageClass != nil {
+		t.Fatalf("unexpected storage class for standard file: %v", resp.StorageClass)
+	}
+
+	rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
+	resp, err = rrDriverUnwrapped.S3.GetObject(&s3.GetObjectInput{
+		Bucket: aws.String(rrDriverUnwrapped.Bucket),
+		Key:    aws.String(rrDriverUnwrapped.s3Path(rrFilename)),
+	})
+	if err != nil {
+		t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StorageClass == nil {
+		t.Fatalf("unexpected storage class for reduced-redundancy file: %v", s3.StorageClassStandard)
+	} else if *resp.StorageClass != s3.StorageClassReducedRedundancy {
+		t.Fatalf("unexpected storage class for reduced-redundancy file: %v", *resp.StorageClass)
+	}
+
+}
+
+func TestOverThousandBlobs(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	rootDir, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(rootDir)
+
+	standardDriver, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with standard storage: %v", err)
+	}
+
+	ctx := context.Background()
+	for i := 0; i < 1005; i++ {
+		filename := "/thousandfiletest/file" + strconv.Itoa(i)
+		contents := []byte("contents")
+		err = standardDriver.PutContent(ctx, filename, contents)
+		if err != nil {
+			t.Fatalf("unexpected error creating content: %v", err)
+		}
+	}
+
+	// cant actually verify deletion because read-after-delete is inconsistent, but can ensure no errors
+	err = standardDriver.Delete(ctx, "/thousandfiletest")
+	if err != nil {
+		t.Fatalf("unexpected error deleting thousand files: %v", err)
+	}
+}
+
+func TestMoveWithMultipartCopy(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	rootDir, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(rootDir)
+
+	d, err := s3DriverConstructor(rootDir, s3.StorageClassStandard)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver: %v", err)
+	}
+
+	ctx := context.Background()
+	sourcePath := "/source"
+	destPath := "/dest"
+
+	defer d.Delete(ctx, sourcePath)
+	defer d.Delete(ctx, destPath)
+
+	// An object larger than d's MultipartCopyThresholdSize will cause d.Move() to perform a multipart copy.
+	multipartCopyThresholdSize := d.baseEmbed.Base.StorageDriver.(*driver).MultipartCopyThresholdSize
+	contents := make([]byte, 2*multipartCopyThresholdSize)
+	rand.Read(contents)
+
+	err = d.PutContent(ctx, sourcePath, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+
+	err = d.Move(ctx, sourcePath, destPath)
+	if err != nil {
+		t.Fatalf("unexpected error moving file: %v", err)
+	}
+
+	received, err := d.GetContent(ctx, destPath)
+	if err != nil {
+		t.Fatalf("unexpected error getting content: %v", err)
+	}
+	if !bytes.Equal(contents, received) {
+		t.Fatal("content differs")
+	}
+
+	_, err = d.GetContent(ctx, sourcePath)
+	switch err.(type) {
+	case storagedriver.PathNotFoundError:
+	default:
+		t.Fatalf("unexpected error getting content: %v", err)
+	}
+}

vendor/github.com/docker/distribution/registry/storage/driver/s3-aws/s3_v2_signer.go

@@ -0,0 +1,219 @@
+package s3
+
+// Source: https://github.com/pivotal-golang/s3cli
+
+// Copyright (c) 2013 Damien Le Berrigaud and Nick Wade
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/aws/aws-sdk-go/aws/corehandlers"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+const (
+	signatureVersion = "2"
+	signatureMethod  = "HmacSHA1"
+	timeFormat       = "2006-01-02T15:04:05Z"
+)
+
+type signer struct {
+	// Values that must be populated from the request
+	Request      *http.Request
+	Time         time.Time
+	Credentials  *credentials.Credentials
+	Query        url.Values
+	stringToSign string
+	signature    string
+}
+
+var s3ParamsToSign = map[string]bool{
+	"acl":                          true,
+	"location":                     true,
+	"logging":                      true,
+	"notification":                 true,
+	"partNumber":                   true,
+	"policy":                       true,
+	"requestPayment":               true,
+	"torrent":                      true,
+	"uploadId":                     true,
+	"uploads":                      true,
+	"versionId":                    true,
+	"versioning":                   true,
+	"versions":                     true,
+	"response-content-type":        true,
+	"response-content-language":    true,
+	"response-expires":             true,
+	"response-cache-control":       true,
+	"response-content-disposition": true,
+	"response-content-encoding":    true,
+	"website":                      true,
+	"delete":                       true,
+}
+
+// setv2Handlers will setup v2 signature signing on the S3 driver
+func setv2Handlers(svc *s3.S3) {
+	svc.Handlers.Build.PushBack(func(r *request.Request) {
+		parsedURL, err := url.Parse(r.HTTPRequest.URL.String())
+		if err != nil {
+			log.Fatalf("Failed to parse URL: %v", err)
+		}
+		r.HTTPRequest.URL.Opaque = parsedURL.Path
+	})
+
+	svc.Handlers.Sign.Clear()
+	svc.Handlers.Sign.PushBack(Sign)
+	svc.Handlers.Sign.PushBackNamed(corehandlers.BuildContentLengthHandler)
+}
+
+// Sign requests with signature version 2.
+//
+// Will sign the requests with the service config's Credentials object
+// Signing is skipped if the credentials is the credentials.AnonymousCredentials
+// object.
+func Sign(req *request.Request) {
+	// If the request does not need to be signed ignore the signing of the
+	// request if the AnonymousCredentials object is used.
+	if req.Config.Credentials == credentials.AnonymousCredentials {
+		return
+	}
+
+	v2 := signer{
+		Request:     req.HTTPRequest,
+		Time:        req.Time,
+		Credentials: req.Config.Credentials,
+	}
+	v2.Sign()
+}
+
+func (v2 *signer) Sign() error {
+	credValue, err := v2.Credentials.Get()
+	if err != nil {
+		return err
+	}
+	accessKey := credValue.AccessKeyID
+	var (
+		md5, ctype, date, xamz string
+		xamzDate               bool
+		sarray                 []string
+		smap                   map[string]string
+		sharray                []string
+	)
+
+	headers := v2.Request.Header
+	params := v2.Request.URL.Query()
+	parsedURL, err := url.Parse(v2.Request.URL.String())
+	if err != nil {
+		return err
+	}
+	host, canonicalPath := parsedURL.Host, parsedURL.Path
+	v2.Request.Header["Host"] = []string{host}
+	v2.Request.Header["date"] = []string{v2.Time.In(time.UTC).Format(time.RFC1123)}
+
+	smap = make(map[string]string)
+	for k, v := range headers {
+		k = strings.ToLower(k)
+		switch k {
+		case "content-md5":
+			md5 = v[0]
+		case "content-type":
+			ctype = v[0]
+		case "date":
+			if !xamzDate {
+				date = v[0]
+			}
+		default:
+			if strings.HasPrefix(k, "x-amz-") {
+				vall := strings.Join(v, ",")
+				smap[k] = k + ":" + vall
+				if k == "x-amz-date" {
+					xamzDate = true
+					date = ""
+				}
+				sharray = append(sharray, k)
+			}
+		}
+	}
+	if len(sharray) > 0 {
+		sort.StringSlice(sharray).Sort()
+		for _, h := range sharray {
+			sarray = append(sarray, smap[h])
+		}
+		xamz = strings.Join(sarray, "\n") + "\n"
+	}
+
+	expires := false
+	if v, ok := params["Expires"]; ok {
+		expires = true
+		date = v[0]
+		params["AWSAccessKeyId"] = []string{accessKey}
+	}
+
+	sarray = sarray[0:0]
+	for k, v := range params {
+		if s3ParamsToSign[k] {
+			for _, vi := range v {
+				if vi == "" {
+					sarray = append(sarray, k)
+				} else {
+					sarray = append(sarray, k+"="+vi)
+				}
+			}
+		}
+	}
+	if len(sarray) > 0 {
+		sort.StringSlice(sarray).Sort()
+		canonicalPath = canonicalPath + "?" + strings.Join(sarray, "&")
+	}
+
+	v2.stringToSign = strings.Join([]string{
+		v2.Request.Method,
+		md5,
+		ctype,
+		date,
+		xamz + canonicalPath,
+	}, "\n")
+	hash := hmac.New(sha1.New, []byte(credValue.SecretAccessKey))
+	hash.Write([]byte(v2.stringToSign))
+	v2.signature = base64.StdEncoding.EncodeToString(hash.Sum(nil))
+
+	if expires {
+		params["Signature"] = []string{string(v2.signature)}
+	} else {
+		headers["Authorization"] = []string{"AWS " + accessKey + ":" + string(v2.signature)}
+	}
+
+	log.WithFields(log.Fields{
+		"string-to-sign": v2.stringToSign,
+		"signature":      v2.signature,
+	}).Debugln("request signature")
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/driver/s3-goamz/s3_test.go

@@ -0,0 +1,201 @@
+package s3
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+	"testing"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+	"github.com/docker/goamz/aws"
+	"github.com/docker/goamz/s3"
+
+	"gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { check.TestingT(t) }
+
+var s3DriverConstructor func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error)
+var skipS3 func() string
+
+func init() {
+	accessKey := os.Getenv("AWS_ACCESS_KEY")
+	secretKey := os.Getenv("AWS_SECRET_KEY")
+	bucket := os.Getenv("S3_BUCKET")
+	encrypt := os.Getenv("S3_ENCRYPT")
+	secure := os.Getenv("S3_SECURE")
+	v4auth := os.Getenv("S3_USE_V4_AUTH")
+	region := os.Getenv("AWS_REGION")
+	root, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(root)
+
+	s3DriverConstructor = func(rootDirectory string, storageClass s3.StorageClass) (*Driver, error) {
+		encryptBool := false
+		if encrypt != "" {
+			encryptBool, err = strconv.ParseBool(encrypt)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		secureBool := true
+		if secure != "" {
+			secureBool, err = strconv.ParseBool(secure)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		v4AuthBool := false
+		if v4auth != "" {
+			v4AuthBool, err = strconv.ParseBool(v4auth)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		parameters := DriverParameters{
+			accessKey,
+			secretKey,
+			bucket,
+			aws.GetRegion(region),
+			encryptBool,
+			secureBool,
+			v4AuthBool,
+			minChunkSize,
+			rootDirectory,
+			storageClass,
+			driverName + "-test",
+		}
+
+		return New(parameters)
+	}
+
+	// Skip S3 storage driver tests if environment variable parameters are not provided
+	skipS3 = func() string {
+		if accessKey == "" || secretKey == "" || region == "" || bucket == "" || encrypt == "" {
+			return "Must set AWS_ACCESS_KEY, AWS_SECRET_KEY, AWS_REGION, S3_BUCKET, and S3_ENCRYPT to run S3 tests"
+		}
+		return ""
+	}
+
+	testsuites.RegisterSuite(func() (storagedriver.StorageDriver, error) {
+		return s3DriverConstructor(root, s3.StandardStorage)
+	}, skipS3)
+}
+
+func TestEmptyRootList(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	rootedDriver, err := s3DriverConstructor(validRoot, s3.StandardStorage)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	emptyRootDriver, err := s3DriverConstructor("", s3.StandardStorage)
+	if err != nil {
+		t.Fatalf("unexpected error creating empty root driver: %v", err)
+	}
+
+	slashRootDriver, err := s3DriverConstructor("/", s3.StandardStorage)
+	if err != nil {
+		t.Fatalf("unexpected error creating slash root driver: %v", err)
+	}
+
+	filename := "/test"
+	contents := []byte("contents")
+	ctx := context.Background()
+	err = rootedDriver.PutContent(ctx, filename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer rootedDriver.Delete(ctx, filename)
+
+	keys, err := emptyRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+
+	keys, err = slashRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+}
+
+func TestStorageClass(t *testing.T) {
+	if skipS3() != "" {
+		t.Skip(skipS3())
+	}
+
+	rootDir, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(rootDir)
+
+	standardDriver, err := s3DriverConstructor(rootDir, s3.StandardStorage)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with standard storage: %v", err)
+	}
+
+	rrDriver, err := s3DriverConstructor(rootDir, s3.ReducedRedundancy)
+	if err != nil {
+		t.Fatalf("unexpected error creating driver with reduced redundancy storage: %v", err)
+	}
+
+	standardFilename := "/test-standard"
+	rrFilename := "/test-rr"
+	contents := []byte("contents")
+	ctx := context.Background()
+
+	err = standardDriver.PutContent(ctx, standardFilename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer standardDriver.Delete(ctx, standardFilename)
+
+	err = rrDriver.PutContent(ctx, rrFilename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+	defer rrDriver.Delete(ctx, rrFilename)
+
+	standardDriverUnwrapped := standardDriver.Base.StorageDriver.(*driver)
+	resp, err := standardDriverUnwrapped.Bucket.GetResponse(standardDriverUnwrapped.s3Path(standardFilename))
+	if err != nil {
+		t.Fatalf("unexpected error retrieving standard storage file: %v", err)
+	}
+	defer resp.Body.Close()
+	// Amazon only populates this header value for non-standard storage classes
+	if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != "" {
+		t.Fatalf("unexpected storage class for standard file: %v", storageClass)
+	}
+
+	rrDriverUnwrapped := rrDriver.Base.StorageDriver.(*driver)
+	resp, err = rrDriverUnwrapped.Bucket.GetResponse(rrDriverUnwrapped.s3Path(rrFilename))
+	if err != nil {
+		t.Fatalf("unexpected error retrieving reduced-redundancy storage file: %v", err)
+	}
+	defer resp.Body.Close()
+	if storageClass := resp.Header.Get("x-amz-storage-class"); storageClass != string(s3.ReducedRedundancy) {
+		t.Fatalf("unexpected storage class for reduced-redundancy file: %v", storageClass)
+	}
+}

vendor/github.com/docker/distribution/registry/storage/driver/storagedriver.go

@@ -0,0 +1,165 @@
+package driver
+
+import (
+	"fmt"
+	"io"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/docker/distribution/context"
+)
+
+// Version is a string representing the storage driver version, of the form
+// Major.Minor.
+// The registry must accept storage drivers with equal major version and greater
+// minor version, but may not be compatible with older storage driver versions.
+type Version string
+
+// Major returns the major (primary) component of a version.
+func (version Version) Major() uint {
+	majorPart := strings.Split(string(version), ".")[0]
+	major, _ := strconv.ParseUint(majorPart, 10, 0)
+	return uint(major)
+}
+
+// Minor returns the minor (secondary) component of a version.
+func (version Version) Minor() uint {
+	minorPart := strings.Split(string(version), ".")[1]
+	minor, _ := strconv.ParseUint(minorPart, 10, 0)
+	return uint(minor)
+}
+
+// CurrentVersion is the current storage driver Version.
+const CurrentVersion Version = "0.1"
+
+// StorageDriver defines methods that a Storage Driver must implement for a
+// filesystem-like key/value object storage. Storage Drivers are automatically
+// registered via an internal registration mechanism, and generally created
+// via the StorageDriverFactory interface (https://godoc.org/github.com/docker/distribution/registry/storage/driver/factory).
+// Please see the aforementioned factory package for example code showing how to get an instance
+// of a StorageDriver
+type StorageDriver interface {
+	// Name returns the human-readable "name" of the driver, useful in error
+	// messages and logging. By convention, this will just be the registration
+	// name, but drivers may provide other information here.
+	Name() string
+
+	// GetContent retrieves the content stored at "path" as a []byte.
+	// This should primarily be used for small objects.
+	GetContent(ctx context.Context, path string) ([]byte, error)
+
+	// PutContent stores the []byte content at a location designated by "path".
+	// This should primarily be used for small objects.
+	PutContent(ctx context.Context, path string, content []byte) error
+
+	// Reader retrieves an io.ReadCloser for the content stored at "path"
+	// with a given byte offset.
+	// May be used to resume reading a stream by providing a nonzero offset.
+	Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error)
+
+	// Writer returns a FileWriter which will store the content written to it
+	// at the location designated by "path" after the call to Commit.
+	Writer(ctx context.Context, path string, append bool) (FileWriter, error)
+
+	// Stat retrieves the FileInfo for the given path, including the current
+	// size in bytes and the creation time.
+	Stat(ctx context.Context, path string) (FileInfo, error)
+
+	// List returns a list of the objects that are direct descendants of the
+	//given path.
+	List(ctx context.Context, path string) ([]string, error)
+
+	// Move moves an object stored at sourcePath to destPath, removing the
+	// original object.
+	// Note: This may be no more efficient than a copy followed by a delete for
+	// many implementations.
+	Move(ctx context.Context, sourcePath string, destPath string) error
+
+	// Delete recursively deletes all objects stored at "path" and its subpaths.
+	Delete(ctx context.Context, path string) error
+
+	// URLFor returns a URL which may be used to retrieve the content stored at
+	// the given path, possibly using the given options.
+	// May return an ErrUnsupportedMethod in certain StorageDriver
+	// implementations.
+	URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error)
+}
+
+// FileWriter provides an abstraction for an opened writable file-like object in
+// the storage backend. The FileWriter must flush all content written to it on
+// the call to Close, but is only required to make its content readable on a
+// call to Commit.
+type FileWriter interface {
+	io.WriteCloser
+
+	// Size returns the number of bytes written to this FileWriter.
+	Size() int64
+
+	// Cancel removes any written content from this FileWriter.
+	Cancel() error
+
+	// Commit flushes all content written to this FileWriter and makes it
+	// available for future calls to StorageDriver.GetContent and
+	// StorageDriver.Reader.
+	Commit() error
+}
+
+// PathRegexp is the regular expression which each file path must match. A
+// file path is absolute, beginning with a slash and containing a positive
+// number of path components separated by slashes, where each component is
+// restricted to alphanumeric characters or a period, underscore, or
+// hyphen.
+var PathRegexp = regexp.MustCompile(`^(/[A-Za-z0-9._-]+)+$`)
+
+// ErrUnsupportedMethod may be returned in the case where a StorageDriver implementation does not support an optional method.
+type ErrUnsupportedMethod struct {
+	DriverName string
+}
+
+func (err ErrUnsupportedMethod) Error() string {
+	return fmt.Sprintf("%s: unsupported method", err.DriverName)
+}
+
+// PathNotFoundError is returned when operating on a nonexistent path.
+type PathNotFoundError struct {
+	Path       string
+	DriverName string
+}
+
+func (err PathNotFoundError) Error() string {
+	return fmt.Sprintf("%s: Path not found: %s", err.DriverName, err.Path)
+}
+
+// InvalidPathError is returned when the provided path is malformed.
+type InvalidPathError struct {
+	Path       string
+	DriverName string
+}
+
+func (err InvalidPathError) Error() string {
+	return fmt.Sprintf("%s: invalid path: %s", err.DriverName, err.Path)
+}
+
+// InvalidOffsetError is returned when attempting to read or write from an
+// invalid offset.
+type InvalidOffsetError struct {
+	Path       string
+	Offset     int64
+	DriverName string
+}
+
+func (err InvalidOffsetError) Error() string {
+	return fmt.Sprintf("%s: invalid offset: %d for path: %s", err.DriverName, err.Offset, err.Path)
+}
+
+// Error is a catch-all error type which captures an error string and
+// the driver type on which it occurred.
+type Error struct {
+	DriverName string
+	Enclosed   error
+}
+
+func (err Error) Error() string {
+	return fmt.Sprintf("%s: %s", err.DriverName, err.Enclosed)
+}

vendor/github.com/docker/distribution/registry/storage/driver/swift/swift_test.go

@@ -0,0 +1,245 @@
+package swift
+
+import (
+	"io/ioutil"
+	"os"
+	"reflect"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/ncw/swift/swifttest"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/testsuites"
+
+	"gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { check.TestingT(t) }
+
+var swiftDriverConstructor func(prefix string) (*Driver, error)
+
+func init() {
+	var (
+		username           string
+		password           string
+		authURL            string
+		tenant             string
+		tenantID           string
+		domain             string
+		domainID           string
+		tenantDomain       string
+		tenantDomainID     string
+		trustID            string
+		container          string
+		region             string
+		AuthVersion        int
+		endpointType       string
+		insecureSkipVerify bool
+		secretKey          string
+		accessKey          string
+		containerKey       bool
+		tempURLMethods     []string
+
+		swiftServer *swifttest.SwiftServer
+		err         error
+	)
+	username = os.Getenv("SWIFT_USERNAME")
+	password = os.Getenv("SWIFT_PASSWORD")
+	authURL = os.Getenv("SWIFT_AUTH_URL")
+	tenant = os.Getenv("SWIFT_TENANT_NAME")
+	tenantID = os.Getenv("SWIFT_TENANT_ID")
+	domain = os.Getenv("SWIFT_DOMAIN_NAME")
+	domainID = os.Getenv("SWIFT_DOMAIN_ID")
+	tenantDomain = os.Getenv("SWIFT_DOMAIN_NAME")
+	tenantDomainID = os.Getenv("SWIFT_DOMAIN_ID")
+	trustID = os.Getenv("SWIFT_TRUST_ID")
+	container = os.Getenv("SWIFT_CONTAINER_NAME")
+	region = os.Getenv("SWIFT_REGION_NAME")
+	AuthVersion, _ = strconv.Atoi(os.Getenv("SWIFT_AUTH_VERSION"))
+	endpointType = os.Getenv("SWIFT_ENDPOINT_TYPE")
+	insecureSkipVerify, _ = strconv.ParseBool(os.Getenv("SWIFT_INSECURESKIPVERIFY"))
+	secretKey = os.Getenv("SWIFT_SECRET_KEY")
+	accessKey = os.Getenv("SWIFT_ACCESS_KEY")
+	containerKey, _ = strconv.ParseBool(os.Getenv("SWIFT_TEMPURL_CONTAINERKEY"))
+	tempURLMethods = strings.Split(os.Getenv("SWIFT_TEMPURL_METHODS"), ",")
+
+	if username == "" || password == "" || authURL == "" || container == "" {
+		if swiftServer, err = swifttest.NewSwiftServer("localhost"); err != nil {
+			panic(err)
+		}
+		username = "swifttest"
+		password = "swifttest"
+		authURL = swiftServer.AuthURL
+		container = "test"
+	}
+
+	prefix, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		panic(err)
+	}
+	defer os.Remove(prefix)
+
+	swiftDriverConstructor = func(root string) (*Driver, error) {
+		parameters := Parameters{
+			username,
+			password,
+			authURL,
+			tenant,
+			tenantID,
+			domain,
+			domainID,
+			tenantDomain,
+			tenantDomainID,
+			trustID,
+			region,
+			AuthVersion,
+			container,
+			root,
+			endpointType,
+			insecureSkipVerify,
+			defaultChunkSize,
+			secretKey,
+			accessKey,
+			containerKey,
+			tempURLMethods,
+		}
+
+		return New(parameters)
+	}
+
+	driverConstructor := func() (storagedriver.StorageDriver, error) {
+		return swiftDriverConstructor(prefix)
+	}
+
+	testsuites.RegisterSuite(driverConstructor, testsuites.NeverSkip)
+}
+
+func TestEmptyRootList(t *testing.T) {
+	validRoot, err := ioutil.TempDir("", "driver-")
+	if err != nil {
+		t.Fatalf("unexpected error creating temporary directory: %v", err)
+	}
+	defer os.Remove(validRoot)
+
+	rootedDriver, err := swiftDriverConstructor(validRoot)
+	if err != nil {
+		t.Fatalf("unexpected error creating rooted driver: %v", err)
+	}
+
+	emptyRootDriver, err := swiftDriverConstructor("")
+	if err != nil {
+		t.Fatalf("unexpected error creating empty root driver: %v", err)
+	}
+
+	slashRootDriver, err := swiftDriverConstructor("/")
+	if err != nil {
+		t.Fatalf("unexpected error creating slash root driver: %v", err)
+	}
+
+	filename := "/test"
+	contents := []byte("contents")
+	ctx := context.Background()
+	err = rootedDriver.PutContent(ctx, filename, contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+
+	keys, err := emptyRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+
+	keys, err = slashRootDriver.List(ctx, "/")
+	for _, path := range keys {
+		if !storagedriver.PathRegexp.MatchString(path) {
+			t.Fatalf("unexpected string in path: %q != %q", path, storagedriver.PathRegexp)
+		}
+	}
+
+	// Create an object with a path nested under the existing object
+	err = rootedDriver.PutContent(ctx, filename+"/file1", contents)
+	if err != nil {
+		t.Fatalf("unexpected error creating content: %v", err)
+	}
+
+	err = rootedDriver.Delete(ctx, filename)
+	if err != nil {
+		t.Fatalf("failed to delete: %v", err)
+	}
+
+	keys, err = rootedDriver.List(ctx, "/")
+	if err != nil {
+		t.Fatalf("failed to list objects after deletion: %v", err)
+	}
+
+	if len(keys) != 0 {
+		t.Fatal("delete did not remove nested objects")
+	}
+}
+
+func TestFilenameChunking(t *testing.T) {
+	// Test valid input and sizes
+	input := []string{"a", "b", "c", "d", "e"}
+	expecteds := [][][]string{
+		{
+			{"a"},
+			{"b"},
+			{"c"},
+			{"d"},
+			{"e"},
+		},
+		{
+			{"a", "b"},
+			{"c", "d"},
+			{"e"},
+		},
+		{
+			{"a", "b", "c"},
+			{"d", "e"},
+		},
+		{
+			{"a", "b", "c", "d"},
+			{"e"},
+		},
+		{
+			{"a", "b", "c", "d", "e"},
+		},
+		{
+			{"a", "b", "c", "d", "e"},
+		},
+	}
+	for i, expected := range expecteds {
+		actual, err := chunkFilenames(input, i+1)
+		if !reflect.DeepEqual(actual, expected) {
+			t.Fatalf("chunk %v didn't match expected value %v", actual, expected)
+		}
+		if err != nil {
+			t.Fatalf("unexpected error chunking filenames: %v", err)
+		}
+	}
+
+	// Test nil input
+	actual, err := chunkFilenames(nil, 5)
+	if len(actual) != 0 {
+		t.Fatal("chunks were returned when passed nil")
+	}
+	if err != nil {
+		t.Fatalf("unexpected error chunking filenames: %v", err)
+	}
+
+	// Test 0 and < 0 sizes
+	actual, err = chunkFilenames(nil, 0)
+	if err == nil {
+		t.Fatal("expected error for size = 0")
+	}
+	actual, err = chunkFilenames(nil, -1)
+	if err == nil {
+		t.Fatal("expected error for size = -1")
+	}
+}

vendor/github.com/docker/distribution/registry/storage/driver/testdriver/testdriver.go

@@ -0,0 +1,71 @@
+package testdriver
+
+import (
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/factory"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+const driverName = "testdriver"
+
+func init() {
+	factory.Register(driverName, &testDriverFactory{})
+}
+
+// testDriverFactory implements the factory.StorageDriverFactory interface.
+type testDriverFactory struct{}
+
+func (factory *testDriverFactory) Create(parameters map[string]interface{}) (storagedriver.StorageDriver, error) {
+	return New(), nil
+}
+
+// TestDriver is a StorageDriver for testing purposes. The Writer returned by this driver
+// simulates the case where Write operations are buffered. This causes the value returned by Size to lag
+// behind until Close (or Commit, or Cancel) is called.
+type TestDriver struct {
+	storagedriver.StorageDriver
+}
+
+type testFileWriter struct {
+	storagedriver.FileWriter
+	prevchunk []byte
+}
+
+var _ storagedriver.StorageDriver = &TestDriver{}
+
+// New constructs a new StorageDriver for testing purposes. The Writer returned by this driver
+// simulates the case where Write operations are buffered. This causes the value returned by Size to lag
+// behind until Close (or Commit, or Cancel) is called.
+func New() *TestDriver {
+	return &TestDriver{StorageDriver: inmemory.New()}
+}
+
+// Writer returns a FileWriter which will store the content written to it
+// at the location designated by "path" after the call to Commit.
+func (td *TestDriver) Writer(ctx context.Context, path string, append bool) (storagedriver.FileWriter, error) {
+	fw, err := td.StorageDriver.Writer(ctx, path, append)
+	return &testFileWriter{FileWriter: fw}, err
+}
+
+func (tfw *testFileWriter) Write(p []byte) (int, error) {
+	_, err := tfw.FileWriter.Write(tfw.prevchunk)
+	tfw.prevchunk = make([]byte, len(p))
+	copy(tfw.prevchunk, p)
+	return len(p), err
+}
+
+func (tfw *testFileWriter) Close() error {
+	tfw.Write(nil)
+	return tfw.FileWriter.Close()
+}
+
+func (tfw *testFileWriter) Cancel() error {
+	tfw.Write(nil)
+	return tfw.FileWriter.Cancel()
+}
+
+func (tfw *testFileWriter) Commit() error {
+	tfw.Write(nil)
+	return tfw.FileWriter.Commit()
+}

vendor/github.com/docker/distribution/registry/storage/filereader.go

@@ -0,0 +1,177 @@
+package storage
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+
+	"github.com/docker/distribution/context"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// TODO(stevvooe): Set an optimal buffer size here. We'll have to
+// understand the latency characteristics of the underlying network to
+// set this correctly, so we may want to leave it to the driver. For
+// out of process drivers, we'll have to optimize this buffer size for
+// local communication.
+const fileReaderBufferSize = 4 << 20
+
+// remoteFileReader provides a read seeker interface to files stored in
+// storagedriver. Used to implement part of layer interface and will be used
+// to implement read side of LayerUpload.
+type fileReader struct {
+	driver storagedriver.StorageDriver
+
+	ctx context.Context
+
+	// identifying fields
+	path string
+	size int64 // size is the total size, must be set.
+
+	// mutable fields
+	rc     io.ReadCloser // remote read closer
+	brd    *bufio.Reader // internal buffered io
+	offset int64         // offset is the current read offset
+	err    error         // terminal error, if set, reader is closed
+}
+
+// newFileReader initializes a file reader for the remote file. The reader
+// takes on the size and path that must be determined externally with a stat
+// call. The reader operates optimistically, assuming that the file is already
+// there.
+func newFileReader(ctx context.Context, driver storagedriver.StorageDriver, path string, size int64) (*fileReader, error) {
+	return &fileReader{
+		ctx:    ctx,
+		driver: driver,
+		path:   path,
+		size:   size,
+	}, nil
+}
+
+func (fr *fileReader) Read(p []byte) (n int, err error) {
+	if fr.err != nil {
+		return 0, fr.err
+	}
+
+	rd, err := fr.reader()
+	if err != nil {
+		return 0, err
+	}
+
+	n, err = rd.Read(p)
+	fr.offset += int64(n)
+
+	// Simulate io.EOR error if we reach filesize.
+	if err == nil && fr.offset >= fr.size {
+		err = io.EOF
+	}
+
+	return n, err
+}
+
+func (fr *fileReader) Seek(offset int64, whence int) (int64, error) {
+	if fr.err != nil {
+		return 0, fr.err
+	}
+
+	var err error
+	newOffset := fr.offset
+
+	switch whence {
+	case os.SEEK_CUR:
+		newOffset += int64(offset)
+	case os.SEEK_END:
+		newOffset = fr.size + int64(offset)
+	case os.SEEK_SET:
+		newOffset = int64(offset)
+	}
+
+	if newOffset < 0 {
+		err = fmt.Errorf("cannot seek to negative position")
+	} else {
+		if fr.offset != newOffset {
+			fr.reset()
+		}
+
+		// No problems, set the offset.
+		fr.offset = newOffset
+	}
+
+	return fr.offset, err
+}
+
+func (fr *fileReader) Close() error {
+	return fr.closeWithErr(fmt.Errorf("fileReader: closed"))
+}
+
+// reader prepares the current reader at the lrs offset, ensuring its buffered
+// and ready to go.
+func (fr *fileReader) reader() (io.Reader, error) {
+	if fr.err != nil {
+		return nil, fr.err
+	}
+
+	if fr.rc != nil {
+		return fr.brd, nil
+	}
+
+	// If we don't have a reader, open one up.
+	rc, err := fr.driver.Reader(fr.ctx, fr.path, fr.offset)
+	if err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			// NOTE(stevvooe): If the path is not found, we simply return a
+			// reader that returns io.EOF. However, we do not set fr.rc,
+			// allowing future attempts at getting a reader to possibly
+			// succeed if the file turns up later.
+			return ioutil.NopCloser(bytes.NewReader([]byte{})), nil
+		default:
+			return nil, err
+		}
+	}
+
+	fr.rc = rc
+
+	if fr.brd == nil {
+		fr.brd = bufio.NewReaderSize(fr.rc, fileReaderBufferSize)
+	} else {
+		fr.brd.Reset(fr.rc)
+	}
+
+	return fr.brd, nil
+}
+
+// resetReader resets the reader, forcing the read method to open up a new
+// connection and rebuild the buffered reader. This should be called when the
+// offset and the reader will become out of sync, such as during a seek
+// operation.
+func (fr *fileReader) reset() {
+	if fr.err != nil {
+		return
+	}
+	if fr.rc != nil {
+		fr.rc.Close()
+		fr.rc = nil
+	}
+}
+
+func (fr *fileReader) closeWithErr(err error) error {
+	if fr.err != nil {
+		return fr.err
+	}
+
+	fr.err = err
+
+	// close and release reader chain
+	if fr.rc != nil {
+		fr.rc.Close()
+	}
+
+	fr.rc = nil
+	fr.brd = nil
+
+	return fr.err
+}

vendor/github.com/docker/distribution/registry/storage/filereader_test.go

@@ -0,0 +1,198 @@
+package storage
+
+import (
+	"bytes"
+	"io"
+	mrand "math/rand"
+	"os"
+	"testing"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+func TestSimpleRead(t *testing.T) {
+	ctx := context.Background()
+	content := make([]byte, 1<<20)
+	n, err := mrand.Read(content)
+	if err != nil {
+		t.Fatalf("unexpected error building random data: %v", err)
+	}
+
+	if n != len(content) {
+		t.Fatalf("random read didn't fill buffer")
+	}
+
+	dgst, err := digest.FromReader(bytes.NewReader(content))
+	if err != nil {
+		t.Fatalf("unexpected error digesting random content: %v", err)
+	}
+
+	driver := inmemory.New()
+	path := "/random"
+
+	if err := driver.PutContent(ctx, path, content); err != nil {
+		t.Fatalf("error putting patterned content: %v", err)
+	}
+
+	fr, err := newFileReader(ctx, driver, path, int64(len(content)))
+	if err != nil {
+		t.Fatalf("error allocating file reader: %v", err)
+	}
+
+	verifier, err := digest.NewDigestVerifier(dgst)
+	if err != nil {
+		t.Fatalf("error getting digest verifier: %s", err)
+	}
+
+	io.Copy(verifier, fr)
+
+	if !verifier.Verified() {
+		t.Fatalf("unable to verify read data")
+	}
+}
+
+func TestFileReaderSeek(t *testing.T) {
+	driver := inmemory.New()
+	pattern := "01234567890ab" // prime length block
+	repititions := 1024
+	path := "/patterned"
+	content := bytes.Repeat([]byte(pattern), repititions)
+	ctx := context.Background()
+
+	if err := driver.PutContent(ctx, path, content); err != nil {
+		t.Fatalf("error putting patterned content: %v", err)
+	}
+
+	fr, err := newFileReader(ctx, driver, path, int64(len(content)))
+
+	if err != nil {
+		t.Fatalf("unexpected error creating file reader: %v", err)
+	}
+
+	// Seek all over the place, in blocks of pattern size and make sure we get
+	// the right data.
+	for _, repitition := range mrand.Perm(repititions - 1) {
+		targetOffset := int64(len(pattern) * repitition)
+		// Seek to a multiple of pattern size and read pattern size bytes
+		offset, err := fr.Seek(targetOffset, os.SEEK_SET)
+		if err != nil {
+			t.Fatalf("unexpected error seeking: %v", err)
+		}
+
+		if offset != targetOffset {
+			t.Fatalf("did not seek to correct offset: %d != %d", offset, targetOffset)
+		}
+
+		p := make([]byte, len(pattern))
+
+		n, err := fr.Read(p)
+		if err != nil {
+			t.Fatalf("error reading pattern: %v", err)
+		}
+
+		if n != len(pattern) {
+			t.Fatalf("incorrect read length: %d != %d", n, len(pattern))
+		}
+
+		if string(p) != pattern {
+			t.Fatalf("incorrect read content: %q != %q", p, pattern)
+		}
+
+		// Check offset
+		current, err := fr.Seek(0, os.SEEK_CUR)
+		if err != nil {
+			t.Fatalf("error checking current offset: %v", err)
+		}
+
+		if current != targetOffset+int64(len(pattern)) {
+			t.Fatalf("unexpected offset after read: %v", err)
+		}
+	}
+
+	start, err := fr.Seek(0, os.SEEK_SET)
+	if err != nil {
+		t.Fatalf("error seeking to start: %v", err)
+	}
+
+	if start != 0 {
+		t.Fatalf("expected to seek to start: %v != 0", start)
+	}
+
+	end, err := fr.Seek(0, os.SEEK_END)
+	if err != nil {
+		t.Fatalf("error checking current offset: %v", err)
+	}
+
+	if end != int64(len(content)) {
+		t.Fatalf("expected to seek to end: %v != %v", end, len(content))
+	}
+
+	// 4. Seek before start, ensure error.
+
+	// seek before start
+	before, err := fr.Seek(-1, os.SEEK_SET)
+	if err == nil {
+		t.Fatalf("error expected, returned offset=%v", before)
+	}
+
+	// 5. Seek after end,
+	after, err := fr.Seek(1, os.SEEK_END)
+	if err != nil {
+		t.Fatalf("unexpected error expected, returned offset=%v", after)
+	}
+
+	p := make([]byte, 16)
+	n, err := fr.Read(p)
+
+	if n != 0 {
+		t.Fatalf("bytes reads %d != %d", n, 0)
+	}
+
+	if err != io.EOF {
+		t.Fatalf("expected io.EOF, got %v", err)
+	}
+}
+
+// TestFileReaderNonExistentFile ensures the reader behaves as expected with a
+// missing or zero-length remote file. While the file may not exist, the
+// reader should not error out on creation and should return 0-bytes from the
+// read method, with an io.EOF error.
+func TestFileReaderNonExistentFile(t *testing.T) {
+	driver := inmemory.New()
+	fr, err := newFileReader(context.Background(), driver, "/doesnotexist", 10)
+	if err != nil {
+		t.Fatalf("unexpected error initializing reader: %v", err)
+	}
+
+	var buf [1024]byte
+
+	n, err := fr.Read(buf[:])
+	if n != 0 {
+		t.Fatalf("non-zero byte read reported: %d != 0", n)
+	}
+
+	if err != io.EOF {
+		t.Fatalf("read on missing file should return io.EOF, got %v", err)
+	}
+}
+
+// TestLayerReadErrors covers the various error return type for different
+// conditions that can arise when reading a layer.
+func TestFileReaderErrors(t *testing.T) {
+	// TODO(stevvooe): We need to cover error return types, driven by the
+	// errors returned via the HTTP API. For now, here is an incomplete list:
+	//
+	// 	1. Layer Not Found: returned when layer is not found or access is
+	//        denied.
+	//	2. Layer Unavailable: returned when link references are unresolved,
+	//     but layer is known to the registry.
+	//  3. Layer Invalid: This may more split into more errors, but should be
+	//     returned when name or tarsum does not reference a valid error. We
+	//     may also need something to communication layer verification errors
+	//     for the inline tarsum check.
+	//	4. Timeout: timeouts to backend. Need to better understand these
+	//     failure cases and how the storage driver propagates these errors
+	//     up the stack.
+}

vendor/github.com/docker/distribution/registry/storage/garbagecollect.go

@@ -0,0 +1,114 @@
+package storage
+
+import (
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+func emit(format string, a ...interface{}) {
+	fmt.Printf(format+"\n", a...)
+}
+
+// MarkAndSweep performs a mark and sweep of registry data
+func MarkAndSweep(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, dryRun bool) error {
+	repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
+	if !ok {
+		return fmt.Errorf("unable to convert Namespace to RepositoryEnumerator")
+	}
+
+	// mark
+	markSet := make(map[digest.Digest]struct{})
+	err := repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
+		emit(repoName)
+
+		var err error
+		named, err := reference.ParseNamed(repoName)
+		if err != nil {
+			return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
+		}
+		repository, err := registry.Repository(ctx, named)
+		if err != nil {
+			return fmt.Errorf("failed to construct repository: %v", err)
+		}
+
+		manifestService, err := repository.Manifests(ctx)
+		if err != nil {
+			return fmt.Errorf("failed to construct manifest service: %v", err)
+		}
+
+		manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
+		if !ok {
+			return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
+		}
+
+		err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
+			// Mark the manifest's blob
+			emit("%s: marking manifest %s ", repoName, dgst)
+			markSet[dgst] = struct{}{}
+
+			manifest, err := manifestService.Get(ctx, dgst)
+			if err != nil {
+				return fmt.Errorf("failed to retrieve manifest for digest %v: %v", dgst, err)
+			}
+
+			descriptors := manifest.References()
+			for _, descriptor := range descriptors {
+				markSet[descriptor.Digest] = struct{}{}
+				emit("%s: marking blob %s", repoName, descriptor.Digest)
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			// In certain situations such as unfinished uploads, deleting all
+			// tags in S3 or removing the _manifests folder manually, this
+			// error may be of type PathNotFound.
+			//
+			// In these cases we can continue marking other manifests safely.
+			if _, ok := err.(driver.PathNotFoundError); ok {
+				return nil
+			}
+		}
+
+		return err
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to mark: %v", err)
+	}
+
+	// sweep
+	blobService := registry.Blobs()
+	deleteSet := make(map[digest.Digest]struct{})
+	err = blobService.Enumerate(ctx, func(dgst digest.Digest) error {
+		// check if digest is in markSet. If not, delete it!
+		if _, ok := markSet[dgst]; !ok {
+			deleteSet[dgst] = struct{}{}
+		}
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("error enumerating blobs: %v", err)
+	}
+	emit("\n%d blobs marked, %d blobs eligible for deletion", len(markSet), len(deleteSet))
+	// Construct vacuum
+	vacuum := NewVacuum(ctx, storageDriver)
+	for dgst := range deleteSet {
+		emit("blob eligible for deletion: %s", dgst)
+		if dryRun {
+			continue
+		}
+		err = vacuum.RemoveBlob(string(dgst))
+		if err != nil {
+			return fmt.Errorf("failed to delete blob %s: %v", dgst, err)
+		}
+	}
+
+	return err
+}

vendor/github.com/docker/distribution/registry/storage/garbagecollect_test.go

@@ -0,0 +1,377 @@
+package storage
+
+import (
+	"io"
+	"path"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/testutil"
+	"github.com/docker/libtrust"
+)
+
+type image struct {
+	manifest       distribution.Manifest
+	manifestDigest digest.Digest
+	layers         map[digest.Digest]io.ReadSeeker
+}
+
+func createRegistry(t *testing.T, driver driver.StorageDriver, options ...RegistryOption) distribution.Namespace {
+	ctx := context.Background()
+	k, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	options = append([]RegistryOption{EnableDelete, Schema1SigningKey(k)}, options...)
+	registry, err := NewRegistry(ctx, driver, options...)
+	if err != nil {
+		t.Fatalf("Failed to construct namespace")
+	}
+	return registry
+}
+
+func makeRepository(t *testing.T, registry distribution.Namespace, name string) distribution.Repository {
+	ctx := context.Background()
+
+	// Initialize a dummy repository
+	named, err := reference.ParseNamed(name)
+	if err != nil {
+		t.Fatalf("Failed to parse name %s:  %v", name, err)
+	}
+
+	repo, err := registry.Repository(ctx, named)
+	if err != nil {
+		t.Fatalf("Failed to construct repository: %v", err)
+	}
+	return repo
+}
+
+func makeManifestService(t *testing.T, repository distribution.Repository) distribution.ManifestService {
+	ctx := context.Background()
+
+	manifestService, err := repository.Manifests(ctx)
+	if err != nil {
+		t.Fatalf("Failed to construct manifest store: %v", err)
+	}
+	return manifestService
+}
+
+func allBlobs(t *testing.T, registry distribution.Namespace) map[digest.Digest]struct{} {
+	ctx := context.Background()
+	blobService := registry.Blobs()
+	allBlobsMap := make(map[digest.Digest]struct{})
+	err := blobService.Enumerate(ctx, func(dgst digest.Digest) error {
+		allBlobsMap[dgst] = struct{}{}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Error getting all blobs: %v", err)
+	}
+	return allBlobsMap
+}
+
+func uploadImage(t *testing.T, repository distribution.Repository, im image) digest.Digest {
+	// upload layers
+	err := testutil.UploadBlobs(repository, im.layers)
+	if err != nil {
+		t.Fatalf("layer upload failed: %v", err)
+	}
+
+	// upload manifest
+	ctx := context.Background()
+	manifestService := makeManifestService(t, repository)
+	manifestDigest, err := manifestService.Put(ctx, im.manifest)
+	if err != nil {
+		t.Fatalf("manifest upload failed: %v", err)
+	}
+
+	return manifestDigest
+}
+
+func uploadRandomSchema1Image(t *testing.T, repository distribution.Repository) image {
+	randomLayers, err := testutil.CreateRandomLayers(2)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	digests := []digest.Digest{}
+	for digest := range randomLayers {
+		digests = append(digests, digest)
+	}
+
+	manifest, err := testutil.MakeSchema1Manifest(digests)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	manifestDigest := uploadImage(t, repository, image{manifest: manifest, layers: randomLayers})
+	return image{
+		manifest:       manifest,
+		manifestDigest: manifestDigest,
+		layers:         randomLayers,
+	}
+}
+
+func uploadRandomSchema2Image(t *testing.T, repository distribution.Repository) image {
+	randomLayers, err := testutil.CreateRandomLayers(2)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	digests := []digest.Digest{}
+	for digest := range randomLayers {
+		digests = append(digests, digest)
+	}
+
+	manifest, err := testutil.MakeSchema2Manifest(repository, digests)
+	if err != nil {
+		t.Fatalf("%v", err)
+	}
+
+	manifestDigest := uploadImage(t, repository, image{manifest: manifest, layers: randomLayers})
+	return image{
+		manifest:       manifest,
+		manifestDigest: manifestDigest,
+		layers:         randomLayers,
+	}
+}
+
+func TestNoDeletionNoEffect(t *testing.T) {
+	ctx := context.Background()
+	inmemoryDriver := inmemory.New()
+
+	registry := createRegistry(t, inmemoryDriver)
+	repo := makeRepository(t, registry, "palailogos")
+	manifestService, err := repo.Manifests(ctx)
+
+	image1 := uploadRandomSchema1Image(t, repo)
+	image2 := uploadRandomSchema1Image(t, repo)
+	uploadRandomSchema2Image(t, repo)
+
+	// construct manifestlist for fun.
+	blobstatter := registry.BlobStatter()
+	manifestList, err := testutil.MakeManifestList(blobstatter, []digest.Digest{
+		image1.manifestDigest, image2.manifestDigest})
+	if err != nil {
+		t.Fatalf("Failed to make manifest list: %v", err)
+	}
+
+	_, err = manifestService.Put(ctx, manifestList)
+	if err != nil {
+		t.Fatalf("Failed to add manifest list: %v", err)
+	}
+
+	before := allBlobs(t, registry)
+
+	// Run GC
+	err = MarkAndSweep(context.Background(), inmemoryDriver, registry, false)
+	if err != nil {
+		t.Fatalf("Failed mark and sweep: %v", err)
+	}
+
+	after := allBlobs(t, registry)
+	if len(before) != len(after) {
+		t.Fatalf("Garbage collection affected storage: %d != %d", len(before), len(after))
+	}
+}
+
+func TestGCWithMissingManifests(t *testing.T) {
+	ctx := context.Background()
+	d := inmemory.New()
+
+	registry := createRegistry(t, d)
+	repo := makeRepository(t, registry, "testrepo")
+	uploadRandomSchema1Image(t, repo)
+
+	// Simulate a missing _manifests directory
+	revPath, err := pathFor(manifestRevisionsPathSpec{"testrepo"})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_manifestsPath := path.Dir(revPath)
+	err = d.Delete(ctx, _manifestsPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = MarkAndSweep(context.Background(), d, registry, false)
+	if err != nil {
+		t.Fatalf("Failed mark and sweep: %v", err)
+	}
+
+	blobs := allBlobs(t, registry)
+	if len(blobs) > 0 {
+		t.Errorf("unexpected blobs after gc")
+	}
+}
+
+func TestDeletionHasEffect(t *testing.T) {
+	ctx := context.Background()
+	inmemoryDriver := inmemory.New()
+
+	registry := createRegistry(t, inmemoryDriver)
+	repo := makeRepository(t, registry, "komnenos")
+	manifests, err := repo.Manifests(ctx)
+
+	image1 := uploadRandomSchema1Image(t, repo)
+	image2 := uploadRandomSchema1Image(t, repo)
+	image3 := uploadRandomSchema2Image(t, repo)
+
+	manifests.Delete(ctx, image2.manifestDigest)
+	manifests.Delete(ctx, image3.manifestDigest)
+
+	// Run GC
+	err = MarkAndSweep(context.Background(), inmemoryDriver, registry, false)
+	if err != nil {
+		t.Fatalf("Failed mark and sweep: %v", err)
+	}
+
+	blobs := allBlobs(t, registry)
+
+	// check that the image1 manifest and all the layers are still in blobs
+	if _, ok := blobs[image1.manifestDigest]; !ok {
+		t.Fatalf("First manifest is missing")
+	}
+
+	for layer := range image1.layers {
+		if _, ok := blobs[layer]; !ok {
+			t.Fatalf("manifest 1 layer is missing: %v", layer)
+		}
+	}
+
+	// check that image2 and image3 layers are not still around
+	for layer := range image2.layers {
+		if _, ok := blobs[layer]; ok {
+			t.Fatalf("manifest 2 layer is present: %v", layer)
+		}
+	}
+
+	for layer := range image3.layers {
+		if _, ok := blobs[layer]; ok {
+			t.Fatalf("manifest 3 layer is present: %v", layer)
+		}
+	}
+}
+
+func getAnyKey(digests map[digest.Digest]io.ReadSeeker) (d digest.Digest) {
+	for d = range digests {
+		break
+	}
+	return
+}
+
+func getKeys(digests map[digest.Digest]io.ReadSeeker) (ds []digest.Digest) {
+	for d := range digests {
+		ds = append(ds, d)
+	}
+	return
+}
+
+func TestDeletionWithSharedLayer(t *testing.T) {
+	ctx := context.Background()
+	inmemoryDriver := inmemory.New()
+
+	registry := createRegistry(t, inmemoryDriver)
+	repo := makeRepository(t, registry, "tzimiskes")
+
+	// Create random layers
+	randomLayers1, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	randomLayers2, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	// Upload all layers
+	err = testutil.UploadBlobs(repo, randomLayers1)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	err = testutil.UploadBlobs(repo, randomLayers2)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	// Construct manifests
+	manifest1, err := testutil.MakeSchema1Manifest(getKeys(randomLayers1))
+	if err != nil {
+		t.Fatalf("failed to make manifest: %v", err)
+	}
+
+	sharedKey := getAnyKey(randomLayers1)
+	manifest2, err := testutil.MakeSchema2Manifest(repo, append(getKeys(randomLayers2), sharedKey))
+	if err != nil {
+		t.Fatalf("failed to make manifest: %v", err)
+	}
+
+	manifestService := makeManifestService(t, repo)
+
+	// Upload manifests
+	_, err = manifestService.Put(ctx, manifest1)
+	if err != nil {
+		t.Fatalf("manifest upload failed: %v", err)
+	}
+
+	manifestDigest2, err := manifestService.Put(ctx, manifest2)
+	if err != nil {
+		t.Fatalf("manifest upload failed: %v", err)
+	}
+
+	// delete
+	err = manifestService.Delete(ctx, manifestDigest2)
+	if err != nil {
+		t.Fatalf("manifest deletion failed: %v", err)
+	}
+
+	// check that all of the layers in layer 1 are still there
+	blobs := allBlobs(t, registry)
+	for dgst := range randomLayers1 {
+		if _, ok := blobs[dgst]; !ok {
+			t.Fatalf("random layer 1 blob missing: %v", dgst)
+		}
+	}
+}
+
+func TestOrphanBlobDeleted(t *testing.T) {
+	inmemoryDriver := inmemory.New()
+
+	registry := createRegistry(t, inmemoryDriver)
+	repo := makeRepository(t, registry, "michael_z_doukas")
+
+	digests, err := testutil.CreateRandomLayers(1)
+	if err != nil {
+		t.Fatalf("Failed to create random digest: %v", err)
+	}
+
+	if err = testutil.UploadBlobs(repo, digests); err != nil {
+		t.Fatalf("Failed to upload blob: %v", err)
+	}
+
+	// formality to create the necessary directories
+	uploadRandomSchema2Image(t, repo)
+
+	// Run GC
+	err = MarkAndSweep(context.Background(), inmemoryDriver, registry, false)
+	if err != nil {
+		t.Fatalf("Failed mark and sweep: %v", err)
+	}
+
+	blobs := allBlobs(t, registry)
+
+	// check that orphan blob layers are not still around
+	for dgst := range digests {
+		if _, ok := blobs[dgst]; ok {
+			t.Fatalf("Orphan layer is present: %v", dgst)
+		}
+	}
+}

vendor/github.com/docker/distribution/registry/storage/linkedblobstore.go

@@ -0,0 +1,470 @@
+package storage
+
+import (
+	"fmt"
+	"net/http"
+	"path"
+	"time"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/uuid"
+)
+
+// linkPathFunc describes a function that can resolve a link based on the
+// repository name and digest.
+type linkPathFunc func(name string, dgst digest.Digest) (string, error)
+
+// linkedBlobStore provides a full BlobService that namespaces the blobs to a
+// given repository. Effectively, it manages the links in a given repository
+// that grant access to the global blob store.
+type linkedBlobStore struct {
+	*blobStore
+	registry               *registry
+	blobServer             distribution.BlobServer
+	blobAccessController   distribution.BlobDescriptorService
+	repository             distribution.Repository
+	ctx                    context.Context // only to be used where context can't come through method args
+	deleteEnabled          bool
+	resumableDigestEnabled bool
+
+	// linkPathFns specifies one or more path functions allowing one to
+	// control the repository blob link set to which the blob store
+	// dispatches. This is required because manifest and layer blobs have not
+	// yet been fully merged. At some point, this functionality should be
+	// removed the blob links folder should be merged. The first entry is
+	// treated as the "canonical" link location and will be used for writes.
+	linkPathFns []linkPathFunc
+
+	// linkDirectoryPathSpec locates the root directories in which one might find links
+	linkDirectoryPathSpec pathSpec
+}
+
+var _ distribution.BlobStore = &linkedBlobStore{}
+
+func (lbs *linkedBlobStore) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	return lbs.blobAccessController.Stat(ctx, dgst)
+}
+
+func (lbs *linkedBlobStore) Get(ctx context.Context, dgst digest.Digest) ([]byte, error) {
+	canonical, err := lbs.Stat(ctx, dgst) // access check
+	if err != nil {
+		return nil, err
+	}
+
+	return lbs.blobStore.Get(ctx, canonical.Digest)
+}
+
+func (lbs *linkedBlobStore) Open(ctx context.Context, dgst digest.Digest) (distribution.ReadSeekCloser, error) {
+	canonical, err := lbs.Stat(ctx, dgst) // access check
+	if err != nil {
+		return nil, err
+	}
+
+	return lbs.blobStore.Open(ctx, canonical.Digest)
+}
+
+func (lbs *linkedBlobStore) ServeBlob(ctx context.Context, w http.ResponseWriter, r *http.Request, dgst digest.Digest) error {
+	canonical, err := lbs.Stat(ctx, dgst) // access check
+	if err != nil {
+		return err
+	}
+
+	if canonical.MediaType != "" {
+		// Set the repository local content type.
+		w.Header().Set("Content-Type", canonical.MediaType)
+	}
+
+	return lbs.blobServer.ServeBlob(ctx, w, r, canonical.Digest)
+}
+
+func (lbs *linkedBlobStore) Put(ctx context.Context, mediaType string, p []byte) (distribution.Descriptor, error) {
+	dgst := digest.FromBytes(p)
+	// Place the data in the blob store first.
+	desc, err := lbs.blobStore.Put(ctx, mediaType, p)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting into main store: %v", err)
+		return distribution.Descriptor{}, err
+	}
+
+	if err := lbs.blobAccessController.SetDescriptor(ctx, dgst, desc); err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	// TODO(stevvooe): Write out mediatype if incoming differs from what is
+	// returned by Put above. Note that we should allow updates for a given
+	// repository.
+
+	return desc, lbs.linkBlob(ctx, desc)
+}
+
+type optionFunc func(interface{}) error
+
+func (f optionFunc) Apply(v interface{}) error {
+	return f(v)
+}
+
+// WithMountFrom returns a BlobCreateOption which designates that the blob should be
+// mounted from the given canonical reference.
+func WithMountFrom(ref reference.Canonical) distribution.BlobCreateOption {
+	return optionFunc(func(v interface{}) error {
+		opts, ok := v.(*distribution.CreateOptions)
+		if !ok {
+			return fmt.Errorf("unexpected options type: %T", v)
+		}
+
+		opts.Mount.ShouldMount = true
+		opts.Mount.From = ref
+
+		return nil
+	})
+}
+
+// Writer begins a blob write session, returning a handle.
+func (lbs *linkedBlobStore) Create(ctx context.Context, options ...distribution.BlobCreateOption) (distribution.BlobWriter, error) {
+	context.GetLogger(ctx).Debug("(*linkedBlobStore).Writer")
+
+	var opts distribution.CreateOptions
+
+	for _, option := range options {
+		err := option.Apply(&opts)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if opts.Mount.ShouldMount {
+		desc, err := lbs.mount(ctx, opts.Mount.From, opts.Mount.From.Digest(), opts.Mount.Stat)
+		if err == nil {
+			// Mount successful, no need to initiate an upload session
+			return nil, distribution.ErrBlobMounted{From: opts.Mount.From, Descriptor: desc}
+		}
+	}
+
+	uuid := uuid.Generate().String()
+	startedAt := time.Now().UTC()
+
+	path, err := pathFor(uploadDataPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   uuid,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	startedAtPath, err := pathFor(uploadStartedAtPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   uuid,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	// Write a startedat file for this upload
+	if err := lbs.blobStore.driver.PutContent(ctx, startedAtPath, []byte(startedAt.Format(time.RFC3339))); err != nil {
+		return nil, err
+	}
+
+	return lbs.newBlobUpload(ctx, uuid, path, startedAt, false)
+}
+
+func (lbs *linkedBlobStore) Resume(ctx context.Context, id string) (distribution.BlobWriter, error) {
+	context.GetLogger(ctx).Debug("(*linkedBlobStore).Resume")
+
+	startedAtPath, err := pathFor(uploadStartedAtPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   id,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	startedAtBytes, err := lbs.blobStore.driver.GetContent(ctx, startedAtPath)
+	if err != nil {
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			return nil, distribution.ErrBlobUploadUnknown
+		default:
+			return nil, err
+		}
+	}
+
+	startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes))
+	if err != nil {
+		return nil, err
+	}
+
+	path, err := pathFor(uploadDataPathSpec{
+		name: lbs.repository.Named().Name(),
+		id:   id,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return lbs.newBlobUpload(ctx, id, path, startedAt, true)
+}
+
+func (lbs *linkedBlobStore) Delete(ctx context.Context, dgst digest.Digest) error {
+	if !lbs.deleteEnabled {
+		return distribution.ErrUnsupported
+	}
+
+	// Ensure the blob is available for deletion
+	_, err := lbs.blobAccessController.Stat(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	err = lbs.blobAccessController.Clear(ctx, dgst)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (lbs *linkedBlobStore) Enumerate(ctx context.Context, ingestor func(digest.Digest) error) error {
+	rootPath, err := pathFor(lbs.linkDirectoryPathSpec)
+	if err != nil {
+		return err
+	}
+	err = Walk(ctx, lbs.blobStore.driver, rootPath, func(fileInfo driver.FileInfo) error {
+		// exit early if directory...
+		if fileInfo.IsDir() {
+			return nil
+		}
+		filePath := fileInfo.Path()
+
+		// check if it's a link
+		_, fileName := path.Split(filePath)
+		if fileName != "link" {
+			return nil
+		}
+
+		// read the digest found in link
+		digest, err := lbs.blobStore.readlink(ctx, filePath)
+		if err != nil {
+			return err
+		}
+
+		// ensure this conforms to the linkPathFns
+		_, err = lbs.Stat(ctx, digest)
+		if err != nil {
+			// we expect this error to occur so we move on
+			if err == distribution.ErrBlobUnknown {
+				return nil
+			}
+			return err
+		}
+
+		err = ingestor(digest)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (lbs *linkedBlobStore) mount(ctx context.Context, sourceRepo reference.Named, dgst digest.Digest, sourceStat *distribution.Descriptor) (distribution.Descriptor, error) {
+	var stat distribution.Descriptor
+	if sourceStat == nil {
+		// look up the blob info from the sourceRepo if not already provided
+		repo, err := lbs.registry.Repository(ctx, sourceRepo)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+		stat, err = repo.Blobs(ctx).Stat(ctx, dgst)
+		if err != nil {
+			return distribution.Descriptor{}, err
+		}
+	} else {
+		// use the provided blob info
+		stat = *sourceStat
+	}
+
+	desc := distribution.Descriptor{
+		Size: stat.Size,
+
+		// NOTE(stevvooe): The central blob store firewalls media types from
+		// other users. The caller should look this up and override the value
+		// for the specific repository.
+		MediaType: "application/octet-stream",
+		Digest:    dgst,
+	}
+	return desc, lbs.linkBlob(ctx, desc)
+}
+
+// newBlobUpload allocates a new upload controller with the given state.
+func (lbs *linkedBlobStore) newBlobUpload(ctx context.Context, uuid, path string, startedAt time.Time, append bool) (distribution.BlobWriter, error) {
+	fw, err := lbs.driver.Writer(ctx, path, append)
+	if err != nil {
+		return nil, err
+	}
+
+	bw := &blobWriter{
+		ctx:        ctx,
+		blobStore:  lbs,
+		id:         uuid,
+		startedAt:  startedAt,
+		digester:   digest.Canonical.New(),
+		fileWriter: fw,
+		driver:     lbs.driver,
+		path:       path,
+		resumableDigestEnabled: lbs.resumableDigestEnabled,
+	}
+
+	return bw, nil
+}
+
+// linkBlob links a valid, written blob into the registry under the named
+// repository for the upload controller.
+func (lbs *linkedBlobStore) linkBlob(ctx context.Context, canonical distribution.Descriptor, aliases ...digest.Digest) error {
+	dgsts := append([]digest.Digest{canonical.Digest}, aliases...)
+
+	// TODO(stevvooe): Need to write out mediatype for only canonical hash
+	// since we don't care about the aliases. They are generally unused except
+	// for tarsum but those versions don't care about mediatype.
+
+	// Don't make duplicate links.
+	seenDigests := make(map[digest.Digest]struct{}, len(dgsts))
+
+	// only use the first link
+	linkPathFn := lbs.linkPathFns[0]
+
+	for _, dgst := range dgsts {
+		if _, seen := seenDigests[dgst]; seen {
+			continue
+		}
+		seenDigests[dgst] = struct{}{}
+
+		blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst)
+		if err != nil {
+			return err
+		}
+
+		if err := lbs.blobStore.link(ctx, blobLinkPath, canonical.Digest); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+type linkedBlobStatter struct {
+	*blobStore
+	repository distribution.Repository
+
+	// linkPathFns specifies one or more path functions allowing one to
+	// control the repository blob link set to which the blob store
+	// dispatches. This is required because manifest and layer blobs have not
+	// yet been fully merged. At some point, this functionality should be
+	// removed an the blob links folder should be merged. The first entry is
+	// treated as the "canonical" link location and will be used for writes.
+	linkPathFns []linkPathFunc
+}
+
+var _ distribution.BlobDescriptorService = &linkedBlobStatter{}
+
+func (lbs *linkedBlobStatter) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	var (
+		found  bool
+		target digest.Digest
+	)
+
+	// try the many link path functions until we get success or an error that
+	// is not PathNotFoundError.
+	for _, linkPathFn := range lbs.linkPathFns {
+		var err error
+		target, err = lbs.resolveWithLinkFunc(ctx, dgst, linkPathFn)
+
+		if err == nil {
+			found = true
+			break // success!
+		}
+
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			// do nothing, just move to the next linkPathFn
+		default:
+			return distribution.Descriptor{}, err
+		}
+	}
+
+	if !found {
+		return distribution.Descriptor{}, distribution.ErrBlobUnknown
+	}
+
+	if target != dgst {
+		// Track when we are doing cross-digest domain lookups. ie, sha512 to sha256.
+		context.GetLogger(ctx).Warnf("looking up blob with canonical target: %v -> %v", dgst, target)
+	}
+
+	// TODO(stevvooe): Look up repository local mediatype and replace that on
+	// the returned descriptor.
+
+	return lbs.blobStore.statter.Stat(ctx, target)
+}
+
+func (lbs *linkedBlobStatter) Clear(ctx context.Context, dgst digest.Digest) (err error) {
+	// clear any possible existence of a link described in linkPathFns
+	for _, linkPathFn := range lbs.linkPathFns {
+		blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst)
+		if err != nil {
+			return err
+		}
+
+		err = lbs.blobStore.driver.Delete(ctx, blobLinkPath)
+		if err != nil {
+			switch err := err.(type) {
+			case driver.PathNotFoundError:
+				continue // just ignore this error and continue
+			default:
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// resolveTargetWithFunc allows us to read a link to a resource with different
+// linkPathFuncs to let us try a few different paths before returning not
+// found.
+func (lbs *linkedBlobStatter) resolveWithLinkFunc(ctx context.Context, dgst digest.Digest, linkPathFn linkPathFunc) (digest.Digest, error) {
+	blobLinkPath, err := linkPathFn(lbs.repository.Named().Name(), dgst)
+	if err != nil {
+		return "", err
+	}
+
+	return lbs.blobStore.readlink(ctx, blobLinkPath)
+}
+
+func (lbs *linkedBlobStatter) SetDescriptor(ctx context.Context, dgst digest.Digest, desc distribution.Descriptor) error {
+	// The canonical descriptor for a blob is set at the commit phase of upload
+	return nil
+}
+
+// blobLinkPath provides the path to the blob link, also known as layers.
+func blobLinkPath(name string, dgst digest.Digest) (string, error) {
+	return pathFor(layerLinkPathSpec{name: name, digest: dgst})
+}
+
+// manifestRevisionLinkPath provides the path to the manifest revision link.
+func manifestRevisionLinkPath(name string, dgst digest.Digest) (string, error) {
+	return pathFor(manifestRevisionLinkPathSpec{name: name, revision: dgst})
+}

vendor/github.com/docker/distribution/registry/storage/linkedblobstore_test.go

@@ -0,0 +1,217 @@
+package storage
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"strconv"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/testutil"
+)
+
+func TestLinkedBlobStoreCreateWithMountFrom(t *testing.T) {
+	fooRepoName, _ := reference.ParseNamed("nm/foo")
+	fooEnv := newManifestStoreTestEnv(t, fooRepoName, "thetag")
+	ctx := context.Background()
+	stats, err := mockRegistry(t, fooEnv.registry)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Build up some test layers and add them to the manifest, saving the
+	// readseekers for upload later.
+	testLayers := map[digest.Digest]io.ReadSeeker{}
+	for i := 0; i < 2; i++ {
+		rs, ds, err := testutil.CreateRandomTarFile()
+		if err != nil {
+			t.Fatalf("unexpected error generating test layer file")
+		}
+		dgst := digest.Digest(ds)
+
+		testLayers[digest.Digest(dgst)] = rs
+	}
+
+	// upload the layers to foo/bar
+	for dgst, rs := range testLayers {
+		wr, err := fooEnv.repository.Blobs(fooEnv.ctx).Create(fooEnv.ctx)
+		if err != nil {
+			t.Fatalf("unexpected error creating test upload: %v", err)
+		}
+
+		if _, err := io.Copy(wr, rs); err != nil {
+			t.Fatalf("unexpected error copying to upload: %v", err)
+		}
+
+		if _, err := wr.Commit(fooEnv.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
+			t.Fatalf("unexpected error finishing upload: %v", err)
+		}
+	}
+
+	// create another repository nm/bar
+	barRepoName, _ := reference.ParseNamed("nm/bar")
+	barRepo, err := fooEnv.registry.Repository(ctx, barRepoName)
+	if err != nil {
+		t.Fatalf("unexpected error getting repo: %v", err)
+	}
+
+	// cross-repo mount the test layers into a nm/bar
+	for dgst := range testLayers {
+		fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
+		option := WithMountFrom(fooCanonical)
+		// ensure we can instrospect it
+		createOpts := distribution.CreateOptions{}
+		if err := option.Apply(&createOpts); err != nil {
+			t.Fatalf("failed to apply MountFrom option: %v", err)
+		}
+		if !createOpts.Mount.ShouldMount || createOpts.Mount.From.String() != fooCanonical.String() {
+			t.Fatalf("unexpected create options: %#+v", createOpts.Mount)
+		}
+
+		_, err := barRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical))
+		if err == nil {
+			t.Fatalf("unexpected non-error while mounting from %q: %v", fooRepoName.String(), err)
+		}
+		if _, ok := err.(distribution.ErrBlobMounted); !ok {
+			t.Fatalf("expected ErrMountFrom error, not %T: %v", err, err)
+		}
+	}
+	for dgst := range testLayers {
+		fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
+		count, exists := stats[fooCanonical.String()]
+		if !exists {
+			t.Errorf("expected entry %q not found among handled stat calls", fooCanonical.String())
+		} else if count != 1 {
+			t.Errorf("expected exactly one stat call for entry %q, not %d", fooCanonical.String(), count)
+		}
+	}
+
+	clearStats(stats)
+
+	// create yet another repository nm/baz
+	bazRepoName, _ := reference.ParseNamed("nm/baz")
+	bazRepo, err := fooEnv.registry.Repository(ctx, bazRepoName)
+	if err != nil {
+		t.Fatalf("unexpected error getting repo: %v", err)
+	}
+
+	// cross-repo mount them into a nm/baz and provide a prepopulated blob descriptor
+	for dgst := range testLayers {
+		fooCanonical, _ := reference.WithDigest(fooRepoName, dgst)
+		size, err := strconv.ParseInt("0x"+dgst.Hex()[:8], 0, 64)
+		if err != nil {
+			t.Fatal(err)
+		}
+		prepolutatedDescriptor := distribution.Descriptor{
+			Digest:    dgst,
+			Size:      size,
+			MediaType: "application/octet-stream",
+		}
+		_, err = bazRepo.Blobs(ctx).Create(ctx, WithMountFrom(fooCanonical), &statCrossMountCreateOption{
+			desc: prepolutatedDescriptor,
+		})
+		blobMounted, ok := err.(distribution.ErrBlobMounted)
+		if !ok {
+			t.Errorf("expected ErrMountFrom error, not %T: %v", err, err)
+			continue
+		}
+		if !reflect.DeepEqual(blobMounted.Descriptor, prepolutatedDescriptor) {
+			t.Errorf("unexpected descriptor: %#+v != %#+v", blobMounted.Descriptor, prepolutatedDescriptor)
+		}
+	}
+	// this time no stat calls will be made
+	if len(stats) != 0 {
+		t.Errorf("unexpected number of stats made: %d != %d", len(stats), len(testLayers))
+	}
+}
+
+func clearStats(stats map[string]int) {
+	for k := range stats {
+		delete(stats, k)
+	}
+}
+
+// mockRegistry sets a mock blob descriptor service factory that overrides
+// statter's Stat method to note each attempt to stat a blob in any repository.
+// Returned stats map contains canonical references to blobs with a number of
+// attempts.
+func mockRegistry(t *testing.T, nm distribution.Namespace) (map[string]int, error) {
+	registry, ok := nm.(*registry)
+	if !ok {
+		return nil, fmt.Errorf("not an expected type of registry: %T", nm)
+	}
+	stats := make(map[string]int)
+
+	registry.blobDescriptorServiceFactory = &mockBlobDescriptorServiceFactory{
+		t:     t,
+		stats: stats,
+	}
+
+	return stats, nil
+}
+
+type mockBlobDescriptorServiceFactory struct {
+	t     *testing.T
+	stats map[string]int
+}
+
+func (f *mockBlobDescriptorServiceFactory) BlobAccessController(svc distribution.BlobDescriptorService) distribution.BlobDescriptorService {
+	return &mockBlobDescriptorService{
+		BlobDescriptorService: svc,
+		t:     f.t,
+		stats: f.stats,
+	}
+}
+
+type mockBlobDescriptorService struct {
+	distribution.BlobDescriptorService
+	t     *testing.T
+	stats map[string]int
+}
+
+var _ distribution.BlobDescriptorService = &mockBlobDescriptorService{}
+
+func (bs *mockBlobDescriptorService) Stat(ctx context.Context, dgst digest.Digest) (distribution.Descriptor, error) {
+	statter, ok := bs.BlobDescriptorService.(*linkedBlobStatter)
+	if !ok {
+		return distribution.Descriptor{}, fmt.Errorf("unexpected blob descriptor service: %T", bs.BlobDescriptorService)
+	}
+
+	name := statter.repository.Named()
+	canonical, err := reference.WithDigest(name, dgst)
+	if err != nil {
+		return distribution.Descriptor{}, fmt.Errorf("failed to make canonical reference: %v", err)
+	}
+
+	bs.stats[canonical.String()]++
+	bs.t.Logf("calling Stat on %s", canonical.String())
+
+	return bs.BlobDescriptorService.Stat(ctx, dgst)
+}
+
+// statCrossMountCreateOptions ensures the expected options type is passed, and optionally pre-fills the cross-mount stat info
+type statCrossMountCreateOption struct {
+	desc distribution.Descriptor
+}
+
+var _ distribution.BlobCreateOption = statCrossMountCreateOption{}
+
+func (f statCrossMountCreateOption) Apply(v interface{}) error {
+	opts, ok := v.(*distribution.CreateOptions)
+	if !ok {
+		return fmt.Errorf("Unexpected create options: %#v", v)
+	}
+
+	if !opts.Mount.ShouldMount {
+		return nil
+	}
+
+	opts.Mount.Stat = &f.desc
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/manifestlisthandler.go

@@ -0,0 +1,92 @@
+package storage
+
+import (
+	"fmt"
+
+	"encoding/json"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest/manifestlist"
+)
+
+// manifestListHandler is a ManifestHandler that covers schema2 manifest lists.
+type manifestListHandler struct {
+	repository distribution.Repository
+	blobStore  distribution.BlobStore
+	ctx        context.Context
+}
+
+var _ ManifestHandler = &manifestListHandler{}
+
+func (ms *manifestListHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Unmarshal")
+
+	var m manifestlist.DeserializedManifestList
+	if err := json.Unmarshal(content, &m); err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (ms *manifestListHandler) Put(ctx context.Context, manifestList distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestListHandler).Put")
+
+	m, ok := manifestList.(*manifestlist.DeserializedManifestList)
+	if !ok {
+		return "", fmt.Errorf("wrong type put to manifestListHandler: %T", manifestList)
+	}
+
+	if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil {
+		return "", err
+	}
+
+	mt, payload, err := m.Payload()
+	if err != nil {
+		return "", err
+	}
+
+	revision, err := ms.blobStore.Put(ctx, mt, payload)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+		return "", err
+	}
+
+	return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. As a policy, the registry only tries to
+// store valid content, leaving trust policies of that content up to
+// consumers.
+func (ms *manifestListHandler) verifyManifest(ctx context.Context, mnfst manifestlist.DeserializedManifestList, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if !skipDependencyVerification {
+		// This manifest service is different from the blob service
+		// returned by Blob. It uses a linked blob store to ensure that
+		// only manifests are accessible.
+
+		manifestService, err := ms.repository.Manifests(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, manifestDescriptor := range mnfst.References() {
+			exists, err := manifestService.Exists(ctx, manifestDescriptor.Digest)
+			if err != nil && err != distribution.ErrBlobUnknown {
+				errs = append(errs, err)
+			}
+			if err != nil || !exists {
+				// On error here, we always append unknown blob errors.
+				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: manifestDescriptor.Digest})
+			}
+		}
+	}
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/manifeststore.go

@@ -0,0 +1,141 @@
+package storage
+
+import (
+	"fmt"
+
+	"encoding/json"
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/manifestlist"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+)
+
+// A ManifestHandler gets and puts manifests of a particular type.
+type ManifestHandler interface {
+	// Unmarshal unmarshals the manifest from a byte slice.
+	Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error)
+
+	// Put creates or updates the given manifest returning the manifest digest.
+	Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error)
+}
+
+// SkipLayerVerification allows a manifest to be Put before its
+// layers are on the filesystem
+func SkipLayerVerification() distribution.ManifestServiceOption {
+	return skipLayerOption{}
+}
+
+type skipLayerOption struct{}
+
+func (o skipLayerOption) Apply(m distribution.ManifestService) error {
+	if ms, ok := m.(*manifestStore); ok {
+		ms.skipDependencyVerification = true
+		return nil
+	}
+	return fmt.Errorf("skip layer verification only valid for manifestStore")
+}
+
+type manifestStore struct {
+	repository *repository
+	blobStore  *linkedBlobStore
+	ctx        context.Context
+
+	skipDependencyVerification bool
+
+	schema1Handler      ManifestHandler
+	schema2Handler      ManifestHandler
+	manifestListHandler ManifestHandler
+}
+
+var _ distribution.ManifestService = &manifestStore{}
+
+func (ms *manifestStore) Exists(ctx context.Context, dgst digest.Digest) (bool, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Exists")
+
+	_, err := ms.blobStore.Stat(ms.ctx, dgst)
+	if err != nil {
+		if err == distribution.ErrBlobUnknown {
+			return false, nil
+		}
+
+		return false, err
+	}
+
+	return true, nil
+}
+
+func (ms *manifestStore) Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Get")
+
+	// TODO(stevvooe): Need to check descriptor from above to ensure that the
+	// mediatype is as we expect for the manifest store.
+
+	content, err := ms.blobStore.Get(ctx, dgst)
+	if err != nil {
+		if err == distribution.ErrBlobUnknown {
+			return nil, distribution.ErrManifestUnknownRevision{
+				Name:     ms.repository.Named().Name(),
+				Revision: dgst,
+			}
+		}
+
+		return nil, err
+	}
+
+	var versioned manifest.Versioned
+	if err = json.Unmarshal(content, &versioned); err != nil {
+		return nil, err
+	}
+
+	switch versioned.SchemaVersion {
+	case 1:
+		return ms.schema1Handler.Unmarshal(ctx, dgst, content)
+	case 2:
+		// This can be an image manifest or a manifest list
+		switch versioned.MediaType {
+		case schema2.MediaTypeManifest:
+			return ms.schema2Handler.Unmarshal(ctx, dgst, content)
+		case manifestlist.MediaTypeManifestList:
+			return ms.manifestListHandler.Unmarshal(ctx, dgst, content)
+		default:
+			return nil, distribution.ErrManifestVerification{fmt.Errorf("unrecognized manifest content type %s", versioned.MediaType)}
+		}
+	}
+
+	return nil, fmt.Errorf("unrecognized manifest schema version %d", versioned.SchemaVersion)
+}
+
+func (ms *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, options ...distribution.ManifestServiceOption) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Put")
+
+	switch manifest.(type) {
+	case *schema1.SignedManifest:
+		return ms.schema1Handler.Put(ctx, manifest, ms.skipDependencyVerification)
+	case *schema2.DeserializedManifest:
+		return ms.schema2Handler.Put(ctx, manifest, ms.skipDependencyVerification)
+	case *manifestlist.DeserializedManifestList:
+		return ms.manifestListHandler.Put(ctx, manifest, ms.skipDependencyVerification)
+	}
+
+	return "", fmt.Errorf("unrecognized manifest type %T", manifest)
+}
+
+// Delete removes the revision of the specified manifest.
+func (ms *manifestStore) Delete(ctx context.Context, dgst digest.Digest) error {
+	context.GetLogger(ms.ctx).Debug("(*manifestStore).Delete")
+	return ms.blobStore.Delete(ctx, dgst)
+}
+
+func (ms *manifestStore) Enumerate(ctx context.Context, ingester func(digest.Digest) error) error {
+	err := ms.blobStore.Enumerate(ctx, func(dgst digest.Digest) error {
+		err := ingester(dgst)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+	return err
+}

vendor/github.com/docker/distribution/registry/storage/manifeststore_test.go

@@ -0,0 +1,391 @@
+package storage
+
+import (
+	"bytes"
+	"io"
+	"reflect"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/cache/memory"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/testutil"
+	"github.com/docker/libtrust"
+)
+
+type manifestStoreTestEnv struct {
+	ctx        context.Context
+	driver     driver.StorageDriver
+	registry   distribution.Namespace
+	repository distribution.Repository
+	name       reference.Named
+	tag        string
+}
+
+func newManifestStoreTestEnv(t *testing.T, name reference.Named, tag string, options ...RegistryOption) *manifestStoreTestEnv {
+	ctx := context.Background()
+	driver := inmemory.New()
+	registry, err := NewRegistry(ctx, driver, options...)
+	if err != nil {
+		t.Fatalf("error creating registry: %v", err)
+	}
+
+	repo, err := registry.Repository(ctx, name)
+	if err != nil {
+		t.Fatalf("unexpected error getting repo: %v", err)
+	}
+
+	return &manifestStoreTestEnv{
+		ctx:        ctx,
+		driver:     driver,
+		registry:   registry,
+		repository: repo,
+		name:       name,
+		tag:        tag,
+	}
+}
+
+func TestManifestStorage(t *testing.T) {
+	k, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	testManifestStorage(t, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableDelete, EnableRedirect, Schema1SigningKey(k))
+}
+
+func testManifestStorage(t *testing.T, options ...RegistryOption) {
+	repoName, _ := reference.ParseNamed("foo/bar")
+	env := newManifestStoreTestEnv(t, repoName, "thetag", options...)
+	ctx := context.Background()
+	ms, err := env.repository.Manifests(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := schema1.Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 1,
+		},
+		Name: env.name.Name(),
+		Tag:  env.tag,
+	}
+
+	// Build up some test layers and add them to the manifest, saving the
+	// readseekers for upload later.
+	testLayers := map[digest.Digest]io.ReadSeeker{}
+	for i := 0; i < 2; i++ {
+		rs, ds, err := testutil.CreateRandomTarFile()
+		if err != nil {
+			t.Fatalf("unexpected error generating test layer file")
+		}
+		dgst := digest.Digest(ds)
+
+		testLayers[digest.Digest(dgst)] = rs
+		m.FSLayers = append(m.FSLayers, schema1.FSLayer{
+			BlobSum: dgst,
+		})
+		m.History = append(m.History, schema1.History{
+			V1Compatibility: "",
+		})
+
+	}
+
+	pk, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatalf("unexpected error generating private key: %v", err)
+	}
+
+	sm, merr := schema1.Sign(&m, pk)
+	if merr != nil {
+		t.Fatalf("error signing manifest: %v", err)
+	}
+
+	_, err = ms.Put(ctx, sm)
+	if err == nil {
+		t.Fatalf("expected errors putting manifest with full verification")
+	}
+
+	switch err := err.(type) {
+	case distribution.ErrManifestVerification:
+		if len(err) != 2 {
+			t.Fatalf("expected 2 verification errors: %#v", err)
+		}
+
+		for _, err := range err {
+			if _, ok := err.(distribution.ErrManifestBlobUnknown); !ok {
+				t.Fatalf("unexpected error type: %v", err)
+			}
+		}
+	default:
+		t.Fatalf("unexpected error verifying manifest: %v", err)
+	}
+
+	// Now, upload the layers that were missing!
+	for dgst, rs := range testLayers {
+		wr, err := env.repository.Blobs(env.ctx).Create(env.ctx)
+		if err != nil {
+			t.Fatalf("unexpected error creating test upload: %v", err)
+		}
+
+		if _, err := io.Copy(wr, rs); err != nil {
+			t.Fatalf("unexpected error copying to upload: %v", err)
+		}
+
+		if _, err := wr.Commit(env.ctx, distribution.Descriptor{Digest: dgst}); err != nil {
+			t.Fatalf("unexpected error finishing upload: %v", err)
+		}
+	}
+
+	var manifestDigest digest.Digest
+	if manifestDigest, err = ms.Put(ctx, sm); err != nil {
+		t.Fatalf("unexpected error putting manifest: %v", err)
+	}
+
+	exists, err := ms.Exists(ctx, manifestDigest)
+	if err != nil {
+		t.Fatalf("unexpected error checking manifest existence: %#v", err)
+	}
+
+	if !exists {
+		t.Fatalf("manifest should exist")
+	}
+
+	fromStore, err := ms.Get(ctx, manifestDigest)
+	if err != nil {
+		t.Fatalf("unexpected error fetching manifest: %v", err)
+	}
+
+	fetchedManifest, ok := fromStore.(*schema1.SignedManifest)
+	if !ok {
+		t.Fatalf("unexpected manifest type from signedstore")
+	}
+
+	if !bytes.Equal(fetchedManifest.Canonical, sm.Canonical) {
+		t.Fatalf("fetched payload does not match original payload: %q != %q", fetchedManifest.Canonical, sm.Canonical)
+	}
+
+	_, pl, err := fetchedManifest.Payload()
+	if err != nil {
+		t.Fatalf("error getting payload %#v", err)
+	}
+
+	fetchedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
+	if err != nil {
+		t.Fatalf("unexpected error parsing jws: %v", err)
+	}
+
+	payload, err := fetchedJWS.Payload()
+	if err != nil {
+		t.Fatalf("unexpected error extracting payload: %v", err)
+	}
+
+	// Now that we have a payload, take a moment to check that the manifest is
+	// return by the payload digest.
+
+	dgst := digest.FromBytes(payload)
+	exists, err = ms.Exists(ctx, dgst)
+	if err != nil {
+		t.Fatalf("error checking manifest existence by digest: %v", err)
+	}
+
+	if !exists {
+		t.Fatalf("manifest %s should exist", dgst)
+	}
+
+	fetchedByDigest, err := ms.Get(ctx, dgst)
+	if err != nil {
+		t.Fatalf("unexpected error fetching manifest by digest: %v", err)
+	}
+
+	byDigestManifest, ok := fetchedByDigest.(*schema1.SignedManifest)
+	if !ok {
+		t.Fatalf("unexpected manifest type from signedstore")
+	}
+
+	if !bytes.Equal(byDigestManifest.Canonical, fetchedManifest.Canonical) {
+		t.Fatalf("fetched manifest not equal: %q != %q", byDigestManifest.Canonical, fetchedManifest.Canonical)
+	}
+
+	sigs, err := fetchedJWS.Signatures()
+	if err != nil {
+		t.Fatalf("unable to extract signatures: %v", err)
+	}
+
+	if len(sigs) != 1 {
+		t.Fatalf("unexpected number of signatures: %d != %d", len(sigs), 1)
+	}
+
+	// Now, push the same manifest with a different key
+	pk2, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatalf("unexpected error generating private key: %v", err)
+	}
+
+	sm2, err := schema1.Sign(&m, pk2)
+	if err != nil {
+		t.Fatalf("unexpected error signing manifest: %v", err)
+	}
+	_, pl, err = sm2.Payload()
+	if err != nil {
+		t.Fatalf("error getting payload %#v", err)
+	}
+
+	jws2, err := libtrust.ParsePrettySignature(pl, "signatures")
+	if err != nil {
+		t.Fatalf("error parsing signature: %v", err)
+	}
+
+	sigs2, err := jws2.Signatures()
+	if err != nil {
+		t.Fatalf("unable to extract signatures: %v", err)
+	}
+
+	if len(sigs2) != 1 {
+		t.Fatalf("unexpected number of signatures: %d != %d", len(sigs2), 1)
+	}
+
+	if manifestDigest, err = ms.Put(ctx, sm2); err != nil {
+		t.Fatalf("unexpected error putting manifest: %v", err)
+	}
+
+	fromStore, err = ms.Get(ctx, manifestDigest)
+	if err != nil {
+		t.Fatalf("unexpected error fetching manifest: %v", err)
+	}
+
+	fetched, ok := fromStore.(*schema1.SignedManifest)
+	if !ok {
+		t.Fatalf("unexpected type from signed manifeststore : %T", fetched)
+	}
+
+	if _, err := schema1.Verify(fetched); err != nil {
+		t.Fatalf("unexpected error verifying manifest: %v", err)
+	}
+
+	_, pl, err = fetched.Payload()
+	if err != nil {
+		t.Fatalf("error getting payload %#v", err)
+	}
+
+	receivedJWS, err := libtrust.ParsePrettySignature(pl, "signatures")
+	if err != nil {
+		t.Fatalf("unexpected error parsing jws: %v", err)
+	}
+
+	receivedPayload, err := receivedJWS.Payload()
+	if err != nil {
+		t.Fatalf("unexpected error extracting received payload: %v", err)
+	}
+
+	if !bytes.Equal(receivedPayload, payload) {
+		t.Fatalf("payloads are not equal")
+	}
+
+	// Test deleting manifests
+	err = ms.Delete(ctx, dgst)
+	if err != nil {
+		t.Fatalf("unexpected an error deleting manifest by digest: %v", err)
+	}
+
+	exists, err = ms.Exists(ctx, dgst)
+	if err != nil {
+		t.Fatalf("Error querying manifest existence")
+	}
+	if exists {
+		t.Errorf("Deleted manifest should not exist")
+	}
+
+	deletedManifest, err := ms.Get(ctx, dgst)
+	if err == nil {
+		t.Errorf("Unexpected success getting deleted manifest")
+	}
+	switch err.(type) {
+	case distribution.ErrManifestUnknownRevision:
+		break
+	default:
+		t.Errorf("Unexpected error getting deleted manifest: %s", reflect.ValueOf(err).Type())
+	}
+
+	if deletedManifest != nil {
+		t.Errorf("Deleted manifest get returned non-nil")
+	}
+
+	// Re-upload should restore manifest to a good state
+	_, err = ms.Put(ctx, sm)
+	if err != nil {
+		t.Errorf("Error re-uploading deleted manifest")
+	}
+
+	exists, err = ms.Exists(ctx, dgst)
+	if err != nil {
+		t.Fatalf("Error querying manifest existence")
+	}
+	if !exists {
+		t.Errorf("Restored manifest should exist")
+	}
+
+	deletedManifest, err = ms.Get(ctx, dgst)
+	if err != nil {
+		t.Errorf("Unexpected error getting manifest")
+	}
+	if deletedManifest == nil {
+		t.Errorf("Deleted manifest get returned non-nil")
+	}
+
+	r, err := NewRegistry(ctx, env.driver, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
+	if err != nil {
+		t.Fatalf("error creating registry: %v", err)
+	}
+	repo, err := r.Repository(ctx, env.name)
+	if err != nil {
+		t.Fatalf("unexpected error getting repo: %v", err)
+	}
+	ms, err = repo.Manifests(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ms.Delete(ctx, dgst)
+	if err == nil {
+		t.Errorf("Unexpected success deleting while disabled")
+	}
+}
+
+// TestLinkPathFuncs ensures that the link path functions behavior are locked
+// down and implemented as expected.
+func TestLinkPathFuncs(t *testing.T) {
+	for _, testcase := range []struct {
+		repo       string
+		digest     digest.Digest
+		linkPathFn linkPathFunc
+		expected   string
+	}{
+		{
+			repo:       "foo/bar",
+			digest:     "sha256:deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+			linkPathFn: blobLinkPath,
+			expected:   "/docker/registry/v2/repositories/foo/bar/_layers/sha256/deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/link",
+		},
+		{
+			repo:       "foo/bar",
+			digest:     "sha256:deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+			linkPathFn: manifestRevisionLinkPath,
+			expected:   "/docker/registry/v2/repositories/foo/bar/_manifests/revisions/sha256/deadbeaf98fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855/link",
+		},
+	} {
+		p, err := testcase.linkPathFn(testcase.repo, testcase.digest)
+		if err != nil {
+			t.Fatalf("unexpected error calling linkPathFn(pm, %q, %q): %v", testcase.repo, testcase.digest, err)
+		}
+
+		if p != testcase.expected {
+			t.Fatalf("incorrect path returned: %q != %q", p, testcase.expected)
+		}
+	}
+
+}

vendor/github.com/docker/distribution/registry/storage/paths.go

@@ -0,0 +1,490 @@
+package storage
+
+import (
+	"fmt"
+	"path"
+	"strings"
+
+	"github.com/docker/distribution/digest"
+)
+
+const (
+	storagePathVersion = "v2"                // fixed storage layout version
+	storagePathRoot    = "/docker/registry/" // all driver paths have a prefix
+
+	// TODO(stevvooe): Get rid of the "storagePathRoot". Initially, we though
+	// storage path root would configurable for all drivers through this
+	// package. In reality, we've found it simpler to do this on a per driver
+	// basis.
+)
+
+// pathFor maps paths based on "object names" and their ids. The "object
+// names" mapped by are internal to the storage system.
+//
+// The path layout in the storage backend is roughly as follows:
+//
+//		<root>/v2
+//			-> repositories/
+// 				-><name>/
+// 					-> _manifests/
+// 						revisions
+//							-> <manifest digest path>
+//								-> link
+// 						tags/<tag>
+//							-> current/link
+// 							-> index
+//								-> <algorithm>/<hex digest>/link
+// 					-> _layers/
+// 						<layer links to blob store>
+// 					-> _uploads/<id>
+// 						data
+// 						startedat
+// 						hashstates/<algorithm>/<offset>
+//			-> blob/<algorithm>
+//				<split directory content addressable storage>
+//
+// The storage backend layout is broken up into a content-addressable blob
+// store and repositories. The content-addressable blob store holds most data
+// throughout the backend, keyed by algorithm and digests of the underlying
+// content. Access to the blob store is controlled through links from the
+// repository to blobstore.
+//
+// A repository is made up of layers, manifests and tags. The layers component
+// is just a directory of layers which are "linked" into a repository. A layer
+// can only be accessed through a qualified repository name if it is linked in
+// the repository. Uploads of layers are managed in the uploads directory,
+// which is key by upload id. When all data for an upload is received, the
+// data is moved into the blob store and the upload directory is deleted.
+// Abandoned uploads can be garbage collected by reading the startedat file
+// and removing uploads that have been active for longer than a certain time.
+//
+// The third component of the repository directory is the manifests store,
+// which is made up of a revision store and tag store. Manifests are stored in
+// the blob store and linked into the revision store.
+// While the registry can save all revisions of a manifest, no relationship is
+// implied as to the ordering of changes to a manifest. The tag store provides
+// support for name, tag lookups of manifests, using "current/link" under a
+// named tag directory. An index is maintained to support deletions of all
+// revisions of a given manifest tag.
+//
+// We cover the path formats implemented by this path mapper below.
+//
+//	Manifests:
+//
+// 	manifestRevisionsPathSpec:      <root>/v2/repositories/<name>/_manifests/revisions/
+// 	manifestRevisionPathSpec:      <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/
+// 	manifestRevisionLinkPathSpec:  <root>/v2/repositories/<name>/_manifests/revisions/<algorithm>/<hex digest>/link
+//
+//	Tags:
+//
+// 	manifestTagsPathSpec:                  <root>/v2/repositories/<name>/_manifests/tags/
+// 	manifestTagPathSpec:                   <root>/v2/repositories/<name>/_manifests/tags/<tag>/
+// 	manifestTagCurrentPathSpec:            <root>/v2/repositories/<name>/_manifests/tags/<tag>/current/link
+// 	manifestTagIndexPathSpec:              <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/
+// 	manifestTagIndexEntryPathSpec:         <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/
+// 	manifestTagIndexEntryLinkPathSpec:     <root>/v2/repositories/<name>/_manifests/tags/<tag>/index/<algorithm>/<hex digest>/link
+//
+// 	Blobs:
+//
+// 	layerLinkPathSpec:            <root>/v2/repositories/<name>/_layers/<algorithm>/<hex digest>/link
+//
+//	Uploads:
+//
+// 	uploadDataPathSpec:             <root>/v2/repositories/<name>/_uploads/<id>/data
+// 	uploadStartedAtPathSpec:        <root>/v2/repositories/<name>/_uploads/<id>/startedat
+// 	uploadHashStatePathSpec:        <root>/v2/repositories/<name>/_uploads/<id>/hashstates/<algorithm>/<offset>
+//
+//	Blob Store:
+//
+//	blobsPathSpec:                  <root>/v2/blobs/
+// 	blobPathSpec:                   <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>
+// 	blobDataPathSpec:               <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
+// 	blobMediaTypePathSpec:               <root>/v2/blobs/<algorithm>/<first two hex bytes of digest>/<hex digest>/data
+//
+// For more information on the semantic meaning of each path and their
+// contents, please see the path spec documentation.
+func pathFor(spec pathSpec) (string, error) {
+
+	// Switch on the path object type and return the appropriate path. At
+	// first glance, one may wonder why we don't use an interface to
+	// accomplish this. By keep the formatting separate from the pathSpec, we
+	// keep separate the path generation componentized. These specs could be
+	// passed to a completely different mapper implementation and generate a
+	// different set of paths.
+	//
+	// For example, imagine migrating from one backend to the other: one could
+	// build a filesystem walker that converts a string path in one version,
+	// to an intermediate path object, than can be consumed and mapped by the
+	// other version.
+
+	rootPrefix := []string{storagePathRoot, storagePathVersion}
+	repoPrefix := append(rootPrefix, "repositories")
+
+	switch v := spec.(type) {
+
+	case manifestRevisionsPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_manifests", "revisions")...), nil
+
+	case manifestRevisionPathSpec:
+		components, err := digestPathComponents(v.revision, false)
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(append(append(repoPrefix, v.name, "_manifests", "revisions"), components...)...), nil
+	case manifestRevisionLinkPathSpec:
+		root, err := pathFor(manifestRevisionPathSpec{
+			name:     v.name,
+			revision: v.revision,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "link"), nil
+	case manifestTagsPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_manifests", "tags")...), nil
+	case manifestTagPathSpec:
+		root, err := pathFor(manifestTagsPathSpec{
+			name: v.name,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, v.tag), nil
+	case manifestTagCurrentPathSpec:
+		root, err := pathFor(manifestTagPathSpec{
+			name: v.name,
+			tag:  v.tag,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "current", "link"), nil
+	case manifestTagIndexPathSpec:
+		root, err := pathFor(manifestTagPathSpec{
+			name: v.name,
+			tag:  v.tag,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "index"), nil
+	case manifestTagIndexEntryLinkPathSpec:
+		root, err := pathFor(manifestTagIndexEntryPathSpec{
+			name:     v.name,
+			tag:      v.tag,
+			revision: v.revision,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, "link"), nil
+	case manifestTagIndexEntryPathSpec:
+		root, err := pathFor(manifestTagIndexPathSpec{
+			name: v.name,
+			tag:  v.tag,
+		})
+
+		if err != nil {
+			return "", err
+		}
+
+		components, err := digestPathComponents(v.revision, false)
+		if err != nil {
+			return "", err
+		}
+
+		return path.Join(root, path.Join(components...)), nil
+	case layerLinkPathSpec:
+		components, err := digestPathComponents(v.digest, false)
+		if err != nil {
+			return "", err
+		}
+
+		// TODO(stevvooe): Right now, all blobs are linked under "_layers". If
+		// we have future migrations, we may want to rename this to "_blobs".
+		// A migration strategy would simply leave existing items in place and
+		// write the new paths, commit a file then delete the old files.
+
+		blobLinkPathComponents := append(repoPrefix, v.name, "_layers")
+
+		return path.Join(path.Join(append(blobLinkPathComponents, components...)...), "link"), nil
+	case blobsPathSpec:
+		blobsPathPrefix := append(rootPrefix, "blobs")
+		return path.Join(blobsPathPrefix...), nil
+	case blobPathSpec:
+		components, err := digestPathComponents(v.digest, true)
+		if err != nil {
+			return "", err
+		}
+
+		blobPathPrefix := append(rootPrefix, "blobs")
+		return path.Join(append(blobPathPrefix, components...)...), nil
+	case blobDataPathSpec:
+		components, err := digestPathComponents(v.digest, true)
+		if err != nil {
+			return "", err
+		}
+
+		components = append(components, "data")
+		blobPathPrefix := append(rootPrefix, "blobs")
+		return path.Join(append(blobPathPrefix, components...)...), nil
+
+	case uploadDataPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "data")...), nil
+	case uploadStartedAtPathSpec:
+		return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "startedat")...), nil
+	case uploadHashStatePathSpec:
+		offset := fmt.Sprintf("%d", v.offset)
+		if v.list {
+			offset = "" // Limit to the prefix for listing offsets.
+		}
+		return path.Join(append(repoPrefix, v.name, "_uploads", v.id, "hashstates", string(v.alg), offset)...), nil
+	case repositoriesRootPathSpec:
+		return path.Join(repoPrefix...), nil
+	default:
+		// TODO(sday): This is an internal error. Ensure it doesn't escape (panic?).
+		return "", fmt.Errorf("unknown path spec: %#v", v)
+	}
+}
+
+// pathSpec is a type to mark structs as path specs. There is no
+// implementation because we'd like to keep the specs and the mappers
+// decoupled.
+type pathSpec interface {
+	pathSpec()
+}
+
+// manifestRevisionsPathSpec describes the directory path for
+// a manifest revision.
+type manifestRevisionsPathSpec struct {
+	name string
+}
+
+func (manifestRevisionsPathSpec) pathSpec() {}
+
+// manifestRevisionPathSpec describes the components of the directory path for
+// a manifest revision.
+type manifestRevisionPathSpec struct {
+	name     string
+	revision digest.Digest
+}
+
+func (manifestRevisionPathSpec) pathSpec() {}
+
+// manifestRevisionLinkPathSpec describes the path components required to look
+// up the data link for a revision of a manifest. If this file is not present,
+// the manifest blob is not available in the given repo. The contents of this
+// file should just be the digest.
+type manifestRevisionLinkPathSpec struct {
+	name     string
+	revision digest.Digest
+}
+
+func (manifestRevisionLinkPathSpec) pathSpec() {}
+
+// manifestTagsPathSpec describes the path elements required to point to the
+// manifest tags directory.
+type manifestTagsPathSpec struct {
+	name string
+}
+
+func (manifestTagsPathSpec) pathSpec() {}
+
+// manifestTagPathSpec describes the path elements required to point to the
+// manifest tag links files under a repository. These contain a blob id that
+// can be used to look up the data and signatures.
+type manifestTagPathSpec struct {
+	name string
+	tag  string
+}
+
+func (manifestTagPathSpec) pathSpec() {}
+
+// manifestTagCurrentPathSpec describes the link to the current revision for a
+// given tag.
+type manifestTagCurrentPathSpec struct {
+	name string
+	tag  string
+}
+
+func (manifestTagCurrentPathSpec) pathSpec() {}
+
+// manifestTagCurrentPathSpec describes the link to the index of revisions
+// with the given tag.
+type manifestTagIndexPathSpec struct {
+	name string
+	tag  string
+}
+
+func (manifestTagIndexPathSpec) pathSpec() {}
+
+// manifestTagIndexEntryPathSpec contains the entries of the index by revision.
+type manifestTagIndexEntryPathSpec struct {
+	name     string
+	tag      string
+	revision digest.Digest
+}
+
+func (manifestTagIndexEntryPathSpec) pathSpec() {}
+
+// manifestTagIndexEntryLinkPathSpec describes the link to a revisions of a
+// manifest with given tag within the index.
+type manifestTagIndexEntryLinkPathSpec struct {
+	name     string
+	tag      string
+	revision digest.Digest
+}
+
+func (manifestTagIndexEntryLinkPathSpec) pathSpec() {}
+
+// blobLinkPathSpec specifies a path for a blob link, which is a file with a
+// blob id. The blob link will contain a content addressable blob id reference
+// into the blob store. The format of the contents is as follows:
+//
+// 	<algorithm>:<hex digest of layer data>
+//
+// The following example of the file contents is more illustrative:
+//
+// 	sha256:96443a84ce518ac22acb2e985eda402b58ac19ce6f91980bde63726a79d80b36
+//
+// This  indicates that there is a blob with the id/digest, calculated via
+// sha256 that can be fetched from the blob store.
+type layerLinkPathSpec struct {
+	name   string
+	digest digest.Digest
+}
+
+func (layerLinkPathSpec) pathSpec() {}
+
+// blobAlgorithmReplacer does some very simple path sanitization for user
+// input. Paths should be "safe" before getting this far due to strict digest
+// requirements but we can add further path conversion here, if needed.
+var blobAlgorithmReplacer = strings.NewReplacer(
+	"+", "/",
+	".", "/",
+	";", "/",
+)
+
+// blobsPathSpec contains the path for the blobs directory
+type blobsPathSpec struct{}
+
+func (blobsPathSpec) pathSpec() {}
+
+// blobPathSpec contains the path for the registry global blob store.
+type blobPathSpec struct {
+	digest digest.Digest
+}
+
+func (blobPathSpec) pathSpec() {}
+
+// blobDataPathSpec contains the path for the registry global blob store. For
+// now, this contains layer data, exclusively.
+type blobDataPathSpec struct {
+	digest digest.Digest
+}
+
+func (blobDataPathSpec) pathSpec() {}
+
+// uploadDataPathSpec defines the path parameters of the data file for
+// uploads.
+type uploadDataPathSpec struct {
+	name string
+	id   string
+}
+
+func (uploadDataPathSpec) pathSpec() {}
+
+// uploadDataPathSpec defines the path parameters for the file that stores the
+// start time of an uploads. If it is missing, the upload is considered
+// unknown. Admittedly, the presence of this file is an ugly hack to make sure
+// we have a way to cleanup old or stalled uploads that doesn't rely on driver
+// FileInfo behavior. If we come up with a more clever way to do this, we
+// should remove this file immediately and rely on the startetAt field from
+// the client to enforce time out policies.
+type uploadStartedAtPathSpec struct {
+	name string
+	id   string
+}
+
+func (uploadStartedAtPathSpec) pathSpec() {}
+
+// uploadHashStatePathSpec defines the path parameters for the file that stores
+// the hash function state of an upload at a specific byte offset. If `list` is
+// set, then the path mapper will generate a list prefix for all hash state
+// offsets for the upload identified by the name, id, and alg.
+type uploadHashStatePathSpec struct {
+	name   string
+	id     string
+	alg    digest.Algorithm
+	offset int64
+	list   bool
+}
+
+func (uploadHashStatePathSpec) pathSpec() {}
+
+// repositoriesRootPathSpec returns the root of repositories
+type repositoriesRootPathSpec struct {
+}
+
+func (repositoriesRootPathSpec) pathSpec() {}
+
+// digestPathComponents provides a consistent path breakdown for a given
+// digest. For a generic digest, it will be as follows:
+//
+// 	<algorithm>/<hex digest>
+//
+// If multilevel is true, the first two bytes of the digest will separate
+// groups of digest folder. It will be as follows:
+//
+// 	<algorithm>/<first two bytes of digest>/<full digest>
+//
+func digestPathComponents(dgst digest.Digest, multilevel bool) ([]string, error) {
+	if err := dgst.Validate(); err != nil {
+		return nil, err
+	}
+
+	algorithm := blobAlgorithmReplacer.Replace(string(dgst.Algorithm()))
+	hex := dgst.Hex()
+	prefix := []string{algorithm}
+
+	var suffix []string
+
+	if multilevel {
+		suffix = append(suffix, hex[:2])
+	}
+
+	suffix = append(suffix, hex)
+
+	return append(prefix, suffix...), nil
+}
+
+// Reconstructs a digest from a path
+func digestFromPath(digestPath string) (digest.Digest, error) {
+
+	digestPath = strings.TrimSuffix(digestPath, "/data")
+	dir, hex := path.Split(digestPath)
+	dir = path.Dir(dir)
+	dir, next := path.Split(dir)
+
+	// next is either the algorithm OR the first two characters in the hex string
+	var algo string
+	if next == hex[:2] {
+		algo = path.Base(dir)
+	} else {
+		algo = next
+	}
+
+	dgst := digest.NewDigestFromHex(algo, hex)
+	return dgst, dgst.Validate()
+}

vendor/github.com/docker/distribution/registry/storage/paths_test.go

@@ -0,0 +1,135 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/docker/distribution/digest"
+)
+
+func TestPathMapper(t *testing.T) {
+	for _, testcase := range []struct {
+		spec     pathSpec
+		expected string
+		err      error
+	}{
+		{
+			spec: manifestRevisionPathSpec{
+				name:     "foo/bar",
+				revision: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/revisions/sha256/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+		},
+		{
+			spec: manifestRevisionLinkPathSpec{
+				name:     "foo/bar",
+				revision: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/revisions/sha256/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789/link",
+		},
+		{
+			spec: manifestTagsPathSpec{
+				name: "foo/bar",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/tags",
+		},
+		{
+			spec: manifestTagPathSpec{
+				name: "foo/bar",
+				tag:  "thetag",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/tags/thetag",
+		},
+		{
+			spec: manifestTagCurrentPathSpec{
+				name: "foo/bar",
+				tag:  "thetag",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/tags/thetag/current/link",
+		},
+		{
+			spec: manifestTagIndexPathSpec{
+				name: "foo/bar",
+				tag:  "thetag",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/tags/thetag/index",
+		},
+		{
+			spec: manifestTagIndexEntryPathSpec{
+				name:     "foo/bar",
+				tag:      "thetag",
+				revision: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/tags/thetag/index/sha256/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+		},
+		{
+			spec: manifestTagIndexEntryLinkPathSpec{
+				name:     "foo/bar",
+				tag:      "thetag",
+				revision: "sha256:abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_manifests/tags/thetag/index/sha256/abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789/link",
+		},
+
+		{
+			spec: uploadDataPathSpec{
+				name: "foo/bar",
+				id:   "asdf-asdf-asdf-adsf",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_uploads/asdf-asdf-asdf-adsf/data",
+		},
+		{
+			spec: uploadStartedAtPathSpec{
+				name: "foo/bar",
+				id:   "asdf-asdf-asdf-adsf",
+			},
+			expected: "/docker/registry/v2/repositories/foo/bar/_uploads/asdf-asdf-asdf-adsf/startedat",
+		},
+	} {
+		p, err := pathFor(testcase.spec)
+		if err != nil {
+			t.Fatalf("unexpected generating path (%T): %v", testcase.spec, err)
+		}
+
+		if p != testcase.expected {
+			t.Fatalf("unexpected path generated (%T): %q != %q", testcase.spec, p, testcase.expected)
+		}
+	}
+
+	// Add a few test cases to ensure we cover some errors
+
+	// Specify a path that requires a revision and get a digest validation error.
+	badpath, err := pathFor(manifestRevisionPathSpec{
+		name: "foo/bar",
+	})
+
+	if err == nil {
+		t.Fatalf("expected an error when mapping an invalid revision: %s", badpath)
+	}
+
+}
+
+func TestDigestFromPath(t *testing.T) {
+	for _, testcase := range []struct {
+		path       string
+		expected   digest.Digest
+		multilevel bool
+		err        error
+	}{
+		{
+			path:       "/docker/registry/v2/blobs/sha256/99/9943fffae777400c0344c58869c4c2619c329ca3ad4df540feda74d291dd7c86/data",
+			multilevel: true,
+			expected:   "sha256:9943fffae777400c0344c58869c4c2619c329ca3ad4df540feda74d291dd7c86",
+			err:        nil,
+		},
+	} {
+		result, err := digestFromPath(testcase.path)
+		if err != testcase.err {
+			t.Fatalf("Unexpected error value %v when we wanted %v", err, testcase.err)
+		}
+
+		if result != testcase.expected {
+			t.Fatalf("Unexpected result value %v when we wanted %v", result, testcase.expected)
+
+		}
+	}
+}

vendor/github.com/docker/distribution/registry/storage/purgeuploads.go

@@ -0,0 +1,139 @@
+package storage
+
+import (
+	"path"
+	"strings"
+	"time"
+
+	log "github.com/Sirupsen/logrus"
+	"github.com/docker/distribution/context"
+	storageDriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/uuid"
+)
+
+// uploadData stored the location of temporary files created during a layer upload
+// along with the date the upload was started
+type uploadData struct {
+	containingDir string
+	startedAt     time.Time
+}
+
+func newUploadData() uploadData {
+	return uploadData{
+		containingDir: "",
+		// default to far in future to protect against missing startedat
+		startedAt: time.Now().Add(time.Duration(10000 * time.Hour)),
+	}
+}
+
+// PurgeUploads deletes files from the upload directory
+// created before olderThan.  The list of files deleted and errors
+// encountered are returned
+func PurgeUploads(ctx context.Context, driver storageDriver.StorageDriver, olderThan time.Time, actuallyDelete bool) ([]string, []error) {
+	log.Infof("PurgeUploads starting: olderThan=%s, actuallyDelete=%t", olderThan, actuallyDelete)
+	uploadData, errors := getOutstandingUploads(ctx, driver)
+	var deleted []string
+	for _, uploadData := range uploadData {
+		if uploadData.startedAt.Before(olderThan) {
+			var err error
+			log.Infof("Upload files in %s have older date (%s) than purge date (%s).  Removing upload directory.",
+				uploadData.containingDir, uploadData.startedAt, olderThan)
+			if actuallyDelete {
+				err = driver.Delete(ctx, uploadData.containingDir)
+			}
+			if err == nil {
+				deleted = append(deleted, uploadData.containingDir)
+			} else {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	log.Infof("Purge uploads finished.  Num deleted=%d, num errors=%d", len(deleted), len(errors))
+	return deleted, errors
+}
+
+// getOutstandingUploads walks the upload directory, collecting files
+// which could be eligible for deletion.  The only reliable way to
+// classify the age of a file is with the date stored in the startedAt
+// file, so gather files by UUID with a date from startedAt.
+func getOutstandingUploads(ctx context.Context, driver storageDriver.StorageDriver) (map[string]uploadData, []error) {
+	var errors []error
+	uploads := make(map[string]uploadData, 0)
+
+	inUploadDir := false
+	root, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return uploads, append(errors, err)
+	}
+
+	err = Walk(ctx, driver, root, func(fileInfo storageDriver.FileInfo) error {
+		filePath := fileInfo.Path()
+		_, file := path.Split(filePath)
+		if file[0] == '_' {
+			// Reserved directory
+			inUploadDir = (file == "_uploads")
+
+			if fileInfo.IsDir() && !inUploadDir {
+				return ErrSkipDir
+			}
+
+		}
+
+		uuid, isContainingDir := uUIDFromPath(filePath)
+		if uuid == "" {
+			// Cannot reliably delete
+			return nil
+		}
+		ud, ok := uploads[uuid]
+		if !ok {
+			ud = newUploadData()
+		}
+		if isContainingDir {
+			ud.containingDir = filePath
+		}
+		if file == "startedat" {
+			if t, err := readStartedAtFile(driver, filePath); err == nil {
+				ud.startedAt = t
+			} else {
+				errors = pushError(errors, filePath, err)
+			}
+
+		}
+
+		uploads[uuid] = ud
+		return nil
+	})
+
+	if err != nil {
+		errors = pushError(errors, root, err)
+	}
+	return uploads, errors
+}
+
+// uUIDFromPath extracts the upload UUID from a given path
+// If the UUID is the last path component, this is the containing
+// directory for all upload files
+func uUIDFromPath(path string) (string, bool) {
+	components := strings.Split(path, "/")
+	for i := len(components) - 1; i >= 0; i-- {
+		if u, err := uuid.Parse(components[i]); err == nil {
+			return u.String(), i == len(components)-1
+		}
+	}
+	return "", false
+}
+
+// readStartedAtFile reads the date from an upload's startedAtFile
+func readStartedAtFile(driver storageDriver.StorageDriver, path string) (time.Time, error) {
+	// todo:(richardscothern) - pass in a context
+	startedAtBytes, err := driver.GetContent(context.Background(), path)
+	if err != nil {
+		return time.Now(), err
+	}
+	startedAt, err := time.Parse(time.RFC3339, string(startedAtBytes))
+	if err != nil {
+		return time.Now(), err
+	}
+	return startedAt, nil
+}

vendor/github.com/docker/distribution/registry/storage/purgeuploads_test.go

@@ -0,0 +1,166 @@
+package storage
+
+import (
+	"path"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/uuid"
+)
+
+func testUploadFS(t *testing.T, numUploads int, repoName string, startedAt time.Time) (driver.StorageDriver, context.Context) {
+	d := inmemory.New()
+	ctx := context.Background()
+	for i := 0; i < numUploads; i++ {
+		addUploads(ctx, t, d, uuid.Generate().String(), repoName, startedAt)
+	}
+	return d, ctx
+}
+
+func addUploads(ctx context.Context, t *testing.T, d driver.StorageDriver, uploadID, repo string, startedAt time.Time) {
+	dataPath, err := pathFor(uploadDataPathSpec{name: repo, id: uploadID})
+	if err != nil {
+		t.Fatalf("Unable to resolve path")
+	}
+	if err := d.PutContent(ctx, dataPath, []byte("")); err != nil {
+		t.Fatalf("Unable to write data file")
+	}
+
+	startedAtPath, err := pathFor(uploadStartedAtPathSpec{name: repo, id: uploadID})
+	if err != nil {
+		t.Fatalf("Unable to resolve path")
+	}
+
+	if d.PutContent(ctx, startedAtPath, []byte(startedAt.Format(time.RFC3339))); err != nil {
+		t.Fatalf("Unable to write startedAt file")
+	}
+
+}
+
+func TestPurgeGather(t *testing.T) {
+	uploadCount := 5
+	fs, ctx := testUploadFS(t, uploadCount, "test-repo", time.Now())
+	uploadData, errs := getOutstandingUploads(ctx, fs)
+	if len(errs) != 0 {
+		t.Errorf("Unexepected errors: %q", errs)
+	}
+	if len(uploadData) != uploadCount {
+		t.Errorf("Unexpected upload file count: %d != %d", uploadCount, len(uploadData))
+	}
+}
+
+func TestPurgeNone(t *testing.T) {
+	fs, ctx := testUploadFS(t, 10, "test-repo", time.Now())
+	oneHourAgo := time.Now().Add(-1 * time.Hour)
+	deleted, errs := PurgeUploads(ctx, fs, oneHourAgo, true)
+	if len(errs) != 0 {
+		t.Error("Unexpected errors", errs)
+	}
+	if len(deleted) != 0 {
+		t.Errorf("Unexpectedly deleted files for time: %s", oneHourAgo)
+	}
+}
+
+func TestPurgeAll(t *testing.T) {
+	uploadCount := 10
+	oneHourAgo := time.Now().Add(-1 * time.Hour)
+	fs, ctx := testUploadFS(t, uploadCount, "test-repo", oneHourAgo)
+
+	// Ensure > 1 repos are purged
+	addUploads(ctx, t, fs, uuid.Generate().String(), "test-repo2", oneHourAgo)
+	uploadCount++
+
+	deleted, errs := PurgeUploads(ctx, fs, time.Now(), true)
+	if len(errs) != 0 {
+		t.Error("Unexpected errors:", errs)
+	}
+	fileCount := uploadCount
+	if len(deleted) != fileCount {
+		t.Errorf("Unexpectedly deleted file count %d != %d",
+			len(deleted), fileCount)
+	}
+}
+
+func TestPurgeSome(t *testing.T) {
+	oldUploadCount := 5
+	oneHourAgo := time.Now().Add(-1 * time.Hour)
+	fs, ctx := testUploadFS(t, oldUploadCount, "library/test-repo", oneHourAgo)
+
+	newUploadCount := 4
+
+	for i := 0; i < newUploadCount; i++ {
+		addUploads(ctx, t, fs, uuid.Generate().String(), "test-repo", time.Now().Add(1*time.Hour))
+	}
+
+	deleted, errs := PurgeUploads(ctx, fs, time.Now(), true)
+	if len(errs) != 0 {
+		t.Error("Unexpected errors:", errs)
+	}
+	if len(deleted) != oldUploadCount {
+		t.Errorf("Unexpectedly deleted file count %d != %d",
+			len(deleted), oldUploadCount)
+	}
+}
+
+func TestPurgeOnlyUploads(t *testing.T) {
+	oldUploadCount := 5
+	oneHourAgo := time.Now().Add(-1 * time.Hour)
+	fs, ctx := testUploadFS(t, oldUploadCount, "test-repo", oneHourAgo)
+
+	// Create a directory tree outside _uploads and ensure
+	// these files aren't deleted.
+	dataPath, err := pathFor(uploadDataPathSpec{name: "test-repo", id: uuid.Generate().String()})
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	nonUploadPath := strings.Replace(dataPath, "_upload", "_important", -1)
+	if strings.Index(nonUploadPath, "_upload") != -1 {
+		t.Fatalf("Non-upload path not created correctly")
+	}
+
+	nonUploadFile := path.Join(nonUploadPath, "file")
+	if err = fs.PutContent(ctx, nonUploadFile, []byte("")); err != nil {
+		t.Fatalf("Unable to write data file")
+	}
+
+	deleted, errs := PurgeUploads(ctx, fs, time.Now(), true)
+	if len(errs) != 0 {
+		t.Error("Unexpected errors", errs)
+	}
+	for _, file := range deleted {
+		if strings.Index(file, "_upload") == -1 {
+			t.Errorf("Non-upload file deleted")
+		}
+	}
+}
+
+func TestPurgeMissingStartedAt(t *testing.T) {
+	oneHourAgo := time.Now().Add(-1 * time.Hour)
+	fs, ctx := testUploadFS(t, 1, "test-repo", oneHourAgo)
+
+	err := Walk(ctx, fs, "/", func(fileInfo driver.FileInfo) error {
+		filePath := fileInfo.Path()
+		_, file := path.Split(filePath)
+
+		if file == "startedat" {
+			if err := fs.Delete(ctx, filePath); err != nil {
+				t.Fatalf("Unable to delete startedat file: %s", filePath)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Unexpected error during Walk: %s ", err.Error())
+	}
+	deleted, errs := PurgeUploads(ctx, fs, time.Now(), true)
+	if len(errs) > 0 {
+		t.Errorf("Unexpected errors")
+	}
+	if len(deleted) > 0 {
+		t.Errorf("Files unexpectedly deleted: %s", deleted)
+	}
+}

vendor/github.com/docker/distribution/registry/storage/registry.go

@@ -0,0 +1,306 @@
+package storage
+
+import (
+	"regexp"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/cache"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/libtrust"
+)
+
+// registry is the top-level implementation of Registry for use in the storage
+// package. All instances should descend from this object.
+type registry struct {
+	blobStore                    *blobStore
+	blobServer                   *blobServer
+	statter                      *blobStatter // global statter service.
+	blobDescriptorCacheProvider  cache.BlobDescriptorCacheProvider
+	deleteEnabled                bool
+	resumableDigestEnabled       bool
+	schema1SigningKey            libtrust.PrivateKey
+	blobDescriptorServiceFactory distribution.BlobDescriptorServiceFactory
+	manifestURLs                 manifestURLs
+}
+
+// manifestURLs holds regular expressions for controlling manifest URL whitelisting
+type manifestURLs struct {
+	allow *regexp.Regexp
+	deny  *regexp.Regexp
+}
+
+// RegistryOption is the type used for functional options for NewRegistry.
+type RegistryOption func(*registry) error
+
+// EnableRedirect is a functional option for NewRegistry. It causes the backend
+// blob server to attempt using (StorageDriver).URLFor to serve all blobs.
+func EnableRedirect(registry *registry) error {
+	registry.blobServer.redirect = true
+	return nil
+}
+
+// EnableDelete is a functional option for NewRegistry. It enables deletion on
+// the registry.
+func EnableDelete(registry *registry) error {
+	registry.deleteEnabled = true
+	return nil
+}
+
+// DisableDigestResumption is a functional option for NewRegistry. It should be
+// used if the registry is acting as a caching proxy.
+func DisableDigestResumption(registry *registry) error {
+	registry.resumableDigestEnabled = false
+	return nil
+}
+
+// ManifestURLsAllowRegexp is a functional option for NewRegistry.
+func ManifestURLsAllowRegexp(r *regexp.Regexp) RegistryOption {
+	return func(registry *registry) error {
+		registry.manifestURLs.allow = r
+		return nil
+	}
+}
+
+// ManifestURLsDenyRegexp is a functional option for NewRegistry.
+func ManifestURLsDenyRegexp(r *regexp.Regexp) RegistryOption {
+	return func(registry *registry) error {
+		registry.manifestURLs.deny = r
+		return nil
+	}
+}
+
+// Schema1SigningKey returns a functional option for NewRegistry. It sets the
+// key for signing  all schema1 manifests.
+func Schema1SigningKey(key libtrust.PrivateKey) RegistryOption {
+	return func(registry *registry) error {
+		registry.schema1SigningKey = key
+		return nil
+	}
+}
+
+// BlobDescriptorServiceFactory returns a functional option for NewRegistry. It sets the
+// factory to create BlobDescriptorServiceFactory middleware.
+func BlobDescriptorServiceFactory(factory distribution.BlobDescriptorServiceFactory) RegistryOption {
+	return func(registry *registry) error {
+		registry.blobDescriptorServiceFactory = factory
+		return nil
+	}
+}
+
+// BlobDescriptorCacheProvider returns a functional option for
+// NewRegistry. It creates a cached blob statter for use by the
+// registry.
+func BlobDescriptorCacheProvider(blobDescriptorCacheProvider cache.BlobDescriptorCacheProvider) RegistryOption {
+	// TODO(aaronl): The duplication of statter across several objects is
+	// ugly, and prevents us from using interface types in the registry
+	// struct. Ideally, blobStore and blobServer should be lazily
+	// initialized, and use the current value of
+	// blobDescriptorCacheProvider.
+	return func(registry *registry) error {
+		if blobDescriptorCacheProvider != nil {
+			statter := cache.NewCachedBlobStatter(blobDescriptorCacheProvider, registry.statter)
+			registry.blobStore.statter = statter
+			registry.blobServer.statter = statter
+			registry.blobDescriptorCacheProvider = blobDescriptorCacheProvider
+		}
+		return nil
+	}
+}
+
+// NewRegistry creates a new registry instance from the provided driver. The
+// resulting registry may be shared by multiple goroutines but is cheap to
+// allocate. If the Redirect option is specified, the backend blob server will
+// attempt to use (StorageDriver).URLFor to serve all blobs.
+func NewRegistry(ctx context.Context, driver storagedriver.StorageDriver, options ...RegistryOption) (distribution.Namespace, error) {
+	// create global statter
+	statter := &blobStatter{
+		driver: driver,
+	}
+
+	bs := &blobStore{
+		driver:  driver,
+		statter: statter,
+	}
+
+	registry := &registry{
+		blobStore: bs,
+		blobServer: &blobServer{
+			driver:  driver,
+			statter: statter,
+			pathFn:  bs.path,
+		},
+		statter:                statter,
+		resumableDigestEnabled: true,
+	}
+
+	for _, option := range options {
+		if err := option(registry); err != nil {
+			return nil, err
+		}
+	}
+
+	return registry, nil
+}
+
+// Scope returns the namespace scope for a registry. The registry
+// will only serve repositories contained within this scope.
+func (reg *registry) Scope() distribution.Scope {
+	return distribution.GlobalScope
+}
+
+// Repository returns an instance of the repository tied to the registry.
+// Instances should not be shared between goroutines but are cheap to
+// allocate. In general, they should be request scoped.
+func (reg *registry) Repository(ctx context.Context, canonicalName reference.Named) (distribution.Repository, error) {
+	var descriptorCache distribution.BlobDescriptorService
+	if reg.blobDescriptorCacheProvider != nil {
+		var err error
+		descriptorCache, err = reg.blobDescriptorCacheProvider.RepositoryScoped(canonicalName.Name())
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &repository{
+		ctx:             ctx,
+		registry:        reg,
+		name:            canonicalName,
+		descriptorCache: descriptorCache,
+	}, nil
+}
+
+func (reg *registry) Blobs() distribution.BlobEnumerator {
+	return reg.blobStore
+}
+
+func (reg *registry) BlobStatter() distribution.BlobStatter {
+	return reg.statter
+}
+
+// repository provides name-scoped access to various services.
+type repository struct {
+	*registry
+	ctx             context.Context
+	name            reference.Named
+	descriptorCache distribution.BlobDescriptorService
+}
+
+// Name returns the name of the repository.
+func (repo *repository) Named() reference.Named {
+	return repo.name
+}
+
+func (repo *repository) Tags(ctx context.Context) distribution.TagService {
+	tags := &tagStore{
+		repository: repo,
+		blobStore:  repo.registry.blobStore,
+	}
+
+	return tags
+}
+
+// Manifests returns an instance of ManifestService. Instantiation is cheap and
+// may be context sensitive in the future. The instance should be used similar
+// to a request local.
+func (repo *repository) Manifests(ctx context.Context, options ...distribution.ManifestServiceOption) (distribution.ManifestService, error) {
+	manifestLinkPathFns := []linkPathFunc{
+		// NOTE(stevvooe): Need to search through multiple locations since
+		// 2.1.0 unintentionally linked into  _layers.
+		manifestRevisionLinkPath,
+		blobLinkPath,
+	}
+
+	manifestDirectoryPathSpec := manifestRevisionsPathSpec{name: repo.name.Name()}
+
+	var statter distribution.BlobDescriptorService = &linkedBlobStatter{
+		blobStore:   repo.blobStore,
+		repository:  repo,
+		linkPathFns: manifestLinkPathFns,
+	}
+
+	if repo.registry.blobDescriptorServiceFactory != nil {
+		statter = repo.registry.blobDescriptorServiceFactory.BlobAccessController(statter)
+	}
+
+	blobStore := &linkedBlobStore{
+		ctx:                  ctx,
+		blobStore:            repo.blobStore,
+		repository:           repo,
+		deleteEnabled:        repo.registry.deleteEnabled,
+		blobAccessController: statter,
+
+		// TODO(stevvooe): linkPath limits this blob store to only
+		// manifests. This instance cannot be used for blob checks.
+		linkPathFns:           manifestLinkPathFns,
+		linkDirectoryPathSpec: manifestDirectoryPathSpec,
+	}
+
+	ms := &manifestStore{
+		ctx:        ctx,
+		repository: repo,
+		blobStore:  blobStore,
+		schema1Handler: &signedManifestHandler{
+			ctx:               ctx,
+			schema1SigningKey: repo.schema1SigningKey,
+			repository:        repo,
+			blobStore:         blobStore,
+		},
+		schema2Handler: &schema2ManifestHandler{
+			ctx:          ctx,
+			repository:   repo,
+			blobStore:    blobStore,
+			manifestURLs: repo.registry.manifestURLs,
+		},
+		manifestListHandler: &manifestListHandler{
+			ctx:        ctx,
+			repository: repo,
+			blobStore:  blobStore,
+		},
+	}
+
+	// Apply options
+	for _, option := range options {
+		err := option.Apply(ms)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return ms, nil
+}
+
+// Blobs returns an instance of the BlobStore. Instantiation is cheap and
+// may be context sensitive in the future. The instance should be used similar
+// to a request local.
+func (repo *repository) Blobs(ctx context.Context) distribution.BlobStore {
+	var statter distribution.BlobDescriptorService = &linkedBlobStatter{
+		blobStore:   repo.blobStore,
+		repository:  repo,
+		linkPathFns: []linkPathFunc{blobLinkPath},
+	}
+
+	if repo.descriptorCache != nil {
+		statter = cache.NewCachedBlobStatter(repo.descriptorCache, statter)
+	}
+
+	if repo.registry.blobDescriptorServiceFactory != nil {
+		statter = repo.registry.blobDescriptorServiceFactory.BlobAccessController(statter)
+	}
+
+	return &linkedBlobStore{
+		registry:             repo.registry,
+		blobStore:            repo.blobStore,
+		blobServer:           repo.blobServer,
+		blobAccessController: statter,
+		repository:           repo,
+		ctx:                  ctx,
+
+		// TODO(stevvooe): linkPath limits this blob store to only layers.
+		// This instance cannot be used for manifest checks.
+		linkPathFns:            []linkPathFunc{blobLinkPath},
+		deleteEnabled:          repo.registry.deleteEnabled,
+		resumableDigestEnabled: repo.resumableDigestEnabled,
+	}
+}

vendor/github.com/docker/distribution/registry/storage/schema2manifesthandler.go

@@ -0,0 +1,136 @@
+package storage
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/manifest/schema2"
+)
+
+var (
+	errUnexpectedURL = errors.New("unexpected URL on layer")
+	errMissingURL    = errors.New("missing URL on layer")
+	errInvalidURL    = errors.New("invalid URL on layer")
+)
+
+//schema2ManifestHandler is a ManifestHandler that covers schema2 manifests.
+type schema2ManifestHandler struct {
+	repository   distribution.Repository
+	blobStore    distribution.BlobStore
+	ctx          context.Context
+	manifestURLs manifestURLs
+}
+
+var _ ManifestHandler = &schema2ManifestHandler{}
+
+func (ms *schema2ManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Unmarshal")
+
+	var m schema2.DeserializedManifest
+	if err := json.Unmarshal(content, &m); err != nil {
+		return nil, err
+	}
+
+	return &m, nil
+}
+
+func (ms *schema2ManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*schema2ManifestHandler).Put")
+
+	m, ok := manifest.(*schema2.DeserializedManifest)
+	if !ok {
+		return "", fmt.Errorf("non-schema2 manifest put to schema2ManifestHandler: %T", manifest)
+	}
+
+	if err := ms.verifyManifest(ms.ctx, *m, skipDependencyVerification); err != nil {
+		return "", err
+	}
+
+	mt, payload, err := m.Payload()
+	if err != nil {
+		return "", err
+	}
+
+	revision, err := ms.blobStore.Put(ctx, mt, payload)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+		return "", err
+	}
+
+	return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. As a policy, the registry only tries to store
+// valid content, leaving trust policies of that content up to consumers.
+func (ms *schema2ManifestHandler) verifyManifest(ctx context.Context, mnfst schema2.DeserializedManifest, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if skipDependencyVerification {
+		return nil
+	}
+
+	manifestService, err := ms.repository.Manifests(ctx)
+	if err != nil {
+		return err
+	}
+
+	blobsService := ms.repository.Blobs(ctx)
+
+	for _, descriptor := range mnfst.References() {
+		var err error
+
+		switch descriptor.MediaType {
+		case schema2.MediaTypeForeignLayer:
+			// Clients download this layer from an external URL, so do not check for
+			// its presense.
+			if len(descriptor.URLs) == 0 {
+				err = errMissingURL
+			}
+			allow := ms.manifestURLs.allow
+			deny := ms.manifestURLs.deny
+			for _, u := range descriptor.URLs {
+				var pu *url.URL
+				pu, err = url.Parse(u)
+				if err != nil || (pu.Scheme != "http" && pu.Scheme != "https") || pu.Fragment != "" || (allow != nil && !allow.MatchString(u)) || (deny != nil && deny.MatchString(u)) {
+					err = errInvalidURL
+					break
+				}
+			}
+		case schema2.MediaTypeManifest, schema1.MediaTypeManifest:
+			var exists bool
+			exists, err = manifestService.Exists(ctx, descriptor.Digest)
+			if err != nil || !exists {
+				err = distribution.ErrBlobUnknown // just coerce to unknown.
+			}
+
+			fallthrough // double check the blob store.
+		default:
+			// forward all else to blob storage
+			if len(descriptor.URLs) == 0 {
+				_, err = blobsService.Stat(ctx, descriptor.Digest)
+			}
+		}
+
+		if err != nil {
+			if err != distribution.ErrBlobUnknown {
+				errs = append(errs, err)
+			}
+
+			// On error here, we always append unknown blob errors.
+			errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: descriptor.Digest})
+		}
+	}
+
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/schema2manifesthandler_test.go

@@ -0,0 +1,136 @@
+package storage
+
+import (
+	"regexp"
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/manifest"
+	"github.com/docker/distribution/manifest/schema2"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+func TestVerifyManifestForeignLayer(t *testing.T) {
+	ctx := context.Background()
+	inmemoryDriver := inmemory.New()
+	registry := createRegistry(t, inmemoryDriver,
+		ManifestURLsAllowRegexp(regexp.MustCompile("^https?://foo")),
+		ManifestURLsDenyRegexp(regexp.MustCompile("^https?://foo/nope")))
+	repo := makeRepository(t, registry, "test")
+	manifestService := makeManifestService(t, repo)
+
+	config, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeConfig, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	layer, err := repo.Blobs(ctx).Put(ctx, schema2.MediaTypeLayer, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	foreignLayer := distribution.Descriptor{
+		Digest:    "sha256:463435349086340864309863409683460843608348608934092322395278926a",
+		Size:      6323,
+		MediaType: schema2.MediaTypeForeignLayer,
+	}
+
+	template := schema2.Manifest{
+		Versioned: manifest.Versioned{
+			SchemaVersion: 2,
+			MediaType:     schema2.MediaTypeManifest,
+		},
+		Config: config,
+	}
+
+	type testcase struct {
+		BaseLayer distribution.Descriptor
+		URLs      []string
+		Err       error
+	}
+
+	cases := []testcase{
+		{
+			foreignLayer,
+			nil,
+			errMissingURL,
+		},
+		{
+			// regular layers may have foreign urls
+			layer,
+			[]string{"http://foo/bar"},
+			nil,
+		},
+		{
+			foreignLayer,
+			[]string{"file:///local/file"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"http://foo/bar#baz"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{""},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"https://foo/bar", ""},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"", "https://foo/bar"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"http://nope/bar"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"http://foo/nope"},
+			errInvalidURL,
+		},
+		{
+			foreignLayer,
+			[]string{"http://foo/bar"},
+			nil,
+		},
+		{
+			foreignLayer,
+			[]string{"https://foo/bar"},
+			nil,
+		},
+	}
+
+	for _, c := range cases {
+		m := template
+		l := c.BaseLayer
+		l.URLs = c.URLs
+		m.Layers = []distribution.Descriptor{l}
+		dm, err := schema2.FromStruct(m)
+		if err != nil {
+			t.Error(err)
+			continue
+		}
+
+		_, err = manifestService.Put(ctx, dm)
+		if verr, ok := err.(distribution.ErrManifestVerification); ok {
+			// Extract the first error
+			if len(verr) == 2 {
+				if _, ok = verr[1].(distribution.ErrManifestBlobUnknown); ok {
+					err = verr[0]
+				}
+			}
+		}
+		if err != c.Err {
+			t.Errorf("%#v: expected %v, got %v", l, c.Err, err)
+		}
+	}
+}

vendor/github.com/docker/distribution/registry/storage/signedmanifesthandler.go

@@ -0,0 +1,141 @@
+package storage
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/manifest/schema1"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/libtrust"
+)
+
+// signedManifestHandler is a ManifestHandler that covers schema1 manifests. It
+// can unmarshal and put schema1 manifests that have been signed by libtrust.
+type signedManifestHandler struct {
+	repository        distribution.Repository
+	schema1SigningKey libtrust.PrivateKey
+	blobStore         distribution.BlobStore
+	ctx               context.Context
+}
+
+var _ ManifestHandler = &signedManifestHandler{}
+
+func (ms *signedManifestHandler) Unmarshal(ctx context.Context, dgst digest.Digest, content []byte) (distribution.Manifest, error) {
+	context.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Unmarshal")
+
+	var (
+		signatures [][]byte
+		err        error
+	)
+
+	jsig, err := libtrust.NewJSONSignature(content, signatures...)
+	if err != nil {
+		return nil, err
+	}
+
+	if ms.schema1SigningKey != nil {
+		if err := jsig.Sign(ms.schema1SigningKey); err != nil {
+			return nil, err
+		}
+	}
+
+	// Extract the pretty JWS
+	raw, err := jsig.PrettySignature("signatures")
+	if err != nil {
+		return nil, err
+	}
+
+	var sm schema1.SignedManifest
+	if err := json.Unmarshal(raw, &sm); err != nil {
+		return nil, err
+	}
+	return &sm, nil
+}
+
+func (ms *signedManifestHandler) Put(ctx context.Context, manifest distribution.Manifest, skipDependencyVerification bool) (digest.Digest, error) {
+	context.GetLogger(ms.ctx).Debug("(*signedManifestHandler).Put")
+
+	sm, ok := manifest.(*schema1.SignedManifest)
+	if !ok {
+		return "", fmt.Errorf("non-schema1 manifest put to signedManifestHandler: %T", manifest)
+	}
+
+	if err := ms.verifyManifest(ms.ctx, *sm, skipDependencyVerification); err != nil {
+		return "", err
+	}
+
+	mt := schema1.MediaTypeManifest
+	payload := sm.Canonical
+
+	revision, err := ms.blobStore.Put(ctx, mt, payload)
+	if err != nil {
+		context.GetLogger(ctx).Errorf("error putting payload into blobstore: %v", err)
+		return "", err
+	}
+
+	return revision.Digest, nil
+}
+
+// verifyManifest ensures that the manifest content is valid from the
+// perspective of the registry. It ensures that the signature is valid for the
+// enclosed payload. As a policy, the registry only tries to store valid
+// content, leaving trust policies of that content up to consumers.
+func (ms *signedManifestHandler) verifyManifest(ctx context.Context, mnfst schema1.SignedManifest, skipDependencyVerification bool) error {
+	var errs distribution.ErrManifestVerification
+
+	if len(mnfst.Name) > reference.NameTotalLengthMax {
+		errs = append(errs,
+			distribution.ErrManifestNameInvalid{
+				Name:   mnfst.Name,
+				Reason: fmt.Errorf("manifest name must not be more than %v characters", reference.NameTotalLengthMax),
+			})
+	}
+
+	if !reference.NameRegexp.MatchString(mnfst.Name) {
+		errs = append(errs,
+			distribution.ErrManifestNameInvalid{
+				Name:   mnfst.Name,
+				Reason: fmt.Errorf("invalid manifest name format"),
+			})
+	}
+
+	if len(mnfst.History) != len(mnfst.FSLayers) {
+		errs = append(errs, fmt.Errorf("mismatched history and fslayer cardinality %d != %d",
+			len(mnfst.History), len(mnfst.FSLayers)))
+	}
+
+	if _, err := schema1.Verify(&mnfst); err != nil {
+		switch err {
+		case libtrust.ErrMissingSignatureKey, libtrust.ErrInvalidJSONContent, libtrust.ErrMissingSignatureKey:
+			errs = append(errs, distribution.ErrManifestUnverified{})
+		default:
+			if err.Error() == "invalid signature" { // TODO(stevvooe): This should be exported by libtrust
+				errs = append(errs, distribution.ErrManifestUnverified{})
+			} else {
+				errs = append(errs, err)
+			}
+		}
+	}
+
+	if !skipDependencyVerification {
+		for _, fsLayer := range mnfst.References() {
+			_, err := ms.repository.Blobs(ctx).Stat(ctx, fsLayer.Digest)
+			if err != nil {
+				if err != distribution.ErrBlobUnknown {
+					errs = append(errs, err)
+				}
+
+				// On error here, we always append unknown blob errors.
+				errs = append(errs, distribution.ErrManifestBlobUnknown{Digest: fsLayer.Digest})
+			}
+		}
+	}
+	if len(errs) != 0 {
+		return errs
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/tagstore.go

@@ -0,0 +1,191 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+var _ distribution.TagService = &tagStore{}
+
+// tagStore provides methods to manage manifest tags in a backend storage driver.
+// This implementation uses the same on-disk layout as the (now deleted) tag
+// store.  This provides backward compatibility with current registry deployments
+// which only makes use of the Digest field of the returned distribution.Descriptor
+// but does not enable full roundtripping of Descriptor objects
+type tagStore struct {
+	repository *repository
+	blobStore  *blobStore
+}
+
+// All returns all tags
+func (ts *tagStore) All(ctx context.Context) ([]string, error) {
+	var tags []string
+
+	pathSpec, err := pathFor(manifestTagPathSpec{
+		name: ts.repository.Named().Name(),
+	})
+	if err != nil {
+		return tags, err
+	}
+
+	entries, err := ts.blobStore.driver.List(ctx, pathSpec)
+	if err != nil {
+		switch err := err.(type) {
+		case storagedriver.PathNotFoundError:
+			return tags, distribution.ErrRepositoryUnknown{Name: ts.repository.Named().Name()}
+		default:
+			return tags, err
+		}
+	}
+
+	for _, entry := range entries {
+		_, filename := path.Split(entry)
+		tags = append(tags, filename)
+	}
+
+	return tags, nil
+}
+
+// exists returns true if the specified manifest tag exists in the repository.
+func (ts *tagStore) exists(ctx context.Context, tag string) (bool, error) {
+	tagPath, err := pathFor(manifestTagCurrentPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	if err != nil {
+		return false, err
+	}
+
+	exists, err := exists(ctx, ts.blobStore.driver, tagPath)
+	if err != nil {
+		return false, err
+	}
+
+	return exists, nil
+}
+
+// Tag tags the digest with the given tag, updating the the store to point at
+// the current tag. The digest must point to a manifest.
+func (ts *tagStore) Tag(ctx context.Context, tag string, desc distribution.Descriptor) error {
+	currentPath, err := pathFor(manifestTagCurrentPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	if err != nil {
+		return err
+	}
+
+	lbs := ts.linkedBlobStore(ctx, tag)
+
+	// Link into the index
+	if err := lbs.linkBlob(ctx, desc); err != nil {
+		return err
+	}
+
+	// Overwrite the current link
+	return ts.blobStore.link(ctx, currentPath, desc.Digest)
+}
+
+// resolve the current revision for name and tag.
+func (ts *tagStore) Get(ctx context.Context, tag string) (distribution.Descriptor, error) {
+	currentPath, err := pathFor(manifestTagCurrentPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	if err != nil {
+		return distribution.Descriptor{}, err
+	}
+
+	revision, err := ts.blobStore.readlink(ctx, currentPath)
+	if err != nil {
+		switch err.(type) {
+		case storagedriver.PathNotFoundError:
+			return distribution.Descriptor{}, distribution.ErrTagUnknown{Tag: tag}
+		}
+
+		return distribution.Descriptor{}, err
+	}
+
+	return distribution.Descriptor{Digest: revision}, nil
+}
+
+// Untag removes the tag association
+func (ts *tagStore) Untag(ctx context.Context, tag string) error {
+	tagPath, err := pathFor(manifestTagPathSpec{
+		name: ts.repository.Named().Name(),
+		tag:  tag,
+	})
+
+	switch err.(type) {
+	case storagedriver.PathNotFoundError:
+		return distribution.ErrTagUnknown{Tag: tag}
+	case nil:
+		break
+	default:
+		return err
+	}
+
+	return ts.blobStore.driver.Delete(ctx, tagPath)
+}
+
+// linkedBlobStore returns the linkedBlobStore for the named tag, allowing one
+// to index manifest blobs by tag name. While the tag store doesn't map
+// precisely to the linked blob store, using this ensures the links are
+// managed via the same code path.
+func (ts *tagStore) linkedBlobStore(ctx context.Context, tag string) *linkedBlobStore {
+	return &linkedBlobStore{
+		blobStore:  ts.blobStore,
+		repository: ts.repository,
+		ctx:        ctx,
+		linkPathFns: []linkPathFunc{func(name string, dgst digest.Digest) (string, error) {
+			return pathFor(manifestTagIndexEntryLinkPathSpec{
+				name:     name,
+				tag:      tag,
+				revision: dgst,
+			})
+
+		}},
+	}
+}
+
+// Lookup recovers a list of tags which refer to this digest.  When a manifest is deleted by
+// digest, tag entries which point to it need to be recovered to avoid dangling tags.
+func (ts *tagStore) Lookup(ctx context.Context, desc distribution.Descriptor) ([]string, error) {
+	allTags, err := ts.All(ctx)
+	switch err.(type) {
+	case distribution.ErrRepositoryUnknown:
+		// This tag store has been initialized but not yet populated
+		break
+	case nil:
+		break
+	default:
+		return nil, err
+	}
+
+	var tags []string
+	for _, tag := range allTags {
+		tagLinkPathSpec := manifestTagCurrentPathSpec{
+			name: ts.repository.Named().Name(),
+			tag:  tag,
+		}
+
+		tagLinkPath, err := pathFor(tagLinkPathSpec)
+		tagDigest, err := ts.blobStore.readlink(ctx, tagLinkPath)
+		if err != nil {
+			return nil, err
+		}
+
+		if tagDigest == desc.Digest {
+			tags = append(tags, tag)
+		}
+	}
+
+	return tags, nil
+}

vendor/github.com/docker/distribution/registry/storage/tagstore_test.go

@@ -0,0 +1,209 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+type tagsTestEnv struct {
+	ts  distribution.TagService
+	ctx context.Context
+}
+
+func testTagStore(t *testing.T) *tagsTestEnv {
+	ctx := context.Background()
+	d := inmemory.New()
+	reg, err := NewRegistry(ctx, d)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	repoRef, _ := reference.ParseNamed("a/b")
+	repo, err := reg.Repository(ctx, repoRef)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return &tagsTestEnv{
+		ctx: ctx,
+		ts:  repo.Tags(ctx),
+	}
+}
+
+func TestTagStoreTag(t *testing.T) {
+	env := testTagStore(t)
+	tags := env.ts
+	ctx := env.ctx
+
+	d := distribution.Descriptor{}
+	err := tags.Tag(ctx, "latest", d)
+	if err == nil {
+		t.Errorf("unexpected error putting malformed descriptor : %s", err)
+	}
+
+	d.Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+	err = tags.Tag(ctx, "latest", d)
+	if err != nil {
+		t.Error(err)
+	}
+
+	d1, err := tags.Get(ctx, "latest")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if d1.Digest != d.Digest {
+		t.Error("put and get digest differ")
+	}
+
+	// Overwrite existing
+	d.Digest = "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
+	err = tags.Tag(ctx, "latest", d)
+	if err != nil {
+		t.Error(err)
+	}
+
+	d1, err = tags.Get(ctx, "latest")
+	if err != nil {
+		t.Error(err)
+	}
+
+	if d1.Digest != d.Digest {
+		t.Error("put and get digest differ")
+	}
+}
+
+func TestTagStoreUnTag(t *testing.T) {
+	env := testTagStore(t)
+	tags := env.ts
+	ctx := env.ctx
+	desc := distribution.Descriptor{Digest: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}
+
+	err := tags.Untag(ctx, "latest")
+	if err == nil {
+		t.Errorf("Expected error untagging non-existant tag")
+	}
+
+	err = tags.Tag(ctx, "latest", desc)
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = tags.Untag(ctx, "latest")
+	if err != nil {
+		t.Error(err)
+	}
+
+	errExpect := distribution.ErrTagUnknown{Tag: "latest"}.Error()
+	_, err = tags.Get(ctx, "latest")
+	if err == nil || err.Error() != errExpect {
+		t.Error("Expected error getting untagged tag")
+	}
+}
+
+func TestTagStoreAll(t *testing.T) {
+	env := testTagStore(t)
+	tagStore := env.ts
+	ctx := env.ctx
+
+	alpha := "abcdefghijklmnopqrstuvwxyz"
+	for i := 0; i < len(alpha); i++ {
+		tag := alpha[i]
+		desc := distribution.Descriptor{Digest: "sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}
+		err := tagStore.Tag(ctx, string(tag), desc)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+
+	all, err := tagStore.All(ctx)
+	if err != nil {
+		t.Error(err)
+	}
+	if len(all) != len(alpha) {
+		t.Errorf("Unexpected count returned from enumerate")
+	}
+
+	for i, c := range all {
+		if c != string(alpha[i]) {
+			t.Errorf("unexpected tag in enumerate %s", c)
+		}
+	}
+
+	removed := "a"
+	err = tagStore.Untag(ctx, removed)
+	if err != nil {
+		t.Error(err)
+	}
+
+	all, err = tagStore.All(ctx)
+	if err != nil {
+		t.Error(err)
+	}
+	for _, tag := range all {
+		if tag == removed {
+			t.Errorf("unexpected tag in enumerate %s", removed)
+		}
+	}
+
+}
+
+func TestTagLookup(t *testing.T) {
+	env := testTagStore(t)
+	tagStore := env.ts
+	ctx := env.ctx
+
+	descA := distribution.Descriptor{Digest: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+	desc0 := distribution.Descriptor{Digest: "sha256:0000000000000000000000000000000000000000000000000000000000000000"}
+
+	tags, err := tagStore.Lookup(ctx, descA)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(tags) != 0 {
+		t.Fatalf("Lookup returned > 0 tags from empty store")
+	}
+
+	err = tagStore.Tag(ctx, "a", descA)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = tagStore.Tag(ctx, "b", descA)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = tagStore.Tag(ctx, "0", desc0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = tagStore.Tag(ctx, "1", desc0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tags, err = tagStore.Lookup(ctx, descA)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(tags) != 2 {
+		t.Errorf("Lookup of descA returned %d tags, expected 2", len(tags))
+	}
+
+	tags, err = tagStore.Lookup(ctx, desc0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(tags) != 2 {
+		t.Errorf("Lookup of descB returned %d tags, expected 2", len(tags))
+	}
+
+}

vendor/github.com/docker/distribution/registry/storage/util.go

@@ -0,0 +1,21 @@
+package storage
+
+import (
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// Exists provides a utility method to test whether or not a path exists in
+// the given driver.
+func exists(ctx context.Context, drv driver.StorageDriver, path string) (bool, error) {
+	if _, err := drv.Stat(ctx, path); err != nil {
+		switch err := err.(type) {
+		case driver.PathNotFoundError:
+			return false, nil
+		default:
+			return false, err
+		}
+	}
+
+	return true, nil
+}

vendor/github.com/docker/distribution/registry/storage/vacuum.go

@@ -0,0 +1,67 @@
+package storage
+
+import (
+	"path"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/digest"
+	"github.com/docker/distribution/registry/storage/driver"
+)
+
+// vacuum contains functions for cleaning up repositories and blobs
+// These functions will only reliably work on strongly consistent
+// storage systems.
+// https://en.wikipedia.org/wiki/Consistency_model
+
+// NewVacuum creates a new Vacuum
+func NewVacuum(ctx context.Context, driver driver.StorageDriver) Vacuum {
+	return Vacuum{
+		ctx:    ctx,
+		driver: driver,
+	}
+}
+
+// Vacuum removes content from the filesystem
+type Vacuum struct {
+	driver driver.StorageDriver
+	ctx    context.Context
+}
+
+// RemoveBlob removes a blob from the filesystem
+func (v Vacuum) RemoveBlob(dgst string) error {
+	d, err := digest.ParseDigest(dgst)
+	if err != nil {
+		return err
+	}
+
+	blobPath, err := pathFor(blobPathSpec{digest: d})
+	if err != nil {
+		return err
+	}
+
+	context.GetLogger(v.ctx).Infof("Deleting blob: %s", blobPath)
+
+	err = v.driver.Delete(v.ctx, blobPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// RemoveRepository removes a repository directory from the
+// filesystem
+func (v Vacuum) RemoveRepository(repoName string) error {
+	rootForRepository, err := pathFor(repositoriesRootPathSpec{})
+	if err != nil {
+		return err
+	}
+	repoDir := path.Join(rootForRepository, repoName)
+	context.GetLogger(v.ctx).Infof("Deleting repo: %s", repoDir)
+	err = v.driver.Delete(v.ctx, repoDir)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

vendor/github.com/docker/distribution/registry/storage/walk.go

@@ -0,0 +1,59 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+
+	"github.com/docker/distribution/context"
+	storageDriver "github.com/docker/distribution/registry/storage/driver"
+)
+
+// ErrSkipDir is used as a return value from onFileFunc to indicate that
+// the directory named in the call is to be skipped. It is not returned
+// as an error by any function.
+var ErrSkipDir = errors.New("skip this directory")
+
+// WalkFn is called once per file by Walk
+// If the returned error is ErrSkipDir and fileInfo refers
+// to a directory, the directory will not be entered and Walk
+// will continue the traversal.  Otherwise Walk will return
+type WalkFn func(fileInfo storageDriver.FileInfo) error
+
+// Walk traverses a filesystem defined within driver, starting
+// from the given path, calling f on each file
+func Walk(ctx context.Context, driver storageDriver.StorageDriver, from string, f WalkFn) error {
+	children, err := driver.List(ctx, from)
+	if err != nil {
+		return err
+	}
+	sort.Stable(sort.StringSlice(children))
+	for _, child := range children {
+		// TODO(stevvooe): Calling driver.Stat for every entry is quite
+		// expensive when running against backends with a slow Stat
+		// implementation, such as s3. This is very likely a serious
+		// performance bottleneck.
+		fileInfo, err := driver.Stat(ctx, child)
+		if err != nil {
+			return err
+		}
+		err = f(fileInfo)
+		skipDir := (err == ErrSkipDir)
+		if err != nil && !skipDir {
+			return err
+		}
+
+		if fileInfo.IsDir() && !skipDir {
+			if err := Walk(ctx, driver, child, f); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// pushError formats an error type given a path and an error
+// and pushes it to a slice of errors
+func pushError(errors []error, path string, err error) []error {
+	return append(errors, fmt.Errorf("%s: %s", path, err))
+}

vendor/github.com/docker/distribution/registry/storage/walk_test.go

@@ -0,0 +1,152 @@
+package storage
+
+import (
+	"fmt"
+	"sort"
+	"testing"
+
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+)
+
+func testFS(t *testing.T) (driver.StorageDriver, map[string]string, context.Context) {
+	d := inmemory.New()
+	ctx := context.Background()
+
+	expected := map[string]string{
+		"/a":       "dir",
+		"/a/b":     "dir",
+		"/a/b/c":   "dir",
+		"/a/b/c/d": "file",
+		"/a/b/c/e": "file",
+		"/a/b/f":   "dir",
+		"/a/b/f/g": "file",
+		"/a/b/f/h": "file",
+		"/a/b/f/i": "file",
+		"/z":       "dir",
+		"/z/y":     "file",
+	}
+
+	for p, typ := range expected {
+		if typ != "file" {
+			continue
+		}
+
+		if err := d.PutContent(ctx, p, []byte(p)); err != nil {
+			t.Fatalf("unable to put content into fixture: %v", err)
+		}
+	}
+
+	return d, expected, ctx
+}
+
+func TestWalkErrors(t *testing.T) {
+	d, expected, ctx := testFS(t)
+	fileCount := len(expected)
+	err := Walk(ctx, d, "", func(fileInfo driver.FileInfo) error {
+		return nil
+	})
+	if err == nil {
+		t.Error("Expected invalid root err")
+	}
+
+	errEarlyExpected := fmt.Errorf("Early termination")
+
+	err = Walk(ctx, d, "/", func(fileInfo driver.FileInfo) error {
+		// error on the 2nd file
+		if fileInfo.Path() == "/a/b" {
+			return errEarlyExpected
+		}
+
+		delete(expected, fileInfo.Path())
+		return nil
+	})
+	if len(expected) != fileCount-1 {
+		t.Error("Walk failed to terminate with error")
+	}
+	if err != errEarlyExpected {
+		if err == nil {
+			t.Fatalf("expected an error due to early termination")
+		} else {
+			t.Error(err.Error())
+		}
+	}
+
+	err = Walk(ctx, d, "/nonexistent", func(fileInfo driver.FileInfo) error {
+		return nil
+	})
+	if err == nil {
+		t.Errorf("Expected missing file err")
+	}
+
+}
+
+func TestWalk(t *testing.T) {
+	d, expected, ctx := testFS(t)
+	var traversed []string
+	err := Walk(ctx, d, "/", func(fileInfo driver.FileInfo) error {
+		filePath := fileInfo.Path()
+		filetype, ok := expected[filePath]
+		if !ok {
+			t.Fatalf("Unexpected file in walk: %q", filePath)
+		}
+
+		if fileInfo.IsDir() {
+			if filetype != "dir" {
+				t.Errorf("Unexpected file type: %q", filePath)
+			}
+		} else {
+			if filetype != "file" {
+				t.Errorf("Unexpected file type: %q", filePath)
+			}
+
+			// each file has its own path as the contents. If the length
+			// doesn't match the path length, fail.
+			if fileInfo.Size() != int64(len(fileInfo.Path())) {
+				t.Fatalf("unexpected size for %q: %v != %v",
+					fileInfo.Path(), fileInfo.Size(), len(fileInfo.Path()))
+			}
+		}
+		delete(expected, filePath)
+		traversed = append(traversed, filePath)
+		return nil
+	})
+	if len(expected) > 0 {
+		t.Errorf("Missed files in walk: %q", expected)
+	}
+
+	if !sort.StringsAreSorted(traversed) {
+		t.Errorf("result should be sorted: %v", traversed)
+	}
+
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+}
+
+func TestWalkSkipDir(t *testing.T) {
+	d, expected, ctx := testFS(t)
+	err := Walk(ctx, d, "/", func(fileInfo driver.FileInfo) error {
+		filePath := fileInfo.Path()
+		if filePath == "/a/b" {
+			// skip processing /a/b/c and /a/b/c/d
+			return ErrSkipDir
+		}
+		delete(expected, filePath)
+		return nil
+	})
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	if _, ok := expected["/a/b/c"]; !ok {
+		t.Errorf("/a/b/c not skipped")
+	}
+	if _, ok := expected["/a/b/c/d"]; !ok {
+		t.Errorf("/a/b/c/d not skipped")
+	}
+	if _, ok := expected["/a/b/c/e"]; !ok {
+		t.Errorf("/a/b/c/e not skipped")
+	}
+
+}