Add glide.yaml and vendor deps

vendor/k8s.io/kubernetes/pkg/volume/rbd/BUILD

@@ -0,0 +1,52 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "disk_manager.go",
+        "doc.go",
+        "rbd.go",
+        "rbd_util.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api/resource:go_default_library",
+        "//pkg/api/v1:go_default_library",
+        "//pkg/client/clientset_generated/release_1_5:go_default_library",
+        "//pkg/types:go_default_library",
+        "//pkg/util/exec:go_default_library",
+        "//pkg/util/mount:go_default_library",
+        "//pkg/util/node:go_default_library",
+        "//pkg/util/strings:go_default_library",
+        "//pkg/util/uuid:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//pkg/volume/util:go_default_library",
+        "//vendor:github.com/golang/glog",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["rbd_test.go"],
+    library = "go_default_library",
+    tags = ["automanaged"],
+    deps = [
+        "//pkg/api/v1:go_default_library",
+        "//pkg/client/clientset_generated/release_1_5/fake:go_default_library",
+        "//pkg/types:go_default_library",
+        "//pkg/util/mount:go_default_library",
+        "//pkg/util/testing:go_default_library",
+        "//pkg/volume:go_default_library",
+        "//pkg/volume/testing:go_default_library",
+    ],
+)

vendor/k8s.io/kubernetes/pkg/volume/rbd/OWNERS

@@ -0,0 +1,2 @@
+assignees:
+- rootfs

vendor/k8s.io/kubernetes/pkg/volume/rbd/disk_manager.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//
+// diskManager interface and diskSetup/TearDown functions abstract commonly used procedures to setup a block volume
+// rbd volume implements diskManager, calls diskSetup when creating a volume, and calls diskTearDown inside volume unmounter.
+// TODO: consolidate, refactor, and share diskManager among iSCSI, GCE PD, and RBD
+//
+
+package rbd
+
+import (
+	"os"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+// Abstract interface to disk operations.
+type diskManager interface {
+	MakeGlobalPDName(disk rbd) string
+	// Attaches the disk to the kubelet's host machine.
+	AttachDisk(disk rbdMounter) error
+	// Detaches the disk from the kubelet's host machine.
+	DetachDisk(disk rbdUnmounter, mntPath string) error
+	// Creates a rbd image
+	CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, volumeSizeGB int, err error)
+	// Deletes a rbd image
+	DeleteImage(deleter *rbdVolumeDeleter) error
+}
+
+// utility to mount a disk based filesystem
+func diskSetUp(manager diskManager, b rbdMounter, volPath string, mounter mount.Interface, fsGroup *int64) error {
+	globalPDPath := manager.MakeGlobalPDName(*b.rbd)
+	// TODO: handle failed mounts here.
+	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
+
+	if err != nil && !os.IsNotExist(err) {
+		glog.Errorf("cannot validate mountpoint: %s", volPath)
+		return err
+	}
+	if !notMnt {
+		return nil
+	}
+	if err := manager.AttachDisk(b); err != nil {
+		glog.Errorf("failed to attach disk")
+		return err
+	}
+
+	if err := os.MkdirAll(volPath, 0750); err != nil {
+		glog.Errorf("failed to mkdir:%s", volPath)
+		return err
+	}
+	// Perform a bind mount to the full path to allow duplicate mounts of the same disk.
+	options := []string{"bind"}
+	if (&b).GetAttributes().ReadOnly {
+		options = append(options, "ro")
+	}
+	err = mounter.Mount(globalPDPath, volPath, "", options)
+	if err != nil {
+		glog.Errorf("failed to bind mount:%s", globalPDPath)
+		return err
+	}
+
+	if !b.ReadOnly {
+		volume.SetVolumeOwnership(&b, fsGroup)
+	}
+
+	return nil
+}
+
+// utility to tear down a disk based filesystem
+func diskTearDown(manager diskManager, c rbdUnmounter, volPath string, mounter mount.Interface) error {
+	notMnt, err := mounter.IsLikelyNotMountPoint(volPath)
+	if err != nil {
+		glog.Errorf("cannot validate mountpoint %s", volPath)
+		return err
+	}
+	if notMnt {
+		return os.Remove(volPath)
+	}
+
+	refs, err := mount.GetMountRefs(mounter, volPath)
+	if err != nil {
+		glog.Errorf("failed to get reference count %s", volPath)
+		return err
+	}
+	if err := mounter.Unmount(volPath); err != nil {
+		glog.Errorf("failed to umount %s", volPath)
+		return err
+	}
+	// If len(refs) is 1, then all bind mounts have been removed, and the
+	// remaining reference is the global mount. It is safe to detach.
+	if len(refs) == 1 {
+		mntPath := refs[0]
+		if err := manager.DetachDisk(c, mntPath); err != nil {
+			glog.Errorf("failed to detach disk from %s", mntPath)
+			return err
+		}
+	}
+
+	notMnt, mntErr := mounter.IsLikelyNotMountPoint(volPath)
+	if mntErr != nil {
+		glog.Errorf("IsLikelyNotMountPoint check failed: %v", mntErr)
+		return err
+	}
+	if notMnt {
+		if err := os.Remove(volPath); err != nil {
+			return err
+		}
+	}
+	return nil
+
+}

vendor/k8s.io/kubernetes/pkg/volume/rbd/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package rbd contains the internal representation of Rados Block Store (Ceph)
+// volumes.
+package rbd // import "k8s.io/kubernetes/pkg/volume/rbd"

vendor/k8s.io/kubernetes/pkg/volume/rbd/rbd.go

@@ -0,0 +1,470 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"fmt"
+	dstrings "strings"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api/resource"
+	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
+	"k8s.io/kubernetes/pkg/types"
+	"k8s.io/kubernetes/pkg/util/exec"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/strings"
+	"k8s.io/kubernetes/pkg/util/uuid"
+	"k8s.io/kubernetes/pkg/volume"
+	volutil "k8s.io/kubernetes/pkg/volume/util"
+)
+
+// This is the primary entrypoint for volume plugins.
+func ProbeVolumePlugins() []volume.VolumePlugin {
+	return []volume.VolumePlugin{&rbdPlugin{nil, exec.New()}}
+}
+
+type rbdPlugin struct {
+	host volume.VolumeHost
+	exe  exec.Interface
+}
+
+var _ volume.VolumePlugin = &rbdPlugin{}
+var _ volume.PersistentVolumePlugin = &rbdPlugin{}
+var _ volume.DeletableVolumePlugin = &rbdPlugin{}
+var _ volume.ProvisionableVolumePlugin = &rbdPlugin{}
+
+const (
+	rbdPluginName = "kubernetes.io/rbd"
+	secretKeyName = "key" // key name used in secret
+)
+
+func (plugin *rbdPlugin) Init(host volume.VolumeHost) error {
+	plugin.host = host
+	return nil
+}
+
+func (plugin *rbdPlugin) GetPluginName() string {
+	return rbdPluginName
+}
+
+func (plugin *rbdPlugin) GetVolumeName(spec *volume.Spec) (string, error) {
+	volumeSource, _, err := getVolumeSource(spec)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf(
+		"%v:%v",
+		volumeSource.CephMonitors,
+		volumeSource.RBDImage), nil
+}
+
+func (plugin *rbdPlugin) CanSupport(spec *volume.Spec) bool {
+	if (spec.Volume != nil && spec.Volume.RBD == nil) || (spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil) {
+		return false
+	}
+
+	return true
+}
+
+func (plugin *rbdPlugin) RequiresRemount() bool {
+	return false
+}
+
+func (plugin *rbdPlugin) GetAccessModes() []v1.PersistentVolumeAccessMode {
+	return []v1.PersistentVolumeAccessMode{
+		v1.ReadWriteOnce,
+		v1.ReadOnlyMany,
+	}
+}
+
+func (plugin *rbdPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, _ volume.VolumeOptions) (volume.Mounter, error) {
+	var secret string
+	var err error
+	source, _ := plugin.getRBDVolumeSource(spec)
+
+	if source.SecretRef != nil {
+		if secret, err = parsePodSecret(pod, source.SecretRef.Name, plugin.host.GetKubeClient()); err != nil {
+			glog.Errorf("Couldn't get secret from %v/%v", pod.Namespace, source.SecretRef)
+			return nil, err
+		}
+	}
+
+	// Inject real implementations here, test through the internal function.
+	return plugin.newMounterInternal(spec, pod.UID, &RBDUtil{}, plugin.host.GetMounter(), secret)
+}
+
+func (plugin *rbdPlugin) getRBDVolumeSource(spec *volume.Spec) (*v1.RBDVolumeSource, bool) {
+	// rbd volumes used directly in a pod have a ReadOnly flag set by the pod author.
+	// rbd volumes used as a PersistentVolume gets the ReadOnly flag indirectly through the persistent-claim volume used to mount the PV
+	if spec.Volume != nil && spec.Volume.RBD != nil {
+		return spec.Volume.RBD, spec.Volume.RBD.ReadOnly
+	} else {
+		return spec.PersistentVolume.Spec.RBD, spec.ReadOnly
+	}
+}
+
+func (plugin *rbdPlugin) newMounterInternal(spec *volume.Spec, podUID types.UID, manager diskManager, mounter mount.Interface, secret string) (volume.Mounter, error) {
+	source, readOnly := plugin.getRBDVolumeSource(spec)
+	pool := source.RBDPool
+	id := source.RadosUser
+	keyring := source.Keyring
+
+	return &rbdMounter{
+		rbd: &rbd{
+			podUID:   podUID,
+			volName:  spec.Name(),
+			Image:    source.RBDImage,
+			Pool:     pool,
+			ReadOnly: readOnly,
+			manager:  manager,
+			mounter:  &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()},
+			plugin:   plugin,
+		},
+		Mon:     source.CephMonitors,
+		Id:      id,
+		Keyring: keyring,
+		Secret:  secret,
+		fsType:  source.FSType,
+	}, nil
+}
+
+func (plugin *rbdPlugin) NewUnmounter(volName string, podUID types.UID) (volume.Unmounter, error) {
+	// Inject real implementations here, test through the internal function.
+	return plugin.newUnmounterInternal(volName, podUID, &RBDUtil{}, plugin.host.GetMounter())
+}
+
+func (plugin *rbdPlugin) newUnmounterInternal(volName string, podUID types.UID, manager diskManager, mounter mount.Interface) (volume.Unmounter, error) {
+	return &rbdUnmounter{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				podUID:  podUID,
+				volName: volName,
+				manager: manager,
+				mounter: &mount.SafeFormatAndMount{Interface: mounter, Runner: exec.New()},
+				plugin:  plugin,
+			},
+			Mon: make([]string, 0),
+		},
+	}, nil
+}
+
+func (plugin *rbdPlugin) ConstructVolumeSpec(volumeName, mountPath string) (*volume.Spec, error) {
+	rbdVolume := &v1.Volume{
+		Name: volumeName,
+		VolumeSource: v1.VolumeSource{
+			RBD: &v1.RBDVolumeSource{
+				CephMonitors: []string{},
+			},
+		},
+	}
+	return volume.NewSpecFromVolume(rbdVolume), nil
+}
+
+func (plugin *rbdPlugin) NewDeleter(spec *volume.Spec) (volume.Deleter, error) {
+	if spec.PersistentVolume != nil && spec.PersistentVolume.Spec.RBD == nil {
+		return nil, fmt.Errorf("spec.PersistentVolumeSource.Spec.RBD is nil")
+	}
+	class, err := volutil.GetClassForVolume(plugin.host.GetKubeClient(), spec.PersistentVolume)
+	if err != nil {
+		return nil, err
+	}
+	adminSecretName := ""
+	adminSecretNamespace := "default"
+	admin := ""
+
+	for k, v := range class.Parameters {
+		switch dstrings.ToLower(k) {
+		case "adminid":
+			admin = v
+		case "adminsecretname":
+			adminSecretName = v
+		case "adminsecretnamespace":
+			adminSecretNamespace = v
+		}
+	}
+
+	secret, err := parsePVSecret(adminSecretNamespace, adminSecretName, plugin.host.GetKubeClient())
+	if err != nil {
+		// log error but don't return yet
+		glog.Errorf("failed to get admin secret from [%q/%q]: %v", adminSecretNamespace, adminSecretName, err)
+	}
+	return plugin.newDeleterInternal(spec, admin, secret, &RBDUtil{})
+}
+
+func (plugin *rbdPlugin) newDeleterInternal(spec *volume.Spec, admin, secret string, manager diskManager) (volume.Deleter, error) {
+	return &rbdVolumeDeleter{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				volName: spec.Name(),
+				Image:   spec.PersistentVolume.Spec.RBD.RBDImage,
+				Pool:    spec.PersistentVolume.Spec.RBD.RBDPool,
+				manager: manager,
+				plugin:  plugin,
+			},
+			Mon:         spec.PersistentVolume.Spec.RBD.CephMonitors,
+			adminId:     admin,
+			adminSecret: secret,
+		}}, nil
+}
+
+func (plugin *rbdPlugin) NewProvisioner(options volume.VolumeOptions) (volume.Provisioner, error) {
+	return plugin.newProvisionerInternal(options, &RBDUtil{})
+}
+
+func (plugin *rbdPlugin) newProvisionerInternal(options volume.VolumeOptions, manager diskManager) (volume.Provisioner, error) {
+	return &rbdVolumeProvisioner{
+		rbdMounter: &rbdMounter{
+			rbd: &rbd{
+				manager: manager,
+				plugin:  plugin,
+			},
+		},
+		options: options,
+	}, nil
+}
+
+type rbdVolumeProvisioner struct {
+	*rbdMounter
+	options volume.VolumeOptions
+}
+
+func (r *rbdVolumeProvisioner) Provision() (*v1.PersistentVolume, error) {
+	if r.options.PVC.Spec.Selector != nil {
+		return nil, fmt.Errorf("claim Selector is not supported")
+	}
+	var err error
+	adminSecretName := ""
+	adminSecretNamespace := "default"
+	secretName := ""
+	secret := ""
+
+	for k, v := range r.options.Parameters {
+		switch dstrings.ToLower(k) {
+		case "monitors":
+			arr := dstrings.Split(v, ",")
+			for _, m := range arr {
+				r.Mon = append(r.Mon, m)
+			}
+		case "adminid":
+			r.adminId = v
+		case "adminsecretname":
+			adminSecretName = v
+		case "adminsecretnamespace":
+			adminSecretNamespace = v
+		case "userid":
+			r.Id = v
+		case "pool":
+			r.Pool = v
+		case "usersecretname":
+			secretName = v
+		default:
+			return nil, fmt.Errorf("invalid option %q for volume plugin %s", k, r.plugin.GetPluginName())
+		}
+	}
+	// sanity check
+	if adminSecretName == "" {
+		return nil, fmt.Errorf("missing Ceph admin secret name")
+	}
+	if secret, err = parsePVSecret(adminSecretNamespace, adminSecretName, r.plugin.host.GetKubeClient()); err != nil {
+		// log error but don't return yet
+		glog.Errorf("failed to get admin secret from [%q/%q]", adminSecretNamespace, adminSecretName)
+	}
+	r.adminSecret = secret
+	if len(r.Mon) < 1 {
+		return nil, fmt.Errorf("missing Ceph monitors")
+	}
+	if secretName == "" {
+		return nil, fmt.Errorf("missing user secret name")
+	}
+	if r.adminId == "" {
+		r.adminId = "admin"
+	}
+	if r.Pool == "" {
+		r.Pool = "rbd"
+	}
+	if r.Id == "" {
+		r.Id = r.adminId
+	}
+
+	// create random image name
+	image := fmt.Sprintf("kubernetes-dynamic-pvc-%s", uuid.NewUUID())
+	r.rbdMounter.Image = image
+	rbd, sizeMB, err := r.manager.CreateImage(r)
+	if err != nil {
+		glog.Errorf("rbd: create volume failed, err: %v", err)
+		return nil, fmt.Errorf("rbd: create volume failed, err: %v", err)
+	}
+	glog.Infof("successfully created rbd image %q", image)
+	pv := new(v1.PersistentVolume)
+	rbd.SecretRef = new(v1.LocalObjectReference)
+	rbd.SecretRef.Name = secretName
+	rbd.RadosUser = r.Id
+	pv.Spec.PersistentVolumeSource.RBD = rbd
+	pv.Spec.PersistentVolumeReclaimPolicy = r.options.PersistentVolumeReclaimPolicy
+	pv.Spec.AccessModes = r.options.PVC.Spec.AccessModes
+	if len(pv.Spec.AccessModes) == 0 {
+		pv.Spec.AccessModes = r.plugin.GetAccessModes()
+	}
+	pv.Spec.Capacity = v1.ResourceList{
+		v1.ResourceName(v1.ResourceStorage): resource.MustParse(fmt.Sprintf("%dMi", sizeMB)),
+	}
+	return pv, nil
+}
+
+type rbdVolumeDeleter struct {
+	*rbdMounter
+}
+
+func (r *rbdVolumeDeleter) GetPath() string {
+	name := rbdPluginName
+	return r.plugin.host.GetPodVolumeDir(r.podUID, strings.EscapeQualifiedNameForDisk(name), r.volName)
+}
+
+func (r *rbdVolumeDeleter) Delete() error {
+	return r.manager.DeleteImage(r)
+}
+
+type rbd struct {
+	volName  string
+	podUID   types.UID
+	Pool     string
+	Image    string
+	ReadOnly bool
+	plugin   *rbdPlugin
+	mounter  *mount.SafeFormatAndMount
+	// Utility interface that provides API calls to the provider to attach/detach disks.
+	manager diskManager
+	volume.MetricsNil
+}
+
+func (rbd *rbd) GetPath() string {
+	name := rbdPluginName
+	// safe to use PodVolumeDir now: volume teardown occurs before pod is cleaned up
+	return rbd.plugin.host.GetPodVolumeDir(rbd.podUID, strings.EscapeQualifiedNameForDisk(name), rbd.volName)
+}
+
+type rbdMounter struct {
+	*rbd
+	// capitalized so they can be exported in persistRBD()
+	Mon         []string
+	Id          string
+	Keyring     string
+	Secret      string
+	fsType      string
+	adminSecret string
+	adminId     string
+}
+
+var _ volume.Mounter = &rbdMounter{}
+
+func (b *rbd) GetAttributes() volume.Attributes {
+	return volume.Attributes{
+		ReadOnly:        b.ReadOnly,
+		Managed:         !b.ReadOnly,
+		SupportsSELinux: true,
+	}
+}
+
+// Checks prior to mount operations to verify that the required components (binaries, etc.)
+// to mount the volume are available on the underlying node.
+// If not, it returns an error
+func (b *rbdMounter) CanMount() error {
+	return nil
+}
+
+func (b *rbdMounter) SetUp(fsGroup *int64) error {
+	return b.SetUpAt(b.GetPath(), fsGroup)
+}
+
+func (b *rbdMounter) SetUpAt(dir string, fsGroup *int64) error {
+	// diskSetUp checks mountpoints and prevent repeated calls
+	glog.V(4).Infof("rbd: attempting to SetUp and mount %s", dir)
+	err := diskSetUp(b.manager, *b, dir, b.mounter, fsGroup)
+	if err != nil {
+		glog.Errorf("rbd: failed to setup mount %s %v", dir, err)
+	}
+	return err
+}
+
+type rbdUnmounter struct {
+	*rbdMounter
+}
+
+var _ volume.Unmounter = &rbdUnmounter{}
+
+// Unmounts the bind mount, and detaches the disk only if the disk
+// resource was the last reference to that disk on the kubelet.
+func (c *rbdUnmounter) TearDown() error {
+	return c.TearDownAt(c.GetPath())
+}
+
+func (c *rbdUnmounter) TearDownAt(dir string) error {
+	return diskTearDown(c.manager, *c, dir, c.mounter)
+}
+
+func (plugin *rbdPlugin) execCommand(command string, args []string) ([]byte, error) {
+	cmd := plugin.exe.Command(command, args...)
+	return cmd.CombinedOutput()
+}
+
+func getVolumeSource(
+	spec *volume.Spec) (*v1.RBDVolumeSource, bool, error) {
+	if spec.Volume != nil && spec.Volume.RBD != nil {
+		return spec.Volume.RBD, spec.Volume.RBD.ReadOnly, nil
+	} else if spec.PersistentVolume != nil &&
+		spec.PersistentVolume.Spec.RBD != nil {
+		return spec.PersistentVolume.Spec.RBD, spec.ReadOnly, nil
+	}
+
+	return nil, false, fmt.Errorf("Spec does not reference a RBD volume type")
+}
+
+func parsePodSecret(pod *v1.Pod, secretName string, kubeClient clientset.Interface) (string, error) {
+	secret, err := volutil.GetSecretForPod(pod, secretName, kubeClient)
+	if err != nil {
+		glog.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
+		return "", fmt.Errorf("failed to get secret from [%q/%q]", pod.Namespace, secretName)
+	}
+	return parseSecretMap(secret)
+}
+
+func parsePVSecret(namespace, secretName string, kubeClient clientset.Interface) (string, error) {
+	secret, err := volutil.GetSecretForPV(namespace, secretName, rbdPluginName, kubeClient)
+	if err != nil {
+		glog.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
+		return "", fmt.Errorf("failed to get secret from [%q/%q]", namespace, secretName)
+	}
+	return parseSecretMap(secret)
+}
+
+// parseSecretMap locates the secret by key name.
+func parseSecretMap(secretMap map[string]string) (string, error) {
+	if len(secretMap) == 0 {
+		return "", fmt.Errorf("empty secret map")
+	}
+	secret := ""
+	for k, v := range secretMap {
+		if k == secretKeyName {
+			return v, nil
+		}
+		secret = v
+	}
+	// If not found, the last secret in the map wins as done before
+	return secret, nil
+}

vendor/k8s.io/kubernetes/pkg/volume/rbd/rbd_test.go

@@ -0,0 +1,241 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rbd
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5/fake"
+	"k8s.io/kubernetes/pkg/types"
+	"k8s.io/kubernetes/pkg/util/mount"
+	utiltesting "k8s.io/kubernetes/pkg/util/testing"
+	"k8s.io/kubernetes/pkg/volume"
+	volumetest "k8s.io/kubernetes/pkg/volume/testing"
+)
+
+func TestCanSupport(t *testing.T) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil))
+
+	plug, err := plugMgr.FindPluginByName("kubernetes.io/rbd")
+	if err != nil {
+		t.Errorf("Can't find the plugin by name")
+	}
+	if plug.GetPluginName() != "kubernetes.io/rbd" {
+		t.Errorf("Wrong name: %s", plug.GetPluginName())
+	}
+	if plug.CanSupport(&volume.Spec{Volume: &v1.Volume{VolumeSource: v1.VolumeSource{}}}) {
+		t.Errorf("Expected false")
+	}
+}
+
+type fakeDiskManager struct {
+	tmpDir string
+}
+
+func NewFakeDiskManager() *fakeDiskManager {
+	return &fakeDiskManager{
+		tmpDir: utiltesting.MkTmpdirOrDie("rbd_test"),
+	}
+}
+
+func (fake *fakeDiskManager) Cleanup() {
+	os.RemoveAll(fake.tmpDir)
+}
+
+func (fake *fakeDiskManager) MakeGlobalPDName(disk rbd) string {
+	return fake.tmpDir
+}
+func (fake *fakeDiskManager) AttachDisk(b rbdMounter) error {
+	globalPath := b.manager.MakeGlobalPDName(*b.rbd)
+	err := os.MkdirAll(globalPath, 0750)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (fake *fakeDiskManager) DetachDisk(c rbdUnmounter, mntPath string) error {
+	globalPath := c.manager.MakeGlobalPDName(*c.rbd)
+	err := os.RemoveAll(globalPath)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (fake *fakeDiskManager) CreateImage(provisioner *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, volumeSizeGB int, err error) {
+	return nil, 0, fmt.Errorf("not implemented")
+}
+
+func (fake *fakeDiskManager) DeleteImage(deleter *rbdVolumeDeleter) error {
+	return fmt.Errorf("not implemented")
+}
+
+func doTestPlugin(t *testing.T, spec *volume.Spec) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, nil, nil))
+
+	plug, err := plugMgr.FindPluginByName("kubernetes.io/rbd")
+	if err != nil {
+		t.Errorf("Can't find the plugin by name")
+	}
+	fdm := NewFakeDiskManager()
+	defer fdm.Cleanup()
+	mounter, err := plug.(*rbdPlugin).newMounterInternal(spec, types.UID("poduid"), fdm, &mount.FakeMounter{}, "secrets")
+	if err != nil {
+		t.Errorf("Failed to make a new Mounter: %v", err)
+	}
+	if mounter == nil {
+		t.Error("Got a nil Mounter")
+	}
+
+	path := mounter.GetPath()
+	expectedPath := fmt.Sprintf("%s/pods/poduid/volumes/kubernetes.io~rbd/vol1", tmpDir)
+	if path != expectedPath {
+		t.Errorf("Unexpected path, expected %q, got: %q", expectedPath, path)
+	}
+
+	if err := mounter.SetUp(nil); err != nil {
+		t.Errorf("Expected success, got: %v", err)
+	}
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			t.Errorf("SetUp() failed, volume path not created: %s", path)
+		} else {
+			t.Errorf("SetUp() failed: %v", err)
+		}
+	}
+
+	unmounter, err := plug.(*rbdPlugin).newUnmounterInternal("vol1", types.UID("poduid"), fdm, &mount.FakeMounter{})
+	if err != nil {
+		t.Errorf("Failed to make a new Unmounter: %v", err)
+	}
+	if unmounter == nil {
+		t.Error("Got a nil Unmounter")
+	}
+
+	if err := unmounter.TearDown(); err != nil {
+		t.Errorf("Expected success, got: %v", err)
+	}
+	if _, err := os.Stat(path); err == nil {
+		t.Errorf("TearDown() failed, volume path still exists: %s", path)
+	} else if !os.IsNotExist(err) {
+		t.Errorf("SetUp() failed: %v", err)
+	}
+}
+
+func TestPluginVolume(t *testing.T) {
+	vol := &v1.Volume{
+		Name: "vol1",
+		VolumeSource: v1.VolumeSource{
+			RBD: &v1.RBDVolumeSource{
+				CephMonitors: []string{"a", "b"},
+				RBDImage:     "bar",
+				FSType:       "ext4",
+			},
+		},
+	}
+	doTestPlugin(t, volume.NewSpecFromVolume(vol))
+}
+func TestPluginPersistentVolume(t *testing.T) {
+	vol := &v1.PersistentVolume{
+		ObjectMeta: v1.ObjectMeta{
+			Name: "vol1",
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				RBD: &v1.RBDVolumeSource{
+					CephMonitors: []string{"a", "b"},
+					RBDImage:     "bar",
+					FSType:       "ext4",
+				},
+			},
+		},
+	}
+
+	doTestPlugin(t, volume.NewSpecFromPersistentVolume(vol, false))
+}
+
+func TestPersistentClaimReadOnlyFlag(t *testing.T) {
+	tmpDir, err := utiltesting.MkTmpdir("rbd_test")
+	if err != nil {
+		t.Fatalf("error creating temp dir: %v", err)
+	}
+	defer os.RemoveAll(tmpDir)
+
+	pv := &v1.PersistentVolume{
+		ObjectMeta: v1.ObjectMeta{
+			Name: "pvA",
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				RBD: &v1.RBDVolumeSource{
+					CephMonitors: []string{"a", "b"},
+					RBDImage:     "bar",
+					FSType:       "ext4",
+				},
+			},
+			ClaimRef: &v1.ObjectReference{
+				Name: "claimA",
+			},
+		},
+	}
+
+	claim := &v1.PersistentVolumeClaim{
+		ObjectMeta: v1.ObjectMeta{
+			Name:      "claimA",
+			Namespace: "nsA",
+		},
+		Spec: v1.PersistentVolumeClaimSpec{
+			VolumeName: "pvA",
+		},
+		Status: v1.PersistentVolumeClaimStatus{
+			Phase: v1.ClaimBound,
+		},
+	}
+
+	client := fake.NewSimpleClientset(pv, claim)
+
+	plugMgr := volume.VolumePluginMgr{}
+	plugMgr.InitPlugins(ProbeVolumePlugins(), volumetest.NewFakeVolumeHost(tmpDir, client, nil))
+	plug, _ := plugMgr.FindPluginByName(rbdPluginName)
+
+	// readOnly bool is supplied by persistent-claim volume source when its mounter creates other volumes
+	spec := volume.NewSpecFromPersistentVolume(pv, true)
+	pod := &v1.Pod{ObjectMeta: v1.ObjectMeta{UID: types.UID("poduid")}}
+	mounter, _ := plug.NewMounter(spec, pod, volume.VolumeOptions{})
+
+	if !mounter.GetAttributes().ReadOnly {
+		t.Errorf("Expected true for mounter.IsReadOnly")
+	}
+}

vendor/k8s.io/kubernetes/pkg/volume/rbd/rbd_util.go

@@ -0,0 +1,414 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+//
+// utility functions to setup rbd volume
+// mainly implement diskManager interface
+//
+
+package rbd
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/util/exec"
+	"k8s.io/kubernetes/pkg/util/mount"
+	"k8s.io/kubernetes/pkg/util/node"
+	"k8s.io/kubernetes/pkg/volume"
+)
+
+const (
+	imageWatcherStr = "watcher="
+)
+
+// search /sys/bus for rbd device that matches given pool and image
+func getDevFromImageAndPool(pool, image string) (string, bool) {
+	// /sys/bus/rbd/devices/X/name and /sys/bus/rbd/devices/X/pool
+	sys_path := "/sys/bus/rbd/devices"
+	if dirs, err := ioutil.ReadDir(sys_path); err == nil {
+		for _, f := range dirs {
+			// pool and name format:
+			// see rbd_pool_show() and rbd_name_show() at
+			// https://github.com/torvalds/linux/blob/master/drivers/block/rbd.c
+			name := f.Name()
+			// first match pool, then match name
+			po := path.Join(sys_path, name, "pool")
+			img := path.Join(sys_path, name, "name")
+			exe := exec.New()
+			out, err := exe.Command("cat", po, img).CombinedOutput()
+			if err != nil {
+				continue
+			}
+			matched, err := regexp.MatchString("^"+pool+"\n"+image+"\n$", string(out))
+			if err != nil || !matched {
+				continue
+			}
+			// found a match, check if device exists
+			devicePath := "/dev/rbd" + name
+			if _, err := os.Lstat(devicePath); err == nil {
+				return devicePath, true
+			}
+		}
+	}
+	return "", false
+}
+
+// stat a path, if not exists, retry maxRetries times
+func waitForPath(pool, image string, maxRetries int) (string, bool) {
+	for i := 0; i < maxRetries; i++ {
+		devicePath, found := getDevFromImageAndPool(pool, image)
+		if found {
+			return devicePath, true
+		}
+		time.Sleep(time.Second)
+	}
+	return "", false
+}
+
+// make a directory like /var/lib/kubelet/plugins/kubernetes.io/pod/rbd/pool-image-image
+func makePDNameInternal(host volume.VolumeHost, pool string, image string) string {
+	return path.Join(host.GetPluginDir(rbdPluginName), "rbd", pool+"-image-"+image)
+}
+
+type RBDUtil struct{}
+
+func (util *RBDUtil) MakeGlobalPDName(rbd rbd) string {
+	return makePDNameInternal(rbd.plugin.host, rbd.Pool, rbd.Image)
+}
+
+func (util *RBDUtil) rbdLock(b rbdMounter, lock bool) error {
+	var err error
+	var output, locker string
+	var cmd []byte
+	var secret_opt []string
+
+	if b.Secret != "" {
+		secret_opt = []string{"--key=" + b.Secret}
+	} else {
+		secret_opt = []string{"-k", b.Keyring}
+	}
+	// construct lock id using host name and a magic prefix
+	lock_id := "kubelet_lock_magic_" + node.GetHostname("")
+
+	l := len(b.Mon)
+	// avoid mount storm, pick a host randomly
+	start := rand.Int() % l
+	// iterate all hosts until mount succeeds.
+	for i := start; i < start+l; i++ {
+		mon := b.Mon[i%l]
+		// cmd "rbd lock list" serves two purposes:
+		// for fencing, check if lock already held for this host
+		// this edge case happens if host crashes in the middle of acquiring lock and mounting rbd
+		// for defencing, get the locker name, something like "client.1234"
+		cmd, err = b.plugin.execCommand("rbd",
+			append([]string{"lock", "list", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
+		output = string(cmd)
+
+		if err != nil {
+			continue
+		}
+
+		if lock {
+			// check if lock is already held for this host by matching lock_id and rbd lock id
+			if strings.Contains(output, lock_id) {
+				// this host already holds the lock, exit
+				glog.V(1).Infof("rbd: lock already held for %s", lock_id)
+				return nil
+			}
+			// hold a lock: rbd lock add
+			cmd, err = b.plugin.execCommand("rbd",
+				append([]string{"lock", "add", b.Image, lock_id, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
+		} else {
+			// defencing, find locker name
+			ind := strings.LastIndex(output, lock_id) - 1
+			for i := ind; i >= 0; i-- {
+				if output[i] == '\n' {
+					locker = output[(i + 1):ind]
+					break
+				}
+			}
+			// remove a lock: rbd lock remove
+			cmd, err = b.plugin.execCommand("rbd",
+				append([]string{"lock", "remove", b.Image, lock_id, locker, "--pool", b.Pool, "--id", b.Id, "-m", mon}, secret_opt...))
+		}
+
+		if err == nil {
+			//lock is acquired
+			break
+		}
+	}
+	return err
+}
+
+func (util *RBDUtil) persistRBD(rbd rbdMounter, mnt string) error {
+	file := path.Join(mnt, "rbd.json")
+	fp, err := os.Create(file)
+	if err != nil {
+		return fmt.Errorf("rbd: create err %s/%s", file, err)
+	}
+	defer fp.Close()
+
+	encoder := json.NewEncoder(fp)
+	if err = encoder.Encode(rbd); err != nil {
+		return fmt.Errorf("rbd: encode err: %v.", err)
+	}
+
+	return nil
+}
+
+func (util *RBDUtil) loadRBD(mounter *rbdMounter, mnt string) error {
+	file := path.Join(mnt, "rbd.json")
+	fp, err := os.Open(file)
+	if err != nil {
+		return fmt.Errorf("rbd: open err %s/%s", file, err)
+	}
+	defer fp.Close()
+
+	decoder := json.NewDecoder(fp)
+	if err = decoder.Decode(mounter); err != nil {
+		return fmt.Errorf("rbd: decode err: %v.", err)
+	}
+
+	return nil
+}
+
+func (util *RBDUtil) fencing(b rbdMounter) error {
+	// no need to fence readOnly
+	if (&b).GetAttributes().ReadOnly {
+		return nil
+	}
+	return util.rbdLock(b, true)
+}
+
+func (util *RBDUtil) defencing(c rbdUnmounter) error {
+	// no need to fence readOnly
+	if c.ReadOnly {
+		return nil
+	}
+
+	return util.rbdLock(*c.rbdMounter, false)
+}
+
+func (util *RBDUtil) AttachDisk(b rbdMounter) error {
+	var err error
+	var output []byte
+
+	devicePath, found := waitForPath(b.Pool, b.Image, 1)
+	if !found {
+		// modprobe
+		_, err = b.plugin.execCommand("modprobe", []string{"rbd"})
+		if err != nil {
+			return fmt.Errorf("rbd: failed to modprobe rbd error:%v", err)
+		}
+		// rbd map
+		l := len(b.Mon)
+		// avoid mount storm, pick a host randomly
+		start := rand.Int() % l
+		// iterate all hosts until mount succeeds.
+		for i := start; i < start+l; i++ {
+			mon := b.Mon[i%l]
+			glog.V(1).Infof("rbd: map mon %s", mon)
+			if b.Secret != "" {
+				output, err = b.plugin.execCommand("rbd",
+					[]string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "--key=" + b.Secret})
+			} else {
+				output, err = b.plugin.execCommand("rbd",
+					[]string{"map", b.Image, "--pool", b.Pool, "--id", b.Id, "-m", mon, "-k", b.Keyring})
+			}
+			if err == nil {
+				break
+			}
+			glog.V(1).Infof("rbd: map error %v %s", err, string(output))
+		}
+		if err != nil {
+			return fmt.Errorf("rbd: map failed %v %s", err, string(output))
+		}
+		devicePath, found = waitForPath(b.Pool, b.Image, 10)
+		if !found {
+			return errors.New("Could not map image: Timeout after 10s")
+		}
+	}
+	// mount it
+	globalPDPath := b.manager.MakeGlobalPDName(*b.rbd)
+	notMnt, err := b.mounter.IsLikelyNotMountPoint(globalPDPath)
+	// in the first time, the path shouldn't exist and IsLikelyNotMountPoint is expected to get NotExist
+	if err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("rbd: %s failed to check mountpoint", globalPDPath)
+	}
+	if !notMnt {
+		return nil
+	}
+
+	if err := os.MkdirAll(globalPDPath, 0750); err != nil {
+		return fmt.Errorf("rbd: failed to mkdir %s, error", globalPDPath)
+	}
+
+	// fence off other mappers
+	if err := util.fencing(b); err != nil {
+		// rbd unmap before exit
+		b.plugin.execCommand("rbd", []string{"unmap", devicePath})
+		return fmt.Errorf("rbd: image %s is locked by other nodes", b.Image)
+	}
+	// rbd lock remove needs ceph and image config
+	// but kubelet doesn't get them from apiserver during teardown
+	// so persit rbd config so upon disk detach, rbd lock can be removed
+	// since rbd json is persisted in the same local directory that is used as rbd mountpoint later,
+	// the json file remains invisible during rbd mount and thus won't be removed accidentally.
+	util.persistRBD(b, globalPDPath)
+
+	if err = b.mounter.FormatAndMount(devicePath, globalPDPath, b.fsType, nil); err != nil {
+		err = fmt.Errorf("rbd: failed to mount rbd volume %s [%s] to %s, error %v", devicePath, b.fsType, globalPDPath, err)
+	}
+	return err
+}
+
+func (util *RBDUtil) DetachDisk(c rbdUnmounter, mntPath string) error {
+	device, cnt, err := mount.GetDeviceNameFromMount(c.mounter, mntPath)
+	if err != nil {
+		return fmt.Errorf("rbd detach disk: failed to get device from mnt: %s\nError: %v", mntPath, err)
+	}
+	if err = c.mounter.Unmount(mntPath); err != nil {
+		return fmt.Errorf("rbd detach disk: failed to umount: %s\nError: %v", mntPath, err)
+	}
+	// if device is no longer used, see if can unmap
+	if cnt <= 1 {
+		// rbd unmap
+		_, err = c.plugin.execCommand("rbd", []string{"unmap", device})
+		if err != nil {
+			return fmt.Errorf("rbd: failed to unmap device %s:Error: %v", device, err)
+		}
+
+		// load ceph and image/pool info to remove fencing
+		if err := util.loadRBD(c.rbdMounter, mntPath); err == nil {
+			// remove rbd lock
+			util.defencing(c)
+		}
+
+		glog.Infof("rbd: successfully unmap device %s", device)
+	}
+	return nil
+}
+
+func (util *RBDUtil) CreateImage(p *rbdVolumeProvisioner) (r *v1.RBDVolumeSource, size int, err error) {
+	capacity := p.options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
+	volSizeBytes := capacity.Value()
+	// convert to MB that rbd defaults on
+	sz := int(volume.RoundUpSize(volSizeBytes, 1024*1024))
+	volSz := fmt.Sprintf("%d", sz)
+	// rbd create
+	l := len(p.rbdMounter.Mon)
+	// pick a mon randomly
+	start := rand.Int() % l
+	// iterate all monitors until create succeeds.
+	for i := start; i < start+l; i++ {
+		mon := p.Mon[i%l]
+		glog.V(4).Infof("rbd: create %s size %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, volSz, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
+		var output []byte
+		output, err = p.rbdMounter.plugin.execCommand("rbd",
+			[]string{"create", p.rbdMounter.Image, "--size", volSz, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret, "--image-format", "1"})
+		if err == nil {
+			break
+		} else {
+			glog.Warningf("failed to create rbd image, output %v", string(output))
+		}
+	}
+
+	if err != nil {
+		glog.Errorf("rbd: Error creating rbd image: %v", err)
+		return nil, 0, err
+	}
+
+	return &v1.RBDVolumeSource{
+		CephMonitors: p.rbdMounter.Mon,
+		RBDImage:     p.rbdMounter.Image,
+		RBDPool:      p.rbdMounter.Pool,
+	}, sz, nil
+}
+
+func (util *RBDUtil) DeleteImage(p *rbdVolumeDeleter) error {
+	var output []byte
+	found, err := util.rbdStatus(p.rbdMounter)
+	if err != nil {
+		return err
+	}
+	if found {
+		glog.Info("rbd is still being used ", p.rbdMounter.Image)
+		return fmt.Errorf("rbd %s is still being used", p.rbdMounter.Image)
+	}
+	// rbd rm
+	l := len(p.rbdMounter.Mon)
+	// pick a mon randomly
+	start := rand.Int() % l
+	// iterate all monitors until rm succeeds.
+	for i := start; i < start+l; i++ {
+		mon := p.rbdMounter.Mon[i%l]
+		glog.V(4).Infof("rbd: rm %s using mon %s, pool %s id %s key %s", p.rbdMounter.Image, mon, p.rbdMounter.Pool, p.rbdMounter.adminId, p.rbdMounter.adminSecret)
+		output, err = p.plugin.execCommand("rbd",
+			[]string{"rm", p.rbdMounter.Image, "--pool", p.rbdMounter.Pool, "--id", p.rbdMounter.adminId, "-m", mon, "--key=" + p.rbdMounter.adminSecret})
+		if err == nil {
+			return nil
+		} else {
+			glog.Errorf("failed to delete rbd image, error %v output %v", err, string(output))
+		}
+	}
+	return err
+}
+
+// run rbd status command to check if there is watcher on the image
+func (util *RBDUtil) rbdStatus(b *rbdMounter) (bool, error) {
+	var err error
+	var output string
+	var cmd []byte
+
+	l := len(b.Mon)
+	start := rand.Int() % l
+	// iterate all hosts until mount succeeds.
+	for i := start; i < start+l; i++ {
+		mon := b.Mon[i%l]
+		// cmd "rbd status" list the rbd client watch with the following output:
+		// Watchers:
+		//   watcher=10.16.153.105:0/710245699 client.14163 cookie=1
+		glog.V(4).Infof("rbd: status %s using mon %s, pool %s id %s key %s", b.Image, mon, b.Pool, b.adminId, b.adminSecret)
+		cmd, err = b.plugin.execCommand("rbd",
+			[]string{"status", b.Image, "--pool", b.Pool, "-m", mon, "--id", b.adminId, "--key=" + b.adminSecret})
+		output = string(cmd)
+
+		if err != nil {
+			// ignore error code, just checkout output for watcher string
+			glog.Warningf("failed to execute rbd status on mon %s", mon)
+		}
+
+		if strings.Contains(output, imageWatcherStr) {
+			glog.V(4).Infof("rbd: watchers on %s: %s", b.Image, output)
+			return true, nil
+		} else {
+			glog.Warningf("rbd: no watchers on %s", b.Image)
+			return false, nil
+		}
+	}
+	return false, nil
+}