Add glide.yaml and vendor deps

vendor/k8s.io/kubernetes/test/e2e/framework/BUILD

@@ -0,0 +1,107 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+    "go_test",
+    "cgo_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cleanup.go",
+        "exec_util.go",
+        "federation_util.go",
+        "framework.go",
+        "kubelet_stats.go",
+        "log_size_monitoring.go",
+        "metrics_util.go",
+        "networking_utils.go",
+        "nodes_util.go",
+        "perf_util.go",
+        "pods.go",
+        "resource_usage_gatherer.go",
+        "test_context.go",
+        "util.go",
+    ],
+    tags = ["automanaged"],
+    deps = [
+        "//federation/client/clientset_generated/federation_release_1_5:go_default_library",
+        "//pkg/api:go_default_library",
+        "//pkg/api/errors:go_default_library",
+        "//pkg/api/v1:go_default_library",
+        "//pkg/api/validation:go_default_library",
+        "//pkg/apimachinery/registered:go_default_library",
+        "//pkg/apis/apps/v1beta1:go_default_library",
+        "//pkg/apis/batch/v1:go_default_library",
+        "//pkg/apis/componentconfig:go_default_library",
+        "//pkg/apis/extensions:go_default_library",
+        "//pkg/apis/extensions/v1beta1:go_default_library",
+        "//pkg/apis/meta/v1:go_default_library",
+        "//pkg/client/clientset_generated/internalclientset:go_default_library",
+        "//pkg/client/clientset_generated/release_1_5:go_default_library",
+        "//pkg/client/clientset_generated/release_1_5/typed/core/v1:go_default_library",
+        "//pkg/client/conditions:go_default_library",
+        "//pkg/client/restclient:go_default_library",
+        "//pkg/client/typed/discovery:go_default_library",
+        "//pkg/client/typed/dynamic:go_default_library",
+        "//pkg/client/unversioned/clientcmd:go_default_library",
+        "//pkg/client/unversioned/clientcmd/api:go_default_library",
+        "//pkg/client/unversioned/remotecommand:go_default_library",
+        "//pkg/cloudprovider:go_default_library",
+        "//pkg/cloudprovider/providers/gce:go_default_library",
+        "//pkg/controller:go_default_library",
+        "//pkg/controller/deployment/util:go_default_library",
+        "//pkg/fields:go_default_library",
+        "//pkg/kubectl:go_default_library",
+        "//pkg/kubelet/api/v1alpha1/stats:go_default_library",
+        "//pkg/kubelet/metrics:go_default_library",
+        "//pkg/kubelet/server/remotecommand:go_default_library",
+        "//pkg/kubelet/server/stats:go_default_library",
+        "//pkg/kubelet/util/format:go_default_library",
+        "//pkg/labels:go_default_library",
+        "//pkg/master/ports:go_default_library",
+        "//pkg/metrics:go_default_library",
+        "//pkg/runtime:go_default_library",
+        "//pkg/runtime/schema:go_default_library",
+        "//pkg/ssh:go_default_library",
+        "//pkg/types:go_default_library",
+        "//pkg/util/errors:go_default_library",
+        "//pkg/util/exec:go_default_library",
+        "//pkg/util/intstr:go_default_library",
+        "//pkg/util/labels:go_default_library",
+        "//pkg/util/rand:go_default_library",
+        "//pkg/util/runtime:go_default_library",
+        "//pkg/util/sets:go_default_library",
+        "//pkg/util/system:go_default_library",
+        "//pkg/util/uuid:go_default_library",
+        "//pkg/util/validation:go_default_library",
+        "//pkg/util/wait:go_default_library",
+        "//pkg/version:go_default_library",
+        "//pkg/watch:go_default_library",
+        "//plugin/pkg/scheduler/algorithm/predicates:go_default_library",
+        "//plugin/pkg/scheduler/schedulercache:go_default_library",
+        "//test/e2e/perftype:go_default_library",
+        "//test/utils:go_default_library",
+        "//vendor:github.com/blang/semver",
+        "//vendor:github.com/golang/glog",
+        "//vendor:github.com/google/cadvisor/info/v1",
+        "//vendor:github.com/onsi/ginkgo",
+        "//vendor:github.com/onsi/ginkgo/config",
+        "//vendor:github.com/onsi/gomega",
+        "//vendor:github.com/onsi/gomega/types",
+        "//vendor:github.com/prometheus/common/expfmt",
+        "//vendor:github.com/prometheus/common/model",
+        "//vendor:github.com/spf13/viper",
+        "//vendor:golang.org/x/crypto/ssh",
+        "//vendor:golang.org/x/net/websocket",
+        "//vendor:gopkg.in/yaml.v2",
+        "//vendor:k8s.io/client-go/kubernetes",
+        "//vendor:k8s.io/client-go/pkg/util/sets",
+        "//vendor:k8s.io/client-go/rest",
+    ],
+)

vendor/k8s.io/kubernetes/test/e2e/framework/cleanup.go

@@ -0,0 +1,61 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import "sync"
+
+type CleanupActionHandle *int
+
+var cleanupActionsLock sync.Mutex
+var cleanupActions = map[CleanupActionHandle]func(){}
+
+// AddCleanupAction installs a function that will be called in the event of the
+// whole test being terminated.  This allows arbitrary pieces of the overall
+// test to hook into SynchronizedAfterSuite().
+func AddCleanupAction(fn func()) CleanupActionHandle {
+	p := CleanupActionHandle(new(int))
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	cleanupActions[p] = fn
+	return p
+}
+
+// RemoveCleanupAction removes a function that was installed by
+// AddCleanupAction.
+func RemoveCleanupAction(p CleanupActionHandle) {
+	cleanupActionsLock.Lock()
+	defer cleanupActionsLock.Unlock()
+	delete(cleanupActions, p)
+}
+
+// RunCleanupActions runs all functions installed by AddCleanupAction.  It does
+// not remove them (see RemoveCleanupAction) but it does run unlocked, so they
+// may remove themselves.
+func RunCleanupActions() {
+	list := []func(){}
+	func() {
+		cleanupActionsLock.Lock()
+		defer cleanupActionsLock.Unlock()
+		for _, fn := range cleanupActions {
+			list = append(list, fn)
+		}
+	}()
+	// Run unlocked.
+	for _, fn := range list {
+		fn()
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/exec_util.go

@@ -0,0 +1,148 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"io"
+	"net/url"
+	"strings"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	"k8s.io/kubernetes/pkg/client/restclient"
+	"k8s.io/kubernetes/pkg/client/unversioned/remotecommand"
+	remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
+
+	. "github.com/onsi/gomega"
+)
+
+// ExecOptions passed to ExecWithOptions
+type ExecOptions struct {
+	Command []string
+
+	Namespace     string
+	PodName       string
+	ContainerName string
+
+	Stdin         io.Reader
+	CaptureStdout bool
+	CaptureStderr bool
+	// If false, whitespace in std{err,out} will be removed.
+	PreserveWhitespace bool
+}
+
+// ExecWithOptions executes a command in the specified container,
+// returning stdout, stderr and error. `options` allowed for
+// additional parameters to be passed.
+func (f *Framework) ExecWithOptions(options ExecOptions) (string, string, error) {
+	Logf("ExecWithOptions %+v", options)
+
+	config, err := LoadConfig()
+	Expect(err).NotTo(HaveOccurred(), "failed to load restclient config")
+
+	const tty = false
+
+	req := f.ClientSet.Core().RESTClient().Post().
+		Resource("pods").
+		Name(options.PodName).
+		Namespace(options.Namespace).
+		SubResource("exec").
+		Param("container", options.ContainerName)
+	req.VersionedParams(&v1.PodExecOptions{
+		Container: options.ContainerName,
+		Command:   options.Command,
+		Stdin:     options.Stdin != nil,
+		Stdout:    options.CaptureStdout,
+		Stderr:    options.CaptureStderr,
+		TTY:       tty,
+	}, api.ParameterCodec)
+
+	var stdout, stderr bytes.Buffer
+	err = execute("POST", req.URL(), config, options.Stdin, &stdout, &stderr, tty)
+
+	if options.PreserveWhitespace {
+		return stdout.String(), stderr.String(), err
+	}
+	return strings.TrimSpace(stdout.String()), strings.TrimSpace(stderr.String()), err
+}
+
+// ExecCommandInContainerWithFullOutput executes a command in the
+// specified container and return stdout, stderr and error
+func (f *Framework) ExecCommandInContainerWithFullOutput(podName, containerName string, cmd ...string) (string, string, error) {
+	return f.ExecWithOptions(ExecOptions{
+		Command:       cmd,
+		Namespace:     f.Namespace.Name,
+		PodName:       podName,
+		ContainerName: containerName,
+
+		Stdin:              nil,
+		CaptureStdout:      true,
+		CaptureStderr:      true,
+		PreserveWhitespace: false,
+	})
+}
+
+// ExecCommandInContainer executes a command in the specified container.
+func (f *Framework) ExecCommandInContainer(podName, containerName string, cmd ...string) string {
+	stdout, stderr, err := f.ExecCommandInContainerWithFullOutput(podName, containerName, cmd...)
+	Logf("Exec stderr: %q", stderr)
+	Expect(err).NotTo(HaveOccurred(),
+		"failed to execute command in pod %v, container %v: %v",
+		podName, containerName, err)
+	return stdout
+}
+
+func (f *Framework) ExecShellInContainer(podName, containerName string, cmd string) string {
+	return f.ExecCommandInContainer(podName, containerName, "/bin/sh", "-c", cmd)
+}
+
+func (f *Framework) ExecCommandInPod(podName string, cmd ...string) string {
+	pod, err := f.PodClient().Get(podName)
+	Expect(err).NotTo(HaveOccurred(), "failed to get pod")
+	Expect(pod.Spec.Containers).NotTo(BeEmpty())
+	return f.ExecCommandInContainer(podName, pod.Spec.Containers[0].Name, cmd...)
+}
+
+func (f *Framework) ExecCommandInPodWithFullOutput(podName string, cmd ...string) (string, string, error) {
+	pod, err := f.PodClient().Get(podName)
+	Expect(err).NotTo(HaveOccurred(), "failed to get pod")
+	Expect(pod.Spec.Containers).NotTo(BeEmpty())
+	return f.ExecCommandInContainerWithFullOutput(podName, pod.Spec.Containers[0].Name, cmd...)
+}
+
+func (f *Framework) ExecShellInPod(podName string, cmd string) string {
+	return f.ExecCommandInPod(podName, "/bin/sh", "-c", cmd)
+}
+
+func (f *Framework) ExecShellInPodWithFullOutput(podName string, cmd string) (string, string, error) {
+	return f.ExecCommandInPodWithFullOutput(podName, "/bin/sh", "-c", cmd)
+}
+
+func execute(method string, url *url.URL, config *restclient.Config, stdin io.Reader, stdout, stderr io.Writer, tty bool) error {
+	exec, err := remotecommand.NewExecutor(config, method, url)
+	if err != nil {
+		return err
+	}
+	return exec.Stream(remotecommand.StreamOptions{
+		SupportedProtocols: remotecommandserver.SupportedStreamingProtocols,
+		Stdin:              stdin,
+		Stdout:             stdout,
+		Stderr:             stderr,
+		Tty:                tty,
+	})
+}

vendor/k8s.io/kubernetes/test/e2e/framework/federation_util.go

@@ -0,0 +1,42 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"regexp"
+
+	"k8s.io/kubernetes/pkg/api/validation"
+	validationutil "k8s.io/kubernetes/pkg/util/validation"
+)
+
+// GetValidDNSSubdomainName massages the given name to be a valid dns subdomain name.
+// Most resources (such as secrets, clusters) require the names to be valid dns subdomain.
+// This is a generic function (not specific to federation). Should be moved to a more generic location if others want to use it.
+func GetValidDNSSubdomainName(name string) (string, error) {
+	// "_" are not allowed. Replace them by "-".
+	name = regexp.MustCompile("_").ReplaceAllLiteralString(name, "-")
+	maxLength := validationutil.DNS1123SubdomainMaxLength
+	if len(name) > maxLength {
+		name = name[0 : maxLength-1]
+	}
+	// Verify that name now passes the validation.
+	if errors := validation.NameIsDNSSubdomain(name, false); len(errors) != 0 {
+		return "", fmt.Errorf("errors in converting name to a valid DNS subdomain %s", errors)
+	}
+	return name, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/log_size_monitoring.go

@@ -0,0 +1,269 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
+)
+
+const (
+	// Minimal period between polling log sizes from components
+	pollingPeriod            = 60 * time.Second
+	workersNo                = 5
+	kubeletLogsPath          = "/var/log/kubelet.log"
+	kubeProxyLogsPath        = "/var/log/kube-proxy.log"
+	kubeAddonsLogsPath       = "/var/log/kube-addons.log"
+	kubeMasterAddonsLogsPath = "/var/log/kube-master-addons.log"
+	apiServerLogsPath        = "/var/log/kube-apiserver.log"
+	controllersLogsPath      = "/var/log/kube-controller-manager.log"
+	schedulerLogsPath        = "/var/log/kube-scheduler.log"
+)
+
+var (
+	nodeLogsToCheck   = []string{kubeletLogsPath, kubeProxyLogsPath}
+	masterLogsToCheck = []string{kubeletLogsPath, kubeAddonsLogsPath, kubeMasterAddonsLogsPath,
+		apiServerLogsPath, controllersLogsPath, schedulerLogsPath}
+)
+
+// TimestampedSize contains a size together with a time of measurement.
+type TimestampedSize struct {
+	timestamp time.Time
+	size      int
+}
+
+// LogSizeGatherer is a worker which grabs a WorkItem from the channel and does assigned work.
+type LogSizeGatherer struct {
+	stopChannel chan bool
+	data        *LogsSizeData
+	wg          *sync.WaitGroup
+	workChannel chan WorkItem
+}
+
+// LogsSizeVerifier gathers data about log files sizes from master and node machines.
+// It oversees a <workersNo> workers which do the gathering.
+type LogsSizeVerifier struct {
+	client      clientset.Interface
+	stopChannel chan bool
+	// data stores LogSizeData groupped per IP and log_path
+	data          *LogsSizeData
+	masterAddress string
+	nodeAddresses []string
+	wg            sync.WaitGroup
+	workChannel   chan WorkItem
+	workers       []*LogSizeGatherer
+}
+
+type SingleLogSummary struct {
+	AverageGenerationRate int
+	NumberOfProbes        int
+}
+
+type LogSizeDataTimeseries map[string]map[string][]TimestampedSize
+
+// node -> file -> data
+type LogsSizeDataSummary map[string]map[string]SingleLogSummary
+
+// TODO: make sure that we don't need locking here
+func (s *LogsSizeDataSummary) PrintHumanReadable() string {
+	buf := &bytes.Buffer{}
+	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
+	fmt.Fprintf(w, "host\tlog_file\taverage_rate (B/s)\tnumber_of_probes\n")
+	for k, v := range *s {
+		fmt.Fprintf(w, "%v\t\t\t\n", k)
+		for path, data := range v {
+			fmt.Fprintf(w, "\t%v\t%v\t%v\n", path, data.AverageGenerationRate, data.NumberOfProbes)
+		}
+	}
+	w.Flush()
+	return buf.String()
+}
+
+func (s *LogsSizeDataSummary) PrintJSON() string {
+	return PrettyPrintJSON(*s)
+}
+
+type LogsSizeData struct {
+	data LogSizeDataTimeseries
+	lock sync.Mutex
+}
+
+// WorkItem is a command for a worker that contains an IP of machine from which we want to
+// gather data and paths to all files we're interested in.
+type WorkItem struct {
+	ip                string
+	paths             []string
+	backoffMultiplier int
+}
+
+func prepareData(masterAddress string, nodeAddresses []string) *LogsSizeData {
+	data := make(LogSizeDataTimeseries)
+	ips := append(nodeAddresses, masterAddress)
+	for _, ip := range ips {
+		data[ip] = make(map[string][]TimestampedSize)
+	}
+	return &LogsSizeData{
+		data: data,
+		lock: sync.Mutex{},
+	}
+}
+
+func (d *LogsSizeData) AddNewData(ip, path string, timestamp time.Time, size int) {
+	d.lock.Lock()
+	defer d.lock.Unlock()
+	d.data[ip][path] = append(
+		d.data[ip][path],
+		TimestampedSize{
+			timestamp: timestamp,
+			size:      size,
+		},
+	)
+}
+
+// NewLogsVerifier creates a new LogsSizeVerifier which will stop when stopChannel is closed
+func NewLogsVerifier(c clientset.Interface, stopChannel chan bool) *LogsSizeVerifier {
+	nodeAddresses, err := NodeSSHHosts(c)
+	ExpectNoError(err)
+	masterAddress := GetMasterHost() + ":22"
+
+	workChannel := make(chan WorkItem, len(nodeAddresses)+1)
+	workers := make([]*LogSizeGatherer, workersNo)
+
+	verifier := &LogsSizeVerifier{
+		client:        c,
+		stopChannel:   stopChannel,
+		data:          prepareData(masterAddress, nodeAddresses),
+		masterAddress: masterAddress,
+		nodeAddresses: nodeAddresses,
+		wg:            sync.WaitGroup{},
+		workChannel:   workChannel,
+		workers:       workers,
+	}
+	verifier.wg.Add(workersNo)
+	for i := 0; i < workersNo; i++ {
+		workers[i] = &LogSizeGatherer{
+			stopChannel: stopChannel,
+			data:        verifier.data,
+			wg:          &verifier.wg,
+			workChannel: workChannel,
+		}
+	}
+	return verifier
+}
+
+// GetSummary returns a summary (average generation rate and number of probes) of the data gathered by LogSizeVerifier
+func (s *LogsSizeVerifier) GetSummary() *LogsSizeDataSummary {
+	result := make(LogsSizeDataSummary)
+	for k, v := range s.data.data {
+		result[k] = make(map[string]SingleLogSummary)
+		for path, data := range v {
+			if len(data) > 1 {
+				last := data[len(data)-1]
+				first := data[0]
+				rate := (last.size - first.size) / int(last.timestamp.Sub(first.timestamp)/time.Second)
+				result[k][path] = SingleLogSummary{
+					AverageGenerationRate: rate,
+					NumberOfProbes:        len(data),
+				}
+			}
+		}
+	}
+	return &result
+}
+
+// Run starts log size gathering. It starts a gorouting for every worker and then blocks until stopChannel is closed
+func (v *LogsSizeVerifier) Run() {
+	v.workChannel <- WorkItem{
+		ip:                v.masterAddress,
+		paths:             masterLogsToCheck,
+		backoffMultiplier: 1,
+	}
+	for _, node := range v.nodeAddresses {
+		v.workChannel <- WorkItem{
+			ip:                node,
+			paths:             nodeLogsToCheck,
+			backoffMultiplier: 1,
+		}
+	}
+	for _, worker := range v.workers {
+		go worker.Run()
+	}
+	<-v.stopChannel
+	v.wg.Wait()
+}
+
+func (g *LogSizeGatherer) Run() {
+	for g.Work() {
+	}
+}
+
+func (g *LogSizeGatherer) pushWorkItem(workItem WorkItem) {
+	select {
+	case <-time.After(time.Duration(workItem.backoffMultiplier) * pollingPeriod):
+		g.workChannel <- workItem
+	case <-g.stopChannel:
+		return
+	}
+}
+
+// Work does a single unit of work: tries to take out a WorkItem from the queue, ssh-es into a given machine,
+// gathers data, writes it to the shared <data> map, and creates a gorouting which reinserts work item into
+// the queue with a <pollingPeriod> delay. Returns false if worker should exit.
+func (g *LogSizeGatherer) Work() bool {
+	var workItem WorkItem
+	select {
+	case <-g.stopChannel:
+		g.wg.Done()
+		return false
+	case workItem = <-g.workChannel:
+	}
+	sshResult, err := SSH(
+		fmt.Sprintf("ls -l %v | awk '{print $9, $5}' | tr '\n' ' '", strings.Join(workItem.paths, " ")),
+		workItem.ip,
+		TestContext.Provider,
+	)
+	if err != nil {
+		Logf("Error while trying to SSH to %v, skipping probe. Error: %v", workItem.ip, err)
+		if workItem.backoffMultiplier < 128 {
+			workItem.backoffMultiplier *= 2
+		}
+		go g.pushWorkItem(workItem)
+		return true
+	}
+	workItem.backoffMultiplier = 1
+	results := strings.Split(sshResult.Stdout, " ")
+
+	now := time.Now()
+	for i := 0; i+1 < len(results); i = i + 2 {
+		path := results[i]
+		size, err := strconv.Atoi(results[i+1])
+		if err != nil {
+			Logf("Error during conversion to int: %v, skipping data. Error: %v", results[i+1], err)
+			continue
+		}
+		g.data.AddNewData(workItem.ip, path, now, size)
+	}
+	go g.pushWorkItem(workItem)
+	return true
+}

vendor/k8s.io/kubernetes/test/e2e/framework/metrics_util.go

@@ -0,0 +1,489 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
+	"k8s.io/kubernetes/pkg/master/ports"
+	"k8s.io/kubernetes/pkg/metrics"
+	"k8s.io/kubernetes/pkg/util/sets"
+
+	"github.com/prometheus/common/expfmt"
+	"github.com/prometheus/common/model"
+)
+
+const (
+	// NodeStartupThreshold is a rough estimate of the time allocated for a pod to start on a node.
+	NodeStartupThreshold = 4 * time.Second
+
+	podStartupThreshold time.Duration = 5 * time.Second
+	// We are setting 1s threshold for apicalls even in small clusters to avoid flakes.
+	// The problem is that if long GC is happening in small clusters (where we have e.g.
+	// 1-core master machines) and tests are pretty short, it may consume significant
+	// portion of CPU and basically stop all the real work.
+	// Increasing threshold to 1s is within our SLO and should solve this problem.
+	apiCallLatencyThreshold time.Duration = 1 * time.Second
+)
+
+type MetricsForE2E metrics.MetricsCollection
+
+func (m *MetricsForE2E) filterMetrics() {
+	interestingApiServerMetrics := make(metrics.ApiServerMetrics)
+	for _, metric := range InterestingApiServerMetrics {
+		interestingApiServerMetrics[metric] = (*m).ApiServerMetrics[metric]
+	}
+	interestingControllerManagerMetrics := make(metrics.ControllerManagerMetrics)
+	for _, metric := range InterestingControllerManagerMetrics {
+		interestingControllerManagerMetrics[metric] = (*m).ControllerManagerMetrics[metric]
+	}
+	interestingKubeletMetrics := make(map[string]metrics.KubeletMetrics)
+	for kubelet, grabbed := range (*m).KubeletMetrics {
+		interestingKubeletMetrics[kubelet] = make(metrics.KubeletMetrics)
+		for _, metric := range InterestingKubeletMetrics {
+			interestingKubeletMetrics[kubelet][metric] = grabbed[metric]
+		}
+	}
+	(*m).ApiServerMetrics = interestingApiServerMetrics
+	(*m).ControllerManagerMetrics = interestingControllerManagerMetrics
+	(*m).KubeletMetrics = interestingKubeletMetrics
+}
+
+func (m *MetricsForE2E) PrintHumanReadable() string {
+	buf := bytes.Buffer{}
+	for _, interestingMetric := range InterestingApiServerMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", interestingMetric))
+		for _, sample := range (*m).ApiServerMetrics[interestingMetric] {
+			buf.WriteString(fmt.Sprintf("\t%v\n", metrics.PrintSample(sample)))
+		}
+	}
+	for _, interestingMetric := range InterestingControllerManagerMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", interestingMetric))
+		for _, sample := range (*m).ControllerManagerMetrics[interestingMetric] {
+			buf.WriteString(fmt.Sprintf("\t%v\n", metrics.PrintSample(sample)))
+		}
+	}
+	for kubelet, grabbed := range (*m).KubeletMetrics {
+		buf.WriteString(fmt.Sprintf("For %v:\n", kubelet))
+		for _, interestingMetric := range InterestingKubeletMetrics {
+			buf.WriteString(fmt.Sprintf("\tFor %v:\n", interestingMetric))
+			for _, sample := range grabbed[interestingMetric] {
+				buf.WriteString(fmt.Sprintf("\t\t%v\n", metrics.PrintSample(sample)))
+			}
+		}
+	}
+	return buf.String()
+}
+
+func (m *MetricsForE2E) PrintJSON() string {
+	m.filterMetrics()
+	return PrettyPrintJSON(*m)
+}
+
+var InterestingApiServerMetrics = []string{
+	"apiserver_request_count",
+	"apiserver_request_latencies_summary",
+	"etcd_helper_cache_entry_count",
+	"etcd_helper_cache_hit_count",
+	"etcd_helper_cache_miss_count",
+	"etcd_request_cache_add_latencies_summary",
+	"etcd_request_cache_get_latencies_summary",
+	"etcd_request_latencies_summary",
+}
+
+var InterestingControllerManagerMetrics = []string{
+	"garbage_collector_event_processing_latency_microseconds",
+	"garbage_collector_dirty_processing_latency_microseconds",
+	"garbage_collector_orphan_processing_latency_microseconds",
+}
+
+var InterestingKubeletMetrics = []string{
+	"kubelet_container_manager_latency_microseconds",
+	"kubelet_docker_errors",
+	"kubelet_docker_operations_latency_microseconds",
+	"kubelet_generate_pod_status_latency_microseconds",
+	"kubelet_pod_start_latency_microseconds",
+	"kubelet_pod_worker_latency_microseconds",
+	"kubelet_pod_worker_start_latency_microseconds",
+	"kubelet_sync_pods_latency_microseconds",
+}
+
+// Dashboard metrics
+type LatencyMetric struct {
+	Perc50  time.Duration `json:"Perc50"`
+	Perc90  time.Duration `json:"Perc90"`
+	Perc99  time.Duration `json:"Perc99"`
+	Perc100 time.Duration `json:"Perc100"`
+}
+
+type PodStartupLatency struct {
+	Latency LatencyMetric `json:"latency"`
+}
+
+type SchedulingLatency struct {
+	Scheduling LatencyMetric `json:"scheduling"`
+	Binding    LatencyMetric `json:"binding"`
+	Total      LatencyMetric `json:"total"`
+}
+
+type SaturationTime struct {
+	TimeToSaturate time.Duration `json:"timeToStaturate"`
+	NumberOfNodes  int           `json:"numberOfNodes"`
+	NumberOfPods   int           `json:"numberOfPods"`
+	Throughput     float32       `json:"throughput"`
+}
+
+type APICall struct {
+	Resource string        `json:"resource"`
+	Verb     string        `json:"verb"`
+	Latency  LatencyMetric `json:"latency"`
+}
+
+type APIResponsiveness struct {
+	APICalls []APICall `json:"apicalls"`
+}
+
+func (a APIResponsiveness) Len() int      { return len(a.APICalls) }
+func (a APIResponsiveness) Swap(i, j int) { a.APICalls[i], a.APICalls[j] = a.APICalls[j], a.APICalls[i] }
+func (a APIResponsiveness) Less(i, j int) bool {
+	return a.APICalls[i].Latency.Perc99 < a.APICalls[j].Latency.Perc99
+}
+
+// 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
+// Only 0.5, 0.9 and 0.99 quantiles are supported.
+func (a *APIResponsiveness) addMetric(resource, verb string, quantile float64, latency time.Duration) {
+	for i, apicall := range a.APICalls {
+		if apicall.Resource == resource && apicall.Verb == verb {
+			a.APICalls[i] = setQuantileAPICall(apicall, quantile, latency)
+			return
+		}
+	}
+	apicall := setQuantileAPICall(APICall{Resource: resource, Verb: verb}, quantile, latency)
+	a.APICalls = append(a.APICalls, apicall)
+}
+
+// 0 <= quantile <=1 (e.g. 0.95 is 95%tile, 0.5 is median)
+// Only 0.5, 0.9 and 0.99 quantiles are supported.
+func setQuantileAPICall(apicall APICall, quantile float64, latency time.Duration) APICall {
+	setQuantile(&apicall.Latency, quantile, latency)
+	return apicall
+}
+
+// Only 0.5, 0.9 and 0.99 quantiles are supported.
+func setQuantile(metric *LatencyMetric, quantile float64, latency time.Duration) {
+	switch quantile {
+	case 0.5:
+		metric.Perc50 = latency
+	case 0.9:
+		metric.Perc90 = latency
+	case 0.99:
+		metric.Perc99 = latency
+	}
+}
+
+func readLatencyMetrics(c clientset.Interface) (APIResponsiveness, error) {
+	var a APIResponsiveness
+
+	body, err := getMetrics(c)
+	if err != nil {
+		return a, err
+	}
+
+	samples, err := extractMetricSamples(body)
+	if err != nil {
+		return a, err
+	}
+
+	ignoredResources := sets.NewString("events")
+	// TODO: figure out why we're getting non-capitalized proxy and fix this.
+	ignoredVerbs := sets.NewString("WATCHLIST", "PROXY", "proxy", "CONNECT")
+
+	for _, sample := range samples {
+		// Example line:
+		// apiserver_request_latencies_summary{resource="namespaces",verb="LIST",quantile="0.99"} 908
+		if sample.Metric[model.MetricNameLabel] != "apiserver_request_latencies_summary" {
+			continue
+		}
+
+		resource := string(sample.Metric["resource"])
+		verb := string(sample.Metric["verb"])
+		if ignoredResources.Has(resource) || ignoredVerbs.Has(verb) {
+			continue
+		}
+		latency := sample.Value
+		quantile, err := strconv.ParseFloat(string(sample.Metric[model.QuantileLabel]), 64)
+		if err != nil {
+			return a, err
+		}
+		a.addMetric(resource, verb, quantile, time.Duration(int64(latency))*time.Microsecond)
+	}
+
+	return a, err
+}
+
+// Prints top five summary metrics for request types with latency and returns
+// number of such request types above threshold.
+func HighLatencyRequests(c clientset.Interface) (int, error) {
+	metrics, err := readLatencyMetrics(c)
+	if err != nil {
+		return 0, err
+	}
+	sort.Sort(sort.Reverse(metrics))
+	badMetrics := 0
+	top := 5
+	for _, metric := range metrics.APICalls {
+		isBad := false
+		if metric.Latency.Perc99 > apiCallLatencyThreshold {
+			badMetrics++
+			isBad = true
+		}
+		if top > 0 || isBad {
+			top--
+			prefix := ""
+			if isBad {
+				prefix = "WARNING "
+			}
+			Logf("%vTop latency metric: %+v", prefix, metric)
+		}
+	}
+
+	// TODO(random-liu): Remove the log when we migrate to new perfdash
+	Logf("API calls latencies: %s", PrettyPrintJSON(metrics))
+	// Log perf data
+	PrintPerfData(ApiCallToPerfData(metrics))
+
+	return badMetrics, nil
+}
+
+// Verifies whether 50, 90 and 99th percentiles of PodStartupLatency are
+// within the threshold.
+func VerifyPodStartupLatency(latency PodStartupLatency) error {
+	Logf("Pod startup latency: %s", PrettyPrintJSON(latency))
+
+	if latency.Latency.Perc50 > podStartupThreshold {
+		return fmt.Errorf("too high pod startup latency 50th percentile: %v", latency.Latency.Perc50)
+	}
+	if latency.Latency.Perc90 > podStartupThreshold {
+		return fmt.Errorf("too high pod startup latency 90th percentile: %v", latency.Latency.Perc90)
+	}
+	if latency.Latency.Perc99 > podStartupThreshold {
+		return fmt.Errorf("too high pod startup latency 99th percentil: %v", latency.Latency.Perc99)
+	}
+	return nil
+}
+
+// Resets latency metrics in apiserver.
+func ResetMetrics(c clientset.Interface) error {
+	Logf("Resetting latency metrics in apiserver...")
+	body, err := c.Core().RESTClient().Delete().AbsPath("/metrics").DoRaw()
+	if err != nil {
+		return err
+	}
+	if string(body) != "metrics reset\n" {
+		return fmt.Errorf("Unexpected response: %q", string(body))
+	}
+	return nil
+}
+
+// Retrieves metrics information.
+func getMetrics(c clientset.Interface) (string, error) {
+	body, err := c.Core().RESTClient().Get().AbsPath("/metrics").DoRaw()
+	if err != nil {
+		return "", err
+	}
+	return string(body), nil
+}
+
+// Retrieves scheduler metrics information.
+func getSchedulingLatency(c clientset.Interface) (SchedulingLatency, error) {
+	result := SchedulingLatency{}
+
+	// Check if master Node is registered
+	nodes, err := c.Core().Nodes().List(v1.ListOptions{})
+	ExpectNoError(err)
+
+	var data string
+	var masterRegistered = false
+	for _, node := range nodes.Items {
+		if strings.HasSuffix(node.Name, "master") {
+			masterRegistered = true
+		}
+	}
+	if masterRegistered {
+		rawData, err := c.Core().RESTClient().Get().
+			Prefix("proxy").
+			Namespace(api.NamespaceSystem).
+			Resource("pods").
+			Name(fmt.Sprintf("kube-scheduler-%v:%v", TestContext.CloudConfig.MasterName, ports.SchedulerPort)).
+			Suffix("metrics").
+			Do().Raw()
+
+		ExpectNoError(err)
+		data = string(rawData)
+	} else {
+		// If master is not registered fall back to old method of using SSH.
+		cmd := "curl http://localhost:10251/metrics"
+		sshResult, err := SSH(cmd, GetMasterHost()+":22", TestContext.Provider)
+		if err != nil || sshResult.Code != 0 {
+			return result, fmt.Errorf("unexpected error (code: %d) in ssh connection to master: %#v", sshResult.Code, err)
+		}
+		data = sshResult.Stdout
+	}
+	samples, err := extractMetricSamples(data)
+	if err != nil {
+		return result, err
+	}
+
+	for _, sample := range samples {
+		var metric *LatencyMetric = nil
+		switch sample.Metric[model.MetricNameLabel] {
+		case "scheduler_scheduling_algorithm_latency_microseconds":
+			metric = &result.Scheduling
+		case "scheduler_binding_latency_microseconds":
+			metric = &result.Binding
+		case "scheduler_e2e_scheduling_latency_microseconds":
+			metric = &result.Total
+		}
+		if metric == nil {
+			continue
+		}
+
+		latency := sample.Value
+		quantile, err := strconv.ParseFloat(string(sample.Metric[model.QuantileLabel]), 64)
+		if err != nil {
+			return result, err
+		}
+		setQuantile(metric, quantile, time.Duration(int64(latency))*time.Microsecond)
+	}
+	return result, nil
+}
+
+// Verifies (currently just by logging them) the scheduling latencies.
+func VerifySchedulerLatency(c clientset.Interface) error {
+	latency, err := getSchedulingLatency(c)
+	if err != nil {
+		return err
+	}
+	Logf("Scheduling latency: %s", PrettyPrintJSON(latency))
+
+	// TODO: Add some reasonable checks once we know more about the values.
+	return nil
+}
+
+func PrettyPrintJSON(metrics interface{}) string {
+	output := &bytes.Buffer{}
+	if err := json.NewEncoder(output).Encode(metrics); err != nil {
+		Logf("Error building encoder: %v", err)
+		return ""
+	}
+	formatted := &bytes.Buffer{}
+	if err := json.Indent(formatted, output.Bytes(), "", "  "); err != nil {
+		Logf("Error indenting: %v", err)
+		return ""
+	}
+	return string(formatted.Bytes())
+}
+
+// extractMetricSamples parses the prometheus metric samples from the input string.
+func extractMetricSamples(metricsBlob string) ([]*model.Sample, error) {
+	dec := expfmt.NewDecoder(strings.NewReader(metricsBlob), expfmt.FmtText)
+	decoder := expfmt.SampleDecoder{
+		Dec:  dec,
+		Opts: &expfmt.DecodeOptions{},
+	}
+
+	var samples []*model.Sample
+	for {
+		var v model.Vector
+		if err := decoder.Decode(&v); err != nil {
+			if err == io.EOF {
+				// Expected loop termination condition.
+				return samples, nil
+			}
+			return nil, err
+		}
+		samples = append(samples, v...)
+	}
+}
+
+// PodLatencyData encapsulates pod startup latency information.
+type PodLatencyData struct {
+	// Name of the pod
+	Name string
+	// Node this pod was running on
+	Node string
+	// Latency information related to pod startuptime
+	Latency time.Duration
+}
+
+type LatencySlice []PodLatencyData
+
+func (a LatencySlice) Len() int           { return len(a) }
+func (a LatencySlice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a LatencySlice) Less(i, j int) bool { return a[i].Latency < a[j].Latency }
+
+func ExtractLatencyMetrics(latencies []PodLatencyData) LatencyMetric {
+	length := len(latencies)
+	perc50 := latencies[int(math.Ceil(float64(length*50)/100))-1].Latency
+	perc90 := latencies[int(math.Ceil(float64(length*90)/100))-1].Latency
+	perc99 := latencies[int(math.Ceil(float64(length*99)/100))-1].Latency
+	perc100 := latencies[length-1].Latency
+	return LatencyMetric{Perc50: perc50, Perc90: perc90, Perc99: perc99, Perc100: perc100}
+}
+
+// LogSuspiciousLatency logs metrics/docker errors from all nodes that had slow startup times
+// If latencyDataLag is nil then it will be populated from latencyData
+func LogSuspiciousLatency(latencyData []PodLatencyData, latencyDataLag []PodLatencyData, nodeCount int, c clientset.Interface) {
+	if latencyDataLag == nil {
+		latencyDataLag = latencyData
+	}
+	for _, l := range latencyData {
+		if l.Latency > NodeStartupThreshold {
+			HighLatencyKubeletOperations(c, 1*time.Second, l.Node, Logf)
+		}
+	}
+	Logf("Approx throughput: %v pods/min",
+		float64(nodeCount)/(latencyDataLag[len(latencyDataLag)-1].Latency.Minutes()))
+}
+
+// testMaximumLatencyValue verifies the highest latency value is less than or equal to
+// the given time.Duration. Since the arrays are sorted we are looking at the last
+// element which will always be the highest. If the latency is higher than the max Failf
+// is called.
+func testMaximumLatencyValue(latencies []PodLatencyData, max time.Duration, name string) {
+	highestLatency := latencies[len(latencies)-1]
+	if !(highestLatency.Latency <= max) {
+		Failf("%s were not all under %s: %#v", name, max.String(), latencies)
+	}
+}
+
+func PrintLatencies(latencies []PodLatencyData, header string) {
+	metrics := ExtractLatencyMetrics(latencies)
+	Logf("10%% %s: %v", header, latencies[(len(latencies)*9)/10:])
+	Logf("perc50: %v, perc90: %v, perc99: %v", metrics.Perc50, metrics.Perc90, metrics.Perc99)
+}

vendor/k8s.io/kubernetes/test/e2e/framework/nodes_util.go

@@ -0,0 +1,232 @@
+/*
+Copyright 2014 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
+	"k8s.io/kubernetes/pkg/fields"
+	"k8s.io/kubernetes/pkg/util/wait"
+)
+
+// The following upgrade functions are passed into the framework below and used
+// to do the actual upgrades.
+var MasterUpgrade = func(v string) error {
+	switch TestContext.Provider {
+	case "gce":
+		return masterUpgradeGCE(v)
+	case "gke":
+		return masterUpgradeGKE(v)
+	default:
+		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+}
+
+func masterUpgradeGCE(rawV string) error {
+	v := "v" + rawV
+	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", v)
+	return err
+}
+
+func masterUpgradeGKE(v string) error {
+	Logf("Upgrading master to %q", v)
+	_, _, err := RunCmd("gcloud", "container",
+		"clusters",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
+		"upgrade",
+		TestContext.CloudConfig.Cluster,
+		"--master",
+		fmt.Sprintf("--cluster-version=%s", v),
+		"--quiet")
+	return err
+}
+
+var NodeUpgrade = func(f *Framework, v string, img string) error {
+	// Perform the upgrade.
+	var err error
+	switch TestContext.Provider {
+	case "gce":
+		// TODO(maisem): add GCE support for upgrading to different images.
+		err = nodeUpgradeGCE(v)
+	case "gke":
+		err = nodeUpgradeGKE(v, img)
+	default:
+		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
+	}
+	if err != nil {
+		return err
+	}
+
+	// Wait for it to complete and validate nodes are healthy.
+	//
+	// TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in
+	// GKE; the operation shouldn't return until they all are.
+	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
+	if _, err := CheckNodesReady(f.ClientSet, RestartNodeReadyAgainTimeout, TestContext.CloudConfig.NumNodes); err != nil {
+		return err
+	}
+	return nil
+}
+
+func nodeUpgradeGCE(rawV string) error {
+	v := "v" + rawV
+	_, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-N", v)
+	return err
+}
+
+func cleanupNodeUpgradeGCE(tmplBefore string) {
+	Logf("Cleaning up any unused node templates")
+	tmplAfter, err := MigTemplate()
+	if err != nil {
+		Logf("Could not get node template post-upgrade; may have leaked template %s", tmplBefore)
+		return
+	}
+	if tmplBefore == tmplAfter {
+		// The node upgrade failed so there's no need to delete
+		// anything.
+		Logf("Node template %s is still in use; not cleaning up", tmplBefore)
+		return
+	}
+	Logf("Deleting node template %s", tmplBefore)
+	if _, _, err := retryCmd("gcloud", "compute", "instance-templates",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		"delete",
+		tmplBefore); err != nil {
+		Logf("gcloud compute instance-templates delete %s call failed with err: %v", tmplBefore, err)
+		Logf("May have leaked instance template %q", tmplBefore)
+	}
+}
+
+func nodeUpgradeGKE(v string, img string) error {
+	Logf("Upgrading nodes to version %q and image %q", v, img)
+	args := []string{
+		"container",
+		"clusters",
+		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+		fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
+		"upgrade",
+		TestContext.CloudConfig.Cluster,
+		fmt.Sprintf("--cluster-version=%s", v),
+		"--quiet",
+	}
+	if len(img) > 0 {
+		args = append(args, fmt.Sprintf("--image-type=%s", img))
+	}
+	_, _, err := RunCmd("gcloud", args...)
+	return err
+}
+
+// CheckNodesReady waits up to nt for expect nodes accessed by c to be ready,
+// returning an error if this doesn't happen in time. It returns the names of
+// nodes it finds.
+func CheckNodesReady(c clientset.Interface, nt time.Duration, expect int) ([]string, error) {
+	// First, keep getting all of the nodes until we get the number we expect.
+	var nodeList *v1.NodeList
+	var errLast error
+	start := time.Now()
+	found := wait.Poll(Poll, nt, func() (bool, error) {
+		// A rolling-update (GCE/GKE implementation of restart) can complete before the apiserver
+		// knows about all of the nodes. Thus, we retry the list nodes call
+		// until we get the expected number of nodes.
+		nodeList, errLast = c.Core().Nodes().List(v1.ListOptions{
+			FieldSelector: fields.Set{"spec.unschedulable": "false"}.AsSelector().String()})
+		if errLast != nil {
+			return false, nil
+		}
+		if len(nodeList.Items) != expect {
+			errLast = fmt.Errorf("expected to find %d nodes but found only %d (%v elapsed)",
+				expect, len(nodeList.Items), time.Since(start))
+			Logf("%v", errLast)
+			return false, nil
+		}
+		return true, nil
+	}) == nil
+	nodeNames := make([]string, len(nodeList.Items))
+	for i, n := range nodeList.Items {
+		nodeNames[i] = n.ObjectMeta.Name
+	}
+	if !found {
+		return nodeNames, fmt.Errorf("couldn't find %d nodes within %v; last error: %v",
+			expect, nt, errLast)
+	}
+	Logf("Successfully found %d nodes", expect)
+
+	// Next, ensure in parallel that all the nodes are ready. We subtract the
+	// time we spent waiting above.
+	timeout := nt - time.Since(start)
+	result := make(chan bool, len(nodeList.Items))
+	for _, n := range nodeNames {
+		n := n
+		go func() { result <- WaitForNodeToBeReady(c, n, timeout) }()
+	}
+	failed := false
+	// TODO(mbforbes): Change to `for range` syntax once we support only Go
+	// >= 1.4.
+	for i := range nodeList.Items {
+		_ = i
+		if !<-result {
+			failed = true
+		}
+	}
+	if failed {
+		return nodeNames, fmt.Errorf("at least one node failed to be ready")
+	}
+	return nodeNames, nil
+}
+
+// MigTemplate (GCE-only) returns the name of the MIG template that the
+// nodes of the cluster use.
+func MigTemplate() (string, error) {
+	var errLast error
+	var templ string
+	key := "instanceTemplate"
+	if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
+		// TODO(mikedanese): make this hit the compute API directly instead of
+		// shelling out to gcloud.
+		// An `instance-groups managed describe` call outputs what we want to stdout.
+		output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
+			fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
+			"describe",
+			fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
+			TestContext.CloudConfig.NodeInstanceGroup)
+		if err != nil {
+			errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
+			return false, nil
+		}
+
+		// The 'describe' call probably succeeded; parse the output and try to
+		// find the line that looks like "instanceTemplate: url/to/<templ>" and
+		// return <templ>.
+		if val := ParseKVLines(output, key); len(val) > 0 {
+			url := strings.Split(val, "/")
+			templ = url[len(url)-1]
+			Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
+			return true, nil
+		}
+		errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
+		return false, nil
+	}) != nil {
+		return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
+	}
+	return templ, nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/perf_util.go

@@ -0,0 +1,135 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+
+	"k8s.io/kubernetes/test/e2e/perftype"
+)
+
+// TODO(random-liu): Change the tests to actually use PerfData from the beginning instead of
+// translating one to the other here.
+
+// currentApiCallMetricsVersion is the current apicall performance metrics version. We should
+// bump up the version each time we make incompatible change to the metrics.
+const currentApiCallMetricsVersion = "v1"
+
+// ApiCallToPerfData transforms APIResponsiveness to PerfData.
+func ApiCallToPerfData(apicalls APIResponsiveness) *perftype.PerfData {
+	perfData := &perftype.PerfData{Version: currentApiCallMetricsVersion}
+	for _, apicall := range apicalls.APICalls {
+		item := perftype.DataItem{
+			Data: map[string]float64{
+				"Perc50": float64(apicall.Latency.Perc50) / 1000000, // us -> ms
+				"Perc90": float64(apicall.Latency.Perc90) / 1000000,
+				"Perc99": float64(apicall.Latency.Perc99) / 1000000,
+			},
+			Unit: "ms",
+			Labels: map[string]string{
+				"Verb":     apicall.Verb,
+				"Resource": apicall.Resource,
+			},
+		}
+		perfData.DataItems = append(perfData.DataItems, item)
+	}
+	return perfData
+}
+
+// currentKubeletPerfMetricsVersion is the current kubelet performance metrics version. We should
+// bump up the version each time we make incompatible change to the metrics.
+const currentKubeletPerfMetricsVersion = "v2"
+
+// ResourceUsageToPerfData transforms ResourceUsagePerNode to PerfData. Notice that this function
+// only cares about memory usage, because cpu usage information will be extracted from NodesCPUSummary.
+func ResourceUsageToPerfData(usagePerNode ResourceUsagePerNode) *perftype.PerfData {
+	return ResourceUsageToPerfDataWithLabels(usagePerNode, nil)
+}
+
+// CPUUsageToPerfData transforms NodesCPUSummary to PerfData.
+func CPUUsageToPerfData(usagePerNode NodesCPUSummary) *perftype.PerfData {
+	return CPUUsageToPerfDataWithLabels(usagePerNode, nil)
+}
+
+// PrintPerfData prints the perfdata in json format with PerfResultTag prefix.
+// If an error occurs, nothing will be printed.
+func PrintPerfData(p *perftype.PerfData) {
+	// Notice that we must make sure the perftype.PerfResultEnd is in a new line.
+	if str := PrettyPrintJSON(p); str != "" {
+		Logf("%s %s\n%s", perftype.PerfResultTag, str, perftype.PerfResultEnd)
+	}
+}
+
+// ResourceUsageToPerfDataWithLabels transforms ResourceUsagePerNode to PerfData with additional labels.
+// Notice that this function only cares about memory usage, because cpu usage information will be extracted from NodesCPUSummary.
+func ResourceUsageToPerfDataWithLabels(usagePerNode ResourceUsagePerNode, labels map[string]string) *perftype.PerfData {
+	items := []perftype.DataItem{}
+	for node, usages := range usagePerNode {
+		for c, usage := range usages {
+			item := perftype.DataItem{
+				Data: map[string]float64{
+					"memory":     float64(usage.MemoryUsageInBytes) / (1024 * 1024),
+					"workingset": float64(usage.MemoryWorkingSetInBytes) / (1024 * 1024),
+					"rss":        float64(usage.MemoryRSSInBytes) / (1024 * 1024),
+				},
+				Unit: "MB",
+				Labels: map[string]string{
+					"node":      node,
+					"container": c,
+					"datatype":  "resource",
+					"resource":  "memory",
+				},
+			}
+			items = append(items, item)
+		}
+	}
+	return &perftype.PerfData{
+		Version:   currentKubeletPerfMetricsVersion,
+		DataItems: items,
+		Labels:    labels,
+	}
+}
+
+// CPUUsageToPerfDataWithLabels transforms NodesCPUSummary to PerfData with additional labels.
+func CPUUsageToPerfDataWithLabels(usagePerNode NodesCPUSummary, labels map[string]string) *perftype.PerfData {
+	items := []perftype.DataItem{}
+	for node, usages := range usagePerNode {
+		for c, usage := range usages {
+			data := map[string]float64{}
+			for perc, value := range usage {
+				data[fmt.Sprintf("Perc%02.0f", perc*100)] = value * 1000
+			}
+
+			item := perftype.DataItem{
+				Data: data,
+				Unit: "mCPU",
+				Labels: map[string]string{
+					"node":      node,
+					"container": c,
+					"datatype":  "resource",
+					"resource":  "cpu",
+				},
+			}
+			items = append(items, item)
+		}
+	}
+	return &perftype.PerfData{
+		Version:   currentKubeletPerfMetricsVersion,
+		DataItems: items,
+		Labels:    labels,
+	}
+}

vendor/k8s.io/kubernetes/test/e2e/framework/pods.go

@@ -0,0 +1,193 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"fmt"
+	"regexp"
+	"sync"
+	"time"
+
+	"k8s.io/kubernetes/pkg/api/errors"
+	"k8s.io/kubernetes/pkg/api/v1"
+	v1core "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5/typed/core/v1"
+	"k8s.io/kubernetes/pkg/labels"
+	"k8s.io/kubernetes/pkg/util/sets"
+	"k8s.io/kubernetes/pkg/util/wait"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// ImageWhiteList is the images used in the current test suite. It should be initialized in test suite and
+// the images in the white list should be pre-pulled in the test suite.  Currently, this is only used by
+// node e2e test.
+var ImageWhiteList sets.String
+
+// Convenience method for getting a pod client interface in the framework's namespace,
+// possibly applying test-suite specific transformations to the pod spec, e.g. for
+// node e2e pod scheduling.
+func (f *Framework) PodClient() *PodClient {
+	return &PodClient{
+		f:            f,
+		PodInterface: f.ClientSet.Core().Pods(f.Namespace.Name),
+	}
+}
+
+type PodClient struct {
+	f *Framework
+	v1core.PodInterface
+}
+
+// Create creates a new pod according to the framework specifications (don't wait for it to start).
+func (c *PodClient) Create(pod *v1.Pod) *v1.Pod {
+	c.mungeSpec(pod)
+	p, err := c.PodInterface.Create(pod)
+	ExpectNoError(err, "Error creating Pod")
+	return p
+}
+
+// CreateSync creates a new pod according to the framework specifications, and wait for it to start.
+func (c *PodClient) CreateSync(pod *v1.Pod) *v1.Pod {
+	p := c.Create(pod)
+	ExpectNoError(c.f.WaitForPodRunning(p.Name))
+	// Get the newest pod after it becomes running, some status may change after pod created, such as pod ip.
+	p, err := c.Get(p.Name)
+	ExpectNoError(err)
+	return p
+}
+
+// CreateBatch create a batch of pods. All pods are created before waiting.
+func (c *PodClient) CreateBatch(pods []*v1.Pod) []*v1.Pod {
+	ps := make([]*v1.Pod, len(pods))
+	var wg sync.WaitGroup
+	for i, pod := range pods {
+		wg.Add(1)
+		go func(i int, pod *v1.Pod) {
+			defer wg.Done()
+			defer GinkgoRecover()
+			ps[i] = c.CreateSync(pod)
+		}(i, pod)
+	}
+	wg.Wait()
+	return ps
+}
+
+// Update updates the pod object. It retries if there is a conflict, throw out error if
+// there is any other errors. name is the pod name, updateFn is the function updating the
+// pod object.
+func (c *PodClient) Update(name string, updateFn func(pod *v1.Pod)) {
+	ExpectNoError(wait.Poll(time.Millisecond*500, time.Second*30, func() (bool, error) {
+		pod, err := c.PodInterface.Get(name)
+		if err != nil {
+			return false, fmt.Errorf("failed to get pod %q: %v", name, err)
+		}
+		updateFn(pod)
+		_, err = c.PodInterface.Update(pod)
+		if err == nil {
+			Logf("Successfully updated pod %q", name)
+			return true, nil
+		}
+		if errors.IsConflict(err) {
+			Logf("Conflicting update to pod %q, re-get and re-update: %v", name, err)
+			return false, nil
+		}
+		return false, fmt.Errorf("failed to update pod %q: %v", name, err)
+	}))
+}
+
+// DeleteSync deletes the pod and wait for the pod to disappear for `timeout`. If the pod doesn't
+// disappear before the timeout, it will fail the test.
+func (c *PodClient) DeleteSync(name string, options *v1.DeleteOptions, timeout time.Duration) {
+	err := c.Delete(name, options)
+	if err != nil && !errors.IsNotFound(err) {
+		Failf("Failed to delete pod %q: %v", name, err)
+	}
+	Expect(WaitForPodToDisappear(c.f.ClientSet, c.f.Namespace.Name, name, labels.Everything(),
+		2*time.Second, timeout)).To(Succeed(), "wait for pod %q to disappear", name)
+}
+
+// mungeSpec apply test-suite specific transformations to the pod spec.
+func (c *PodClient) mungeSpec(pod *v1.Pod) {
+	if !TestContext.NodeE2E {
+		return
+	}
+
+	Expect(pod.Spec.NodeName).To(Or(BeZero(), Equal(TestContext.NodeName)), "Test misconfigured")
+	pod.Spec.NodeName = TestContext.NodeName
+	// Node e2e does not support the default DNSClusterFirst policy. Set
+	// the policy to DNSDefault, which is configured per node.
+	pod.Spec.DNSPolicy = v1.DNSDefault
+
+	// PrepullImages only works for node e2e now. For cluster e2e, image prepull is not enforced,
+	// we should not munge ImagePullPolicy for cluster e2e pods.
+	if !TestContext.PrepullImages {
+		return
+	}
+	// If prepull is enabled, munge the container spec to make sure the images are not pulled
+	// during the test.
+	for i := range pod.Spec.Containers {
+		c := &pod.Spec.Containers[i]
+		if c.ImagePullPolicy == v1.PullAlways {
+			// If the image pull policy is PullAlways, the image doesn't need to be in
+			// the white list or pre-pulled, because the image is expected to be pulled
+			// in the test anyway.
+			continue
+		}
+		// If the image policy is not PullAlways, the image must be in the white list and
+		// pre-pulled.
+		Expect(ImageWhiteList.Has(c.Image)).To(BeTrue(), "Image %q is not in the white list, consider adding it to CommonImageWhiteList in test/e2e/common/util.go or NodeImageWhiteList in test/e2e_node/image_list.go", c.Image)
+		// Do not pull images during the tests because the images in white list should have
+		// been prepulled.
+		c.ImagePullPolicy = v1.PullNever
+	}
+}
+
+// TODO(random-liu): Move pod wait function into this file
+// WaitForSuccess waits for pod to success.
+func (c *PodClient) WaitForSuccess(name string, timeout time.Duration) {
+	f := c.f
+	Expect(WaitForPodCondition(f.ClientSet, f.Namespace.Name, name, "success or failure", timeout,
+		func(pod *v1.Pod) (bool, error) {
+			switch pod.Status.Phase {
+			case v1.PodFailed:
+				return true, fmt.Errorf("pod %q failed with reason: %q, message: %q", name, pod.Status.Reason, pod.Status.Message)
+			case v1.PodSucceeded:
+				return true, nil
+			default:
+				return false, nil
+			}
+		},
+	)).To(Succeed(), "wait for pod %q to success", name)
+}
+
+// MatchContainerOutput gest output of a container and match expected regexp in the output.
+func (c *PodClient) MatchContainerOutput(name string, containerName string, expectedRegexp string) error {
+	f := c.f
+	output, err := GetPodLogs(f.ClientSet, f.Namespace.Name, name, containerName)
+	if err != nil {
+		return fmt.Errorf("failed to get output for container %q of pod %q", containerName, name)
+	}
+	regex, err := regexp.Compile(expectedRegexp)
+	if err != nil {
+		return fmt.Errorf("failed to compile regexp %q: %v", expectedRegexp, err)
+	}
+	if !regex.MatchString(output) {
+		return fmt.Errorf("failed to match regexp %q in output %q", expectedRegexp, output)
+	}
+	return nil
+}

vendor/k8s.io/kubernetes/test/e2e/framework/resource_usage_gatherer.go

@@ -0,0 +1,378 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"text/tabwriter"
+	"time"
+
+	. "github.com/onsi/gomega"
+	"k8s.io/kubernetes/pkg/api/v1"
+	clientset "k8s.io/kubernetes/pkg/client/clientset_generated/release_1_5"
+	utilruntime "k8s.io/kubernetes/pkg/util/runtime"
+	"k8s.io/kubernetes/pkg/util/system"
+)
+
+const (
+	resourceDataGatheringPeriod = 60 * time.Second
+	probeDuration               = 15 * time.Second
+)
+
+type ResourceConstraint struct {
+	CPUConstraint    float64
+	MemoryConstraint uint64
+}
+
+type SingleContainerSummary struct {
+	Name string
+	Cpu  float64
+	Mem  uint64
+}
+
+// we can't have int here, as JSON does not accept integer keys.
+type ResourceUsageSummary map[string][]SingleContainerSummary
+
+func (s *ResourceUsageSummary) PrintHumanReadable() string {
+	buf := &bytes.Buffer{}
+	w := tabwriter.NewWriter(buf, 1, 0, 1, ' ', 0)
+	for perc, summaries := range *s {
+		buf.WriteString(fmt.Sprintf("%v percentile:\n", perc))
+		fmt.Fprintf(w, "container\tcpu(cores)\tmemory(MB)\n")
+		for _, summary := range summaries {
+			fmt.Fprintf(w, "%q\t%.3f\t%.2f\n", summary.Name, summary.Cpu, float64(summary.Mem)/(1024*1024))
+		}
+		w.Flush()
+	}
+	return buf.String()
+}
+
+func (s *ResourceUsageSummary) PrintJSON() string {
+	return PrettyPrintJSON(*s)
+}
+
+func computePercentiles(timeSeries []ResourceUsagePerContainer, percentilesToCompute []int) map[int]ResourceUsagePerContainer {
+	if len(timeSeries) == 0 {
+		return make(map[int]ResourceUsagePerContainer)
+	}
+	dataMap := make(map[string]*usageDataPerContainer)
+	for i := range timeSeries {
+		for name, data := range timeSeries[i] {
+			if dataMap[name] == nil {
+				dataMap[name] = &usageDataPerContainer{
+					cpuData:        make([]float64, len(timeSeries)),
+					memUseData:     make([]uint64, len(timeSeries)),
+					memWorkSetData: make([]uint64, len(timeSeries)),
+				}
+			}
+			dataMap[name].cpuData = append(dataMap[name].cpuData, data.CPUUsageInCores)
+			dataMap[name].memUseData = append(dataMap[name].memUseData, data.MemoryUsageInBytes)
+			dataMap[name].memWorkSetData = append(dataMap[name].memWorkSetData, data.MemoryWorkingSetInBytes)
+		}
+	}
+	for _, v := range dataMap {
+		sort.Float64s(v.cpuData)
+		sort.Sort(uint64arr(v.memUseData))
+		sort.Sort(uint64arr(v.memWorkSetData))
+	}
+
+	result := make(map[int]ResourceUsagePerContainer)
+	for _, perc := range percentilesToCompute {
+		data := make(ResourceUsagePerContainer)
+		for k, v := range dataMap {
+			percentileIndex := int(math.Ceil(float64(len(v.cpuData)*perc)/100)) - 1
+			data[k] = &ContainerResourceUsage{
+				Name:                    k,
+				CPUUsageInCores:         v.cpuData[percentileIndex],
+				MemoryUsageInBytes:      v.memUseData[percentileIndex],
+				MemoryWorkingSetInBytes: v.memWorkSetData[percentileIndex],
+			}
+		}
+		result[perc] = data
+	}
+	return result
+}
+
+func leftMergeData(left, right map[int]ResourceUsagePerContainer) map[int]ResourceUsagePerContainer {
+	result := make(map[int]ResourceUsagePerContainer)
+	for percentile, data := range left {
+		result[percentile] = data
+		if _, ok := right[percentile]; !ok {
+			continue
+		}
+		for k, v := range right[percentile] {
+			result[percentile][k] = v
+		}
+	}
+	return result
+}
+
+type resourceGatherWorker struct {
+	c                    clientset.Interface
+	nodeName             string
+	wg                   *sync.WaitGroup
+	containerIDToNameMap map[string]string
+	containerIDs         []string
+	stopCh               chan struct{}
+	dataSeries           []ResourceUsagePerContainer
+	finished             bool
+	inKubemark           bool
+}
+
+func (w *resourceGatherWorker) singleProbe() {
+	var data ResourceUsagePerContainer
+	if w.inKubemark {
+		data = getKubemarkMasterComponentsResourceUsage()
+		if data == nil {
+			return
+		}
+	} else {
+		data = make(ResourceUsagePerContainer)
+		nodeUsage, err := getOneTimeResourceUsageOnNode(w.c, w.nodeName, probeDuration, func() []string { return w.containerIDs }, true)
+		if err != nil {
+			Logf("Error while reading data from %v: %v", w.nodeName, err)
+			return
+		}
+		for k, v := range nodeUsage {
+			data[w.containerIDToNameMap[k]] = v
+		}
+	}
+	w.dataSeries = append(w.dataSeries, data)
+}
+
+func (w *resourceGatherWorker) gather(initialSleep time.Duration) {
+	defer utilruntime.HandleCrash()
+	defer w.wg.Done()
+	defer Logf("Closing worker for %v", w.nodeName)
+	defer func() { w.finished = true }()
+	select {
+	case <-time.After(initialSleep):
+		w.singleProbe()
+		for {
+			select {
+			case <-time.After(resourceDataGatheringPeriod):
+				w.singleProbe()
+			case <-w.stopCh:
+				return
+			}
+		}
+	case <-w.stopCh:
+		return
+	}
+}
+
+func getKubemarkMasterComponentsResourceUsage() ResourceUsagePerContainer {
+	result := make(ResourceUsagePerContainer)
+	sshResult, err := SSH("ps ax -o %cpu,rss,command | tail -n +2 | grep kube", GetMasterHost()+":22", TestContext.Provider)
+	if err != nil {
+		Logf("Error when trying to SSH to master machine. Skipping probe")
+		return nil
+	}
+	scanner := bufio.NewScanner(strings.NewReader(sshResult.Stdout))
+	for scanner.Scan() {
+		var cpu float64
+		var mem uint64
+		var name string
+		fmt.Sscanf(strings.TrimSpace(scanner.Text()), "%f %d kubernetes/server/bin/%s", &cpu, &mem, &name)
+		if name != "" {
+			// Gatherer expects pod_name/container_name format
+			fullName := name + "/" + name
+			result[fullName] = &ContainerResourceUsage{Name: fullName, MemoryWorkingSetInBytes: mem * 1024, CPUUsageInCores: cpu / 100}
+		}
+	}
+	return result
+}
+
+func (g *containerResourceGatherer) getKubeSystemContainersResourceUsage(c clientset.Interface) {
+	if len(g.workers) == 0 {
+		return
+	}
+	delayPeriod := resourceDataGatheringPeriod / time.Duration(len(g.workers))
+	delay := time.Duration(0)
+	for i := range g.workers {
+		go g.workers[i].gather(delay)
+		delay += delayPeriod
+	}
+	g.workerWg.Wait()
+}
+
+type containerResourceGatherer struct {
+	client               clientset.Interface
+	stopCh               chan struct{}
+	workers              []resourceGatherWorker
+	workerWg             sync.WaitGroup
+	containerIDToNameMap map[string]string
+	containerIDs         []string
+	options              ResourceGathererOptions
+}
+
+type ResourceGathererOptions struct {
+	inKubemark bool
+	masterOnly bool
+}
+
+func NewResourceUsageGatherer(c clientset.Interface, options ResourceGathererOptions) (*containerResourceGatherer, error) {
+	g := containerResourceGatherer{
+		client:               c,
+		stopCh:               make(chan struct{}),
+		containerIDToNameMap: make(map[string]string),
+		containerIDs:         make([]string, 0),
+		options:              options,
+	}
+
+	if options.inKubemark {
+		g.workerWg.Add(1)
+		g.workers = append(g.workers, resourceGatherWorker{
+			inKubemark: true,
+			stopCh:     g.stopCh,
+			wg:         &g.workerWg,
+			finished:   false,
+		})
+	} else {
+		pods, err := c.Core().Pods("kube-system").List(v1.ListOptions{})
+		if err != nil {
+			Logf("Error while listing Pods: %v", err)
+			return nil, err
+		}
+		for _, pod := range pods.Items {
+			for _, container := range pod.Status.ContainerStatuses {
+				containerID := strings.TrimPrefix(container.ContainerID, "docker:/")
+				g.containerIDToNameMap[containerID] = pod.Name + "/" + container.Name
+				g.containerIDs = append(g.containerIDs, containerID)
+			}
+		}
+		nodeList, err := c.Core().Nodes().List(v1.ListOptions{})
+		if err != nil {
+			Logf("Error while listing Nodes: %v", err)
+			return nil, err
+		}
+
+		for _, node := range nodeList.Items {
+			if !options.masterOnly || system.IsMasterNode(node.Name) {
+				g.workerWg.Add(1)
+				g.workers = append(g.workers, resourceGatherWorker{
+					c:                    c,
+					nodeName:             node.Name,
+					wg:                   &g.workerWg,
+					containerIDToNameMap: g.containerIDToNameMap,
+					containerIDs:         g.containerIDs,
+					stopCh:               g.stopCh,
+					finished:             false,
+					inKubemark:           false,
+				})
+				if options.masterOnly {
+					break
+				}
+			}
+		}
+	}
+	return &g, nil
+}
+
+// startGatheringData blocks until stopAndSummarize is called.
+func (g *containerResourceGatherer) startGatheringData() {
+	g.getKubeSystemContainersResourceUsage(g.client)
+}
+
+func (g *containerResourceGatherer) stopAndSummarize(percentiles []int, constraints map[string]ResourceConstraint) *ResourceUsageSummary {
+	close(g.stopCh)
+	Logf("Closed stop channel. Waiting for %v workers", len(g.workers))
+	finished := make(chan struct{})
+	go func() {
+		g.workerWg.Wait()
+		finished <- struct{}{}
+	}()
+	select {
+	case <-finished:
+		Logf("Waitgroup finished.")
+	case <-time.After(2 * time.Minute):
+		unfinished := make([]string, 0)
+		for i := range g.workers {
+			if !g.workers[i].finished {
+				unfinished = append(unfinished, g.workers[i].nodeName)
+			}
+		}
+		Logf("Timed out while waiting for waitgroup, some workers failed to finish: %v", unfinished)
+	}
+
+	if len(percentiles) == 0 {
+		Logf("Warning! Empty percentile list for stopAndPrintData.")
+		return &ResourceUsageSummary{}
+	}
+	data := make(map[int]ResourceUsagePerContainer)
+	for i := range g.workers {
+		if g.workers[i].finished {
+			stats := computePercentiles(g.workers[i].dataSeries, percentiles)
+			data = leftMergeData(stats, data)
+		}
+	}
+
+	// Workers has been stopped. We need to gather data stored in them.
+	sortedKeys := []string{}
+	for name := range data[percentiles[0]] {
+		sortedKeys = append(sortedKeys, name)
+	}
+	sort.Strings(sortedKeys)
+	violatedConstraints := make([]string, 0)
+	summary := make(ResourceUsageSummary)
+	for _, perc := range percentiles {
+		for _, name := range sortedKeys {
+			usage := data[perc][name]
+			summary[strconv.Itoa(perc)] = append(summary[strconv.Itoa(perc)], SingleContainerSummary{
+				Name: name,
+				Cpu:  usage.CPUUsageInCores,
+				Mem:  usage.MemoryWorkingSetInBytes,
+			})
+			// Verifying 99th percentile of resource usage
+			if perc == 99 {
+				// Name has a form: <pod_name>/<container_name>
+				containerName := strings.Split(name, "/")[1]
+				if constraint, ok := constraints[containerName]; ok {
+					if usage.CPUUsageInCores > constraint.CPUConstraint {
+						violatedConstraints = append(
+							violatedConstraints,
+							fmt.Sprintf("Container %v is using %v/%v CPU",
+								name,
+								usage.CPUUsageInCores,
+								constraint.CPUConstraint,
+							),
+						)
+					}
+					if usage.MemoryWorkingSetInBytes > constraint.MemoryConstraint {
+						violatedConstraints = append(
+							violatedConstraints,
+							fmt.Sprintf("Container %v is using %v/%v MB of memory",
+								name,
+								float64(usage.MemoryWorkingSetInBytes)/(1024*1024),
+								float64(constraint.MemoryConstraint)/(1024*1024),
+							),
+						)
+					}
+				}
+			}
+		}
+	}
+	Expect(violatedConstraints).To(BeEmpty())
+	return &summary
+}

vendor/k8s.io/kubernetes/test/e2e/framework/test_context.go

@@ -0,0 +1,240 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package framework
+
+import (
+	"flag"
+	"os"
+	"time"
+
+	"github.com/onsi/ginkgo/config"
+	"github.com/spf13/viper"
+	"k8s.io/kubernetes/pkg/apis/componentconfig"
+	"k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
+	"k8s.io/kubernetes/pkg/cloudprovider"
+)
+
+type TestContextType struct {
+	KubeConfig         string
+	KubeContext        string
+	KubeAPIContentType string
+	KubeVolumeDir      string
+	CertDir            string
+	Host               string
+	// TODO: Deprecating this over time... instead just use gobindata_util.go , see #23987.
+	RepoRoot string
+
+	Provider       string
+	CloudConfig    CloudConfig
+	KubectlPath    string
+	OutputDir      string
+	ReportDir      string
+	ReportPrefix   string
+	Prefix         string
+	MinStartupPods int
+	// Timeout for waiting for system pods to be running
+	SystemPodsStartupTimeout time.Duration
+	UpgradeTarget            string
+	UpgradeImage             string
+	PrometheusPushGateway    string
+	ContainerRuntime         string
+	MasterOSDistro           string
+	NodeOSDistro             string
+	VerifyServiceAccount     bool
+	DeleteNamespace          bool
+	DeleteNamespaceOnFailure bool
+	AllowedNotReadyNodes     int
+	CleanStart               bool
+	// If set to 'true' or 'all' framework will start a goroutine monitoring resource usage of system add-ons.
+	// It will read the data every 30 seconds from all Nodes and print summary during afterEach. If set to 'master'
+	// only master Node will be monitored.
+	GatherKubeSystemResourceUsageData string
+	GatherLogsSizes                   bool
+	GatherMetricsAfterTest            bool
+	// Currently supported values are 'hr' for human-readable and 'json'. It's a comma separated list.
+	OutputPrintType string
+	// CreateTestingNS is responsible for creating namespace used for executing e2e tests.
+	// It accepts namespace base name, which will be prepended with e2e prefix, kube client
+	// and labels to be applied to a namespace.
+	CreateTestingNS CreateTestingNSFn
+	// If set to true test will dump data about the namespace in which test was running.
+	DumpLogsOnFailure bool
+	// If the garbage collector is enabled in the kube-apiserver and kube-controller-manager.
+	GarbageCollectorEnabled bool
+	// FeatureGates is a set of key=value pairs that describe feature gates for alpha/experimental features.
+	FeatureGates string
+	// Node e2e specific test context
+	NodeTestContextType
+
+	// Viper-only parameters.  These will in time replace all flags.
+
+	// Example: Create a file 'e2e.json' with the following:
+	// 	"Cadvisor":{
+	// 		"MaxRetries":"6"
+	// 	}
+
+	Viper    string
+	Cadvisor struct {
+		MaxRetries      int
+		SleepDurationMS int
+	}
+
+	LoggingSoak struct {
+		Scale                    int
+		MilliSecondsBetweenWaves int
+	}
+}
+
+// NodeTestContextType is part of TestContextType, it is shared by all node e2e test.
+type NodeTestContextType struct {
+	// NodeE2E indicates whether it is running node e2e.
+	NodeE2E bool
+	// Name of the node to run tests on.
+	NodeName string
+	// NodeConformance indicates whether the test is running in node conformance mode.
+	NodeConformance bool
+	// PrepullImages indicates whether node e2e framework should prepull images.
+	PrepullImages bool
+	// KubeletConfig is the kubelet configuration the test is running against.
+	KubeletConfig componentconfig.KubeletConfiguration
+}
+
+type CloudConfig struct {
+	ProjectID         string
+	Zone              string
+	Cluster           string
+	MasterName        string
+	NodeInstanceGroup string
+	NumNodes          int
+	ClusterTag        string
+	Network           string
+
+	Provider cloudprovider.Interface
+}
+
+var TestContext TestContextType
+var federatedKubeContext string
+
+// Register flags common to all e2e test suites.
+func RegisterCommonFlags() {
+	// Turn on verbose by default to get spec names
+	config.DefaultReporterConfig.Verbose = true
+
+	// Turn on EmitSpecProgress to get spec progress (especially on interrupt)
+	config.GinkgoConfig.EmitSpecProgress = true
+
+	// Randomize specs as well as suites
+	config.GinkgoConfig.RandomizeAllSpecs = true
+
+	flag.StringVar(&TestContext.GatherKubeSystemResourceUsageData, "gather-resource-usage", "false", "If set to 'true' or 'all' framework will be monitoring resource usage of system all add-ons in (some) e2e tests, if set to 'master' framework will be monitoring master node only, if set to 'none' of 'false' monitoring will be turned off.")
+	flag.BoolVar(&TestContext.GatherLogsSizes, "gather-logs-sizes", false, "If set to true framework will be monitoring logs sizes on all machines running e2e tests.")
+	flag.BoolVar(&TestContext.GatherMetricsAfterTest, "gather-metrics-at-teardown", false, "If set to true framwork will gather metrics from all components after each test.")
+	flag.StringVar(&TestContext.OutputPrintType, "output-print-type", "hr", "Comma separated list: 'hr' for human readable summaries 'json' for JSON ones.")
+	flag.BoolVar(&TestContext.DumpLogsOnFailure, "dump-logs-on-failure", true, "If set to true test will dump data about the namespace in which test was running.")
+	flag.BoolVar(&TestContext.DeleteNamespace, "delete-namespace", true, "If true tests will delete namespace after completion. It is only designed to make debugging easier, DO NOT turn it off by default.")
+	flag.BoolVar(&TestContext.DeleteNamespaceOnFailure, "delete-namespace-on-failure", true, "If true, framework will delete test namespace on failure. Used only during test debugging.")
+	flag.IntVar(&TestContext.AllowedNotReadyNodes, "allowed-not-ready-nodes", 0, "If non-zero, framework will allow for that many non-ready nodes when checking for all ready nodes.")
+	flag.StringVar(&TestContext.Host, "host", "http://127.0.0.1:8080", "The host, or apiserver, to connect to")
+	flag.StringVar(&TestContext.ReportPrefix, "report-prefix", "", "Optional prefix for JUnit XML reports. Default is empty, which doesn't prepend anything to the default name.")
+	flag.StringVar(&TestContext.ReportDir, "report-dir", "", "Path to the directory where the JUnit XML reports should be saved. Default is empty, which doesn't generate these reports.")
+	flag.StringVar(&TestContext.FeatureGates, "feature-gates", "", "A set of key=value pairs that describe feature gates for alpha/experimental features.")
+	flag.StringVar(&TestContext.Viper, "viper-config", "e2e", "The name of the viper config i.e. 'e2e' will read values from 'e2e.json' locally.  All e2e parameters are meant to be configurable by viper.")
+}
+
+// Register flags specific to the cluster e2e test suite.
+func RegisterClusterFlags() {
+	flag.BoolVar(&TestContext.VerifyServiceAccount, "e2e-verify-service-account", true, "If true tests will verify the service account before running.")
+	flag.StringVar(&TestContext.KubeConfig, clientcmd.RecommendedConfigPathFlag, os.Getenv(clientcmd.RecommendedConfigPathEnvVar), "Path to kubeconfig containing embedded authinfo.")
+	flag.StringVar(&TestContext.KubeContext, clientcmd.FlagContext, "", "kubeconfig context to use/override. If unset, will use value from 'current-context'")
+	flag.StringVar(&TestContext.KubeAPIContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType used to communicate with apiserver")
+	flag.StringVar(&federatedKubeContext, "federated-kube-context", "federation-cluster", "kubeconfig context for federation-cluster.")
+
+	flag.StringVar(&TestContext.KubeVolumeDir, "volume-dir", "/var/lib/kubelet", "Path to the directory containing the kubelet volumes.")
+	flag.StringVar(&TestContext.CertDir, "cert-dir", "", "Path to the directory containing the certs. Default is empty, which doesn't use certs.")
+	flag.StringVar(&TestContext.RepoRoot, "repo-root", "../../", "Root directory of kubernetes repository, for finding test files.")
+	flag.StringVar(&TestContext.Provider, "provider", "", "The name of the Kubernetes provider (gce, gke, local, vagrant, etc.)")
+	flag.StringVar(&TestContext.KubectlPath, "kubectl-path", "kubectl", "The kubectl binary to use. For development, you might use 'cluster/kubectl.sh' here.")
+	flag.StringVar(&TestContext.OutputDir, "e2e-output-dir", "/tmp", "Output directory for interesting/useful test data, like performance data, benchmarks, and other metrics.")
+	flag.StringVar(&TestContext.Prefix, "prefix", "e2e", "A prefix to be added to cloud resources created during testing.")
+	flag.StringVar(&TestContext.ContainerRuntime, "container-runtime", "docker", "The container runtime of cluster VM instances (docker or rkt).")
+	flag.StringVar(&TestContext.MasterOSDistro, "master-os-distro", "debian", "The OS distribution of cluster master (debian, trusty, or coreos).")
+	flag.StringVar(&TestContext.NodeOSDistro, "node-os-distro", "debian", "The OS distribution of cluster VM instances (debian, trusty, or coreos).")
+
+	// TODO: Flags per provider?  Rename gce-project/gce-zone?
+	cloudConfig := &TestContext.CloudConfig
+	flag.StringVar(&cloudConfig.MasterName, "kube-master", "", "Name of the kubernetes master. Only required if provider is gce or gke")
+	flag.StringVar(&cloudConfig.ProjectID, "gce-project", "", "The GCE project being used, if applicable")
+	flag.StringVar(&cloudConfig.Zone, "gce-zone", "", "GCE zone being used, if applicable")
+	flag.StringVar(&cloudConfig.Cluster, "gke-cluster", "", "GKE name of cluster being used, if applicable")
+	flag.StringVar(&cloudConfig.NodeInstanceGroup, "node-instance-group", "", "Name of the managed instance group for nodes. Valid only for gce, gke or aws. If there is more than one group: comma separated list of groups.")
+	flag.StringVar(&cloudConfig.Network, "network", "e2e", "The cloud provider network for this e2e cluster.")
+	flag.IntVar(&cloudConfig.NumNodes, "num-nodes", -1, "Number of nodes in the cluster")
+
+	flag.StringVar(&cloudConfig.ClusterTag, "cluster-tag", "", "Tag used to identify resources.  Only required if provider is aws.")
+	flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.")
+	flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.")
+	flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.")
+	flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.")
+	flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.")
+	flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.")
+	flag.BoolVar(&TestContext.GarbageCollectorEnabled, "garbage-collector-enabled", true, "Set to true if the garbage collector is enabled in the kube-apiserver and kube-controller-manager, then some tests will rely on the garbage collector to delete dependent resources.")
+}
+
+// Register flags specific to the node e2e test suite.
+func RegisterNodeFlags() {
+	// Mark the test as node e2e when node flags are registered.
+	TestContext.NodeE2E = true
+	flag.StringVar(&TestContext.NodeName, "node-name", "", "Name of the node to run tests on.")
+	// TODO(random-liu): Move kubelet start logic out of the test.
+	// TODO(random-liu): Move log fetch logic out of the test.
+	// There are different ways to start kubelet (systemd, initd, docker, rkt, manually started etc.)
+	// and manage logs (journald, upstart etc.).
+	// For different situation we need to mount different things into the container, run different commands.
+	// It is hard and unnecessary to deal with the complexity inside the test suite.
+	flag.BoolVar(&TestContext.NodeConformance, "conformance", false, "If true, the test suite will not start kubelet, and fetch system log (kernel, docker, kubelet log etc.) to the report directory.")
+	flag.BoolVar(&TestContext.PrepullImages, "prepull-images", true, "If true, prepull images so image pull failures do not cause test failures.")
+}
+
+// overwriteFlagsWithViperConfig finds and writes values to flags using viper as input.
+func overwriteFlagsWithViperConfig() {
+	viperFlagSetter := func(f *flag.Flag) {
+		if viper.IsSet(f.Name) {
+			f.Value.Set(viper.GetString(f.Name))
+		}
+	}
+	flag.VisitAll(viperFlagSetter)
+}
+
+// ViperizeFlags sets up all flag and config processing. Future configuration info should be added to viper, not to flags.
+func ViperizeFlags() {
+
+	// Part 1: Set regular flags.
+	// TODO: Future, lets eliminate e2e 'flag' deps entirely in favor of viper only,
+	// since go test 'flag's are sort of incompatible w/ flag, glog, etc.
+	RegisterCommonFlags()
+	RegisterClusterFlags()
+	flag.Parse()
+
+	// Part 2: Set Viper provided flags.
+	// This must be done after common flags are registered, since Viper is a flag option.
+	viper.SetConfigName(TestContext.Viper)
+	viper.AddConfigPath(".")
+	viper.ReadInConfig()
+
+	// TODO Consider wether or not we want to use overwriteFlagsWithViperConfig().
+	viper.Unmarshal(&TestContext)
+}