tf_wide_deep.go

Documentation: k8s.io/kubernetes/test/e2e_node/perf/workloads

     1  /*
     2  Copyright 2018 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package workloads
    18  
    19  import (
    20  	"fmt"
    21  	"strings"
    22  	"time"
    23  
    24  	v1 "k8s.io/api/core/v1"
    25  	"k8s.io/apimachinery/pkg/api/resource"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
    28  	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager"
    29  	imageutils "k8s.io/kubernetes/test/utils/image"
    30  )
    31  
    32  // tfWideDeepWorkload defines a workload to run
    33  // https://github.com/tensorflow/models/tree/master/official/r1/wide_deep.
    34  type tfWideDeepWorkload struct{}
    35  
    36  // Ensure tfWideDeepWorkload implements NodePerfWorkload interface.
    37  var _ NodePerfWorkload = &tfWideDeepWorkload{}
    38  
    39  func (w tfWideDeepWorkload) Name() string {
    40  	return "tensorflow-wide-deep"
    41  }
    42  
    43  func (w tfWideDeepWorkload) PodSpec() v1.PodSpec {
    44  	var containers []v1.Container
    45  	ctn := v1.Container{
    46  		Name:  fmt.Sprintf("%s-ctn", w.Name()),
    47  		Image: imageutils.GetE2EImage(imageutils.NodePerfTfWideDeep),
    48  		Resources: v1.ResourceRequirements{
    49  			Requests: v1.ResourceList{
    50  				v1.ResourceName(v1.ResourceCPU):    resource.MustParse("15000m"),
    51  				v1.ResourceName(v1.ResourceMemory): resource.MustParse("16Gi"),
    52  			},
    53  			Limits: v1.ResourceList{
    54  				v1.ResourceName(v1.ResourceCPU):    resource.MustParse("15000m"),
    55  				v1.ResourceName(v1.ResourceMemory): resource.MustParse("16Gi"),
    56  			},
    57  		},
    58  		Command: []string{"/bin/sh"},
    59  		Args:    []string{"-c", "time -p python ./wide_deep.py --model_type=wide_deep --train_epochs=300 --epochs_between_evals=300 --batch_size=32561"},
    60  	}
    61  	containers = append(containers, ctn)
    62  
    63  	return v1.PodSpec{
    64  		RestartPolicy: v1.RestartPolicyNever,
    65  		Containers:    containers,
    66  	}
    67  }
    68  
    69  func (w tfWideDeepWorkload) Timeout() time.Duration {
    70  	return 15 * time.Minute
    71  }
    72  
    73  func (w tfWideDeepWorkload) KubeletConfig(oldCfg *kubeletconfig.KubeletConfiguration) (newCfg *kubeletconfig.KubeletConfiguration, err error) {
    74  	// Enable CPU Manager in Kubelet with static policy.
    75  	newCfg = oldCfg.DeepCopy()
    76  	// Set the CPU Manager policy to static.
    77  	newCfg.CPUManagerPolicy = string(cpumanager.PolicyStatic)
    78  	// Set the CPU Manager reconcile period to 10 second.
    79  	newCfg.CPUManagerReconcilePeriod = metav1.Duration{Duration: 10 * time.Second}
    80  
    81  	// The Kubelet panics if either kube-reserved or system-reserved is not set
    82  	// when static CPU Manager is enabled. Set cpu in kube-reserved > 0 so that
    83  	// kubelet doesn't panic.
    84  	if newCfg.KubeReserved == nil {
    85  		newCfg.KubeReserved = map[string]string{}
    86  	}
    87  
    88  	if _, ok := newCfg.KubeReserved["cpu"]; !ok {
    89  		newCfg.KubeReserved["cpu"] = "200m"
    90  	}
    91  
    92  	return newCfg, nil
    93  }
    94  
    95  func (w tfWideDeepWorkload) PreTestExec() error {
    96  	cmd := "/bin/sh"
    97  	args := []string{"-c", "rm -f /var/lib/kubelet/cpu_manager_state"}
    98  	err := runCmd(cmd, args)
    99  
   100  	return err
   101  }
   102  
   103  func (w tfWideDeepWorkload) PostTestExec() error {
   104  	cmd := "/bin/sh"
   105  	args := []string{"-c", "rm -f /var/lib/kubelet/cpu_manager_state"}
   106  	err := runCmd(cmd, args)
   107  
   108  	return err
   109  }
   110  
   111  func (w tfWideDeepWorkload) ExtractPerformanceFromLogs(logs string) (perf time.Duration, err error) {
   112  	perfLine, err := getMatchingLineFromLog(logs, "real")
   113  	if err != nil {
   114  		return perf, err
   115  	}
   116  	perfString := fmt.Sprintf("%ss", strings.TrimSpace(strings.TrimPrefix(perfLine, "real")))
   117  	perf, err = time.ParseDuration(perfString)
   118  
   119  	return perf, err
   120  }
   121
View as plain text