clusterloader2/testing/experimental/storage/pod-startup/config.yaml

# ASSUMPTIONS:
# - Number of nodes should be divisible by NODES_PER_NAMESPACE (default 100).
# - If using Persistent Volumes, the default storage class must have volumeBindingMode: Immediate

# Cluster Variables
  # Cater for the case where the number of nodes is less than nodes per namespace. See https://github.com/kubernetes/perf-tests/issues/887
  {{$NODES_PER_NAMESPACE := MinInt .Nodes (DefaultParam .NODES_PER_NAMESPACE 100)}} 

# Test Variales
  {{$PODS_PER_NODE := DefaultParam .PODS_PER_NODE 1}}
  {{$DEPLOYMENT_TEMPLATE_PATH := .DEPLOYMENT_TEMPLATE_PATH }}
  {{$VOLUMES_PER_POD := DefaultParam .VOLUMES_PER_POD 1}}
  {{$VOLUME_TEMPLATE_PATH := .VOLUME_TEMPLATE_PATH}}
  # Set this to false if metrics data is not needed.
  {{$GATHER_METRICS := DefaultParam .GATHER_METRICS true}}
  {{$START_PODS := DefaultParam .START_PODS true}}
  {{$PROVISION_VOLUME := DefaultParam .PROVISION_VOLUME false}}
  {{$STORAGE_CLASS := DefaultParam .STORAGE_CLASS "csi-gce-pd"}}
  # Shortcut for provisioning volumes with some external provisioner. Without it,
  # the PV controller must update each PVC before the external provisioner
  # starts to work on it, which limits the volume creation rate.
  # With it, volume creation starts immediately. This is best
  # used together with WAIT_FOR_PVS_CREATED=true and
  # WAIT_FOR_PVS_BOUND=false because then test does not to wait for the
  # PV controller during volume creation at all.
  {{$PROVISIONER := DefaultParam .PROVISIONER ""}}
  # When checking PVs, the test by default considers *all* PVs in the cluster,
  # whether they were created by the test or not. In other words, the test
  # only works in clusters with no PVs. To make it a bit more flexible,
  # the value of the expected provisioner can be set here and then only
  # PVs with that annotation will be counted.
  {{$EXPECTED_PROVISIONER := DefaultParam .EXPECTED_PROVISIONER ""}}
  {{$VOL_SIZE := DefaultParam .VOL_SIZE "8Gi"}}
  {{$WAIT_FOR_PVS_CREATED := DefaultParam .WAIT_FOR_PVS_CREATED false}}
  {{$WAIT_FOR_PVS_BOUND := DefaultParam .WAIT_FOR_PVS_BOUND false}}
  {{$WAIT_FOR_PVS_DELETED := DefaultParam .WAIT_FOR_PVS_DELETED false}}
  {{$POD_THROUGHPUT := DefaultParam .POD_THROUGHPUT 10}}
  # TODO(hantaowang): remove knob after deciding on right values
  {{$POD_STARTUP_TIMEOUT := DefaultParam .POD_STARTUP_TIMEOUT "15m"}}
  {{$POD_STARTUP_SLO := DefaultParam .POD_STARTUP_SLO 300}}

# Computed Variables
  {{$namespaces := DivideInt .Nodes $NODES_PER_NAMESPACE | MaxInt 1}}
  {{$podsPerNamespace := MultiplyInt $NODES_PER_NAMESPACE $PODS_PER_NODE}}
  {{$volumesPerNamespace := MultiplyInt $podsPerNamespace $VOLUMES_PER_POD}}
  {{$totalVols := MultiplyInt $volumesPerNamespace $namespaces}}
  {{$guessedStepTime := MultiplyInt $totalVols 2 | MaxInt 60}}

# This is a very conservative estimate of 2 seconds per volume.
  {{$StepTimeSeconds := DefaultParam .STEP_TIME_SECONDS $guessedStepTime}}


name: storage
namespace:
  number: {{$namespaces}}
tuningSets:
- name: UniformQPS
  qpsLoad:
    qps: {{$POD_THROUGHPUT}}
steps:
{{ if $GATHER_METRICS }}
# Start measurements
- name: Starting measurement for the entire test
  measurements:
  - Identifier: APIResponsivenessPrometheus
    Method: APIResponsivenessPrometheus
    Params:
      action: start
  - Identifier: APIResponsivenessPrometheusSimple
    Method: APIResponsivenessPrometheus
    Params:
      action: start
  - Identifier: PodWithVolumesStartupLatency
    Method: PodStartupLatency
    Params:
      action: start
      labelSelector: group = volume-test
      threshold: {{$POD_STARTUP_SLO}}s
{{ end }}
{{ if $PROVISION_VOLUME }}
# Provision volumes
- name: Provisioning volumes
  phases:
  - namespaceRange:
      min: 1
      max: {{$namespaces}}
    replicasPerNamespace: {{$volumesPerNamespace}}
    tuningSet: UniformQPS
    objectBundle:
    - basename: vol
      objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}}
      templateFillMap:
        Group: volume-test
        VolSize: {{$VOL_SIZE}}
        StorageClass: {{$STORAGE_CLASS}}
        Provisioner: {{$PROVISIONER}}
{{ end }}
{{if $WAIT_FOR_PVS_CREATED }}
- name: Waiting for volume creation
  measurements:
    - Identifier: WaitForPVsToBeCreated
      Method: WaitForAvailablePVs
      Params:
        desiredPVCount: {{$totalVols}}
        apiVersion: v1
        provisioner: {{$EXPECTED_PROVISIONER}}
        timeout: {{$StepTimeSeconds}}s
{{ end }}
{{ if $WAIT_FOR_PVS_BOUND }}
- name: Waiting for PVs to be bound
  measurements:
  - Identifier: WaitForPVCsToBeBound
    Method: WaitForBoundPVCs
    Params:
      desiredPVCCount: {{$totalVols}}
      apiVersion: v1
      labelSelector: group = volume-test
      timeout: {{$StepTimeSeconds}}s
{{ end }}
{{ if $START_PODS }}
- name: Starting measurement for waiting for deployments
  measurements:
  - Identifier: WaitForRunningDeployments
    Method: WaitForControlledPodsRunning
    Params:
      action: start
      apiVersion: apps/v1
      kind: Deployment
      labelSelector: group = volume-test
      operationTimeout: {{$POD_STARTUP_TIMEOUT}}
# Create deployments
- name: Creating deployments
  phases:
  - namespaceRange:
      min: 1
      max: {{$namespaces}}
    replicasPerNamespace: {{$podsPerNamespace}}
    tuningSet: UniformQPS
    objectBundle:
    - basename: deployment
      objectTemplatePath: {{$DEPLOYMENT_TEMPLATE_PATH}}
      templateFillMap:
        Group: volume-test
        VolumesPerPod: {{$VOLUMES_PER_POD}}
        VolSize: {{$VOL_SIZE}}
        StorageClass: {{$STORAGE_CLASS}}
        Provisioner: {{$PROVISIONER}}
- name: Waiting for deployments to be running
  measurements:
  - Identifier: WaitForRunningDeployments
    Method: WaitForControlledPodsRunning
    Params:
      action: gather
# Delete deployments
- name: Deleting deployments
  phases:
  - namespaceRange:
      min: 1
      max: {{$namespaces}}
    replicasPerNamespace: 0
    tuningSet: UniformQPS
    objectBundle:
    - basename: deployment
      objectTemplatePath: {{$DEPLOYMENT_TEMPLATE_PATH}}
{{ end }}
{{ if $PROVISION_VOLUME }}
# Delete volumes
- name: Deleting volumes
  phases:
  - namespaceRange:
      min: 1
      max: {{$namespaces}}
    replicasPerNamespace: 0
    tuningSet: UniformQPS
    objectBundle:
      - basename: vol
        objectTemplatePath: {{$VOLUME_TEMPLATE_PATH}}
{{ end }}
{{if $WAIT_FOR_PVS_DELETED }}
- name: Waiting for volume deletion
  measurements:
    - Identifier: WaitForPVsToBeDeleted
      Method: WaitForAvailablePVs
      Params:
        desiredPVCount: 0
        apiVersion: v1
        provisioner: {{$EXPECTED_PROVISIONER}}
        timeout: {{$StepTimeSeconds}}s
{{ end }}
{{ if $GATHER_METRICS }}
# Collect measurements
- name: Gather test measurements
  measurements:
  - Identifier: APIResponsivenessPrometheusSimple
    Method: APIResponsivenessPrometheus
    Params:
      action: gather
      enableViolations: true
      useSimpleLatencyQuery: true
      summaryName: APIResponsivenessPrometheus_simple
  - Identifier: APIResponsivenessPrometheus
    Method: APIResponsivenessPrometheus
    Params:
      action: gather
  - Identifier: PodWithVolumesStartupLatency
    Method: PodStartupLatency
    Params:
      action: gather
{{ end }}