def calculate()

in scripts/capacity/calculator.py [0:0]

• 59 lines of code
• 19 McCabe index (conditional complexity)

def calculate(args):
  """calculate job configuration according to requirements.
     For segcache, returns a dict with:
      cpu, ram, disk,
      hash_power, seg_mem,
      instance, host_limit, rack_limit,
      memory_bound
     For twemcache, returns a dict with:
      cpu, ram, disk,
      hash_power, slab_mem,
      instance, host_limit, rack_limit,
      memory_bound
     For slimcache, return a dict with:
      cpu, ram, disk,
      item_size, nitem,
      instance, host_limit, rack_limit,
      memory_bound
  """
  if args.failure_domain < FAILURE_DOMAIN_LOWER or args.failure_domain > FAILURE_DOMAIN_UPPER:
    print('ERROR: failure domain should be between {:.1f}% and {:.1f}'.format(
      FAILURE_DOMAIN_LOWER, FAILURE_DOMAIN_UPPER))

  # first calculate njob disrecarding memory, note both njob & bottleneck are not yet final
  njob_qps = int(ceil(1.0 * args.qps / KQPS))
  njob_fd = int(ceil(100.0 / args.failure_domain))
  if njob_qps >= njob_fd:
    bottleneck = 'qps'
    njob = njob_qps
  else:
    bottleneck = 'failure domain'
    njob = njob_fd

  # then calculate njob (vector) assuming memory-bound

  # all ram-related values in this function are in MB
  # amount of ram needed to store dataset, factoring in overhead
  item_size = int(KEYVAL_ALIGNMENT * ceil(1.0 * (ITEM_OVERHEAD[args.runnable] + args.size) /
    KEYVAL_ALIGNMENT))
  ram_data = 1.0 * item_size * args.nkey * M / MB
  # per-job memory overhead, in MB
  ram_conn = int(ceil(1.0 * CONN_OVERHEAD * args.nconn / MB))
  ram_fixed = BASE_OVERHEAD + SAFETY_BUF

  njob_mem = []
  sorted_ram = sorted(args.ram)
  for ram in sorted_ram:
    ram = ram * GB / MB  # change unit to MB
    n_low = int(ceil(ram_data / ram))  # number of shards, lower bound
    nkey_per_shard = 1.0 * args.nkey * M / n_low  # number of keys per shard, upper bound
    hash_power, ram_hash = hash_parameters(nkey_per_shard, args.runnable)  # upper bound for both
    n = int(ceil(ram_data / (ram - ram_fixed - ram_conn - ram_hash)))
    njob_mem.append(n)

  # get final njob count; prefer larger ram if it reduces njob, which means:
  # if cluster needs higher job ram AND more instances due to memory, update njob
  # if cluster is memory-bound with smaller job ram but qps-bound with larger ones, use higher ram
  # otherwise, use smaller job ram and keep njob value unchanged
  index = 0  # if qps bound, use smallest ram setting
  for i, n in reversed(list(enumerate(njob_mem))[1:]):
    if n > njob or njob_mem[i - 1] > njob:
      bottleneck = 'memory'
      index = i
      njob = max(njob, n)
      break
  if njob > WARNING_THRESHOLD:
    print('WARNING: more than {} instances needed, please verify input.'.format(WARNING_THRESHOLD))

  # recalculate hash parameters with the final job count
  nkey_per_shard = 1.0 * (sorted_ram[index] * GB - ram_fixed * MB - ram_conn * MB) / item_size
  # used by twemcache and segcache
  hash_power, ram_hash = hash_parameters(nkey_per_shard, args.runnable)
  slab_mem = sorted_ram[index] * GB / MB - ram_fixed - ram_conn - ram_hash
  # only used by slimcache
  nitem = int(NITEM_ALIGNMENT * floor(nkey_per_shard / NITEM_ALIGNMENT))

  rack_limit = int(floor(njob * args.failure_domain / 100))  # >= 1 given how we calculate njob
  host_limit = int(floor(min(MAX_HOST_LIMIT, max(1, rack_limit / RACK_TO_HOST_RATIO))))

  ret = {
      'cpu': CPU_PER_JOB,
      'ram': sorted_ram[index],
      'disk': DISK_PER_JOB,
      'instance': njob,
      'rack_limit': rack_limit,
      'host_limit': host_limit,
      'bottleneck': bottleneck}
  if args.runnable == 'twemcache':
    ret['hash_power'] = hash_power
    ret['slab_mem'] = slab_mem
  elif args.runnable == 'segcache':
      ret['hash_power'] = hash_power
      ret['seg_mem'] = slab_mem
  elif args.runnable == 'slimcache':
    ret['item_size'] = item_size
    ret['nitem'] = nitem

  return ret