static void RefineUsingDistortion()

in Extended/libwebp/src/enc/quant_enc.c [1223:1334]

• 95 lines of code
• 23 McCabe index (conditional complexity)

static void RefineUsingDistortion(VP8EncIterator* const it,
                                  int try_both_modes, int refine_uv_mode,
                                  VP8ModeScore* const rd) {
  score_t best_score = MAX_COST;
  int nz = 0;
  int mode;
  int is_i16 = try_both_modes || (it->mb_->type_ == 1);

  const VP8SegmentInfo* const dqm = &it->enc_->dqm_[it->mb_->segment_];
  // Some empiric constants, of approximate order of magnitude.
  const int lambda_d_i16 = 106;
  const int lambda_d_i4 = 11;
  const int lambda_d_uv = 120;
  score_t score_i4 = dqm->i4_penalty_;
  score_t i4_bit_sum = 0;
  const score_t bit_limit = try_both_modes ? it->enc_->mb_header_limit_
                                           : MAX_COST;  // no early-out allowed

  if (is_i16) {   // First, evaluate Intra16 distortion
    int best_mode = -1;
    const uint8_t* const src = it->yuv_in_ + Y_OFF_ENC;
    for (mode = 0; mode < NUM_PRED_MODES; ++mode) {
      const uint8_t* const ref = it->yuv_p_ + VP8I16ModeOffsets[mode];
      const score_t score = (score_t)VP8SSE16x16(src, ref) * RD_DISTO_MULT
                          + VP8FixedCostsI16[mode] * lambda_d_i16;
      if (mode > 0 && VP8FixedCostsI16[mode] > bit_limit) {
        continue;
      }

      if (score < best_score) {
        best_mode = mode;
        best_score = score;
      }
    }
    if (it->x_ == 0 || it->y_ == 0) {
      // avoid starting a checkerboard resonance from the border. See bug #432.
      if (IsFlatSource16(src)) {
        best_mode = (it->x_ == 0) ? 0 : 2;
        try_both_modes = 0;  // stick to i16
      }
    }
    VP8SetIntra16Mode(it, best_mode);
    // we'll reconstruct later, if i16 mode actually gets selected
  }

  // Next, evaluate Intra4
  if (try_both_modes || !is_i16) {
    // We don't evaluate the rate here, but just account for it through a
    // constant penalty (i4 mode usually needs more bits compared to i16).
    is_i16 = 0;
    VP8IteratorStartI4(it);
    do {
      int best_i4_mode = -1;
      score_t best_i4_score = MAX_COST;
      const uint8_t* const src = it->yuv_in_ + Y_OFF_ENC + VP8Scan[it->i4_];
      const uint16_t* const mode_costs = GetCostModeI4(it, rd->modes_i4);

      VP8MakeIntra4Preds(it);
      for (mode = 0; mode < NUM_BMODES; ++mode) {
        const uint8_t* const ref = it->yuv_p_ + VP8I4ModeOffsets[mode];
        const score_t score = VP8SSE4x4(src, ref) * RD_DISTO_MULT
                            + mode_costs[mode] * lambda_d_i4;
        if (score < best_i4_score) {
          best_i4_mode = mode;
          best_i4_score = score;
        }
      }
      i4_bit_sum += mode_costs[best_i4_mode];
      rd->modes_i4[it->i4_] = best_i4_mode;
      score_i4 += best_i4_score;
      if (score_i4 >= best_score || i4_bit_sum > bit_limit) {
        // Intra4 won't be better than Intra16. Bail out and pick Intra16.
        is_i16 = 1;
        break;
      } else {  // reconstruct partial block inside yuv_out2_ buffer
        uint8_t* const tmp_dst = it->yuv_out2_ + Y_OFF_ENC + VP8Scan[it->i4_];
        nz |= ReconstructIntra4(it, rd->y_ac_levels[it->i4_],
                                src, tmp_dst, best_i4_mode) << it->i4_;
      }
    } while (VP8IteratorRotateI4(it, it->yuv_out2_ + Y_OFF_ENC));
  }

  // Final reconstruction, depending on which mode is selected.
  if (!is_i16) {
    VP8SetIntra4Mode(it, rd->modes_i4);
    SwapOut(it);
    best_score = score_i4;
  } else {
    nz = ReconstructIntra16(it, rd, it->yuv_out_ + Y_OFF_ENC, it->preds_[0]);
  }

  // ... and UV!
  if (refine_uv_mode) {
    int best_mode = -1;
    score_t best_uv_score = MAX_COST;
    const uint8_t* const src = it->yuv_in_ + U_OFF_ENC;
    for (mode = 0; mode < NUM_PRED_MODES; ++mode) {
      const uint8_t* const ref = it->yuv_p_ + VP8UVModeOffsets[mode];
      const score_t score = VP8SSE16x8(src, ref) * RD_DISTO_MULT
                          + VP8FixedCostsUV[mode] * lambda_d_uv;
      if (score < best_uv_score) {
        best_mode = mode;
        best_uv_score = score;
      }
    }
    VP8SetIntraUVMode(it, best_mode);
  }
  nz |= ReconstructUV(it, rd, it->yuv_out_ + U_OFF_ENC, it->mb_->uv_mode_);

  rd->nz = nz;
  rd->score = best_score;
}