fn download()

in src/rust/engine/src/nodes.rs [700:795]


  fn download(
    core: Arc<Core>,
    url: Url,
    file_name: String,
    expected_digest: hashing::Digest,
  ) -> BoxFuture<(), String> {
    // TODO: Retry failures
    core
      .http_client()
      .get(url.clone())
      .send()
      .map_err(|err| format!("Error downloading file: {}", err))
      .and_then(move |response| {
        // Handle common HTTP errors.
        if response.status().is_server_error() {
          Err(format!(
            "Server error ({}) downloading file {} from {}",
            response.status().as_str(),
            file_name,
            url,
          ))
        } else if response.status().is_client_error() {
          Err(format!(
            "Client error ({}) downloading file {} from {}",
            response.status().as_str(),
            file_name,
            url,
          ))
        } else {
          Ok(response)
        }
      })
      .and_then(move |response| {
        struct SizeLimiter<W: std::io::Write> {
          writer: W,
          written: usize,
          size_limit: usize,
        }

        impl<W: std::io::Write> Write for SizeLimiter<W> {
          fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
            let new_size = self.written + buf.len();
            if new_size > self.size_limit {
              Err(std::io::Error::new(
                std::io::ErrorKind::InvalidData,
                "Downloaded file was larger than expected digest",
              ))
            } else {
              self.written = new_size;
              self.writer.write_all(buf)?;
              Ok(buf.len())
            }
          }

          fn flush(&mut self) -> Result<(), std::io::Error> {
            self.writer.flush()
          }
        }

        let hasher = hashing::WriterHasher::new(SizeLimiter {
          writer: bytes::BytesMut::with_capacity(expected_digest.1).writer(),
          written: 0,
          size_limit: expected_digest.1,
        });

        response
          .into_body()
          .map_err(|err| format!("Error reading URL fetch response: {}", err))
          .fold(hasher, |mut hasher, chunk| {
            hasher
              .write_all(&chunk)
              .map(|_| hasher)
              .map_err(|err| format!("Error hashing/writing URL fetch response: {}", err))
          })
          .map(|hasher| {
            let (digest, bytewriter) = hasher.finish();
            (digest, bytewriter.writer.into_inner().freeze())
          })
      })
      .and_then(move |(actual_digest, buf)| {
        if expected_digest != actual_digest {
          return future::err(format!(
            "Wrong digest for downloaded file: want {:?} got {:?}",
            expected_digest, actual_digest
          ))
          .to_boxed();
        }

        core
          .store()
          .store_file_bytes(buf, true)
          .map(|_| ())
          .to_boxed()
      })
      .to_boxed()
  }