in src/rust/engine/src/nodes.rs [700:795]
fn download(
core: Arc<Core>,
url: Url,
file_name: String,
expected_digest: hashing::Digest,
) -> BoxFuture<(), String> {
// TODO: Retry failures
core
.http_client()
.get(url.clone())
.send()
.map_err(|err| format!("Error downloading file: {}", err))
.and_then(move |response| {
// Handle common HTTP errors.
if response.status().is_server_error() {
Err(format!(
"Server error ({}) downloading file {} from {}",
response.status().as_str(),
file_name,
url,
))
} else if response.status().is_client_error() {
Err(format!(
"Client error ({}) downloading file {} from {}",
response.status().as_str(),
file_name,
url,
))
} else {
Ok(response)
}
})
.and_then(move |response| {
struct SizeLimiter<W: std::io::Write> {
writer: W,
written: usize,
size_limit: usize,
}
impl<W: std::io::Write> Write for SizeLimiter<W> {
fn write(&mut self, buf: &[u8]) -> Result<usize, std::io::Error> {
let new_size = self.written + buf.len();
if new_size > self.size_limit {
Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"Downloaded file was larger than expected digest",
))
} else {
self.written = new_size;
self.writer.write_all(buf)?;
Ok(buf.len())
}
}
fn flush(&mut self) -> Result<(), std::io::Error> {
self.writer.flush()
}
}
let hasher = hashing::WriterHasher::new(SizeLimiter {
writer: bytes::BytesMut::with_capacity(expected_digest.1).writer(),
written: 0,
size_limit: expected_digest.1,
});
response
.into_body()
.map_err(|err| format!("Error reading URL fetch response: {}", err))
.fold(hasher, |mut hasher, chunk| {
hasher
.write_all(&chunk)
.map(|_| hasher)
.map_err(|err| format!("Error hashing/writing URL fetch response: {}", err))
})
.map(|hasher| {
let (digest, bytewriter) = hasher.finish();
(digest, bytewriter.writer.into_inner().freeze())
})
})
.and_then(move |(actual_digest, buf)| {
if expected_digest != actual_digest {
return future::err(format!(
"Wrong digest for downloaded file: want {:?} got {:?}",
expected_digest, actual_digest
))
.to_boxed();
}
core
.store()
.store_file_bytes(buf, true)
.map(|_| ())
.to_boxed()
})
.to_boxed()
}