Download and upload

Recipe	Crates	Categories
Download a file to a temporary directory
Make a partial download with HTTP range headers
POST a file to paste.rs

Download a file to a temporary directory

Creates a temporary directory with tempfile::Builder⮳ and downloads a file over HTTP using reqwest::get⮳ asynchronously.

Creates a target std::fs::File⮳ with name obtained from reqwest::Response::url⮳ within tempfile::Builder::tempdir⮳ and copies downloaded data into it with std::io::copy⮳. The temporary directory is automatically removed on program exit.

use std::fs::File;
use std::io::copy;

use anyhow::Result;
use tempfile::Builder;

#[tokio::main]
async fn main() -> Result<()> {
    let tmp_dir = Builder::new().prefix("example").tempdir()?;
    let target = "https://www.rust-lang.org/logos/rust-logo-512x512.png";
    let response = reqwest::get(target).await?;

    let mut dest = {
        let fname = response
            .url()
            .path_segments()
            .and_then(|segments| segments.last())
            .and_then(|name| if name.is_empty() { None } else { Some(name) })
            .unwrap_or("tmp.bin");

        println!("file to download: '{}'", fname);
        let fname = tmp_dir.path().join(fname);
        println!("will be located under: '{:?}'", fname);
        File::create(fname)?
    };
    let content = response.text().await?;
    copy(&mut content.as_bytes(), &mut dest)?;
    Ok(())
}

POST a file to paste.rs

reqwest::Client⮳ establishes a connection to https://paste.rs following the reqwest::RequestBuilder⮳ pattern. Calling reqwest::Client::post⮳ with a URL establishes the destination, reqwest::RequestBuilder::body⮳ sets the content to send by reading the file, and reqwest::RequestBuilder::send⮳ blocks until the file uploads and the response returns. std::io::Read::read_to_string⮳ returns the response and displays in the console.

use std::fs::File;
use std::io::Read;

use anyhow::Result;

#[tokio::main]
async fn main() -> Result<()> {
    let mut file = File::open("temp/message")?;
    let mut contents = String::new();
    file.read_to_string(&mut contents)?;

    let paste_api = "https://paste.rs";
    let client = reqwest::Client::new();
    let res = client.post(paste_api).body(contents).send().await?;
    let response_text = res.text().await?;
    println!("Your paste is located at: {}", response_text);
    Ok(())
}

Make a partial download with HTTP range headers

Uses reqwest::blocking::Client::head⮳ to get the Content-Length⮳ of the response.

The code then uses reqwest::blocking::Client::get⮳ to download the content in chunks of 10240 bytes, while printing progress messages. This exmple uses the synchronous reqwest module. The Range⮳ header specifies the chunk size and position.

The Range header is defined in RFC7233⮳.

use std::fs::File;
use std::str::FromStr;

use anyhow::Result;
use anyhow::anyhow;
use anyhow::bail;
use reqwest::StatusCode;
use reqwest::header::CONTENT_LENGTH;
use reqwest::header::HeaderValue;
use reqwest::header::RANGE;

struct PartialRangeIter {
    start: u64,
    end: u64,
    buffer_size: u32,
}

impl PartialRangeIter {
    pub fn new(start: u64, end: u64, buffer_size: u32) -> Result<Self> {
        if buffer_size == 0 {
            Err(anyhow!(
                "invalid buffer_size, give a value greater than zero."
            ))?;
        }
        Ok(PartialRangeIter {
            start,
            end,
            buffer_size,
        })
    }
}

impl Iterator for PartialRangeIter {
    type Item = HeaderValue;

    fn next(&mut self) -> Option<Self::Item> {
        if self.start > self.end {
            None
        } else {
            let prev_start = self.start;
            self.start += std::cmp::min(
                self.buffer_size as u64,
                self.end - self.start + 1,
            );
            Some(
                HeaderValue::from_str(&format!(
                    "bytes={}-{}",
                    prev_start,
                    self.start - 1
                ))
                .expect("string provided by format!"),
            )
        }
    }
}

fn main() -> Result<()> {
    let url = "https://httpbin.org/range/102400?duration=2";
    const CHUNK_SIZE: u32 = 10240;

    let client = reqwest::blocking::Client::new();
    let response = client.head(url).send()?;
    let length = response
        .headers()
        .get(CONTENT_LENGTH)
        .ok_or(anyhow!("response doesn't include the content length"))?;
    let length = u64::from_str(length.to_str()?)
        .map_err(|_| anyhow!("invalid Content-Length header"))?;

    if !std::fs::exists("temp")? {
        std::fs::create_dir("temp")?;
    }
    let mut output_file = File::create("temp/download.bin")?;

    println!("starting download...");
    for range in PartialRangeIter::new(0, length - 1, CHUNK_SIZE)? {
        println!("range {:?}", range);
        let mut response = client.get(url).header(RANGE, range).send()?;

        let status = response.status();
        if !(status == StatusCode::OK || status == StatusCode::PARTIAL_CONTENT)
        {
            bail!("Unexpected server response: {}", status)
        }
        std::io::copy(&mut response, &mut output_file)?;
    }

    let content = response.text()?;
    std::io::copy(&mut content.as_bytes(), &mut output_file)?;

    println!("Finished with success!");
    Ok(())
}