Downloading Images and Media in Bulk

Stream downloads

download_stream.py

from pathlib import Path
import requests
 
 
def download(url: str, out: Path):
    out.parent.mkdir(parents=True, exist_ok=True)
    with requests.get(url, stream=True, timeout=20) as r:
        r.raise_for_status()
        with open(out, "wb") as f:
            for chunk in r.iter_content(chunk_size=1024 * 64):
                if chunk:
                    f.write(chunk)
 
 
download("https://httpbin.org/image/png", Path("downloads/image.png"))

Tips

• validate content-type
• handle name collisions
• avoid downloading too fast