How to scrape Amazon product data in Python, and what a proxy changes

import os
import re
import time
import uuid

import requests
from bs4 import BeautifulSoup

USER = os.environ["PROXY_USER"]
PASS = os.environ["PROXY_PASS"]

HEADERS = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
                  "(KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}


def proxy(country: str = "us", city: str = "") -> dict:
    # A new session label per call = a new exit IP per request (sticky port).
    user = f"{USER}-country-{country.lower()}"
    if city:
        user += f"-city-{city.lower()}"
    url = f"http://{user}-session-{uuid.uuid4().hex[:8]}-lifetime-3:{PASS}@via.proxies.vision:10000"
    return {"http": url, "https": url}


def fetch(asin: str, country: str = "us", city: str = "", attempts: int = 3) -> str | None:
    for attempt in range(attempts):
        try:
            r = requests.get(f"https://www.amazon.com/dp/{asin}", headers=HEADERS,
                             proxies=proxy(country, city), timeout=40)
        except requests.RequestException:
            time.sleep(attempt + 1)  # a dropped exit: try the next one
            continue
        if r.status_code == 404:
            return None  # the product doesn't exist
        if r.status_code == 200 and 'id="productTitle"' in r.text:
            return r.text
        time.sleep(2 * (attempt + 1))  # a 200 without a product is not a success
    return None


def parse(html: str) -> dict:
    soup = BeautifulSoup(html, "lxml")

    def text(selector: str) -> str:
        el = soup.select_one(selector)
        return el.get_text(" ", strip=True) if el else ""

    rating = re.search(r"([\d.]+) out of", text("#acrPopover .a-icon-alt"))
    return {
        "title": text("#productTitle"),
        "price": text("#corePrice_feature_div .a-offscreen"),
        "rating": rating.group(1) if rating else "",
        "reviews": re.sub(r"\D", "", text("#acrCustomerReviewText")),
        "availability": text("#availability"),
        "location": text("#glow-ingress-block").replace("Update location", "").strip(),
    }


if __name__ == "__main__":
    for country, city in [("us", ""), ("us", "chicago"), ("de", "")]:
        html = fetch("B09B8V1LZ3", country, city)
        print(country, city, parse(html) if html else "no product page")
        time.sleep(2)