From 025e4f3b31debc4b7dfc1fa9e1e969bf0c564346 Mon Sep 17 00:00:00 2001 From: TheBestJohn Date: Sat, 31 Jan 2026 16:52:58 -0500 Subject: [PATCH] Add environment-based config and parallel tile downloads - Use dotenv to load API key, regions, workers, and zoom levels from .env - Add concurrent downloads with ThreadPoolExecutor for faster processing - Organize tiles into per-region subdirectories - Skip already-downloaded tiles to support resume - Add .env_sample, .gitignore, and requirements.txt --- .env_sample | 11 ++++++ .gitignore | 2 ++ README.md | 32 +++++++----------- TileDL.py | 86 ++++++++++++++++++++++++++++++++++------------- requirements.txt | Bin 0 -> 586 bytes 5 files changed, 87 insertions(+), 44 deletions(-) create mode 100644 .env_sample create mode 100644 .gitignore create mode 100644 requirements.txt diff --git a/.env_sample b/.env_sample new file mode 100644 index 0000000..eb51521 --- /dev/null +++ b/.env_sample @@ -0,0 +1,11 @@ +API_KEY= + +# region_name:min_lat,min_lon,max_lat,max_lon; +REGIONS=" +southern_ontario:41.5, -83.5, 45.5, -75.0; +las_vegas:35.5, -116.0, 37.5, -114.0; +grand_canyon:35.5, -113.0, 37.0, -111.0 +" +WORKERS=10 +MINZOOM=1 +MAXZOOM=14 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fa1b177 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +maps/* +.env \ No newline at end of file diff --git a/README.md b/README.md index 838e792..db2ae2d 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ This Python script downloads map tiles from Thunderforest's Mobile Atlas for spe - Python 3.x - `requests` library - `tqdm` library +- `dotenv` library - For KMLtoTiles: `fastkml` library ## Installation @@ -30,7 +31,7 @@ This Python script downloads map tiles from Thunderforest's Mobile Atlas for spe 2. Install the required Python packages: ```bash - pip install requests tqdm fastkml + pip install -r requirements.txt ``` Note: fastkml is not required to use `TileDL.py` @@ -38,25 +39,19 @@ This Python script downloads map tiles from Thunderforest's Mobile Atlas for spe 1. Obtain an API key from [Thunderforest](https://www.thunderforest.com/docs/apikeys/). -2. Edit the script to include your API key: +2. copy .env_sample as .env and edit it to include your API key, desired regions, max workers, and zoom levels: - ```python - api_key = "your_api_key_here" - ``` - -3. Specify the regions and zoom levels you want to download in the script: - - ```python - # Define the bounding boxes and zoom levels - regions = { - "southern_ontario": (41.5, -83.5, 45.5, -75.0), - "las_vegas": (35.5, -116.0, 37.5, -114.0), - "grand_canyon": (35.5, -113.0, 37.0, -111.0) - } - zoom_levels = range(1, 15) # Focusing on zoom levels 1 to 14 + ```bash + API_KEY="your_api_key_here" + REGIONS="southern_ontario:41.5, -83.5, 45.5, -75.0; + las_vegas:35.5, -116.0, 37.5, -114.0; + grand_canyon:35.5, -113.0, 37.0, -111.0" + WORKERS=10 + MINZOOM=1 + MAXZOOM=14 ``` -4. Choose map style +2. Choose map style ```python # mapstyle = "cycle" @@ -70,9 +65,6 @@ This Python script downloads map tiles from Thunderforest's Mobile Atlas for spe # mapstyle = "neighbourhood" # mapstyle = "atlas" ``` - - - ## Usage (TileDL.py) diff --git a/TileDL.py b/TileDL.py index 0989f3e..bb994c0 100644 --- a/TileDL.py +++ b/TileDL.py @@ -1,15 +1,40 @@ import os +from dotenv import load_dotenv import requests from math import log, tan, cos, pi from tqdm import tqdm +from concurrent.futures import ThreadPoolExecutor, as_completed + +load_dotenv() + +def load_regions_from_env(): + regions_str = os.getenv("REGIONS", "") + regions = {} + if regions_str: + for region in regions_str.split(";"): + if region.strip(): + parts = region.split(":") + if len(parts) == 2: + name = parts[0].strip() + values = tuple(float(v.strip()) for v in parts[1].split(",")) + min_lat, min_lon, max_lat, max_lon = values + + # Ensure min values are actually less than max values + if min_lon > max_lon: + print(f"Warning: min_lon ({min_lon}) > max_lon ({max_lon}) for region {name}, swapping") + min_lon, max_lon = max_lon, min_lon + if min_lat > max_lat: + print(f"Warning: min_lat ({min_lat}) > max_lat ({max_lat}) for region {name}, swapping") + min_lat, max_lat = max_lat, min_lat + + regions[name] = (min_lat, min_lon, max_lat, max_lon) + return regions # Define the bounding boxes and zoom levels. Below are random examples. -regions = { - "southern_ontario": (41.5, -83.5, 45.5, -75.0), - "las_vegas": (35.5, -116.0, 37.5, -114.0), - "grand_canyon": (35.5, -113.0, 37.0, -111.0) -} -zoom_levels = range(1, 15) # Focusing on zoom levels 1 to 14 +regions = load_regions_from_env() +print(regions) +WORKERS=os.getenv("WORKERS", 10) +zoom_levels = range(int(os.getenv("MINZOOM", 1)), int(os.getenv("MAXZOOM", 14))+1) # Defaults to zoom levels 1 to 14 # mapstyle = "cycle" # mapstyle = "transport" @@ -23,8 +48,8 @@ # mapstyle = "atlas" # API Key and output directory -api_key = "your_api_key_here" -output_dir = os.path.join(os.path.expanduser("~"), "Desktop", "tiles") +api_key = os.getenv("API_KEY") +output_dir = "./maps" os.makedirs(output_dir, exist_ok=True) def lon2tilex(lon, zoom): @@ -33,43 +58,56 @@ def lon2tilex(lon, zoom): def lat2tiley(lat, zoom): return int((1.0 - log(tan(lat * pi / 180.0) + 1.0 / cos(lat * pi / 180.0)) / pi) / 2.0 * (1 << zoom)) -def download_tile(zoom, x, y): +def download_tile(zoom, region_name, x, y): url = f"https://tile.thunderforest.com/{mapstyle}/{zoom}/{x}/{y}.png?apikey={api_key}" - tile_dir = os.path.join(output_dir, str(zoom), str(x)) + tile_dir = os.path.join(output_dir, region_name, str(zoom), str(x)) tile_path = os.path.join(tile_dir, f"{y}.png") + + if os.path.exists(tile_path): + return + os.makedirs(tile_dir, exist_ok=True) - if not os.path.exists(tile_path): - response = requests.get(url) - if response.status_code == 200: - with open(tile_path, "wb") as file: - file.write(response.content) - else: - print(f"Failed to download tile {zoom}/{x}/{y}: {response.status_code} {response.reason}") + response = requests.get(url) + if response.status_code == 200: + with open(tile_path, "wb") as file: + file.write(response.content) + else: + print(f"Failed to download tile {zoom}/{x}/{y}: {response.status_code} {response.reason}") def main(): total_tiles = 0 for zoom in zoom_levels: - for min_lat, min_lon, max_lat, max_lon in regions.values(): + for region_name, (min_lat, min_lon, max_lat, max_lon) in regions.items(): start_x = lon2tilex(min_lon, zoom) end_x = lon2tilex(max_lon, zoom) start_y = lat2tiley(max_lat, zoom) end_y = lat2tiley(min_lat, zoom) - +# + print(zoom) + print(f"folders={end_x-start_x +1} tiles/folder = {end_y-start_y +1}" ) + print(f"{(end_x - start_x + 1) * (end_y - start_y + 1)} Tiles at Zoom level {zoom}") total_tiles += (end_x - start_x + 1) * (end_y - start_y + 1) + print(total_tiles, "total tiles") + + with tqdm(total=total_tiles, desc="Downloading tiles") as pbar: for zoom in zoom_levels: - for min_lat, min_lon, max_lat, max_lon in regions.values(): + for region_name, (min_lat, min_lon, max_lat, max_lon) in regions.items(): start_x = lon2tilex(min_lon, zoom) end_x = lon2tilex(max_lon, zoom) start_y = lat2tiley(max_lat, zoom) end_y = lat2tiley(min_lat, zoom) - - for x in range(start_x, end_x + 1): - for y in range(start_y, end_y + 1): - download_tile(zoom, x, y) + + tiles = [(x, y) for x in range(start_x, end_x + 1) for y in range(start_y, end_y + 1)] + + with ThreadPoolExecutor(max_workers=WORKERS) as executor: + futures = {executor.submit(download_tile, zoom, region_name, x, y): (x, y) for x, y in tiles} + for future in as_completed(futures): + x, y = futures[future] + pbar.set_description(f"📁:{x - start_x + 1}/{end_x-start_x +1} 🖼️:{y - start_y + 1}/{end_y-start_y +1} of 🔍:{zoom}") pbar.update(1) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..0569aaf940faab99464fc446a5b688ed930afe1b GIT binary patch literal 586 zcmZuu+fKqj6r5)gKSe@YsPVyXF~x#xtlQcyh{sK6^k*Wn+jo7aZvZ{NSCgy|ub_RwM5$ zJYh#XvP(+`W!Uo@@}4s1P@aK2rTAZapI`Z;^&BHzzv#O${bV_XR>oOb)`>fhw{8WV zi2J5QZOi7$ulK4AD@~QyV4yOx=PUh>)*y0EuvWfHssgJ&j;Gw#|CztE@XB2Ge5sBT Nu;z{SDy{yT