In this Python code, the download_perpetual_usdt_trades function iterates over the given trading symbols and downloads the corresponding trade data for the specified date range.
import os, requests, hashlib, time
from datetime import datetime, timedelta
from pathlib import Path
from tqdm.auto import tqdm
from rich import print
incomplete_checksum_files = []
def md5sum(file_data: bytes, file_size: int) -> tuple[str, float]:
m = hashlib.md5()
with tqdm(total=file_size, unit="B", unit_scale=True, desc="Checksum validation") as progress_bar:
for i in range(0, file_size, 1024):
chunk = file_data[i:i + 1024]
m.update(chunk)
progress_bar.update(len(chunk))
progress_percentage = (progress_bar.n / progress_bar.total) * 100
return m.hexdigest(), progress_percentage
def download_file(url: str, local_folder: str, max_retries: int = 100, retry_delay: int = 60, retry_increment: int = 15) -> None:
retry_count = 0
current_retry_delay = retry_delay
while retry_count < max_retries:
try:
response = requests.get(url, stream=True)
if response.status_code == 404:
print(f"[yellow]File not found:[/yellow] [red1]{url}[/red1]")
return
file_size = int(response.headers.get("Content-Length", 0))
filename = Path(local_folder) / url.split("/")[-1]
checksum_url = url + ".CHECKSUM"
checksum_filename = Path(local_folder) / checksum_url.split("/")[-1]
expected_checksum = None
if checksum_filename.exists():
with open(checksum_filename, "r") as f:
expected_checksum = f.read().strip()
if filename.exists():
with open(filename, "rb") as f:
file_data = f.read()
actual_checksum, progress_percentage = md5sum(file_data, file_size)
if progress_percentage < 100:
print(f"[yellow]Checksum progress not reaching 100% for file:[/yellow] [red1]{filename}[/red1]")
incomplete_checksum_files.append(filename)
if actual_checksum == expected_checksum:
print(f"[green]Local copy found for[/green] {filename}")
return
else:
print(f"Checksum failed for {filename}, redownloading...")
if not filename.parent.exists():
filename.parent.mkdir(parents=True, exist_ok=True)
print(f"[bold green]filename.parent:[/bold green] [red]{filename.parent}[/red] [bold green]has been created![/bold green]")
with open(filename, "wb") as f, tqdm(
total=file_size, unit="B", unit_scale=True, desc=filename.name
) as progress_bar:
for data in response.iter_content(chunk_size=1024):
f.write(data)
progress_bar.update(len(data))
with open(filename, "rb") as f:
file_data = f.read()
actual_checksum, progress_percentage = md5sum(file_data, file_size)
if progress_percentage < 100:
print(f"[yellow]Checksum progress not reaching 100% for file:[/yellow] [red1]{filename}[/red1]")
incomplete_checksum_files.append(filename)
with open(checksum_filename, "w") as f:
f.write(actual_checksum)
if expected_checksum is not None and actual_checksum != expected_checksum:
filename.unlink()
checksum_filename.unlink()
print(f"Checksum failed for {filename}, redownloading...")
time.sleep(5)
download_file(url, local_folder)
else:
print(f"{filename.name} downloaded and validated against its checksum.")
return # Download completed successfully, exit the loop
except requests.exceptions.ConnectionError as e:
retry_count += 1
if retry_count < max_retries:
print(f"Connection error: {e}. Retrying in {current_retry_delay} seconds... (attempt {retry_count}/{max_retries})")
time.sleep(current_retry_delay)
current_retry_delay += retry_increment
else:
print(f"Connection error: {e}. Maximum retries ({max_retries}) reached. Aborting download.")
break
def download_perpetual_usdt_trades_daily(
symbols: list[str], start_year: int, start_month: int, start_day: int, end_year: int, end_month: int, end_day: int, local_folder: str, daily: bool, base_asset: str = "USDT",
specific_files: list[Path] = None
) -> None:
start_date = datetime(start_year, start_month, start_day)
end_date = datetime(end_year, end_month, end_day)
if not Path(local_folder).exists():
Path(local_folder).mkdir(parents=True)
time_unit = "daily" if daily else "monthly"
date_format = "%Y-%m-%d" if daily else "%Y-%m"
for symbol in symbols:
while start_date <= end_date:
date_str = start_date.strftime(date_format)
url = f"https://data.binance.vision/data/futures/um/{time_unit}/trades/{symbol}{base_asset}/{symbol}{base_asset}-trades-{date_str}.zip"
if specific_files and filename not in specific_files:
# Skip files not in the specific_files list
continue
download_file(url, Path(local_folder) / f"{symbol}{base_asset}")
start_date += timedelta(days=1 if daily else 32)
if daily:
start_date = start_date.replace(day=start_date.day)
else:
start_date = start_date.replace(day=1)
start_date = datetime(start_year, start_month, start_day)
def remove_incomplete_files(file_list: list[Path]) -> None:
for file_path in file_list:
md5_file_path = file_path.with_suffix(file_path.suffix + ".CHECKSUM")
if file_path.exists():
file_path.unlink()
print(f"[red]Deleted[/red] {file_path}")
if md5_file_path.exists():
md5_file_path.unlink()
print(f"[red]Deleted[/red] {md5_file_path}")
if __name__ == "__main__":
symbols = 'ETH BNB BTC'.split()
symbols = 'ETH'.split()
start_year = 2019
start_month = 12
start_day = 1
end_year = 2023
end_month = 5
end_day = 5
local_folder_monthly = r"D:/binance_raw_trade_data/data/futures/um/monthly/trades"
local_folder_daily = r"D:/binance_raw_trade_data/data/futures/um/daily/trades"
# Set daily to True for daily data, False for monthly data
daily = True
# daily = False
if daily:
download_perpetual_usdt_trades_daily(symbols, start_year, start_month, start_day, end_year, end_month, end_day, local_folder_daily, daily)
else:
download_perpetual_usdt_trades_daily(symbols, start_year, start_month, start_day, end_year, end_month, end_day, local_folder_monthly, daily)
if incomplete_checksum_files:
print("\nFiles that never reached 100% checksum progress:")
for file_path in incomplete_checksum_files:
print(file_path)
print("\nDeleting incomplete files and their corresponding MD5 files...")
remove_incomplete_files(incomplete_checksum_files)
print("\nRe-downloading deleted files...")
if daily:
download_perpetual_usdt_trades_daily(symbols, start_year, start_month, start_day, end_year, end_month, end_day, local_folder_daily, daily, specific_files=incomplete_checksum_files)
else:
download_perpetual_usdt_trades_daily(symbols, start_year, start_month, start_day, end_year, end_month, end_day, local_folder_monthly, daily, specific_files=incomplete_checksum_files)