From 1bf6d4efd81334316cff86eab50e074402b0c254 Mon Sep 17 00:00:00 2001 From: b3nw Date: Sat, 7 Sep 2024 19:12:56 -0500 Subject: [PATCH] clean up curl usage, add consistent debug printing across ovpn_downloader --- nord-checker.py | 12 +++++----- ovpn_downloader.py | 56 ++++++++++++++++++++++++++++------------------ 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/nord-checker.py b/nord-checker.py index 2082e74..78f6d4d 100644 --- a/nord-checker.py +++ b/nord-checker.py @@ -22,7 +22,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "-d": def debug_print(message): if DEBUG_MODE: - print(message) + print(f"{datetime.datetime.now()} + [Debug Mode]:{message}") + else: + log_file.flush() # Open the log file in append mode if not in debug mode log_file = None # Define log_file in the global scope @@ -62,7 +64,7 @@ conn, cursor = connect_to_database() # Download and extract OVPN configurations, and get the list of files to check debug_print(f"{datetime.datetime.now()} [Main Script]: Downloading and extracting OVPN configurations...") -ovpn_files_to_check = download_and_extract_ovpn_configs(cursor, conn) +ovpn_files_to_check = download_and_extract_ovpn_configs(cursor, conn, debug_print=debug_print) total_files = len(ovpn_files_to_check) @@ -77,13 +79,13 @@ for OVPN_FILE in ovpn_files_to_check: debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Processing OVPN file...") # Check if the OVPN file exists, download if needed - file_exists = download_ovpn_if_needed(OVPN_FILENAME, cursor, conn) + file_exists = download_ovpn_if_needed(OVPN_FILENAME, cursor, conn, debug_print=debug_print) if not file_exists: print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Skipping, file not found and removed from database.") FILE_NUM += 1 - continue - + continue + # Read credentials with open(CREDENTIAL_FILE, 'r') as f: username = f.readline().strip() diff --git a/ovpn_downloader.py b/ovpn_downloader.py index 257d2a3..3018a8c 100644 --- a/ovpn_downloader.py +++ b/ovpn_downloader.py @@ -1,5 +1,5 @@ import os -import subprocess +import zipfile import random import datetime import requests @@ -8,7 +8,7 @@ OVPN_DIR = "ovpn_configs" OVPN_ZIP_URL = "https://downloads.nordcdn.com/configs/archives/servers/ovpn.zip" OVPN_LEGACY_URL = "https://downloads.nordcdn.com/configs/files/ovpn_legacy/servers/" -def download_and_extract_ovpn_configs(cursor, conn): +def download_and_extract_ovpn_configs(cursor, conn, debug_print=None): """ Downloads the OVPN ZIP archive, extracts its contents, adds new entries to the database, and cleans up. @@ -18,22 +18,34 @@ def download_and_extract_ovpn_configs(cursor, conn): if not os.path.exists(OVPN_DIR): os.makedirs(OVPN_DIR) - subprocess.run(["curl", "-s", "-L", "-o", f"{OVPN_DIR}/ovpn.zip", OVPN_ZIP_URL]) - subprocess.run(["unzip", "-q", "-o", f"{OVPN_DIR}/ovpn.zip", "-d", OVPN_DIR]) - os.remove(f"{OVPN_DIR}/ovpn.zip") + zip_file_path = os.path.join(OVPN_DIR, "ovpn.zip") - # Find all OVPN files - ovpn_files = [f for f in os.listdir(OVPN_DIR) if f.endswith(".ovpn")] + try: + # Download the ZIP file using requests + debug_print(f"[Ovpn Downloader]: Downloading {OVPN_ZIP_URL}") + response = requests.get(OVPN_ZIP_URL, stream=True) + response.raise_for_status() # Raise an exception for bad status codes - # Add new entries to the database - for filename in ovpn_files: - cursor.execute("SELECT 1 FROM ovpn_files WHERE file_name = %s", (filename,)) - if cursor.fetchone() is None: # If no record found, insert a new one - cursor.execute(""" - INSERT INTO ovpn_files (file_name, last_observed) - VALUES (%s, CURRENT_TIMESTAMP) - """, (filename,)) - conn.commit() + with open(zip_file_path, 'wb') as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + + debug_print(f"[Ovpn Downloader]: Download complete. Extracting...") + + # Extract the ZIP file + with zipfile.ZipFile(zip_file_path, 'r') as zip_ref: + zip_ref.extractall(OVPN_DIR) + + debug_print(f"[Ovpn Downloader]: Extraction complete.") + + except requests.exceptions.RequestException as e: + debug_print(f"[Ovpn Downloader]: Error downloading or extracting: {e}") + return [] # Return an empty list if there's an error + + finally: + # Clean up the ZIP file + if os.path.exists(zip_file_path): + os.remove(zip_file_path) # Fetch the list of VPN servers that need to be checked (last check older than 3 days) three_days_ago = datetime.datetime.now() - datetime.timedelta(days=3) @@ -42,7 +54,7 @@ def download_and_extract_ovpn_configs(cursor, conn): return ovpn_files_to_check -def download_ovpn_if_needed(filename, cursor, conn): +def download_ovpn_if_needed(filename, cursor, conn, debug_print=None): """ Downloads the specified OVPN file from the legacy URL if it doesn't exist in the OVPN_DIR. Removes the entry from the database if the file returns a 404 error. @@ -51,23 +63,23 @@ def download_ovpn_if_needed(filename, cursor, conn): filepath = os.path.join(OVPN_DIR+"/ovpn_tcp/", filename) if os.path.exists(filepath): - print(f"{filename} already exists. Skipping download.") + debug_print(f"[{filename}]: Skipping download, file already exists.") return True url = OVPN_LEGACY_URL + filename - print(f"Attempting to download {filename} from legacy URL...") + debug_print(f" [Ovpn Downloader]: Attempting to download {filename} from legacy URL...") response = requests.get(url) if response.status_code == 200: with open(filepath, 'wb') as f: f.write(response.content) - print(f"Downloaded {filename} from legacy URL.") + print(f"{datetime.datetime.now()} [Ovpn Downloader]: from legacy URL.") return True elif response.status_code == 404: - print(f"{filename} not found on legacy URL: {filepath}. Removing from database.") # this is the updated line + debug_print(f"[Ovpn Downloader]:{filename} not found on legacy URL: {filepath}. Removing from database.") # this is the updated line cursor.execute("DELETE FROM ovpn_files WHERE file_name = %s", (filename,)) conn.commit() return False else: - print(f"Error downloading {filename}: {response.status_code}") + debug_print(f"Error downloading {filename}: {response.status_code}") return False