From c7bd23977c121c4b0b4b1f8e64e156a7cb5432ff Mon Sep 17 00:00:00 2001 From: b3nw Date: Thu, 5 Sep 2024 19:39:15 -0500 Subject: [PATCH] refactor sql, logging, failure mgmt --- .gitignore | 2 +- nord-checker.py | 167 ++++++++++++++++++++++++++++----------------- openvpn_manager.py | 2 +- ovpn_downloader.py | 41 +++++++++++ 4 files changed, 146 insertions(+), 66 deletions(-) create mode 100644 ovpn_downloader.py diff --git a/.gitignore b/.gitignore index d39a14f..521a7b4 100644 --- a/.gitignore +++ b/.gitignore @@ -162,5 +162,5 @@ cython_debug/ # env/configs *.creds -ovpn_* +ovpn_configs vpnlist.txt \ No newline at end of file diff --git a/nord-checker.py b/nord-checker.py index 747d77c..e9164d6 100644 --- a/nord-checker.py +++ b/nord-checker.py @@ -5,7 +5,7 @@ import time import datetime import re import sys -from psql_utils import connect_to_database, should_skip_file, update_or_insert_ip +from psql_utils import connect_to_database, update_or_insert_ip from ovpn_downloader import download_and_extract_ovpn_configs from openvpn_manager import establish_vpn_connection, is_vpn_active, disconnect_vpn, get_external_ip @@ -24,9 +24,26 @@ def debug_print(message): if DEBUG_MODE: print(message) -# Get current IP before starting -CURRENT_IP = subprocess.check_output(["curl", "-s", "ifconfig.me"]).decode().strip() -print(f"{datetime.datetime.now()} [Main Script]: Current IP before VPN connections: {CURRENT_IP}") +# Open the log file in append mode if not in debug mode +log_file = None # Define log_file in the global scope +if not DEBUG_MODE: + log_file = open('/root/nordvpn/nord-checker.log', 'a') + # Redirect stdout and stderr to the log file + sys.stdout = log_file + sys.stderr = log_file + + +# Get current IP before starting (using get_external_ip) +CURRENT_IP = get_external_ip() +if CURRENT_IP is None: + print(f"{datetime.datetime.now()} [Main Script]: Unable to get current IP. Exiting.") + if not DEBUG_MODE: + log_file.close() # Close the log file if it was opened + sys.exit(1) # Exit if unable to get current IP +else: + print(f"{datetime.datetime.now()} [Main Script]: Current IP before VPN connections: {CURRENT_IP}") + if not DEBUG_MODE: + log_file.flush() # Terminate any existing OpenVPN connections try: @@ -35,84 +52,109 @@ try: for pid in pids: subprocess.run(["kill", pid]) print(f"{datetime.datetime.now()} [Main Script]: Existing OpenVPN connections terminated.") + if not DEBUG_MODE: + log_file.flush() except subprocess.CalledProcessError: pass # No openvpn process found -# Download and extract OVPN configurations (using the function from ovpn_downloader.py) -debug_print(f"{datetime.datetime.now()} [Main Script]: Downloading and extracting OVPN configurations...") -ovpn_files = download_and_extract_ovpn_configs() - -total_files = len(ovpn_files) - -debug_print(f"{datetime.datetime.now()} [Main Script]: Found {total_files} OVPN files to process.") -debug_print(ovpn_files) # Print the list of OVPN files - # Connect to PostgreSQL database conn, cursor = connect_to_database() +# Download and extract OVPN configurations, and get the list of files to check +debug_print(f"{datetime.datetime.now()} [Main Script]: Downloading and extracting OVPN configurations...") +ovpn_files_to_check = download_and_extract_ovpn_configs(cursor, conn) + +total_files = len(ovpn_files_to_check) + +debug_print(f"{datetime.datetime.now()} [Main Script]: Found {total_files} OVPN files to check.") +debug_print(ovpn_files_to_check) # Print the list of OVPN files + # Process each OVPN file FILE_NUM = 1 -for OVPN_FILE in ovpn_files: +for OVPN_FILE in ovpn_files_to_check: OVPN_FILENAME = os.path.basename(OVPN_FILE) debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Processing OVPN file...") - # Check if file exists in database and if last check is older than 3 days - skip_file = should_skip_file(cursor, OVPN_FILENAME) - debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: should_skip_file returned: {skip_file}") - - if skip_file: - print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Skipping, last check was less than 3 days ago.") - FILE_NUM += 1 - continue - # Read credentials with open(CREDENTIAL_FILE, 'r') as f: username = f.readline().strip() password = f.readline().strip() # Establish VPN connection (capture stdout and stderr) - openvpn_command = ["openvpn", "--config", f"{OVPN_DIR}/{OVPN_FILE}", - "--auth-user-pass", CREDENTIAL_FILE, "--daemon", - "--log-append", LOG_FILE, "--verb", "3"] + max_retries = 3 # Maximum number of retries + retry_count = 0 + while retry_count < max_retries: + openvpn_command = ["openvpn", "--config", f"{OVPN_DIR}/{OVPN_FILE}", + "--auth-user-pass", CREDENTIAL_FILE, "--daemon", + "--log-append", LOG_FILE, "--verb", "3"] - debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Executing OpenVPN command: {' '.join(openvpn_command)}") - process = subprocess.Popen(openvpn_command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = process.communicate(input=password.encode()) + debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Executing OpenVPN command: {' '.join(openvpn_command)}") + process = subprocess.Popen(openvpn_command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + stdout, stderr = process.communicate(input=password.encode()) - # Check for errors and print output - if process.returncode != 0: - print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Error starting OpenVPN connection:") - print(stderr.decode()) + # Check for errors and print output + if process.returncode != 0 or "AUTH_FAILED" in stderr.decode(): + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Error starting OpenVPN connection or authentication failed:") + print(stderr.decode()) + + # Explicitly disconnect VPN if there was an error or auth failure + disconnect_vpn() + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected (due to error or auth failure).") + + if retry_count < max_retries - 1: # Retry if not the last attempt + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Retrying connection in 10 seconds...") + time.sleep(10) + retry_count += 1 + else: + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Max retries reached. Skipping this OVPN file.") + break # Exit the retry loop if max retries reached + else: + break # Exit the retry loop if connection is successful # Wait for connection to establish (with backoff for auth failure) print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Waiting for VPN connection to establish...") max_attempts = 10 attempt = 0 backoff_time = 0 - while attempt < max_attempts: - try: - last_lines = subprocess.check_output(["tail", "-n", "10", LOG_FILE]).decode().strip() - if "Initialization Sequence Completed" in last_lines: + with open(LOG_FILE, 'r') as f: + while attempt < max_attempts: + line = f.readline() + if not line: + time.sleep(1) + attempt += 1 + continue + + if "Initialization Sequence Completed" in line: print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection established.") break - elif "AUTH: Received control message: AUTH_FAILED" in last_lines: - backoff_time += 1 # Increment backoff time - if backoff_time >= 30: # If backoff reaches 30 seconds, skip + elif "AUTH: Received control message: AUTH_FAILED" in line: + backoff_time += 1 + if backoff_time >= 30: print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Authentication failed. Skipping this OVPN file.") - continue # Skip to the next OVPN file - except subprocess.CalledProcessError: - pass - time.sleep(1) - attempt += 1 - else: - print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection failed to establish (timeout).") - continue + break - # Check if the VPN connection is active + if attempt == max_attempts or backoff_time >= 30: + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection failed to establish (timeout or auth failure).") + + # Disconnect VPN + disconnect_vpn() + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected.") + if not DEBUG_MODE: + log_file.flush() + # Wait for a random time between 1 and 4 seconds + sleep_time = random.randint(1, 4) + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Waiting for {sleep_time} seconds before the next connection...") + time.sleep(sleep_time) + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Resuming...") + if not DEBUG_MODE: + log_file.flush() + continue # Skip to the next OVPN file + +# Check if the VPN connection is active (using the function from openvpn_manager.py) try: subprocess.run(["pgrep", "-f", OVPN_FILENAME], check=True, capture_output=True) print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection established.") @@ -133,21 +175,17 @@ for OVPN_FILE in ovpn_files: print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection failed to establish.") # Disconnect VPN - try: - result = subprocess.run(["pgrep", "openvpn"], check=True, capture_output=True) - pids = result.stdout.decode().strip().split('\n') - for pid in pids: - subprocess.run(["kill", pid]) - print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected.") - except subprocess.CalledProcessError: - print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: No openvpn process found.") - + disconnect_vpn() + print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected.") + if not DEBUG_MODE: + log_file.flush() # Wait for a random time between 5 and 8 seconds sleep_time = random.randint(5, 8) print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Waiting for {sleep_time} seconds before the next connection...") time.sleep(sleep_time) print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Resuming...") - + if not DEBUG_MODE: + log_file.flush() # Increment file number FILE_NUM += 1 @@ -155,5 +193,6 @@ for OVPN_FILE in ovpn_files: cursor.close() conn.close() -# Close the log file -log_file.close() \ No newline at end of file +# Close the log file if it was opened +if not DEBUG_MODE: + log_file.close() \ No newline at end of file diff --git a/openvpn_manager.py b/openvpn_manager.py index 663c7e0..a238288 100644 --- a/openvpn_manager.py +++ b/openvpn_manager.py @@ -17,7 +17,7 @@ def establish_vpn_connection(ovpn_file, credential_file, log_file): # Establish VPN connection (capture stdout and stderr) openvpn_command = ["openvpn", "--config", ovpn_file, "--auth-user-pass", credential_file, "--daemon", - "--log-append", log_file, "--verb", "3"] + "--log-append", log_file, "--verb", "0", "--flush-log"] # Added --flush-log process = subprocess.Popen(openvpn_command, stdin=subprocess.PIPE, diff --git a/ovpn_downloader.py b/ovpn_downloader.py new file mode 100644 index 0000000..4b8a1fd --- /dev/null +++ b/ovpn_downloader.py @@ -0,0 +1,41 @@ +import os +import subprocess +import random +import datetime + +OVPN_DIR = "/root/nordvpn/ovpn_configs/ovpn_tcp" +OVPN_ZIP_URL = "https://downloads.nordcdn.com/configs/archives/servers/ovpn.zip" + +def download_and_extract_ovpn_configs(cursor, conn): + """ + Downloads the OVPN ZIP archive, extracts its contents, + adds new entries to the database, and cleans up. + Returns a list of OVPN filenames that need to be checked. + """ + + if not os.path.exists(OVPN_DIR): + os.makedirs(OVPN_DIR) + + subprocess.run(["curl", "-s", "-L", "-o", f"{OVPN_DIR}/ovpn.zip", OVPN_ZIP_URL]) + subprocess.run(["unzip", "-q", "-o", f"{OVPN_DIR}/ovpn.zip", "-d", OVPN_DIR]) + os.remove(f"{OVPN_DIR}/ovpn.zip") + + # Find all OVPN files + ovpn_files = [f for f in os.listdir(OVPN_DIR) if f.endswith(".ovpn")] + + # Add new entries to the database + for filename in ovpn_files: + cursor.execute("SELECT 1 FROM ovpn_files WHERE file_name = %s", (filename,)) + if cursor.fetchone() is None: # If no record found, insert a new one + cursor.execute(""" + INSERT INTO ovpn_files (file_name, last_observed) + VALUES (%s, CURRENT_TIMESTAMP) + """, (filename,)) + conn.commit() + + # Fetch the list of VPN servers that need to be checked (last check older than 3 days) + three_days_ago = datetime.datetime.now() - datetime.timedelta(days=3) + cursor.execute("SELECT file_name FROM ovpn_files WHERE last_exit_ip_check < %s OR last_exit_ip_check IS NULL", (three_days_ago,)) + ovpn_files_to_check = [row[0] for row in cursor.fetchall()] + + return ovpn_files_to_check \ No newline at end of file