refactor sql, logging, failure mgmt

.gitignore

@@ -162,5 +162,5 @@ cython_debug/
 
 # env/configs
 *.creds
-ovpn_*
+ovpn_configs
 vpnlist.txt

nord-checker.py

@@ -5,7 +5,7 @@ import time
 import datetime
 import re
 import sys
-from psql_utils import connect_to_database, should_skip_file, update_or_insert_ip
+from psql_utils import connect_to_database, update_or_insert_ip
 from ovpn_downloader import download_and_extract_ovpn_configs
 from openvpn_manager import establish_vpn_connection, is_vpn_active, disconnect_vpn, get_external_ip 
 
@@ -24,9 +24,26 @@ def debug_print(message):
     if DEBUG_MODE:
         print(message)
 
-# Get current IP before starting
+# Open the log file in append mode if not in debug mode
-CURRENT_IP = subprocess.check_output(["curl", "-s", "ifconfig.me"]).decode().strip()
+log_file = None  # Define log_file in the global scope
+if not DEBUG_MODE:
+    log_file = open('/root/nordvpn/nord-checker.log', 'a')
+    # Redirect stdout and stderr to the log file
+    sys.stdout = log_file
+    sys.stderr = log_file
+
+
+# Get current IP before starting (using get_external_ip)
+CURRENT_IP = get_external_ip()
+if CURRENT_IP is None:
+    print(f"{datetime.datetime.now()} [Main Script]: Unable to get current IP. Exiting.")
+    if not DEBUG_MODE:
+        log_file.close()  # Close the log file if it was opened
+    sys.exit(1)  # Exit if unable to get current IP
+else:
     print(f"{datetime.datetime.now()} [Main Script]: Current IP before VPN connections: {CURRENT_IP}")
+    if not DEBUG_MODE:
+        log_file.flush() 
 
 # Terminate any existing OpenVPN connections
 try:
@@ -35,43 +52,39 @@ try:
     for pid in pids:
         subprocess.run(["kill", pid])
     print(f"{datetime.datetime.now()} [Main Script]: Existing OpenVPN connections terminated.")
+    if not DEBUG_MODE:
+        log_file.flush()
 except subprocess.CalledProcessError:
     pass  # No openvpn process found
 
-# Download and extract OVPN configurations (using the function from ovpn_downloader.py)
-debug_print(f"{datetime.datetime.now()} [Main Script]: Downloading and extracting OVPN configurations...")
-ovpn_files = download_and_extract_ovpn_configs() 
-
-total_files = len(ovpn_files)
-
-debug_print(f"{datetime.datetime.now()} [Main Script]: Found {total_files} OVPN files to process.")
-debug_print(ovpn_files)  # Print the list of OVPN files
-
 # Connect to PostgreSQL database
 conn, cursor = connect_to_database()
 
+# Download and extract OVPN configurations, and get the list of files to check
+debug_print(f"{datetime.datetime.now()} [Main Script]: Downloading and extracting OVPN configurations...")
+ovpn_files_to_check = download_and_extract_ovpn_configs(cursor, conn) 
+
+total_files = len(ovpn_files_to_check)
+
+debug_print(f"{datetime.datetime.now()} [Main Script]: Found {total_files} OVPN files to check.")
+debug_print(ovpn_files_to_check)  # Print the list of OVPN files
+
 # Process each OVPN file
 FILE_NUM = 1
-for OVPN_FILE in ovpn_files:
+for OVPN_FILE in ovpn_files_to_check:
     OVPN_FILENAME = os.path.basename(OVPN_FILE)
 
     debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Processing OVPN file...")
 
-    # Check if file exists in database and if last check is older than 3 days
-    skip_file = should_skip_file(cursor, OVPN_FILENAME)
-    debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: should_skip_file returned: {skip_file}")
-
-    if skip_file:
-        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Skipping, last check was less than 3 days ago.")
-        FILE_NUM += 1
-        continue
-
     # Read credentials
     with open(CREDENTIAL_FILE, 'r') as f:
         username = f.readline().strip()
         password = f.readline().strip()
 
     # Establish VPN connection (capture stdout and stderr)
+    max_retries = 3  # Maximum number of retries
+    retry_count = 0
+    while retry_count < max_retries:
         openvpn_command = ["openvpn", "--config", f"{OVPN_DIR}/{OVPN_FILE}", 
                            "--auth-user-pass", CREDENTIAL_FILE, "--daemon", 
                            "--log-append", LOG_FILE, "--verb", "3"]
@@ -84,35 +97,64 @@ for OVPN_FILE in ovpn_files:
         stdout, stderr = process.communicate(input=password.encode())
 
         # Check for errors and print output
-    if process.returncode != 0:
+        if process.returncode != 0 or "AUTH_FAILED" in stderr.decode():
-        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Error starting OpenVPN connection:")
+            print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Error starting OpenVPN connection or authentication failed:")
             print(stderr.decode()) 
             
+            # Explicitly disconnect VPN if there was an error or auth failure
+            disconnect_vpn()
+            print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected (due to error or auth failure).")
+
+            if retry_count < max_retries - 1:  # Retry if not the last attempt
+                print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Retrying connection in 10 seconds...")
+                time.sleep(10)
+                retry_count += 1
+            else:
+                print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Max retries reached. Skipping this OVPN file.")
+                break  # Exit the retry loop if max retries reached
+        else:
+            break  # Exit the retry loop if connection is successful
+
     # Wait for connection to establish (with backoff for auth failure)
     print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Waiting for VPN connection to establish...")
     max_attempts = 10
     attempt = 0
     backoff_time = 0
+    with open(LOG_FILE, 'r') as f: 
         while attempt < max_attempts:
-        try:
+            line = f.readline()
-            last_lines = subprocess.check_output(["tail", "-n", "10", LOG_FILE]).decode().strip() 
+            if not line:
-            if "Initialization Sequence Completed" in last_lines:
-                print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection established.")
-                break
-            elif "AUTH: Received control message: AUTH_FAILED" in last_lines:
-                backoff_time += 1  # Increment backoff time
-                if backoff_time >= 30:  # If backoff reaches 30 seconds, skip
-                    print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Authentication failed. Skipping this OVPN file.")
-                    continue  # Skip to the next OVPN file
-        except subprocess.CalledProcessError:
-            pass
                 time.sleep(1) 
                 attempt += 1
-    else:
-        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection failed to establish (timeout).")
                 continue
 
-    # Check if the VPN connection is active
+            if "Initialization Sequence Completed" in line:
+                print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection established.")
+                break
+            elif "AUTH: Received control message: AUTH_FAILED" in line:
+                backoff_time += 1
+                if backoff_time >= 30:
+                    print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Authentication failed. Skipping this OVPN file.")
+                    break 
+
+    if attempt == max_attempts or backoff_time >= 30: 
+        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection failed to establish (timeout or auth failure).")
+
+        # Disconnect VPN 
+        disconnect_vpn()
+        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected.")
+        if not DEBUG_MODE:
+            log_file.flush() 
+        # Wait for a random time between 1 and 4 seconds
+        sleep_time = random.randint(1, 4) 
+        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Waiting for {sleep_time} seconds before the next connection...")
+        time.sleep(sleep_time)
+        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Resuming...")
+        if not DEBUG_MODE:
+            log_file.flush() 
+        continue  # Skip to the next OVPN file
+
+# Check if the VPN connection is active (using the function from openvpn_manager.py)
     try:
         subprocess.run(["pgrep", "-f", OVPN_FILENAME], check=True, capture_output=True)
         print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection established.")
@@ -133,21 +175,17 @@ for OVPN_FILE in ovpn_files:
         print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN connection failed to establish.")
 
     # Disconnect VPN 
-    try:
+    disconnect_vpn()
-        result = subprocess.run(["pgrep", "openvpn"], check=True, capture_output=True)
-        pids = result.stdout.decode().strip().split('\n') 
-        for pid in pids:
-            subprocess.run(["kill", pid])
     print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: VPN disconnected.")
-    except subprocess.CalledProcessError:
+    if not DEBUG_MODE:
-        print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: No openvpn process found.")
+        log_file.flush() 
-
     # Wait for a random time between 5 and 8 seconds
     sleep_time = random.randint(5, 8) 
     print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Waiting for {sleep_time} seconds before the next connection...")
     time.sleep(sleep_time)
     print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Resuming...")
-
+    if not DEBUG_MODE:
+        log_file.flush()
     # Increment file number
     FILE_NUM += 1
 
@@ -155,5 +193,6 @@ for OVPN_FILE in ovpn_files:
 cursor.close()
 conn.close()
 
-# Close the log file
+# Close the log file if it was opened
+if not DEBUG_MODE:
     log_file.close()

openvpn_manager.py

@@ -17,7 +17,7 @@ def establish_vpn_connection(ovpn_file, credential_file, log_file):
     # Establish VPN connection (capture stdout and stderr)
     openvpn_command = ["openvpn", "--config", ovpn_file, 
                        "--auth-user-pass", credential_file, "--daemon", 
-                       "--log-append", log_file, "--verb", "3"]
+                       "--log-append", log_file, "--verb", "0", "--flush-log"]  # Added --flush-log
 
     process = subprocess.Popen(openvpn_command, 
                                 stdin=subprocess.PIPE, 

ovpn_downloader.py

@@ -0,0 +1,41 @@
+import os
+import subprocess
+import random
+import datetime
+
+OVPN_DIR = "/root/nordvpn/ovpn_configs/ovpn_tcp"
+OVPN_ZIP_URL = "https://downloads.nordcdn.com/configs/archives/servers/ovpn.zip"
+
+def download_and_extract_ovpn_configs(cursor, conn):
+    """
+    Downloads the OVPN ZIP archive, extracts its contents, 
+    adds new entries to the database, and cleans up.
+    Returns a list of OVPN filenames that need to be checked.
+    """
+
+    if not os.path.exists(OVPN_DIR):
+        os.makedirs(OVPN_DIR)
+
+    subprocess.run(["curl", "-s", "-L", "-o", f"{OVPN_DIR}/ovpn.zip", OVPN_ZIP_URL])
+    subprocess.run(["unzip", "-q", "-o", f"{OVPN_DIR}/ovpn.zip", "-d", OVPN_DIR])
+    os.remove(f"{OVPN_DIR}/ovpn.zip")
+
+    # Find all OVPN files
+    ovpn_files = [f for f in os.listdir(OVPN_DIR) if f.endswith(".ovpn")]
+
+    # Add new entries to the database
+    for filename in ovpn_files:
+        cursor.execute("SELECT 1 FROM ovpn_files WHERE file_name = %s", (filename,))
+        if cursor.fetchone() is None:  # If no record found, insert a new one
+            cursor.execute("""
+                INSERT INTO ovpn_files (file_name, last_observed)
+                VALUES (%s, CURRENT_TIMESTAMP)
+            """, (filename,))
+    conn.commit()
+
+    # Fetch the list of VPN servers that need to be checked (last check older than 3 days)
+    three_days_ago = datetime.datetime.now() - datetime.timedelta(days=3)
+    cursor.execute("SELECT file_name FROM ovpn_files WHERE last_exit_ip_check < %s OR last_exit_ip_check IS NULL", (three_days_ago,))
+    ovpn_files_to_check = [row[0] for row in cursor.fetchall()]
+
+    return ovpn_files_to_check