clean up curl usage, add consistent debug printing across ovpn_downloader

nord-checker.py

@@ -22,7 +22,9 @@ if len(sys.argv) > 1 and sys.argv[1] == "-d":
 
 def debug_print(message):
     if DEBUG_MODE:
-        print(message)
+        print(f"{datetime.datetime.now()} + [Debug Mode]:{message}")
+    else:
+        log_file.flush()
 
 # Open the log file in append mode if not in debug mode
 log_file = None  # Define log_file in the global scope
@@ -62,7 +64,7 @@ conn, cursor = connect_to_database()
 
 # Download and extract OVPN configurations, and get the list of files to check
 debug_print(f"{datetime.datetime.now()} [Main Script]: Downloading and extracting OVPN configurations...")
-ovpn_files_to_check = download_and_extract_ovpn_configs(cursor, conn)
+ovpn_files_to_check = download_and_extract_ovpn_configs(cursor, conn, debug_print=debug_print)
 
 total_files = len(ovpn_files_to_check)
 
@@ -77,13 +79,13 @@ for OVPN_FILE in ovpn_files_to_check:
     debug_print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Processing OVPN file...")
 
     # Check if the OVPN file exists, download if needed
-    file_exists = download_ovpn_if_needed(OVPN_FILENAME, cursor, conn)
+    file_exists = download_ovpn_if_needed(OVPN_FILENAME, cursor, conn, debug_print=debug_print)
 
     if not file_exists:
         print(f"{datetime.datetime.now()} [{FILE_NUM}/{total_files}] [{OVPN_FILENAME}]: Skipping, file not found and removed from database.")
         FILE_NUM += 1
-        continue 
+        continue
-
+    
     # Read credentials
     with open(CREDENTIAL_FILE, 'r') as f:
         username = f.readline().strip()

ovpn_downloader.py

@@ -1,5 +1,5 @@
 import os
-import subprocess
+import zipfile
 import random
 import datetime
 import requests
@@ -8,7 +8,7 @@ OVPN_DIR = "ovpn_configs"
 OVPN_ZIP_URL = "https://downloads.nordcdn.com/configs/archives/servers/ovpn.zip"
 OVPN_LEGACY_URL = "https://downloads.nordcdn.com/configs/files/ovpn_legacy/servers/"
 
-def download_and_extract_ovpn_configs(cursor, conn):
+def download_and_extract_ovpn_configs(cursor, conn, debug_print=None):
     """
     Downloads the OVPN ZIP archive, extracts its contents, 
     adds new entries to the database, and cleans up.
@@ -18,22 +18,34 @@ def download_and_extract_ovpn_configs(cursor, conn):
     if not os.path.exists(OVPN_DIR):
         os.makedirs(OVPN_DIR)
 
-    subprocess.run(["curl", "-s", "-L", "-o", f"{OVPN_DIR}/ovpn.zip", OVPN_ZIP_URL])
+    zip_file_path = os.path.join(OVPN_DIR, "ovpn.zip")
-    subprocess.run(["unzip", "-q", "-o", f"{OVPN_DIR}/ovpn.zip", "-d", OVPN_DIR])
-    os.remove(f"{OVPN_DIR}/ovpn.zip")
 
-    # Find all OVPN files
+    try:
-    ovpn_files = [f for f in os.listdir(OVPN_DIR) if f.endswith(".ovpn")]
+        # Download the ZIP file using requests
+        debug_print(f"[Ovpn Downloader]: Downloading {OVPN_ZIP_URL}")
+        response = requests.get(OVPN_ZIP_URL, stream=True)
+        response.raise_for_status()  # Raise an exception for bad status codes
 
-    # Add new entries to the database
+        with open(zip_file_path, 'wb') as f:
-    for filename in ovpn_files:
+            for chunk in response.iter_content(chunk_size=8192):
-        cursor.execute("SELECT 1 FROM ovpn_files WHERE file_name = %s", (filename,))
+                f.write(chunk)
-        if cursor.fetchone() is None:  # If no record found, insert a new one
+
-            cursor.execute("""
+        debug_print(f"[Ovpn Downloader]: Download complete. Extracting...")
-                INSERT INTO ovpn_files (file_name, last_observed)
+
-                VALUES (%s, CURRENT_TIMESTAMP)
+        # Extract the ZIP file
-            """, (filename,))
+        with zipfile.ZipFile(zip_file_path, 'r') as zip_ref:
-    conn.commit()
+            zip_ref.extractall(OVPN_DIR)
+
+        debug_print(f"[Ovpn Downloader]: Extraction complete.")
+
+    except requests.exceptions.RequestException as e:
+        debug_print(f"[Ovpn Downloader]: Error downloading or extracting: {e}")
+        return []  # Return an empty list if there's an error
+
+    finally:
+        # Clean up the ZIP file
+        if os.path.exists(zip_file_path):
+            os.remove(zip_file_path)
 
     # Fetch the list of VPN servers that need to be checked (last check older than 3 days)
     three_days_ago = datetime.datetime.now() - datetime.timedelta(days=3)
@@ -42,7 +54,7 @@ def download_and_extract_ovpn_configs(cursor, conn):
 
     return ovpn_files_to_check
 
-def download_ovpn_if_needed(filename, cursor, conn):
+def download_ovpn_if_needed(filename, cursor, conn, debug_print=None):
     """
     Downloads the specified OVPN file from the legacy URL if it doesn't exist in the OVPN_DIR.
     Removes the entry from the database if the file returns a 404 error.
@@ -51,23 +63,23 @@ def download_ovpn_if_needed(filename, cursor, conn):
     filepath = os.path.join(OVPN_DIR+"/ovpn_tcp/", filename)
 
     if os.path.exists(filepath):
-        print(f"{filename} already exists. Skipping download.")
+        debug_print(f"[{filename}]: Skipping download, file already exists.")
         return True
 
     url = OVPN_LEGACY_URL + filename
-    print(f"Attempting to download {filename} from legacy URL...")
+    debug_print(f" [Ovpn Downloader]: Attempting to download {filename} from legacy URL...")
     response = requests.get(url)
 
     if response.status_code == 200:
         with open(filepath, 'wb') as f:
             f.write(response.content)
-        print(f"Downloaded {filename} from legacy URL.")
+        print(f"{datetime.datetime.now()} [Ovpn Downloader]: from legacy URL.")
         return True
     elif response.status_code == 404:
-        print(f"{filename} not found on legacy URL: {filepath}. Removing from database.") # this is the updated line
+        debug_print(f"[Ovpn Downloader]:{filename} not found on legacy URL: {filepath}. Removing from database.") # this is the updated line
         cursor.execute("DELETE FROM ovpn_files WHERE file_name = %s", (filename,))
         conn.commit()
         return False
     else:
-        print(f"Error downloading {filename}: {response.status_code}")
+        debug_print(f"Error downloading {filename}: {response.status_code}")
         return False